From af725f42864c8fb56afcf3ba76d2df7d372534e4 Mon Sep 17 00:00:00 2001
From: AndreasHGK
Date: Thu, 30 Jun 2022 15:59:30 +0000
Subject: [PATCH 01/78] os: fix a typo in path_windows.go
I believe the path_windows.go file has a typo, which is fixed in this PR
Change-Id: Ibf1a7189a6312dbb3b1e6b512beeb6d99da5b5bc
GitHub-Last-Rev: cedac7eaa07d26667e6800c5ac96239d5ccf6ba8
GitHub-Pull-Request: golang/go#53629
Reviewed-on: https://go-review.googlesource.com/c/go/+/415434
Auto-Submit: Ian Lance Taylor
TryBot-Result: Gopher Robot
Reviewed-by: Ian Lance Taylor
Reviewed-by: Dmitri Shuralyov
Run-TryBot: Ian Lance Taylor
---
src/os/path_windows.go | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/src/os/path_windows.go b/src/os/path_windows.go
index a96245f358..3356908a36 100644
--- a/src/os/path_windows.go
+++ b/src/os/path_windows.go
@@ -11,7 +11,7 @@ const (
// IsPathSeparator reports whether c is a directory separator character.
func IsPathSeparator(c uint8) bool {
- // NOTE: Windows accept / as path separator.
+ // NOTE: Windows accepts / as path separator.
return c == '\\' || c == '/'
}
From aad9382e590265efb5e5fa3591c30486dcf215e0 Mon Sep 17 00:00:00 2001
From: Ben Sarah Golightly
Date: Wed, 29 Jun 2022 19:42:46 +0100
Subject: [PATCH 02/78] go/doc/comment: support links in lists in comments
The proposed (#51082) new go doc comment additions supports lists,
links, and doc links, but does not support links and doc links inside
lists, so implemnent this.
Fixes #53610
Change-Id: I4fa17d204fc9efa8f3633133e4a49e56cf1aa9bc
Reviewed-on: https://go-review.googlesource.com/c/go/+/415174
Reviewed-by: Ben Golightly
Reviewed-by: Dmitri Shuralyov
Reviewed-by: Ian Lance Taylor
TryBot-Result: Gopher Robot
Run-TryBot: Ian Lance Taylor
Auto-Submit: Ian Lance Taylor
---
src/go/doc/comment/parse.go | 7 ++++
src/go/doc/comment/testdata/linklist.txt | 18 +++++++++++
src/go/doc/comment/testdata/linklist2.txt | 39 +++++++++++++++++++++++
src/go/doc/comment/testdata/linklist3.txt | 31 ++++++++++++++++++
src/go/doc/comment/testdata/linklist4.txt | 36 +++++++++++++++++++++
5 files changed, 131 insertions(+)
create mode 100644 src/go/doc/comment/testdata/linklist.txt
create mode 100644 src/go/doc/comment/testdata/linklist2.txt
create mode 100644 src/go/doc/comment/testdata/linklist3.txt
create mode 100644 src/go/doc/comment/testdata/linklist4.txt
diff --git a/src/go/doc/comment/parse.go b/src/go/doc/comment/parse.go
index 4de8ce710d..e8d844c491 100644
--- a/src/go/doc/comment/parse.go
+++ b/src/go/doc/comment/parse.go
@@ -326,6 +326,13 @@ func (p *Parser) Parse(text string) *Doc {
switch b := b.(type) {
case *Paragraph:
b.Text = d.parseLinkedText(string(b.Text[0].(Plain)))
+ case *List:
+ for _, i := range b.Items {
+ for _, c := range i.Content {
+ p := c.(*Paragraph)
+ p.Text = d.parseLinkedText(string(p.Text[0].(Plain)))
+ }
+ }
}
}
diff --git a/src/go/doc/comment/testdata/linklist.txt b/src/go/doc/comment/testdata/linklist.txt
new file mode 100644
index 0000000000..baf40624b3
--- /dev/null
+++ b/src/go/doc/comment/testdata/linklist.txt
@@ -0,0 +1,18 @@
+{"DocLinkBaseURL": "https://pkg.go.dev"}
+-- input --
+Did you know?
+
+ - [encoding/json.Marshal] is a doc link. So is [encoding/json.Unmarshal].
+-- text --
+Did you know?
+
+ - encoding/json.Marshal is a doc link. So is encoding/json.Unmarshal.
+-- markdown --
+Did you know?
+
+ - [encoding/json.Marshal](https://pkg.go.dev/encoding/json#Marshal) is a doc link. So is [encoding/json.Unmarshal](https://pkg.go.dev/encoding/json#Unmarshal).
+-- html --
+Did you know?
+
diff --git a/src/go/doc/comment/testdata/linklist2.txt b/src/go/doc/comment/testdata/linklist2.txt
new file mode 100644
index 0000000000..81b306100f
--- /dev/null
+++ b/src/go/doc/comment/testdata/linklist2.txt
@@ -0,0 +1,39 @@
+{"DocLinkBaseURL": "https://pkg.go.dev"}
+-- input --
+Did you know?
+
+ - [testing.T] is one doc link.
+ - So is [testing.M].
+ - So is [testing.B].
+ This is the same list paragraph.
+
+ There is [testing.PB] in this list item, too!
+-- text --
+Did you know?
+
+ - testing.T is one doc link.
+
+ - So is testing.M.
+
+ - So is testing.B. This is the same list paragraph.
+
+ There is testing.PB in this list item, too!
+-- markdown --
+Did you know?
+
+ - [testing.T](https://pkg.go.dev/testing#T) is one doc link.
+
+ - So is [testing.M](https://pkg.go.dev/testing#M).
+
+ - So is [testing.B](https://pkg.go.dev/testing#B). This is the same list paragraph.
+
+ There is [testing.PB](https://pkg.go.dev/testing#PB) in this list item, too!
+-- html --
+Did you know?
+
diff --git a/src/go/doc/comment/testdata/linklist3.txt b/src/go/doc/comment/testdata/linklist3.txt
new file mode 100644
index 0000000000..701a54ecff
--- /dev/null
+++ b/src/go/doc/comment/testdata/linklist3.txt
@@ -0,0 +1,31 @@
+{"DocLinkBaseURL": "https://pkg.go.dev"}
+-- input --
+Cool things:
+
+ - Foo
+ - [Go]
+ - Bar
+
+[Go]: https://go.dev/
+-- text --
+Cool things:
+
+ - Foo
+ - Go
+ - Bar
+
+[Go]: https://go.dev/
+-- markdown --
+Cool things:
+
+ - Foo
+ - [Go](https://go.dev/)
+ - Bar
+
+-- html --
+Cool things:
+
diff --git a/src/go/doc/comment/testdata/linklist4.txt b/src/go/doc/comment/testdata/linklist4.txt
new file mode 100644
index 0000000000..db39ec4ee1
--- /dev/null
+++ b/src/go/doc/comment/testdata/linklist4.txt
@@ -0,0 +1,36 @@
+{"DocLinkBaseURL": "https://pkg.go.dev"}
+-- input --
+Cool things:
+
+ - Foo
+ - [Go] is great
+
+ [Go]: https://go.dev/
+ - Bar
+
+-- text --
+Cool things:
+
+ - Foo
+
+ - Go is great
+
+ - Bar
+
+[Go]: https://go.dev/
+-- markdown --
+Cool things:
+
+ - Foo
+
+ - [Go](https://go.dev/) is great
+
+ - Bar
+
+-- html --
+Cool things:
+
From 405c269b858941c46e35fb9a92b45aa75c61561c Mon Sep 17 00:00:00 2001
From: Robert Griesemer
Date: Wed, 29 Jun 2022 18:21:32 -0700
Subject: [PATCH 03/78] go/types, types2: re-enable a couple of commented out
tests
Change-Id: Ibb27012b18fc0f0f9f9ef74cc120e7ef981e6d43
Reviewed-on: https://go-review.googlesource.com/c/go/+/415156
TryBot-Result: Gopher Robot
Reviewed-by: Ian Lance Taylor
Auto-Submit: Robert Griesemer
Reviewed-by: Robert Griesemer
Run-TryBot: Robert Griesemer
---
.../types2/testdata/fixedbugs/issue39634.go | 13 +++++--------
src/go/types/testdata/fixedbugs/issue39634.go | 13 +++++--------
2 files changed, 10 insertions(+), 16 deletions(-)
diff --git a/src/cmd/compile/internal/types2/testdata/fixedbugs/issue39634.go b/src/cmd/compile/internal/types2/testdata/fixedbugs/issue39634.go
index b408dd7003..b7d99f96c2 100644
--- a/src/cmd/compile/internal/types2/testdata/fixedbugs/issue39634.go
+++ b/src/cmd/compile/internal/types2/testdata/fixedbugs/issue39634.go
@@ -31,10 +31,8 @@ type x7[A any] struct{ foo7 }
func main7() { var _ foo7 = x7[int]{} }
// crash 8
-// Embedding stand-alone type parameters is not permitted for now. Disabled.
-// type foo8[A any] interface { ~A }
-// func bar8[A foo8[A]](a A) {}
-// func main8() {}
+type foo8[A any] interface { ~A /* ERROR cannot be a type parameter */ }
+func bar8[A foo8[A]](a A) {}
// crash 9
type foo9[A any] interface { foo9 /* ERROR illegal cycle */ [A] }
@@ -74,10 +72,9 @@ func F20[t Z20]() { F20(t /* ERROR invalid composite literal type */ {}) }
type Z21 /* ERROR illegal cycle */ interface{ Z21 }
func F21[T Z21]() { ( /* ERROR not used */ F21[Z21]) }
-// For now, a lone type parameter is not permitted as RHS in a type declaration (issue #45639).
-// // crash 24
-// type T24[P any] P
-// func (r T24[P]) m() { T24 /* ERROR without instantiation */ .m() }
+// crash 24
+type T24[P any] P // ERROR cannot use a type parameter as RHS in type declaration
+func (r T24[P]) m() { T24 /* ERROR without instantiation */ .m() }
// crash 25
type T25[A any] int
diff --git a/src/go/types/testdata/fixedbugs/issue39634.go b/src/go/types/testdata/fixedbugs/issue39634.go
index 8cba2e735a..ce84299a61 100644
--- a/src/go/types/testdata/fixedbugs/issue39634.go
+++ b/src/go/types/testdata/fixedbugs/issue39634.go
@@ -31,10 +31,8 @@ type x7[A any] struct{ foo7 }
func main7() { var _ foo7 = x7[int]{} }
// crash 8
-// Embedding stand-alone type parameters is not permitted for now. Disabled.
-// type foo8[A any] interface { ~A }
-// func bar8[A foo8[A]](a A) {}
-// func main8() {}
+type foo8[A any] interface { ~A /* ERROR cannot be a type parameter */ }
+func bar8[A foo8[A]](a A) {}
// crash 9
type foo9[A any] interface { foo9 /* ERROR illegal cycle */ [A] }
@@ -74,10 +72,9 @@ func F20[t Z20]() { F20(t /* ERROR invalid composite literal type */ {}) }
type Z21 /* ERROR illegal cycle */ interface{ Z21 }
func F21[T Z21]() { ( /* ERROR not used */ F21[Z21]) }
-// For now, a lone type parameter is not permitted as RHS in a type declaration (issue #45639).
-// // crash 24
-// type T24[P any] P
-// func (r T24[P]) m() { T24 /* ERROR without instantiation */ .m() }
+// crash 24
+type T24[P any] P // ERROR cannot use a type parameter as RHS in type declaration
+func (r T24[P]) m() { T24 /* ERROR without instantiation */ .m() }
// crash 25
type T25[A any] int
From c847a2c9f024f47eee25c132f2d80e7037adea36 Mon Sep 17 00:00:00 2001
From: Robert Griesemer
Date: Thu, 30 Jun 2022 16:41:31 -0700
Subject: [PATCH 04/78] go/types, types2: document that exported predicates are
unspecified for invalid type arguments
Per discussion on the issue.
For #53595.
Change-Id: Iefd161e5c7e792d454652cbe831a0c2d769f748e
Reviewed-on: https://go-review.googlesource.com/c/go/+/415574
Reviewed-by: Robert Findley
Reviewed-by: Robert Griesemer
Reviewed-by: Ian Lance Taylor
---
src/cmd/compile/internal/types2/api.go | 15 ++++++++-------
src/go/types/api.go | 15 ++++++++-------
2 files changed, 16 insertions(+), 14 deletions(-)
diff --git a/src/cmd/compile/internal/types2/api.go b/src/cmd/compile/internal/types2/api.go
index a22ea5d12f..94c290b9ee 100644
--- a/src/cmd/compile/internal/types2/api.go
+++ b/src/cmd/compile/internal/types2/api.go
@@ -418,7 +418,8 @@ func (conf *Config) Check(path string, files []*syntax.File, info *Info) (*Packa
// AssertableTo reports whether a value of type V can be asserted to have type T.
//
-// The behavior of AssertableTo is undefined in two cases:
+// The behavior of AssertableTo is unspecified in three cases:
+// - if T is Typ[Invalid]
// - if V is a generalized interface; i.e., an interface that may only be used
// as a type constraint in Go code
// - if T is an uninstantiated generic type
@@ -434,8 +435,8 @@ func AssertableTo(V *Interface, T Type) bool {
// AssignableTo reports whether a value of type V is assignable to a variable
// of type T.
//
-// The behavior of AssignableTo is undefined if V or T is an uninstantiated
-// generic type.
+// The behavior of AssignableTo is unspecified if V or T is Typ[Invalid] or an
+// uninstantiated generic type.
func AssignableTo(V, T Type) bool {
x := operand{mode: value, typ: V}
ok, _ := x.assignableTo(nil, T, nil) // check not needed for non-constant x
@@ -445,8 +446,8 @@ func AssignableTo(V, T Type) bool {
// ConvertibleTo reports whether a value of type V is convertible to a value of
// type T.
//
-// The behavior of ConvertibleTo is undefined if V or T is an uninstantiated
-// generic type.
+// The behavior of ConvertibleTo is unspecified if V or T is Typ[Invalid] or an
+// uninstantiated generic type.
func ConvertibleTo(V, T Type) bool {
x := operand{mode: value, typ: V}
return x.convertibleTo(nil, T, nil) // check not needed for non-constant x
@@ -454,8 +455,8 @@ func ConvertibleTo(V, T Type) bool {
// Implements reports whether type V implements interface T.
//
-// The behavior of Implements is undefined if V is an uninstantiated generic
-// type.
+// The behavior of Implements is unspecified if V is Typ[Invalid] or an uninstantiated
+// generic type.
func Implements(V Type, T *Interface) bool {
if T.Empty() {
// All types (even Typ[Invalid]) implement the empty interface.
diff --git a/src/go/types/api.go b/src/go/types/api.go
index 0915d6a6ee..5e7be29b3c 100644
--- a/src/go/types/api.go
+++ b/src/go/types/api.go
@@ -413,7 +413,8 @@ func (conf *Config) Check(path string, fset *token.FileSet, files []*ast.File, i
// AssertableTo reports whether a value of type V can be asserted to have type T.
//
-// The behavior of AssertableTo is undefined in two cases:
+// The behavior of AssertableTo is unspecified in three cases:
+// - if T is Typ[Invalid]
// - if V is a generalized interface; i.e., an interface that may only be used
// as a type constraint in Go code
// - if T is an uninstantiated generic type
@@ -429,8 +430,8 @@ func AssertableTo(V *Interface, T Type) bool {
// AssignableTo reports whether a value of type V is assignable to a variable
// of type T.
//
-// The behavior of AssignableTo is undefined if V or T is an uninstantiated
-// generic type.
+// The behavior of AssignableTo is unspecified if V or T is Typ[Invalid] or an
+// uninstantiated generic type.
func AssignableTo(V, T Type) bool {
x := operand{mode: value, typ: V}
ok, _ := x.assignableTo(nil, T, nil) // check not needed for non-constant x
@@ -440,8 +441,8 @@ func AssignableTo(V, T Type) bool {
// ConvertibleTo reports whether a value of type V is convertible to a value of
// type T.
//
-// The behavior of ConvertibleTo is undefined if V or T is an uninstantiated
-// generic type.
+// The behavior of ConvertibleTo is unspecified if V or T is Typ[Invalid] or an
+// uninstantiated generic type.
func ConvertibleTo(V, T Type) bool {
x := operand{mode: value, typ: V}
return x.convertibleTo(nil, T, nil) // check not needed for non-constant x
@@ -449,8 +450,8 @@ func ConvertibleTo(V, T Type) bool {
// Implements reports whether type V implements interface T.
//
-// The behavior of Implements is undefined if V is an uninstantiated generic
-// type.
+// The behavior of Implements is unspecified if V is Typ[Invalid] or an uninstantiated
+// generic type.
func Implements(V Type, T *Interface) bool {
if T.Empty() {
// All types (even Typ[Invalid]) implement the empty interface.
From 9a4d5357f40c367fcad279184f245290ba0a8fb9 Mon Sep 17 00:00:00 2001
From: Sebastian Gassner
Date: Thu, 30 Jun 2022 13:37:04 +0000
Subject: [PATCH 05/78] flag: highlight support for double dashes in docs
Updating examples, to show that double dashes are also permitted. This has been easy to miss previously.
Change-Id: Ib67b4e39fea90ef4cb9e894709c53baedfc18fc2
GitHub-Last-Rev: f7df57b646d6412c1346e85c3a7353a8df41afc6
GitHub-Pull-Request: golang/go#53628
Reviewed-on: https://go-review.googlesource.com/c/go/+/415374
Reviewed-by: Rob Pike
Reviewed-by: Ian Lance Taylor
Auto-Submit: Ian Lance Taylor
Reviewed-by: David Chase
TryBot-Result: Gopher Robot
Run-TryBot: Ian Lance Taylor
---
src/flag/flag.go | 3 ++-
1 file changed, 2 insertions(+), 1 deletion(-)
diff --git a/src/flag/flag.go b/src/flag/flag.go
index a0762441a5..9abf8d769e 100644
--- a/src/flag/flag.go
+++ b/src/flag/flag.go
@@ -49,10 +49,11 @@ The arguments are indexed from 0 through flag.NArg()-1.
The following forms are permitted:
-flag
+ --flag // double dashes are also permitted
-flag=x
-flag x // non-boolean flags only
-One or two minus signs may be used; they are equivalent.
+One or two dashes may be used; they are equivalent.
The last form is not permitted for boolean flags because the
meaning of the command
From 4a2a3bca180509bc39fe99992e16b5e0a45e0e43 Mon Sep 17 00:00:00 2001
From: Ian Lance Taylor
Date: Fri, 10 Jun 2022 15:58:22 -0700
Subject: [PATCH 06/78] cmd/go, go/build: clarify build constraint docs
Clarify that the //go:build line is an expression of constraints,
not a constraint itself.
Fixes #53308
Change-Id: Ib67243c6ee5cfe3b688c12b943b5e7496f686035
Reviewed-on: https://go-review.googlesource.com/c/go/+/411697
Reviewed-by: Rob Pike
TryBot-Result: Gopher Robot
Run-TryBot: Ian Lance Taylor
Reviewed-by: Ian Lance Taylor
Run-TryBot: Ian Lance Taylor
Auto-Submit: Ian Lance Taylor
Reviewed-by: David Chase
---
src/cmd/go/alldocs.go | 22 +++++++++++-----------
src/cmd/go/internal/help/helpdoc.go | 13 +++++++------
src/cmd/go/internal/work/build.go | 9 ++++-----
src/go/build/doc.go | 7 ++++---
4 files changed, 26 insertions(+), 25 deletions(-)
diff --git a/src/cmd/go/alldocs.go b/src/cmd/go/alldocs.go
index fdb7a085b0..78128dcf23 100644
--- a/src/cmd/go/alldocs.go
+++ b/src/cmd/go/alldocs.go
@@ -195,11 +195,10 @@
// For example, when building with a non-standard configuration,
// use -pkgdir to keep generated packages in a separate location.
// -tags tag,list
-// a comma-separated list of build tags to consider satisfied during the
-// build. For more information about build tags, see the description of
-// build constraints in the documentation for the go/build package.
-// (Earlier versions of Go used a space-separated list, and that form
-// is deprecated but still recognized.)
+// a comma-separated list of additional build tags to consider satisfied
+// during the build. For more information about build tags, see
+// 'go help buildconstraint'. (Earlier versions of Go used a
+// space-separated list, and that form is deprecated but still recognized.)
// -trimpath
// remove all file system paths from the resulting executable.
// Instead of absolute file system paths, the recorded file names
@@ -1797,11 +1796,12 @@
//
// # Build constraints
//
-// A build constraint, also known as a build tag, is a line comment that begins
+// A build constraint, also known as a build tag, is a condition under which a
+// file should be included in the package. Build constraints are given by a
+// line comment that begins
//
// //go:build
//
-// that lists the conditions under which a file should be included in the package.
// Constraints may appear in any kind of source file (not just Go), but
// they must appear near the top of the file, preceded
// only by blank lines and other line comments. These rules mean that in Go
@@ -1810,9 +1810,9 @@
// To distinguish build constraints from package documentation,
// a build constraint should be followed by a blank line.
//
-// A build constraint is evaluated as an expression containing options
-// combined by ||, &&, and ! operators and parentheses. Operators have
-// the same meaning as in Go.
+// A build constraint comment is evaluated as an expression containing
+// build tags combined by ||, &&, and ! operators and parentheses.
+// Operators have the same meaning as in Go.
//
// For example, the following build constraint constrains a file to
// build when the "linux" and "386" constraints are satisfied, or when
@@ -1822,7 +1822,7 @@
//
// It is an error for a file to have more than one //go:build line.
//
-// During a particular build, the following words are satisfied:
+// During a particular build, the following build tags are satisfied:
//
// - the target operating system, as spelled by runtime.GOOS, set with the
// GOOS environment variable.
diff --git a/src/cmd/go/internal/help/helpdoc.go b/src/cmd/go/internal/help/helpdoc.go
index 36bc4f28b7..c38c403006 100644
--- a/src/cmd/go/internal/help/helpdoc.go
+++ b/src/cmd/go/internal/help/helpdoc.go
@@ -812,11 +812,12 @@ var HelpBuildConstraint = &base.Command{
UsageLine: "buildconstraint",
Short: "build constraints",
Long: `
-A build constraint, also known as a build tag, is a line comment that begins
+A build constraint, also known as a build tag, is a condition under which a
+file should be included in the package. Build constraints are given by a
+line comment that begins
//go:build
-that lists the conditions under which a file should be included in the package.
Constraints may appear in any kind of source file (not just Go), but
they must appear near the top of the file, preceded
only by blank lines and other line comments. These rules mean that in Go
@@ -825,9 +826,9 @@ files a build constraint must appear before the package clause.
To distinguish build constraints from package documentation,
a build constraint should be followed by a blank line.
-A build constraint is evaluated as an expression containing options
-combined by ||, &&, and ! operators and parentheses. Operators have
-the same meaning as in Go.
+A build constraint comment is evaluated as an expression containing
+build tags combined by ||, &&, and ! operators and parentheses.
+Operators have the same meaning as in Go.
For example, the following build constraint constrains a file to
build when the "linux" and "386" constraints are satisfied, or when
@@ -837,7 +838,7 @@ build when the "linux" and "386" constraints are satisfied, or when
It is an error for a file to have more than one //go:build line.
-During a particular build, the following words are satisfied:
+During a particular build, the following build tags are satisfied:
- the target operating system, as spelled by runtime.GOOS, set with the
GOOS environment variable.
diff --git a/src/cmd/go/internal/work/build.go b/src/cmd/go/internal/work/build.go
index feb82d8d38..42745d9928 100644
--- a/src/cmd/go/internal/work/build.go
+++ b/src/cmd/go/internal/work/build.go
@@ -151,11 +151,10 @@ and test commands:
For example, when building with a non-standard configuration,
use -pkgdir to keep generated packages in a separate location.
-tags tag,list
- a comma-separated list of build tags to consider satisfied during the
- build. For more information about build tags, see the description of
- build constraints in the documentation for the go/build package.
- (Earlier versions of Go used a space-separated list, and that form
- is deprecated but still recognized.)
+ a comma-separated list of additional build tags to consider satisfied
+ during the build. For more information about build tags, see
+ 'go help buildconstraint'. (Earlier versions of Go used a
+ space-separated list, and that form is deprecated but still recognized.)
-trimpath
remove all file system paths from the resulting executable.
Instead of absolute file system paths, the recorded file names
diff --git a/src/go/build/doc.go b/src/go/build/doc.go
index 262f6709af..cd1d3fd33e 100644
--- a/src/go/build/doc.go
+++ b/src/go/build/doc.go
@@ -57,12 +57,13 @@
//
// # Build Constraints
//
-// A build constraint, also known as a build tag, is a line comment that begins
+// A build constraint, also known as a build tag, is a condition under which a
+// file should be included in the package. Build constraints are given by a
+// line comment that begins
//
// //go:build
//
-// that lists the conditions under which a file should be included in the
-// package. Build constraints may also be part of a file's name
+// Build constraints may also be part of a file's name
// (for example, source_windows.go will only be included if the target
// operating system is windows).
//
From e822b1e26e20ef1c76672c0b77b0fd8a97a1fe84 Mon Sep 17 00:00:00 2001
From: Alexander Yastrebov
Date: Thu, 14 Oct 2021 22:01:49 +0000
Subject: [PATCH 07/78] net/http: omit invalid header value from error message
Updates #43631
Change-Id: I0fe3aafdf7ef889fed1a830128721393f8d020e6
GitHub-Last-Rev: c359542d741b17f4e2cb0d50982bf341246233b0
GitHub-Pull-Request: golang/go#48979
Reviewed-on: https://go-review.googlesource.com/c/go/+/355929
Reviewed-by: Dmitri Shuralyov
Run-TryBot: Cherry Mui
Reviewed-by: Damien Neil
Reviewed-by: David Chase
TryBot-Result: Gopher Robot
---
src/net/http/transport.go | 3 ++-
src/net/http/transport_test.go | 16 ++++++++--------
2 files changed, 10 insertions(+), 9 deletions(-)
diff --git a/src/net/http/transport.go b/src/net/http/transport.go
index f2d538b04a..e470a6c080 100644
--- a/src/net/http/transport.go
+++ b/src/net/http/transport.go
@@ -525,7 +525,8 @@ func (t *Transport) roundTrip(req *Request) (*Response, error) {
for _, v := range vv {
if !httpguts.ValidHeaderFieldValue(v) {
req.closeBody()
- return nil, fmt.Errorf("net/http: invalid header field value %q for key %v", v, k)
+ // Don't include the value in the error, because it may be sensitive.
+ return nil, fmt.Errorf("net/http: invalid header field value for %q", k)
}
}
}
diff --git a/src/net/http/transport_test.go b/src/net/http/transport_test.go
index 84868e2c5e..cba35db257 100644
--- a/src/net/http/transport_test.go
+++ b/src/net/http/transport_test.go
@@ -6085,14 +6085,14 @@ func TestTransportClosesBodyOnInvalidRequests(t *testing.T) {
Method: " ",
URL: u,
},
- wantErr: "invalid method",
+ wantErr: `invalid method " "`,
},
{
name: "nil URL",
req: &Request{
Method: "GET",
},
- wantErr: "nil Request.URL",
+ wantErr: `nil Request.URL`,
},
{
name: "invalid header key",
@@ -6101,7 +6101,7 @@ func TestTransportClosesBodyOnInvalidRequests(t *testing.T) {
Header: Header{"💡": {"emoji"}},
URL: u,
},
- wantErr: "invalid header field name",
+ wantErr: `invalid header field name "💡"`,
},
{
name: "invalid header value",
@@ -6110,7 +6110,7 @@ func TestTransportClosesBodyOnInvalidRequests(t *testing.T) {
Header: Header{"key": {"\x19"}},
URL: u,
},
- wantErr: "invalid header field value",
+ wantErr: `invalid header field value for "key"`,
},
{
name: "non HTTP(s) scheme",
@@ -6118,7 +6118,7 @@ func TestTransportClosesBodyOnInvalidRequests(t *testing.T) {
Method: "POST",
URL: &url.URL{Scheme: "faux"},
},
- wantErr: "unsupported protocol scheme",
+ wantErr: `unsupported protocol scheme "faux"`,
},
{
name: "no Host in URL",
@@ -6126,7 +6126,7 @@ func TestTransportClosesBodyOnInvalidRequests(t *testing.T) {
Method: "POST",
URL: &url.URL{Scheme: "http"},
},
- wantErr: "no Host",
+ wantErr: `no Host in request URL`,
},
}
@@ -6142,8 +6142,8 @@ func TestTransportClosesBodyOnInvalidRequests(t *testing.T) {
if !bc {
t.Fatal("Expected body to have been closed")
}
- if g, w := err.Error(), tt.wantErr; !strings.Contains(g, w) {
- t.Fatalf("Error mismatch\n\t%q\ndoes not contain\n\t%q", g, w)
+ if g, w := err.Error(), tt.wantErr; !strings.HasSuffix(g, w) {
+ t.Fatalf("Error mismatch: %q does not end with %q", g, w)
}
})
}
From 3cf79d96105d890d7097d274804644b2a2093df1 Mon Sep 17 00:00:00 2001
From: Ori Bernstein
Date: Sun, 3 Jul 2022 12:26:30 -0400
Subject: [PATCH 08/78] runtime: pass correct string to exits on Plan 9
In CL 405901 the definition of exit in the Plan 9 go runtime
was changed like so:
- status = append(itoa(tmp[:len(tmp)-1], uint64(e)), 0)
+ sl := itoa(tmp[:len(tmp)-1], uint64(e))
+ // Don't append, rely on the existing data being zero.
+ status = tmp[:len(sl)+1]
However, itoa only puts the converted number "somewhere" in the buffer.
Specifically, it builds it from the end of the buffer towards the start,
meaning the first byte of the buffer is a 0 byte, and the resulting string
that's passed to exits is empty, leading to a falsely successful exit.
This change uses the returned value from itoa, rather than the buffer
that was passed in, so that we start from the correct location in the
string.
Fixes #53669
Change-Id: I63f0c7641fc6f55250857dc17a1eeb12ae0c2e10
Reviewed-on: https://go-review.googlesource.com/c/go/+/415680
Reviewed-by: David Chase
Reviewed-by: Ian Lance Taylor
---
src/runtime/os_plan9.go | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/src/runtime/os_plan9.go b/src/runtime/os_plan9.go
index 13bc3be4ab..f0e7c6ae70 100644
--- a/src/runtime/os_plan9.go
+++ b/src/runtime/os_plan9.go
@@ -439,7 +439,7 @@ func exit(e int32) {
var tmp [32]byte
sl := itoa(tmp[:len(tmp)-1], uint64(e))
// Don't append, rely on the existing data being zero.
- status = tmp[:len(sl)+1]
+ status = sl[:len(sl)+1]
}
goexitsall(&status[0])
exits(&status[0])
From ceda93ed673294f0ce5eb3a723d563091bff0a39 Mon Sep 17 00:00:00 2001
From: Cristian Greco
Date: Sun, 3 Jul 2022 11:04:04 +0000
Subject: [PATCH 09/78] build/constraint: update doc to mention a feature added
in Go 1.17
The pkg documentation mentions that the "//go:build" syntax "will be"
added in Go 1.17. In fact, it has been added in that Go release, so the
documentation can now be updated.
Change-Id: I72f24063c3be62d97ca78bf724d56599f5f19460
GitHub-Last-Rev: 4371886f6ce9f2c2a370df047a5baa1f122c681f
GitHub-Pull-Request: golang/go#53647
Reviewed-on: https://go-review.googlesource.com/c/go/+/415774
Reviewed-by: Ian Lance Taylor
Run-TryBot: Ian Lance Taylor
TryBot-Result: Gopher Robot
Reviewed-by: Benny Siegert
Auto-Submit: Ian Lance Taylor
---
src/go/build/constraint/expr.go | 4 +---
1 file changed, 1 insertion(+), 3 deletions(-)
diff --git a/src/go/build/constraint/expr.go b/src/go/build/constraint/expr.go
index d64eead654..505cbffa4c 100644
--- a/src/go/build/constraint/expr.go
+++ b/src/go/build/constraint/expr.go
@@ -5,9 +5,7 @@
// Package constraint implements parsing and evaluation of build constraint lines.
// See https://golang.org/cmd/go/#hdr-Build_constraints for documentation about build constraints themselves.
//
-// This package parses both the original “// +build” syntax and the “//go:build” syntax that will be added in Go 1.17.
-// The parser is being included in Go 1.16 to allow tools that need to process Go 1.17 source code
-// to still be built against the Go 1.16 release.
+// This package parses both the original “// +build” syntax and the “//go:build” syntax that was added in Go 1.17.
// See https://golang.org/design/draft-gobuild for details about the “//go:build” syntax.
package constraint
From 84e091eef033255b4a3fdb515d677faa135714dc Mon Sep 17 00:00:00 2001
From: Russ Cox
Date: Wed, 8 Jun 2022 23:56:28 -0400
Subject: [PATCH 10/78] cmd/go: record origin metadata during module download
This change adds an "Origin" JSON key to the output of
go list -json -m and go mod download -json. The associated value is a
JSON object with metadata about the source control system. For Git,
that metadata is sufficient to evaluate whether the remote server has
changed in any interesting way that might invalidate the cached data.
In most cases, it will not have, and a fetch could then avoid
downloading a full repo from the server.
This origin metadata is also now recorded in the .info file for a
given module@version, for informational and debugging purposes.
This change only adds the metadata. It does not use it to optimize
away unnecessary git fetch operations. (That's the next change.)
For #53644.
Change-Id: I4a1712a2386d1d8ab4e02ffdf0f72ba75d556115
Reviewed-on: https://go-review.googlesource.com/c/go/+/411397
TryBot-Result: Gopher Robot
Run-TryBot: Russ Cox
Reviewed-by: Bryan Mills
---
src/cmd/go/internal/modcmd/download.go | 2 +-
src/cmd/go/internal/modfetch/cache.go | 44 +++++---
.../go/internal/modfetch/codehost/codehost.go | 90 ++++++++++++++-
src/cmd/go/internal/modfetch/codehost/git.go | 106 ++++++++++++++++--
.../go/internal/modfetch/codehost/git_test.go | 87 ++++++++++----
src/cmd/go/internal/modfetch/codehost/vcs.go | 43 ++++++-
src/cmd/go/internal/modfetch/coderepo.go | 64 ++++++++---
src/cmd/go/internal/modfetch/coderepo_test.go | 12 +-
src/cmd/go/internal/modfetch/proxy.go | 38 +++----
src/cmd/go/internal/modfetch/repo.go | 39 +++++--
src/cmd/go/internal/modload/mvs.go | 4 +-
src/cmd/go/internal/modload/query.go | 27 +++--
12 files changed, 435 insertions(+), 121 deletions(-)
diff --git a/src/cmd/go/internal/modcmd/download.go b/src/cmd/go/internal/modcmd/download.go
index 5bc6cbc4bb..ea4f9f8663 100644
--- a/src/cmd/go/internal/modcmd/download.go
+++ b/src/cmd/go/internal/modcmd/download.go
@@ -149,7 +149,7 @@ func runDownload(ctx context.Context, cmd *base.Command, args []string) {
downloadModule := func(m *moduleJSON) {
var err error
- m.Info, err = modfetch.InfoFile(m.Path, m.Version)
+ _, m.Info, err = modfetch.InfoFile(m.Path, m.Version)
if err != nil {
m.Error = err.Error()
return
diff --git a/src/cmd/go/internal/modfetch/cache.go b/src/cmd/go/internal/modfetch/cache.go
index b0dae1cb3d..417c5598fb 100644
--- a/src/cmd/go/internal/modfetch/cache.go
+++ b/src/cmd/go/internal/modfetch/cache.go
@@ -164,7 +164,7 @@ func SideLock() (unlock func(), err error) {
}
// A cachingRepo is a cache around an underlying Repo,
-// avoiding redundant calls to ModulePath, Versions, Stat, Latest, and GoMod (but not Zip).
+// avoiding redundant calls to ModulePath, Versions, Stat, Latest, and GoMod (but not CheckReuse or Zip).
// It is also safe for simultaneous use by multiple goroutines
// (so that it can be returned from Lookup multiple times).
// It serializes calls to the underlying Repo.
@@ -195,24 +195,32 @@ func (r *cachingRepo) repo() Repo {
return r.r
}
+func (r *cachingRepo) CheckReuse(old *codehost.Origin) error {
+ return r.repo().CheckReuse(old)
+}
+
func (r *cachingRepo) ModulePath() string {
return r.path
}
-func (r *cachingRepo) Versions(prefix string) ([]string, error) {
+func (r *cachingRepo) Versions(prefix string) (*Versions, error) {
type cached struct {
- list []string
- err error
+ v *Versions
+ err error
}
c := r.cache.Do("versions:"+prefix, func() any {
- list, err := r.repo().Versions(prefix)
- return cached{list, err}
+ v, err := r.repo().Versions(prefix)
+ return cached{v, err}
}).(cached)
if c.err != nil {
return nil, c.err
}
- return append([]string(nil), c.list...), nil
+ v := &Versions{
+ Origin: c.v.Origin,
+ List: append([]string(nil), c.v.List...),
+ }
+ return v, nil
}
type cachedInfo struct {
@@ -310,31 +318,35 @@ func (r *cachingRepo) Zip(dst io.Writer, version string) error {
return r.repo().Zip(dst, version)
}
-// InfoFile is like Lookup(path).Stat(version) but returns the name of the file
+// InfoFile is like Lookup(path).Stat(version) but also returns the name of the file
// containing the cached information.
-func InfoFile(path, version string) (string, error) {
+func InfoFile(path, version string) (*RevInfo, string, error) {
if !semver.IsValid(version) {
- return "", fmt.Errorf("invalid version %q", version)
+ return nil, "", fmt.Errorf("invalid version %q", version)
}
- if file, _, err := readDiskStat(path, version); err == nil {
- return file, nil
+ if file, info, err := readDiskStat(path, version); err == nil {
+ return info, file, nil
}
+ var info *RevInfo
err := TryProxies(func(proxy string) error {
- _, err := Lookup(proxy, path).Stat(version)
+ i, err := Lookup(proxy, path).Stat(version)
+ if err == nil {
+ info = i
+ }
return err
})
if err != nil {
- return "", err
+ return nil, "", err
}
// Stat should have populated the disk cache for us.
file, err := CachePath(module.Version{Path: path, Version: version}, "info")
if err != nil {
- return "", err
+ return nil, "", err
}
- return file, nil
+ return info, file, nil
}
// GoMod is like Lookup(path).GoMod(rev) but avoids the
diff --git a/src/cmd/go/internal/modfetch/codehost/codehost.go b/src/cmd/go/internal/modfetch/codehost/codehost.go
index e08a84b32c..3d9eb0c712 100644
--- a/src/cmd/go/internal/modfetch/codehost/codehost.go
+++ b/src/cmd/go/internal/modfetch/codehost/codehost.go
@@ -22,6 +22,9 @@ import (
"cmd/go/internal/cfg"
"cmd/go/internal/lockedfile"
"cmd/go/internal/str"
+
+ "golang.org/x/mod/module"
+ "golang.org/x/mod/semver"
)
// Downloaded size limits.
@@ -36,8 +39,15 @@ const (
// remote version control servers, and code hosting sites.
// A Repo must be safe for simultaneous use by multiple goroutines.
type Repo interface {
+ // CheckReuse checks whether the old origin information
+ // remains up to date. If so, whatever cached object it was
+ // taken from can be reused.
+ // The subdir gives subdirectory name where the module root is expected to be found,
+ // "" for the root or "sub/dir" for a subdirectory (no trailing slash).
+ CheckReuse(old *Origin, subdir string) error
+
// List lists all tags with the given prefix.
- Tags(prefix string) (tags []string, err error)
+ Tags(prefix string) (*Tags, error)
// Stat returns information about the revision rev.
// A revision can be any identifier known to the underlying service:
@@ -74,8 +84,84 @@ type Repo interface {
DescendsFrom(rev, tag string) (bool, error)
}
-// A Rev describes a single revision in a source code repository.
+// An Origin describes the provenance of a given repo method result.
+// It can be passed to CheckReuse (usually in a different go command invocation)
+// to see whether the result remains up-to-date.
+type Origin struct {
+ VCS string `json:",omitempty"` // "git" etc
+ URL string `json:",omitempty"` // URL of repository
+ Subdir string `json:",omitempty"` // subdirectory in repo
+
+ // If TagSum is non-empty, then the resolution of this module version
+ // depends on the set of tags present in the repo, specifically the tags
+ // of the form TagPrefix + a valid semver version.
+ // If the matching repo tags and their commit hashes still hash to TagSum,
+ // the Origin is still valid (at least as far as the tags are concerned).
+ // The exact checksum is up to the Repo implementation; see (*gitRepo).Tags.
+ TagPrefix string `json:",omitempty"`
+ TagSum string `json:",omitempty"`
+
+ // If Ref is non-empty, then the resolution of this module version
+ // depends on Ref resolving to the revision identified by Hash.
+ // If Ref still resolves to Hash, the Origin is still valid (at least as far as Ref is concerned).
+ // For Git, the Ref is a full ref like "refs/heads/main" or "refs/tags/v1.2.3",
+ // and the Hash is the Git object hash the ref maps to.
+ // Other VCS might choose differently, but the idea is that Ref is the name
+ // with a mutable meaning while Hash is a name with an immutable meaning.
+ Ref string `json:",omitempty"`
+ Hash string `json:",omitempty"`
+}
+
+// Checkable reports whether the Origin contains anything that can be checked.
+// If not, it's purely informational and should fail a CheckReuse call.
+func (o *Origin) Checkable() bool {
+ return o.TagSum != "" || o.Ref != "" || o.Hash != ""
+}
+
+func (o *Origin) Merge(other *Origin) {
+ if o.TagSum == "" {
+ o.TagPrefix = other.TagPrefix
+ o.TagSum = other.TagSum
+ }
+ if o.Ref == "" {
+ o.Ref = other.Ref
+ o.Hash = other.Hash
+ }
+}
+
+// A Tags describes the available tags in a code repository.
+type Tags struct {
+ Origin *Origin
+ List []Tag
+}
+
+// A Tag describes a single tag in a code repository.
+type Tag struct {
+ Name string
+ Hash string // content hash identifying tag's content, if available
+}
+
+// isOriginTag reports whether tag should be preserved
+// in the Tags method's Origin calculation.
+// We can safely ignore tags that are not look like pseudo-versions,
+// because ../coderepo.go's (*codeRepo).Versions ignores them too.
+// We can also ignore non-semver tags, but we have to include semver
+// tags with extra suffixes, because the pseudo-version base finder uses them.
+func isOriginTag(tag string) bool {
+ // modfetch.(*codeRepo).Versions uses Canonical == tag,
+ // but pseudo-version calculation has a weaker condition that
+ // the canonical is a prefix of the tag.
+ // Include those too, so that if any new one appears, we'll invalidate the cache entry.
+ // This will lead to spurious invalidation of version list results,
+ // but tags of this form being created should be fairly rare
+ // (and invalidate pseudo-version results anyway).
+ c := semver.Canonical(tag)
+ return c != "" && strings.HasPrefix(tag, c) && !module.IsPseudoVersion(tag)
+}
+
+// A RevInfo describes a single revision in a source code repository.
type RevInfo struct {
+ Origin *Origin
Name string // complete ID in underlying repository
Short string // shortened ID, for use in pseudo-version
Version string // version used in lookup
diff --git a/src/cmd/go/internal/modfetch/codehost/git.go b/src/cmd/go/internal/modfetch/codehost/git.go
index 034abf360b..3129a31786 100644
--- a/src/cmd/go/internal/modfetch/codehost/git.go
+++ b/src/cmd/go/internal/modfetch/codehost/git.go
@@ -6,6 +6,8 @@ package codehost
import (
"bytes"
+ "crypto/sha256"
+ "encoding/base64"
"errors"
"fmt"
"io"
@@ -169,6 +171,53 @@ func (r *gitRepo) loadLocalTags() {
}
}
+func (r *gitRepo) CheckReuse(old *Origin, subdir string) error {
+ if old == nil {
+ return fmt.Errorf("missing origin")
+ }
+ if old.VCS != "git" || old.URL != r.remoteURL {
+ return fmt.Errorf("origin moved from %v %q to %v %q", old.VCS, old.URL, "git", r.remoteURL)
+ }
+ if old.Subdir != subdir {
+ return fmt.Errorf("origin moved from %v %q %q to %v %q %q", old.VCS, old.URL, old.Subdir, "git", r.remoteURL, subdir)
+ }
+
+ // Note: Can have Hash with no Ref and no TagSum,
+ // meaning the Hash simply has to remain in the repo.
+ // In that case we assume it does in the absence of any real way to check.
+ // But if neither Hash nor TagSum is present, we have nothing to check,
+ // which we take to mean we didn't record enough information to be sure.
+ if old.Hash == "" && old.TagSum == "" {
+ return fmt.Errorf("non-specific origin")
+ }
+
+ r.loadRefs()
+ if r.refsErr != nil {
+ return r.refsErr
+ }
+
+ if old.Ref != "" {
+ hash, ok := r.refs[old.Ref]
+ if !ok {
+ return fmt.Errorf("ref %q deleted", old.Ref)
+ }
+ if hash != old.Hash {
+ return fmt.Errorf("ref %q moved from %s to %s", old.Ref, old.Hash, hash)
+ }
+ }
+ if old.TagSum != "" {
+ tags, err := r.Tags(old.TagPrefix)
+ if err != nil {
+ return err
+ }
+ if tags.Origin.TagSum != old.TagSum {
+ return fmt.Errorf("tags changed")
+ }
+ }
+
+ return nil
+}
+
// loadRefs loads heads and tags references from the remote into the map r.refs.
// The result is cached in memory.
func (r *gitRepo) loadRefs() (map[string]string, error) {
@@ -219,14 +268,21 @@ func (r *gitRepo) loadRefs() (map[string]string, error) {
return r.refs, r.refsErr
}
-func (r *gitRepo) Tags(prefix string) ([]string, error) {
+func (r *gitRepo) Tags(prefix string) (*Tags, error) {
refs, err := r.loadRefs()
if err != nil {
return nil, err
}
- tags := []string{}
- for ref := range refs {
+ tags := &Tags{
+ Origin: &Origin{
+ VCS: "git",
+ URL: r.remoteURL,
+ TagPrefix: prefix,
+ },
+ List: []Tag{},
+ }
+ for ref, hash := range refs {
if !strings.HasPrefix(ref, "refs/tags/") {
continue
}
@@ -234,9 +290,20 @@ func (r *gitRepo) Tags(prefix string) ([]string, error) {
if !strings.HasPrefix(tag, prefix) {
continue
}
- tags = append(tags, tag)
+ tags.List = append(tags.List, Tag{tag, hash})
}
- sort.Strings(tags)
+ sort.Slice(tags.List, func(i, j int) bool {
+ return tags.List[i].Name < tags.List[j].Name
+ })
+
+ dir := prefix[:strings.LastIndex(prefix, "/")+1]
+ h := sha256.New()
+ for _, tag := range tags.List {
+ if isOriginTag(strings.TrimPrefix(tag.Name, dir)) {
+ fmt.Fprintf(h, "%q %s\n", tag.Name, tag.Hash)
+ }
+ }
+ tags.Origin.TagSum = "t1:" + base64.StdEncoding.EncodeToString(h.Sum(nil))
return tags, nil
}
@@ -248,7 +315,13 @@ func (r *gitRepo) Latest() (*RevInfo, error) {
if refs["HEAD"] == "" {
return nil, ErrNoCommits
}
- return r.Stat(refs["HEAD"])
+ info, err := r.Stat(refs["HEAD"])
+ if err != nil {
+ return nil, err
+ }
+ info.Origin.Ref = "HEAD"
+ info.Origin.Hash = refs["HEAD"]
+ return info, nil
}
// findRef finds some ref name for the given hash,
@@ -278,7 +351,7 @@ const minHashDigits = 7
// stat stats the given rev in the local repository,
// or else it fetches more info from the remote repository and tries again.
-func (r *gitRepo) stat(rev string) (*RevInfo, error) {
+func (r *gitRepo) stat(rev string) (info *RevInfo, err error) {
if r.local {
return r.statLocal(rev, rev)
}
@@ -348,6 +421,13 @@ func (r *gitRepo) stat(rev string) (*RevInfo, error) {
return nil, &UnknownRevisionError{Rev: rev}
}
+ defer func() {
+ if info != nil {
+ info.Origin.Ref = ref
+ info.Origin.Hash = info.Name
+ }
+ }()
+
// Protect r.fetchLevel and the "fetch more and more" sequence.
unlock, err := r.mu.Lock()
if err != nil {
@@ -465,11 +545,19 @@ func (r *gitRepo) statLocal(version, rev string) (*RevInfo, error) {
}
info := &RevInfo{
+ Origin: &Origin{
+ VCS: "git",
+ URL: r.remoteURL,
+ Hash: hash,
+ },
Name: hash,
Short: ShortenSHA1(hash),
Time: time.Unix(t, 0).UTC(),
Version: hash,
}
+ if !strings.HasPrefix(hash, rev) {
+ info.Origin.Ref = rev
+ }
// Add tags. Output looks like:
// ede458df7cd0fdca520df19a33158086a8a68e81 1523994202 HEAD -> master, tag: v1.2.4-annotated, tag: v1.2.3, origin/master, origin/HEAD
@@ -580,7 +668,7 @@ func (r *gitRepo) RecentTag(rev, prefix string, allowed func(tag string) bool) (
if err != nil {
return "", err
}
- if len(tags) == 0 {
+ if len(tags.List) == 0 {
return "", nil
}
@@ -634,7 +722,7 @@ func (r *gitRepo) DescendsFrom(rev, tag string) (bool, error) {
if err != nil {
return false, err
}
- if len(tags) == 0 {
+ if len(tags.List) == 0 {
return false, nil
}
diff --git a/src/cmd/go/internal/modfetch/codehost/git_test.go b/src/cmd/go/internal/modfetch/codehost/git_test.go
index a684fa1a9b..6a4212fc5a 100644
--- a/src/cmd/go/internal/modfetch/codehost/git_test.go
+++ b/src/cmd/go/internal/modfetch/codehost/git_test.go
@@ -43,7 +43,7 @@ var altRepos = []string{
// For now, at least the hgrepo1 tests check the general vcs.go logic.
// localGitRepo is like gitrepo1 but allows archive access.
-var localGitRepo string
+var localGitRepo, localGitURL string
func testMain(m *testing.M) int {
dir, err := os.MkdirTemp("", "gitrepo-test-")
@@ -65,6 +65,15 @@ func testMain(m *testing.M) int {
if _, err := Run(localGitRepo, "git", "config", "daemon.uploadarch", "true"); err != nil {
log.Fatal(err)
}
+
+ // Convert absolute path to file URL. LocalGitRepo will not accept
+ // Windows absolute paths because they look like a host:path remote.
+ // TODO(golang.org/issue/32456): use url.FromFilePath when implemented.
+ if strings.HasPrefix(localGitRepo, "/") {
+ localGitURL = "file://" + localGitRepo
+ } else {
+ localGitURL = "file:///" + filepath.ToSlash(localGitRepo)
+ }
}
}
@@ -73,17 +82,8 @@ func testMain(m *testing.M) int {
func testRepo(t *testing.T, remote string) (Repo, error) {
if remote == "localGitRepo" {
- // Convert absolute path to file URL. LocalGitRepo will not accept
- // Windows absolute paths because they look like a host:path remote.
- // TODO(golang.org/issue/32456): use url.FromFilePath when implemented.
- var url string
- if strings.HasPrefix(localGitRepo, "/") {
- url = "file://" + localGitRepo
- } else {
- url = "file:///" + filepath.ToSlash(localGitRepo)
- }
testenv.MustHaveExecPath(t, "git")
- return LocalGitRepo(url)
+ return LocalGitRepo(localGitURL)
}
vcs := "git"
for _, k := range []string{"hg"} {
@@ -98,13 +98,28 @@ func testRepo(t *testing.T, remote string) (Repo, error) {
var tagsTests = []struct {
repo string
prefix string
- tags []string
+ tags []Tag
}{
- {gitrepo1, "xxx", []string{}},
- {gitrepo1, "", []string{"v1.2.3", "v1.2.4-annotated", "v2.0.1", "v2.0.2", "v2.3"}},
- {gitrepo1, "v", []string{"v1.2.3", "v1.2.4-annotated", "v2.0.1", "v2.0.2", "v2.3"}},
- {gitrepo1, "v1", []string{"v1.2.3", "v1.2.4-annotated"}},
- {gitrepo1, "2", []string{}},
+ {gitrepo1, "xxx", []Tag{}},
+ {gitrepo1, "", []Tag{
+ {"v1.2.3", "ede458df7cd0fdca520df19a33158086a8a68e81"},
+ {"v1.2.4-annotated", "ede458df7cd0fdca520df19a33158086a8a68e81"},
+ {"v2.0.1", "76a00fb249b7f93091bc2c89a789dab1fc1bc26f"},
+ {"v2.0.2", "9d02800338b8a55be062c838d1f02e0c5780b9eb"},
+ {"v2.3", "76a00fb249b7f93091bc2c89a789dab1fc1bc26f"},
+ }},
+ {gitrepo1, "v", []Tag{
+ {"v1.2.3", "ede458df7cd0fdca520df19a33158086a8a68e81"},
+ {"v1.2.4-annotated", "ede458df7cd0fdca520df19a33158086a8a68e81"},
+ {"v2.0.1", "76a00fb249b7f93091bc2c89a789dab1fc1bc26f"},
+ {"v2.0.2", "9d02800338b8a55be062c838d1f02e0c5780b9eb"},
+ {"v2.3", "76a00fb249b7f93091bc2c89a789dab1fc1bc26f"},
+ }},
+ {gitrepo1, "v1", []Tag{
+ {"v1.2.3", "ede458df7cd0fdca520df19a33158086a8a68e81"},
+ {"v1.2.4-annotated", "ede458df7cd0fdca520df19a33158086a8a68e81"},
+ }},
+ {gitrepo1, "2", []Tag{}},
}
func TestTags(t *testing.T) {
@@ -121,13 +136,24 @@ func TestTags(t *testing.T) {
if err != nil {
t.Fatal(err)
}
- if !reflect.DeepEqual(tags, tt.tags) {
- t.Errorf("Tags: incorrect tags\nhave %v\nwant %v", tags, tt.tags)
+ if tags == nil || !reflect.DeepEqual(tags.List, tt.tags) {
+ t.Errorf("Tags(%q): incorrect tags\nhave %v\nwant %v", tt.prefix, tags, tt.tags)
}
}
t.Run(path.Base(tt.repo)+"/"+tt.prefix, f)
if tt.repo == gitrepo1 {
+ // Clear hashes.
+ clearTags := []Tag{}
+ for _, tag := range tt.tags {
+ clearTags = append(clearTags, Tag{tag.Name, ""})
+ }
+ tags := tt.tags
for _, tt.repo = range altRepos {
+ if strings.Contains(tt.repo, "Git") {
+ tt.tags = tags
+ } else {
+ tt.tags = clearTags
+ }
t.Run(path.Base(tt.repo)+"/"+tt.prefix, f)
}
}
@@ -141,6 +167,12 @@ var latestTests = []struct {
{
gitrepo1,
&RevInfo{
+ Origin: &Origin{
+ VCS: "git",
+ URL: "https://vcs-test.golang.org/git/gitrepo1",
+ Ref: "HEAD",
+ Hash: "ede458df7cd0fdca520df19a33158086a8a68e81",
+ },
Name: "ede458df7cd0fdca520df19a33158086a8a68e81",
Short: "ede458df7cd0",
Version: "ede458df7cd0fdca520df19a33158086a8a68e81",
@@ -151,6 +183,11 @@ var latestTests = []struct {
{
hgrepo1,
&RevInfo{
+ Origin: &Origin{
+ VCS: "hg",
+ URL: "https://vcs-test.golang.org/hg/hgrepo1",
+ Hash: "18518c07eb8ed5c80221e997e518cccaa8c0c287",
+ },
Name: "18518c07eb8ed5c80221e997e518cccaa8c0c287",
Short: "18518c07eb8e",
Version: "18518c07eb8ed5c80221e997e518cccaa8c0c287",
@@ -174,12 +211,17 @@ func TestLatest(t *testing.T) {
t.Fatal(err)
}
if !reflect.DeepEqual(info, tt.info) {
- t.Errorf("Latest: incorrect info\nhave %+v\nwant %+v", *info, *tt.info)
+ t.Errorf("Latest: incorrect info\nhave %+v (origin %+v)\nwant %+v (origin %+v)", info, info.Origin, tt.info, tt.info.Origin)
}
}
t.Run(path.Base(tt.repo), f)
if tt.repo == gitrepo1 {
tt.repo = "localGitRepo"
+ info := *tt.info
+ tt.info = &info
+ o := *info.Origin
+ info.Origin = &o
+ o.URL = localGitURL
t.Run(path.Base(tt.repo), f)
}
}
@@ -590,11 +632,12 @@ func TestStat(t *testing.T) {
if !strings.Contains(err.Error(), tt.err) {
t.Fatalf("Stat: wrong error %q, want %q", err, tt.err)
}
- if info != nil {
- t.Errorf("Stat: non-nil info with error %q", err)
+ if info != nil && info.Origin == nil {
+ t.Errorf("Stat: non-nil info with nil Origin with error %q", err)
}
return
}
+ info.Origin = nil // TestLatest and ../../../testdata/script/reuse_git.txt test Origin well enough
if !reflect.DeepEqual(info, tt.info) {
t.Errorf("Stat: incorrect info\nhave %+v\nwant %+v", *info, *tt.info)
}
diff --git a/src/cmd/go/internal/modfetch/codehost/vcs.go b/src/cmd/go/internal/modfetch/codehost/vcs.go
index de62265efc..f1c40998b2 100644
--- a/src/cmd/go/internal/modfetch/codehost/vcs.go
+++ b/src/cmd/go/internal/modfetch/codehost/vcs.go
@@ -290,7 +290,13 @@ func (r *vcsRepo) loadBranches() {
}
}
-func (r *vcsRepo) Tags(prefix string) ([]string, error) {
+var ErrNoRepoHash = errors.New("RepoHash not supported")
+
+func (r *vcsRepo) CheckReuse(old *Origin, subdir string) error {
+ return fmt.Errorf("vcs %s does not implement CheckReuse", r.cmd.vcs)
+}
+
+func (r *vcsRepo) Tags(prefix string) (*Tags, error) {
unlock, err := r.mu.Lock()
if err != nil {
return nil, err
@@ -298,14 +304,24 @@ func (r *vcsRepo) Tags(prefix string) ([]string, error) {
defer unlock()
r.tagsOnce.Do(r.loadTags)
-
- tags := []string{}
+ tags := &Tags{
+ // None of the other VCS provide a reasonable way to compute TagSum
+ // without downloading the whole repo, so we only include VCS and URL
+ // in the Origin.
+ Origin: &Origin{
+ VCS: r.cmd.vcs,
+ URL: r.remote,
+ },
+ List: []Tag{},
+ }
for tag := range r.tags {
if strings.HasPrefix(tag, prefix) {
- tags = append(tags, tag)
+ tags.List = append(tags.List, Tag{tag, ""})
}
}
- sort.Strings(tags)
+ sort.Slice(tags.List, func(i, j int) bool {
+ return tags.List[i].Name < tags.List[j].Name
+ })
return tags, nil
}
@@ -352,7 +368,16 @@ func (r *vcsRepo) statLocal(rev string) (*RevInfo, error) {
if err != nil {
return nil, &UnknownRevisionError{Rev: rev}
}
- return r.cmd.parseStat(rev, string(out))
+ info, err := r.cmd.parseStat(rev, string(out))
+ if err != nil {
+ return nil, err
+ }
+ if info.Origin == nil {
+ info.Origin = new(Origin)
+ }
+ info.Origin.VCS = r.cmd.vcs
+ info.Origin.URL = r.remote
+ return info, nil
}
func (r *vcsRepo) Latest() (*RevInfo, error) {
@@ -491,6 +516,9 @@ func hgParseStat(rev, out string) (*RevInfo, error) {
sort.Strings(tags)
info := &RevInfo{
+ Origin: &Origin{
+ Hash: hash,
+ },
Name: hash,
Short: ShortenSHA1(hash),
Time: time.Unix(t, 0).UTC(),
@@ -569,6 +597,9 @@ func fossilParseStat(rev, out string) (*RevInfo, error) {
version = hash // extend to full hash
}
info := &RevInfo{
+ Origin: &Origin{
+ Hash: hash,
+ },
Name: hash,
Short: ShortenSHA1(hash),
Time: t,
diff --git a/src/cmd/go/internal/modfetch/coderepo.go b/src/cmd/go/internal/modfetch/coderepo.go
index ff1cea1d94..a994f79d4b 100644
--- a/src/cmd/go/internal/modfetch/coderepo.go
+++ b/src/cmd/go/internal/modfetch/coderepo.go
@@ -130,12 +130,16 @@ func (r *codeRepo) ModulePath() string {
return r.modPath
}
-func (r *codeRepo) Versions(prefix string) ([]string, error) {
+func (r *codeRepo) CheckReuse(old *codehost.Origin) error {
+ return r.code.CheckReuse(old, r.codeDir)
+}
+
+func (r *codeRepo) Versions(prefix string) (*Versions, error) {
// Special case: gopkg.in/macaroon-bakery.v2-unstable
// does not use the v2 tags (those are for macaroon-bakery.v2).
// It has no possible tags at all.
if strings.HasPrefix(r.modPath, "gopkg.in/") && strings.HasSuffix(r.modPath, "-unstable") {
- return nil, nil
+ return &Versions{}, nil
}
p := prefix
@@ -151,14 +155,16 @@ func (r *codeRepo) Versions(prefix string) ([]string, error) {
}
var list, incompatible []string
- for _, tag := range tags {
- if !strings.HasPrefix(tag, p) {
+ for _, tag := range tags.List {
+ if !strings.HasPrefix(tag.Name, p) {
continue
}
- v := tag
+ v := tag.Name
if r.codeDir != "" {
v = v[len(r.codeDir)+1:]
}
+ // Note: ./codehost/codehost.go's isOriginTag knows about these conditions too.
+ // If these are relaxed, isOriginTag will need to be relaxed as well.
if v == "" || v != semver.Canonical(v) {
// Ignore non-canonical tags: Stat rewrites those to canonical
// pseudo-versions. Note that we compare against semver.Canonical here
@@ -186,7 +192,7 @@ func (r *codeRepo) Versions(prefix string) ([]string, error) {
semver.Sort(list)
semver.Sort(incompatible)
- return r.appendIncompatibleVersions(list, incompatible)
+ return r.appendIncompatibleVersions(tags.Origin, list, incompatible)
}
// appendIncompatibleVersions appends "+incompatible" versions to list if
@@ -196,10 +202,14 @@ func (r *codeRepo) Versions(prefix string) ([]string, error) {
// prefix.
//
// Both list and incompatible must be sorted in semantic order.
-func (r *codeRepo) appendIncompatibleVersions(list, incompatible []string) ([]string, error) {
+func (r *codeRepo) appendIncompatibleVersions(origin *codehost.Origin, list, incompatible []string) (*Versions, error) {
+ versions := &Versions{
+ Origin: origin,
+ List: list,
+ }
if len(incompatible) == 0 || r.pathMajor != "" {
// No +incompatible versions are possible, so no need to check them.
- return list, nil
+ return versions, nil
}
versionHasGoMod := func(v string) (bool, error) {
@@ -232,7 +242,7 @@ func (r *codeRepo) appendIncompatibleVersions(list, incompatible []string) ([]st
// (github.com/russross/blackfriday@v2.0.0 and
// github.com/libp2p/go-libp2p@v6.0.23), and (as of 2019-10-29) have no
// concrete examples for which it is undesired.
- return list, nil
+ return versions, nil
}
}
@@ -271,10 +281,10 @@ func (r *codeRepo) appendIncompatibleVersions(list, incompatible []string) ([]st
// bounds.
continue
}
- list = append(list, v+"+incompatible")
+ versions.List = append(versions.List, v+"+incompatible")
}
- return list, nil
+ return versions, nil
}
func (r *codeRepo) Stat(rev string) (*RevInfo, error) {
@@ -439,7 +449,28 @@ func (r *codeRepo) convert(info *codehost.RevInfo, statVers string) (*RevInfo, e
return nil, errIncompatible
}
+ origin := info.Origin
+ if module.IsPseudoVersion(v) {
+ // Add tags that are relevant to pseudo-version calculation to origin.
+ prefix := ""
+ if r.codeDir != "" {
+ prefix = r.codeDir + "/"
+ }
+ if r.pathMajor != "" { // "/v2" or "/.v2"
+ prefix += r.pathMajor[1:] + "." // += "v2."
+ }
+ tags, err := r.code.Tags(prefix)
+ if err != nil {
+ return nil, err
+ }
+ o := *origin
+ origin = &o
+ origin.TagPrefix = tags.Origin.TagPrefix
+ origin.TagSum = tags.Origin.TagSum
+ }
+
return &RevInfo{
+ Origin: origin,
Name: info.Name,
Short: info.Short,
Time: info.Time,
@@ -674,11 +705,11 @@ func (r *codeRepo) validatePseudoVersion(info *codehost.RevInfo, version string)
var lastTag string // Prefer to log some real tag rather than a canonically-equivalent base.
ancestorFound := false
- for _, tag := range tags {
- versionOnly := strings.TrimPrefix(tag, tagPrefix)
+ for _, tag := range tags.List {
+ versionOnly := strings.TrimPrefix(tag.Name, tagPrefix)
if semver.Compare(versionOnly, base) == 0 {
- lastTag = tag
- ancestorFound, err = r.code.DescendsFrom(info.Name, tag)
+ lastTag = tag.Name
+ ancestorFound, err = r.code.DescendsFrom(info.Name, tag.Name)
if ancestorFound {
break
}
@@ -922,10 +953,11 @@ func (r *codeRepo) modPrefix(rev string) string {
}
func (r *codeRepo) retractedVersions() (func(string) bool, error) {
- versions, err := r.Versions("")
+ vs, err := r.Versions("")
if err != nil {
return nil, err
}
+ versions := vs.List
for i, v := range versions {
if strings.HasSuffix(v, "+incompatible") {
diff --git a/src/cmd/go/internal/modfetch/coderepo_test.go b/src/cmd/go/internal/modfetch/coderepo_test.go
index 8d0eb00544..967978cd4d 100644
--- a/src/cmd/go/internal/modfetch/coderepo_test.go
+++ b/src/cmd/go/internal/modfetch/coderepo_test.go
@@ -823,7 +823,7 @@ func TestCodeRepoVersions(t *testing.T) {
if err != nil {
t.Fatalf("Versions(%q): %v", tt.prefix, err)
}
- if !reflect.DeepEqual(list, tt.versions) {
+ if !reflect.DeepEqual(list.List, tt.versions) {
t.Fatalf("Versions(%q):\nhave %v\nwant %v", tt.prefix, list, tt.versions)
}
})
@@ -921,7 +921,13 @@ type fixedTagsRepo struct {
codehost.Repo
}
-func (ch *fixedTagsRepo) Tags(string) ([]string, error) { return ch.tags, nil }
+func (ch *fixedTagsRepo) Tags(string) (*codehost.Tags, error) {
+ tags := &codehost.Tags{}
+ for _, t := range ch.tags {
+ tags.List = append(tags.List, codehost.Tag{Name: t})
+ }
+ return tags, nil
+}
func TestNonCanonicalSemver(t *testing.T) {
root := "golang.org/x/issue24476"
@@ -945,7 +951,7 @@ func TestNonCanonicalSemver(t *testing.T) {
if err != nil {
t.Fatal(err)
}
- if len(v) != 1 || v[0] != "v1.0.1" {
+ if len(v.List) != 1 || v.List[0] != "v1.0.1" {
t.Fatal("unexpected versions returned:", v)
}
}
diff --git a/src/cmd/go/internal/modfetch/proxy.go b/src/cmd/go/internal/modfetch/proxy.go
index 2491b7d185..d2374680d8 100644
--- a/src/cmd/go/internal/modfetch/proxy.go
+++ b/src/cmd/go/internal/modfetch/proxy.go
@@ -225,6 +225,12 @@ func (p *proxyRepo) ModulePath() string {
return p.path
}
+var errProxyReuse = fmt.Errorf("proxy does not support CheckReuse")
+
+func (p *proxyRepo) CheckReuse(old *codehost.Origin) error {
+ return errProxyReuse
+}
+
// versionError returns err wrapped in a ModuleError for p.path.
func (p *proxyRepo) versionError(version string, err error) error {
if version != "" && version != module.CanonicalVersion(version) {
@@ -279,7 +285,7 @@ func (p *proxyRepo) getBody(path string) (r io.ReadCloser, err error) {
return resp.Body, nil
}
-func (p *proxyRepo) Versions(prefix string) ([]string, error) {
+func (p *proxyRepo) Versions(prefix string) (*Versions, error) {
data, err := p.getBytes("@v/list")
if err != nil {
p.listLatestOnce.Do(func() {
@@ -299,7 +305,7 @@ func (p *proxyRepo) Versions(prefix string) ([]string, error) {
p.listLatest, p.listLatestErr = p.latestFromList(allLine)
})
semver.Sort(list)
- return list, nil
+ return &Versions{List: list}, nil
}
func (p *proxyRepo) latest() (*RevInfo, error) {
@@ -317,9 +323,8 @@ func (p *proxyRepo) latest() (*RevInfo, error) {
func (p *proxyRepo) latestFromList(allLine []string) (*RevInfo, error) {
var (
- bestTime time.Time
- bestTimeIsFromPseudo bool
- bestVersion string
+ bestTime time.Time
+ bestVersion string
)
for _, line := range allLine {
f := strings.Fields(line)
@@ -327,14 +332,12 @@ func (p *proxyRepo) latestFromList(allLine []string) (*RevInfo, error) {
// If the proxy includes timestamps, prefer the timestamp it reports.
// Otherwise, derive the timestamp from the pseudo-version.
var (
- ft time.Time
- ftIsFromPseudo = false
+ ft time.Time
)
if len(f) >= 2 {
ft, _ = time.Parse(time.RFC3339, f[1])
} else if module.IsPseudoVersion(f[0]) {
ft, _ = module.PseudoVersionTime(f[0])
- ftIsFromPseudo = true
} else {
// Repo.Latest promises that this method is only called where there are
// no tagged versions. Ignore any tagged versions that were added in the
@@ -343,7 +346,6 @@ func (p *proxyRepo) latestFromList(allLine []string) (*RevInfo, error) {
}
if bestTime.Before(ft) {
bestTime = ft
- bestTimeIsFromPseudo = ftIsFromPseudo
bestVersion = f[0]
}
}
@@ -352,22 +354,8 @@ func (p *proxyRepo) latestFromList(allLine []string) (*RevInfo, error) {
return nil, p.versionError("", codehost.ErrNoCommits)
}
- if bestTimeIsFromPseudo {
- // We parsed bestTime from the pseudo-version, but that's in UTC and we're
- // supposed to report the timestamp as reported by the VCS.
- // Stat the selected version to canonicalize the timestamp.
- //
- // TODO(bcmills): Should we also stat other versions to ensure that we
- // report the correct Name and Short for the revision?
- return p.Stat(bestVersion)
- }
-
- return &RevInfo{
- Version: bestVersion,
- Name: bestVersion,
- Short: bestVersion,
- Time: bestTime,
- }, nil
+ // Call Stat to get all the other fields, including Origin information.
+ return p.Stat(bestVersion)
}
func (p *proxyRepo) Stat(rev string) (*RevInfo, error) {
diff --git a/src/cmd/go/internal/modfetch/repo.go b/src/cmd/go/internal/modfetch/repo.go
index 1b42ecb6ed..d4c57bb300 100644
--- a/src/cmd/go/internal/modfetch/repo.go
+++ b/src/cmd/go/internal/modfetch/repo.go
@@ -29,6 +29,12 @@ type Repo interface {
// ModulePath returns the module path.
ModulePath() string
+ // CheckReuse checks whether the validation criteria in the origin
+ // are still satisfied on the server corresponding to this module.
+ // If so, the caller can reuse any cached Versions or RevInfo containing
+ // this origin rather than redownloading those from the server.
+ CheckReuse(old *codehost.Origin) error
+
// Versions lists all known versions with the given prefix.
// Pseudo-versions are not included.
//
@@ -42,7 +48,7 @@ type Repo interface {
//
// If the underlying repository does not exist,
// Versions returns an error matching errors.Is(_, os.NotExist).
- Versions(prefix string) ([]string, error)
+ Versions(prefix string) (*Versions, error)
// Stat returns information about the revision rev.
// A revision can be any identifier known to the underlying service:
@@ -61,7 +67,14 @@ type Repo interface {
Zip(dst io.Writer, version string) error
}
-// A Rev describes a single revision in a module repository.
+// A Versions describes the available versions in a module repository.
+type Versions struct {
+ Origin *codehost.Origin `json:",omitempty"` // origin information for reuse
+
+ List []string // semver versions
+}
+
+// A RevInfo describes a single revision in a module repository.
type RevInfo struct {
Version string // suggested version string for this revision
Time time.Time // commit time
@@ -70,6 +83,8 @@ type RevInfo struct {
// but they are not recorded when talking about module versions.
Name string `json:"-"` // complete ID in underlying repository
Short string `json:"-"` // shortened ID, for use in pseudo-version
+
+ Origin *codehost.Origin `json:",omitempty"` // provenance for reuse
}
// Re: module paths, import paths, repository roots, and lookups
@@ -320,7 +335,14 @@ func (l *loggingRepo) ModulePath() string {
return l.r.ModulePath()
}
-func (l *loggingRepo) Versions(prefix string) (tags []string, err error) {
+func (l *loggingRepo) CheckReuse(old *codehost.Origin) (err error) {
+ defer func() {
+ logCall("CheckReuse[%s]: %v", l.r.ModulePath(), err)
+ }()
+ return l.r.CheckReuse(old)
+}
+
+func (l *loggingRepo) Versions(prefix string) (*Versions, error) {
defer logCall("Repo[%s]: Versions(%q)", l.r.ModulePath(), prefix)()
return l.r.Versions(prefix)
}
@@ -360,11 +382,12 @@ type errRepo struct {
func (r errRepo) ModulePath() string { return r.modulePath }
-func (r errRepo) Versions(prefix string) (tags []string, err error) { return nil, r.err }
-func (r errRepo) Stat(rev string) (*RevInfo, error) { return nil, r.err }
-func (r errRepo) Latest() (*RevInfo, error) { return nil, r.err }
-func (r errRepo) GoMod(version string) ([]byte, error) { return nil, r.err }
-func (r errRepo) Zip(dst io.Writer, version string) error { return r.err }
+func (r errRepo) CheckReuse(old *codehost.Origin) error { return r.err }
+func (r errRepo) Versions(prefix string) (*Versions, error) { return nil, r.err }
+func (r errRepo) Stat(rev string) (*RevInfo, error) { return nil, r.err }
+func (r errRepo) Latest() (*RevInfo, error) { return nil, r.err }
+func (r errRepo) GoMod(version string) ([]byte, error) { return nil, r.err }
+func (r errRepo) Zip(dst io.Writer, version string) error { return r.err }
// A notExistError is like fs.ErrNotExist, but with a custom message
type notExistError struct {
diff --git a/src/cmd/go/internal/modload/mvs.go b/src/cmd/go/internal/modload/mvs.go
index 588bcf4bdc..2055303efe 100644
--- a/src/cmd/go/internal/modload/mvs.go
+++ b/src/cmd/go/internal/modload/mvs.go
@@ -91,8 +91,8 @@ func versions(ctx context.Context, path string, allowed AllowedFunc) ([]string,
if err != nil {
return err
}
- allowedVersions := make([]string, 0, len(allVersions))
- for _, v := range allVersions {
+ allowedVersions := make([]string, 0, len(allVersions.List))
+ for _, v := range allVersions.List {
if err := allowed(ctx, module.Version{Path: path, Version: v}); err == nil {
allowedVersions = append(allowedVersions, v)
} else if !errors.Is(err, ErrDisallowed) {
diff --git a/src/cmd/go/internal/modload/query.go b/src/cmd/go/internal/modload/query.go
index ae5304f87e..051a4fe822 100644
--- a/src/cmd/go/internal/modload/query.go
+++ b/src/cmd/go/internal/modload/query.go
@@ -177,7 +177,7 @@ func queryProxy(ctx context.Context, proxy, path, query, current string, allowed
if err != nil {
return nil, err
}
- releases, prereleases, err := qm.filterVersions(ctx, versions)
+ releases, prereleases, err := qm.filterVersions(ctx, versions.List)
if err != nil {
return nil, err
}
@@ -991,7 +991,7 @@ func versionHasGoMod(_ context.Context, m module.Version) (bool, error) {
// available versions, but cannot fetch specific source files.
type versionRepo interface {
ModulePath() string
- Versions(prefix string) ([]string, error)
+ Versions(prefix string) (*modfetch.Versions, error)
Stat(rev string) (*modfetch.RevInfo, error)
Latest() (*modfetch.RevInfo, error)
}
@@ -1023,8 +1023,10 @@ type emptyRepo struct {
var _ versionRepo = emptyRepo{}
-func (er emptyRepo) ModulePath() string { return er.path }
-func (er emptyRepo) Versions(prefix string) ([]string, error) { return nil, nil }
+func (er emptyRepo) ModulePath() string { return er.path }
+func (er emptyRepo) Versions(prefix string) (*modfetch.Versions, error) {
+ return &modfetch.Versions{}, nil
+}
func (er emptyRepo) Stat(rev string) (*modfetch.RevInfo, error) { return nil, er.err }
func (er emptyRepo) Latest() (*modfetch.RevInfo, error) { return nil, er.err }
@@ -1044,13 +1046,16 @@ func (rr *replacementRepo) ModulePath() string { return rr.repo.ModulePath() }
// Versions returns the versions from rr.repo augmented with any matching
// replacement versions.
-func (rr *replacementRepo) Versions(prefix string) ([]string, error) {
+func (rr *replacementRepo) Versions(prefix string) (*modfetch.Versions, error) {
repoVersions, err := rr.repo.Versions(prefix)
- if err != nil && !errors.Is(err, os.ErrNotExist) {
- return nil, err
+ if err != nil {
+ if !errors.Is(err, os.ErrNotExist) {
+ return nil, err
+ }
+ repoVersions = new(modfetch.Versions)
}
- versions := repoVersions
+ versions := repoVersions.List
for _, mm := range MainModules.Versions() {
if index := MainModules.Index(mm); index != nil && len(index.replace) > 0 {
path := rr.ModulePath()
@@ -1062,15 +1067,15 @@ func (rr *replacementRepo) Versions(prefix string) ([]string, error) {
}
}
- if len(versions) == len(repoVersions) { // No replacement versions added.
- return versions, nil
+ if len(versions) == len(repoVersions.List) { // replacement versions added
+ return repoVersions, nil
}
sort.Slice(versions, func(i, j int) bool {
return semver.Compare(versions[i], versions[j]) < 0
})
str.Uniq(&versions)
- return versions, nil
+ return &modfetch.Versions{List: versions}, nil
}
func (rr *replacementRepo) Stat(rev string) (*modfetch.RevInfo, error) {
From 5f305ae8e5796ea3821088863a6842117c58da72 Mon Sep 17 00:00:00 2001
From: Russ Cox
Date: Fri, 10 Jun 2022 12:03:06 -0400
Subject: [PATCH 11/78] cmd/go: add -reuse flag to make proxy invocations more
efficient
The go list -m and go mod download commands now have a -reuse flag,
which is passed the name of a file containing the JSON output from a
previous run of the same command. (It is up to the caller to ensure
that flags such as -versions or -retracted, which affect the output,
are consistent between the old and new run.)
The new run uses the old JSON to evaluate whether the answer is
unchanged since the old run. If so, it reuses that information,
avoiding a costly 'git fetch', and sets a new Reuse: true field in its
own JSON output.
This dance with saving the JSON output and passing it back to -reuse
is not necessary on most systems, because the go command caches
version control checkouts in the module cache. That cache means that a
new 'git fetch' would only download the commits that are new since the
previous one (often none at all).
The dance becomes important only on systems that do not preserve the
module cache, for example by running 'go clean -modcache' aggressively
or by running in some environment that starts with an empty file
system.
For #53644.
Change-Id: I447960abf8055f83cc6dbc699a9fde9931130004
Reviewed-on: https://go-review.googlesource.com/c/go/+/411398
Run-TryBot: Russ Cox
Reviewed-by: Bryan Mills
---
src/cmd/go/alldocs.go | 26 +-
src/cmd/go/internal/list/list.go | 27 +-
src/cmd/go/internal/modcmd/download.go | 40 +-
src/cmd/go/internal/modcmd/why.go | 2 +-
src/cmd/go/internal/modfetch/cache.go | 20 +
.../go/internal/modfetch/codehost/codehost.go | 17 +-
src/cmd/go/internal/modfetch/codehost/git.go | 5 +-
src/cmd/go/internal/modfetch/coderepo.go | 36 +-
src/cmd/go/internal/modinfo/info.go | 22 +-
src/cmd/go/internal/modload/build.go | 62 ++-
src/cmd/go/internal/modload/edit.go | 2 +-
src/cmd/go/internal/modload/list.go | 60 ++-
src/cmd/go/internal/modload/mvs.go | 11 +-
src/cmd/go/internal/modload/query.go | 75 +++-
src/cmd/go/testdata/script/reuse_git.txt | 371 ++++++++++++++++++
15 files changed, 711 insertions(+), 65 deletions(-)
create mode 100644 src/cmd/go/testdata/script/reuse_git.txt
diff --git a/src/cmd/go/alldocs.go b/src/cmd/go/alldocs.go
index 78128dcf23..db6372642a 100644
--- a/src/cmd/go/alldocs.go
+++ b/src/cmd/go/alldocs.go
@@ -930,6 +930,7 @@
//
// type Module struct {
// Path string // module path
+// Query string // version query corresponding to this version
// Version string // module version
// Versions []string // available module versions
// Replace *Module // replaced by this module
@@ -943,6 +944,8 @@
// Retracted []string // retraction information, if any (with -retracted or -u)
// Deprecated string // deprecation message, if any (with -u)
// Error *ModuleError // error loading module
+// Origin any // provenance of module
+// Reuse bool // reuse of old module info is safe
// }
//
// type ModuleError struct {
@@ -1019,6 +1022,16 @@
// module as a Module struct. If an error occurs, the result will
// be a Module struct with a non-nil Error field.
//
+// When using -m, the -reuse=old.json flag accepts the name of file containing
+// the JSON output of a previous 'go list -m -json' invocation with the
+// same set of modifier flags (such as -u, -retracted, and -versions).
+// The go command may use this file to determine that a module is unchanged
+// since the previous invocation and avoid redownloading information about it.
+// Modules that are not redownloaded will be marked in the new output by
+// setting the Reuse field to true. Normally the module cache provides this
+// kind of reuse automatically; the -reuse flag can be useful on systems that
+// do not preserve the module cache.
+//
// For more about build flags, see 'go help build'.
//
// For more about specifying packages, see 'go help packages'.
@@ -1055,7 +1068,7 @@
//
// Usage:
//
-// go mod download [-x] [-json] [modules]
+// go mod download [-x] [-json] [-reuse=old.json] [modules]
//
// Download downloads the named modules, which can be module patterns selecting
// dependencies of the main module or module queries of the form path@version.
@@ -1078,6 +1091,7 @@
//
// type Module struct {
// Path string // module path
+// Query string // version query corresponding to this version
// Version string // module version
// Error string // error loading module
// Info string // absolute path to cached .info file
@@ -1086,8 +1100,18 @@
// Dir string // absolute path to cached source root directory
// Sum string // checksum for path, version (as in go.sum)
// GoModSum string // checksum for go.mod (as in go.sum)
+// Origin any // provenance of module
+// Reuse bool // reuse of old module info is safe
// }
//
+// The -reuse flag accepts the name of file containing the JSON output of a
+// previous 'go mod download -json' invocation. The go command may use this
+// file to determine that a module is unchanged since the previous invocation
+// and avoid redownloading it. Modules that are not redownloaded will be marked
+// in the new output by setting the Reuse field to true. Normally the module
+// cache provides this kind of reuse automatically; the -reuse flag can be
+// useful on systems that do not preserve the module cache.
+//
// The -x flag causes download to print the commands download executes.
//
// See https://golang.org/ref/mod#go-mod-download for more about 'go mod download'.
diff --git a/src/cmd/go/internal/list/list.go b/src/cmd/go/internal/list/list.go
index 9c651f2bf3..5f8be6e3c9 100644
--- a/src/cmd/go/internal/list/list.go
+++ b/src/cmd/go/internal/list/list.go
@@ -223,6 +223,7 @@ applied to a Go struct, but now a Module struct:
type Module struct {
Path string // module path
+ Query string // version query corresponding to this version
Version string // module version
Versions []string // available module versions
Replace *Module // replaced by this module
@@ -236,6 +237,8 @@ applied to a Go struct, but now a Module struct:
Retracted []string // retraction information, if any (with -retracted or -u)
Deprecated string // deprecation message, if any (with -u)
Error *ModuleError // error loading module
+ Origin any // provenance of module
+ Reuse bool // reuse of old module info is safe
}
type ModuleError struct {
@@ -312,6 +315,16 @@ that must be a module path or query and returns the specified
module as a Module struct. If an error occurs, the result will
be a Module struct with a non-nil Error field.
+When using -m, the -reuse=old.json flag accepts the name of file containing
+the JSON output of a previous 'go list -m -json' invocation with the
+same set of modifier flags (such as -u, -retracted, and -versions).
+The go command may use this file to determine that a module is unchanged
+since the previous invocation and avoid redownloading information about it.
+Modules that are not redownloaded will be marked in the new output by
+setting the Reuse field to true. Normally the module cache provides this
+kind of reuse automatically; the -reuse flag can be useful on systems that
+do not preserve the module cache.
+
For more about build flags, see 'go help build'.
For more about specifying packages, see 'go help packages'.
@@ -337,6 +350,7 @@ var (
listJsonFields jsonFlag // If not empty, only output these fields.
listM = CmdList.Flag.Bool("m", false, "")
listRetracted = CmdList.Flag.Bool("retracted", false, "")
+ listReuse = CmdList.Flag.String("reuse", "", "")
listTest = CmdList.Flag.Bool("test", false, "")
listU = CmdList.Flag.Bool("u", false, "")
listVersions = CmdList.Flag.Bool("versions", false, "")
@@ -398,6 +412,12 @@ func runList(ctx context.Context, cmd *base.Command, args []string) {
if *listFmt != "" && listJson == true {
base.Fatalf("go list -f cannot be used with -json")
}
+ if *listReuse != "" && !*listM {
+ base.Fatalf("go list -reuse cannot be used without -m")
+ }
+ if *listReuse != "" && modload.HasModRoot() {
+ base.Fatalf("go list -reuse cannot be used inside a module")
+ }
work.BuildInit()
out := newTrackingWriter(os.Stdout)
@@ -532,7 +552,10 @@ func runList(ctx context.Context, cmd *base.Command, args []string) {
mode |= modload.ListRetractedVersions
}
}
- mods, err := modload.ListModules(ctx, args, mode)
+ if *listReuse != "" && len(args) == 0 {
+ base.Fatalf("go: list -m -reuse only has an effect with module@version arguments")
+ }
+ mods, err := modload.ListModules(ctx, args, mode, *listReuse)
if !*listE {
for _, m := range mods {
if m.Error != nil {
@@ -783,7 +806,7 @@ func runList(ctx context.Context, cmd *base.Command, args []string) {
if *listRetracted {
mode |= modload.ListRetracted
}
- rmods, err := modload.ListModules(ctx, args, mode)
+ rmods, err := modload.ListModules(ctx, args, mode, *listReuse)
if err != nil && !*listE {
base.Errorf("go: %v", err)
}
diff --git a/src/cmd/go/internal/modcmd/download.go b/src/cmd/go/internal/modcmd/download.go
index ea4f9f8663..a5fc63ed26 100644
--- a/src/cmd/go/internal/modcmd/download.go
+++ b/src/cmd/go/internal/modcmd/download.go
@@ -13,6 +13,7 @@ import (
"cmd/go/internal/base"
"cmd/go/internal/cfg"
"cmd/go/internal/modfetch"
+ "cmd/go/internal/modfetch/codehost"
"cmd/go/internal/modload"
"golang.org/x/mod/module"
@@ -20,7 +21,7 @@ import (
)
var cmdDownload = &base.Command{
- UsageLine: "go mod download [-x] [-json] [modules]",
+ UsageLine: "go mod download [-x] [-json] [-reuse=old.json] [modules]",
Short: "download modules to local cache",
Long: `
Download downloads the named modules, which can be module patterns selecting
@@ -44,6 +45,7 @@ corresponding to this Go struct:
type Module struct {
Path string // module path
+ Query string // version query corresponding to this version
Version string // module version
Error string // error loading module
Info string // absolute path to cached .info file
@@ -52,8 +54,18 @@ corresponding to this Go struct:
Dir string // absolute path to cached source root directory
Sum string // checksum for path, version (as in go.sum)
GoModSum string // checksum for go.mod (as in go.sum)
+ Origin any // provenance of module
+ Reuse bool // reuse of old module info is safe
}
+The -reuse flag accepts the name of file containing the JSON output of a
+previous 'go mod download -json' invocation. The go command may use this
+file to determine that a module is unchanged since the previous invocation
+and avoid redownloading it. Modules that are not redownloaded will be marked
+in the new output by setting the Reuse field to true. Normally the module
+cache provides this kind of reuse automatically; the -reuse flag can be
+useful on systems that do not preserve the module cache.
+
The -x flag causes download to print the commands download executes.
See https://golang.org/ref/mod#go-mod-download for more about 'go mod download'.
@@ -62,7 +74,10 @@ See https://golang.org/ref/mod#version-queries for more about version queries.
`,
}
-var downloadJSON = cmdDownload.Flag.Bool("json", false, "")
+var (
+ downloadJSON = cmdDownload.Flag.Bool("json", false, "")
+ downloadReuse = cmdDownload.Flag.String("reuse", "", "")
+)
func init() {
cmdDownload.Run = runDownload // break init cycle
@@ -75,6 +90,7 @@ func init() {
type moduleJSON struct {
Path string `json:",omitempty"`
Version string `json:",omitempty"`
+ Query string `json:",omitempty"`
Error string `json:",omitempty"`
Info string `json:",omitempty"`
GoMod string `json:",omitempty"`
@@ -82,6 +98,9 @@ type moduleJSON struct {
Dir string `json:",omitempty"`
Sum string `json:",omitempty"`
GoModSum string `json:",omitempty"`
+
+ Origin *codehost.Origin `json:",omitempty"`
+ Reuse bool `json:",omitempty"`
}
func runDownload(ctx context.Context, cmd *base.Command, args []string) {
@@ -148,12 +167,12 @@ func runDownload(ctx context.Context, cmd *base.Command, args []string) {
}
downloadModule := func(m *moduleJSON) {
- var err error
- _, m.Info, err = modfetch.InfoFile(m.Path, m.Version)
+ _, file, err := modfetch.InfoFile(m.Path, m.Version)
if err != nil {
m.Error = err.Error()
return
}
+ m.Info = file
m.GoMod, err = modfetch.GoModFile(m.Path, m.Version)
if err != nil {
m.Error = err.Error()
@@ -179,9 +198,14 @@ func runDownload(ctx context.Context, cmd *base.Command, args []string) {
}
var mods []*moduleJSON
+
+ if *downloadReuse != "" && modload.HasModRoot() {
+ base.Fatalf("go mod download -reuse cannot be used inside a module")
+ }
+
type token struct{}
sem := make(chan token, runtime.GOMAXPROCS(0))
- infos, infosErr := modload.ListModules(ctx, args, 0)
+ infos, infosErr := modload.ListModules(ctx, args, 0, *downloadReuse)
if !haveExplicitArgs {
// 'go mod download' is sometimes run without arguments to pre-populate the
// module cache. It may fetch modules that aren't needed to build packages
@@ -209,12 +233,18 @@ func runDownload(ctx context.Context, cmd *base.Command, args []string) {
m := &moduleJSON{
Path: info.Path,
Version: info.Version,
+ Query: info.Query,
+ Reuse: info.Reuse,
+ Origin: info.Origin,
}
mods = append(mods, m)
if info.Error != nil {
m.Error = info.Error.Err
continue
}
+ if m.Reuse {
+ continue
+ }
sem <- token{}
go func() {
downloadModule(m)
diff --git a/src/cmd/go/internal/modcmd/why.go b/src/cmd/go/internal/modcmd/why.go
index 2d3f1eb05b..8e929a0001 100644
--- a/src/cmd/go/internal/modcmd/why.go
+++ b/src/cmd/go/internal/modcmd/why.go
@@ -82,7 +82,7 @@ func runWhy(ctx context.Context, cmd *base.Command, args []string) {
}
}
- mods, err := modload.ListModules(ctx, args, 0)
+ mods, err := modload.ListModules(ctx, args, 0, "")
if err != nil {
base.Fatalf("go: %v", err)
}
diff --git a/src/cmd/go/internal/modfetch/cache.go b/src/cmd/go/internal/modfetch/cache.go
index 417c5598fb..7ebe208c12 100644
--- a/src/cmd/go/internal/modfetch/cache.go
+++ b/src/cmd/go/internal/modfetch/cache.go
@@ -573,6 +573,26 @@ func writeDiskStat(file string, info *RevInfo) error {
if file == "" {
return nil
}
+
+ if info.Origin != nil {
+ // Clean the origin information, which might have too many
+ // validation criteria, for example if we are saving the result of
+ // m@master as m@pseudo-version.
+ clean := *info
+ info = &clean
+ o := *info.Origin
+ info.Origin = &o
+
+ // Tags never matter if you are starting with a semver version,
+ // as we would be when finding this cache entry.
+ o.TagSum = ""
+ o.TagPrefix = ""
+ // Ref doesn't matter if you have a pseudoversion.
+ if module.IsPseudoVersion(info.Version) {
+ o.Ref = ""
+ }
+ }
+
js, err := json.Marshal(info)
if err != nil {
return err
diff --git a/src/cmd/go/internal/modfetch/codehost/codehost.go b/src/cmd/go/internal/modfetch/codehost/codehost.go
index 3d9eb0c712..937ac6819a 100644
--- a/src/cmd/go/internal/modfetch/codehost/codehost.go
+++ b/src/cmd/go/internal/modfetch/codehost/codehost.go
@@ -113,20 +113,17 @@ type Origin struct {
}
// Checkable reports whether the Origin contains anything that can be checked.
-// If not, it's purely informational and should fail a CheckReuse call.
+// If not, the Origin is purely informational and should fail a CheckReuse call.
func (o *Origin) Checkable() bool {
return o.TagSum != "" || o.Ref != "" || o.Hash != ""
}
-func (o *Origin) Merge(other *Origin) {
- if o.TagSum == "" {
- o.TagPrefix = other.TagPrefix
- o.TagSum = other.TagSum
- }
- if o.Ref == "" {
- o.Ref = other.Ref
- o.Hash = other.Hash
- }
+// ClearCheckable clears the Origin enough to make Checkable return false.
+func (o *Origin) ClearCheckable() {
+ o.TagSum = ""
+ o.TagPrefix = ""
+ o.Ref = ""
+ o.Hash = ""
}
// A Tags describes the available tags in a code repository.
diff --git a/src/cmd/go/internal/modfetch/codehost/git.go b/src/cmd/go/internal/modfetch/codehost/git.go
index 3129a31786..a225aaf1ed 100644
--- a/src/cmd/go/internal/modfetch/codehost/git.go
+++ b/src/cmd/go/internal/modfetch/codehost/git.go
@@ -423,8 +423,11 @@ func (r *gitRepo) stat(rev string) (info *RevInfo, err error) {
defer func() {
if info != nil {
- info.Origin.Ref = ref
info.Origin.Hash = info.Name
+ // There's a ref = hash below; don't write that hash down as Origin.Ref.
+ if ref != info.Origin.Hash {
+ info.Origin.Ref = ref
+ }
}
}()
diff --git a/src/cmd/go/internal/modfetch/coderepo.go b/src/cmd/go/internal/modfetch/coderepo.go
index a994f79d4b..86e3ee9d1c 100644
--- a/src/cmd/go/internal/modfetch/coderepo.go
+++ b/src/cmd/go/internal/modfetch/coderepo.go
@@ -153,6 +153,9 @@ func (r *codeRepo) Versions(prefix string) (*Versions, error) {
Err: err,
}
}
+ if tags.Origin != nil {
+ tags.Origin.Subdir = r.codeDir
+ }
var list, incompatible []string
for _, tag := range tags.List {
@@ -450,23 +453,26 @@ func (r *codeRepo) convert(info *codehost.RevInfo, statVers string) (*RevInfo, e
}
origin := info.Origin
- if module.IsPseudoVersion(v) {
- // Add tags that are relevant to pseudo-version calculation to origin.
- prefix := ""
- if r.codeDir != "" {
- prefix = r.codeDir + "/"
- }
- if r.pathMajor != "" { // "/v2" or "/.v2"
- prefix += r.pathMajor[1:] + "." // += "v2."
- }
- tags, err := r.code.Tags(prefix)
- if err != nil {
- return nil, err
- }
+ if origin != nil {
o := *origin
origin = &o
- origin.TagPrefix = tags.Origin.TagPrefix
- origin.TagSum = tags.Origin.TagSum
+ origin.Subdir = r.codeDir
+ if module.IsPseudoVersion(v) && (v != statVers || !strings.HasPrefix(v, "v0.0.0-")) {
+ // Add tags that are relevant to pseudo-version calculation to origin.
+ prefix := r.codeDir
+ if prefix != "" {
+ prefix += "/"
+ }
+ if r.pathMajor != "" { // "/v2" or "/.v2"
+ prefix += r.pathMajor[1:] + "." // += "v2."
+ }
+ tags, err := r.code.Tags(prefix)
+ if err != nil {
+ return nil, err
+ }
+ origin.TagPrefix = tags.Origin.TagPrefix
+ origin.TagSum = tags.Origin.TagSum
+ }
}
return &RevInfo{
diff --git a/src/cmd/go/internal/modinfo/info.go b/src/cmd/go/internal/modinfo/info.go
index 19088352f0..b0adcbcfb3 100644
--- a/src/cmd/go/internal/modinfo/info.go
+++ b/src/cmd/go/internal/modinfo/info.go
@@ -4,7 +4,11 @@
package modinfo
-import "time"
+import (
+ "cmd/go/internal/modfetch/codehost"
+ "encoding/json"
+ "time"
+)
// Note that these structs are publicly visible (part of go list's API)
// and the fields are documented in the help text in ../list/list.go
@@ -12,6 +16,7 @@ import "time"
type ModulePublic struct {
Path string `json:",omitempty"` // module path
Version string `json:",omitempty"` // module version
+ Query string `json:",omitempty"` // version query corresponding to this version
Versions []string `json:",omitempty"` // available module versions
Replace *ModulePublic `json:",omitempty"` // replaced by this module
Time *time.Time `json:",omitempty"` // time version was created
@@ -24,12 +29,27 @@ type ModulePublic struct {
Retracted []string `json:",omitempty"` // retraction information, if any (with -retracted or -u)
Deprecated string `json:",omitempty"` // deprecation message, if any (with -u)
Error *ModuleError `json:",omitempty"` // error loading module
+
+ Origin *codehost.Origin `json:",omitempty"` // provenance of module
+ Reuse bool `json:",omitempty"` // reuse of old module info is safe
}
type ModuleError struct {
Err string // error text
}
+type moduleErrorNoMethods ModuleError
+
+// UnmarshalJSON accepts both {"Err":"text"} and "text",
+// so that the output of go mod download -json can still
+// be unmarshalled into a ModulePublic during -reuse processing.
+func (e *ModuleError) UnmarshalJSON(data []byte) error {
+ if len(data) > 0 && data[0] == '"' {
+ return json.Unmarshal(data, &e.Err)
+ }
+ return json.Unmarshal(data, (*moduleErrorNoMethods)(e))
+}
+
func (m *ModulePublic) String() string {
s := m.Path
versionString := func(mm *ModulePublic) string {
diff --git a/src/cmd/go/internal/modload/build.go b/src/cmd/go/internal/modload/build.go
index 0799fec35c..e983e0ae0c 100644
--- a/src/cmd/go/internal/modload/build.go
+++ b/src/cmd/go/internal/modload/build.go
@@ -17,6 +17,7 @@ import (
"cmd/go/internal/base"
"cmd/go/internal/cfg"
"cmd/go/internal/modfetch"
+ "cmd/go/internal/modfetch/codehost"
"cmd/go/internal/modindex"
"cmd/go/internal/modinfo"
"cmd/go/internal/search"
@@ -60,7 +61,7 @@ func PackageModuleInfo(ctx context.Context, pkgpath string) *modinfo.ModulePubli
}
rs := LoadModFile(ctx)
- return moduleInfo(ctx, rs, m, 0)
+ return moduleInfo(ctx, rs, m, 0, nil)
}
// PackageModRoot returns the module root directory for the module that provides
@@ -90,7 +91,7 @@ func ModuleInfo(ctx context.Context, path string) *modinfo.ModulePublic {
if i := strings.Index(path, "@"); i >= 0 {
m := module.Version{Path: path[:i], Version: path[i+1:]}
- return moduleInfo(ctx, nil, m, 0)
+ return moduleInfo(ctx, nil, m, 0, nil)
}
rs := LoadModFile(ctx)
@@ -119,7 +120,7 @@ func ModuleInfo(ctx context.Context, path string) *modinfo.ModulePublic {
}
}
- return moduleInfo(ctx, rs, module.Version{Path: path, Version: v}, 0)
+ return moduleInfo(ctx, rs, module.Version{Path: path, Version: v}, 0, nil)
}
// addUpdate fills in m.Update if an updated version is available.
@@ -156,6 +157,45 @@ func addUpdate(ctx context.Context, m *modinfo.ModulePublic) {
}
}
+// mergeOrigin merges two origins,
+// returning and possibly modifying one of its arguments.
+// If the two origins conflict, mergeOrigin returns a non-specific one
+// that will not pass CheckReuse.
+// If m1 or m2 is nil, the other is returned unmodified.
+// But if m1 or m2 is non-nil and uncheckable, the result is also uncheckable,
+// to preserve uncheckability.
+func mergeOrigin(m1, m2 *codehost.Origin) *codehost.Origin {
+ if m1 == nil {
+ return m2
+ }
+ if m2 == nil {
+ return m1
+ }
+ if !m1.Checkable() {
+ return m1
+ }
+ if !m2.Checkable() {
+ return m2
+ }
+ if m2.TagSum != "" {
+ if m1.TagSum != "" && (m1.TagSum != m2.TagSum || m1.TagPrefix != m2.TagPrefix) {
+ m1.ClearCheckable()
+ return m1
+ }
+ m1.TagSum = m2.TagSum
+ m1.TagPrefix = m2.TagPrefix
+ }
+ if m2.Hash != "" {
+ if m1.Hash != "" && (m1.Hash != m2.Hash || m1.Ref != m2.Ref) {
+ m1.ClearCheckable()
+ return m1
+ }
+ m1.Hash = m2.Hash
+ m1.Ref = m2.Ref
+ }
+ return m1
+}
+
// addVersions fills in m.Versions with the list of known versions.
// Excluded versions will be omitted. If listRetracted is false, retracted
// versions will also be omitted.
@@ -164,11 +204,12 @@ func addVersions(ctx context.Context, m *modinfo.ModulePublic, listRetracted boo
if listRetracted {
allowed = CheckExclusions
}
- var err error
- m.Versions, err = versions(ctx, m.Path, allowed)
+ v, origin, err := versions(ctx, m.Path, allowed)
if err != nil && m.Error == nil {
m.Error = &modinfo.ModuleError{Err: err.Error()}
}
+ m.Versions = v
+ m.Origin = mergeOrigin(m.Origin, origin)
}
// addRetraction fills in m.Retracted if the module was retracted by its author.
@@ -230,7 +271,7 @@ func addDeprecation(ctx context.Context, m *modinfo.ModulePublic) {
// moduleInfo returns information about module m, loaded from the requirements
// in rs (which may be nil to indicate that m was not loaded from a requirement
// graph).
-func moduleInfo(ctx context.Context, rs *Requirements, m module.Version, mode ListMode) *modinfo.ModulePublic {
+func moduleInfo(ctx context.Context, rs *Requirements, m module.Version, mode ListMode, reuse map[module.Version]*modinfo.ModulePublic) *modinfo.ModulePublic {
if m.Version == "" && MainModules.Contains(m.Path) {
info := &modinfo.ModulePublic{
Path: m.Path,
@@ -260,6 +301,15 @@ func moduleInfo(ctx context.Context, rs *Requirements, m module.Version, mode Li
// completeFromModCache fills in the extra fields in m using the module cache.
completeFromModCache := func(m *modinfo.ModulePublic) {
+ if old := reuse[module.Version{Path: m.Path, Version: m.Version}]; old != nil {
+ if err := checkReuse(ctx, m.Path, old.Origin); err == nil {
+ *m = *old
+ m.Query = ""
+ m.Dir = ""
+ return
+ }
+ }
+
checksumOk := func(suffix string) bool {
return rs == nil || m.Version == "" || cfg.BuildMod == "mod" ||
modfetch.HaveSum(module.Version{Path: m.Path, Version: m.Version + suffix})
diff --git a/src/cmd/go/internal/modload/edit.go b/src/cmd/go/internal/modload/edit.go
index c556664c35..f6937a48b4 100644
--- a/src/cmd/go/internal/modload/edit.go
+++ b/src/cmd/go/internal/modload/edit.go
@@ -509,7 +509,7 @@ func (l *versionLimiter) UpgradeToward(ctx context.Context, m module.Version) er
}
if l.check(m, l.pruning).isDisqualified() {
- candidates, err := versions(ctx, m.Path, CheckAllowed)
+ candidates, _, err := versions(ctx, m.Path, CheckAllowed)
if err != nil {
// This is likely a transient error reaching the repository,
// rather than a permanent error with the retrieved version.
diff --git a/src/cmd/go/internal/modload/list.go b/src/cmd/go/internal/modload/list.go
index f782cd93db..e822d06504 100644
--- a/src/cmd/go/internal/modload/list.go
+++ b/src/cmd/go/internal/modload/list.go
@@ -5,15 +5,19 @@
package modload
import (
+ "bytes"
"context"
+ "encoding/json"
"errors"
"fmt"
+ "io"
"os"
"runtime"
"strings"
"cmd/go/internal/base"
"cmd/go/internal/cfg"
+ "cmd/go/internal/modfetch/codehost"
"cmd/go/internal/modinfo"
"cmd/go/internal/search"
@@ -34,13 +38,44 @@ const (
// along with any error preventing additional matches from being identified.
//
// The returned slice can be nonempty even if the error is non-nil.
-func ListModules(ctx context.Context, args []string, mode ListMode) ([]*modinfo.ModulePublic, error) {
- rs, mods, err := listModules(ctx, LoadModFile(ctx), args, mode)
+func ListModules(ctx context.Context, args []string, mode ListMode, reuseFile string) ([]*modinfo.ModulePublic, error) {
+ var reuse map[module.Version]*modinfo.ModulePublic
+ if reuseFile != "" {
+ data, err := os.ReadFile(reuseFile)
+ if err != nil {
+ return nil, err
+ }
+ dec := json.NewDecoder(bytes.NewReader(data))
+ reuse = make(map[module.Version]*modinfo.ModulePublic)
+ for {
+ var m modinfo.ModulePublic
+ if err := dec.Decode(&m); err != nil {
+ if err == io.EOF {
+ break
+ }
+ return nil, fmt.Errorf("parsing %s: %v", reuseFile, err)
+ }
+ if m.Origin == nil || !m.Origin.Checkable() {
+ // Nothing to check to validate reuse.
+ continue
+ }
+ m.Reuse = true
+ reuse[module.Version{Path: m.Path, Version: m.Version}] = &m
+ if m.Query != "" {
+ reuse[module.Version{Path: m.Path, Version: m.Query}] = &m
+ }
+ }
+ }
+
+ rs, mods, err := listModules(ctx, LoadModFile(ctx), args, mode, reuse)
type token struct{}
sem := make(chan token, runtime.GOMAXPROCS(0))
if mode != 0 {
for _, m := range mods {
+ if m.Reuse {
+ continue
+ }
add := func(m *modinfo.ModulePublic) {
sem <- token{}
go func() {
@@ -80,11 +115,11 @@ func ListModules(ctx context.Context, args []string, mode ListMode) ([]*modinfo.
return mods, err
}
-func listModules(ctx context.Context, rs *Requirements, args []string, mode ListMode) (_ *Requirements, mods []*modinfo.ModulePublic, mgErr error) {
+func listModules(ctx context.Context, rs *Requirements, args []string, mode ListMode, reuse map[module.Version]*modinfo.ModulePublic) (_ *Requirements, mods []*modinfo.ModulePublic, mgErr error) {
if len(args) == 0 {
var ms []*modinfo.ModulePublic
for _, m := range MainModules.Versions() {
- ms = append(ms, moduleInfo(ctx, rs, m, mode))
+ ms = append(ms, moduleInfo(ctx, rs, m, mode, reuse))
}
return rs, ms, nil
}
@@ -157,12 +192,17 @@ func listModules(ctx context.Context, rs *Requirements, args []string, mode List
// specific revision or used 'go list -retracted'.
allowed = nil
}
- info, err := Query(ctx, path, vers, current, allowed)
+ info, err := queryReuse(ctx, path, vers, current, allowed, reuse)
if err != nil {
+ var origin *codehost.Origin
+ if info != nil {
+ origin = info.Origin
+ }
mods = append(mods, &modinfo.ModulePublic{
Path: path,
Version: vers,
Error: modinfoError(path, vers, err),
+ Origin: origin,
})
continue
}
@@ -171,7 +211,11 @@ func listModules(ctx context.Context, rs *Requirements, args []string, mode List
// *Requirements instead.
var noRS *Requirements
- mod := moduleInfo(ctx, noRS, module.Version{Path: path, Version: info.Version}, mode)
+ mod := moduleInfo(ctx, noRS, module.Version{Path: path, Version: info.Version}, mode, reuse)
+ if vers != mod.Version {
+ mod.Query = vers
+ }
+ mod.Origin = info.Origin
mods = append(mods, mod)
continue
}
@@ -200,7 +244,7 @@ func listModules(ctx context.Context, rs *Requirements, args []string, mode List
continue
}
if v != "none" {
- mods = append(mods, moduleInfo(ctx, rs, module.Version{Path: arg, Version: v}, mode))
+ mods = append(mods, moduleInfo(ctx, rs, module.Version{Path: arg, Version: v}, mode, reuse))
} else if cfg.BuildMod == "vendor" {
// In vendor mode, we can't determine whether a missing module is “a
// known dependency” because the module graph is incomplete.
@@ -229,7 +273,7 @@ func listModules(ctx context.Context, rs *Requirements, args []string, mode List
matched = true
if !matchedModule[m] {
matchedModule[m] = true
- mods = append(mods, moduleInfo(ctx, rs, m, mode))
+ mods = append(mods, moduleInfo(ctx, rs, m, mode, reuse))
}
}
}
diff --git a/src/cmd/go/internal/modload/mvs.go b/src/cmd/go/internal/modload/mvs.go
index 2055303efe..ea1c21b4f1 100644
--- a/src/cmd/go/internal/modload/mvs.go
+++ b/src/cmd/go/internal/modload/mvs.go
@@ -11,6 +11,7 @@ import (
"sort"
"cmd/go/internal/modfetch"
+ "cmd/go/internal/modfetch/codehost"
"golang.org/x/mod/module"
"golang.org/x/mod/semver"
@@ -78,11 +79,10 @@ func (*mvsReqs) Upgrade(m module.Version) (module.Version, error) {
return m, nil
}
-func versions(ctx context.Context, path string, allowed AllowedFunc) ([]string, error) {
+func versions(ctx context.Context, path string, allowed AllowedFunc) (versions []string, origin *codehost.Origin, err error) {
// Note: modfetch.Lookup and repo.Versions are cached,
// so there's no need for us to add extra caching here.
- var versions []string
- err := modfetch.TryProxies(func(proxy string) error {
+ err = modfetch.TryProxies(func(proxy string) error {
repo, err := lookupRepo(proxy, path)
if err != nil {
return err
@@ -100,9 +100,10 @@ func versions(ctx context.Context, path string, allowed AllowedFunc) ([]string,
}
}
versions = allowedVersions
+ origin = allVersions.Origin
return nil
})
- return versions, err
+ return versions, origin, err
}
// previousVersion returns the tagged version of m.Path immediately prior to
@@ -117,7 +118,7 @@ func previousVersion(m module.Version) (module.Version, error) {
return module.Version{Path: m.Path, Version: "none"}, nil
}
- list, err := versions(context.TODO(), m.Path, CheckAllowed)
+ list, _, err := versions(context.TODO(), m.Path, CheckAllowed)
if err != nil {
if errors.Is(err, os.ErrNotExist) {
return module.Version{Path: m.Path, Version: "none"}, nil
diff --git a/src/cmd/go/internal/modload/query.go b/src/cmd/go/internal/modload/query.go
index 051a4fe822..1d2f5d5e15 100644
--- a/src/cmd/go/internal/modload/query.go
+++ b/src/cmd/go/internal/modload/query.go
@@ -20,6 +20,8 @@ import (
"cmd/go/internal/cfg"
"cmd/go/internal/imports"
"cmd/go/internal/modfetch"
+ "cmd/go/internal/modfetch/codehost"
+ "cmd/go/internal/modinfo"
"cmd/go/internal/search"
"cmd/go/internal/str"
"cmd/go/internal/trace"
@@ -72,18 +74,39 @@ import (
//
// If path is the path of the main module and the query is "latest",
// Query returns Target.Version as the version.
+//
+// Query often returns a non-nil *RevInfo with a non-nil error,
+// to provide an info.Origin that can allow the error to be cached.
func Query(ctx context.Context, path, query, current string, allowed AllowedFunc) (*modfetch.RevInfo, error) {
ctx, span := trace.StartSpan(ctx, "modload.Query "+path)
defer span.Done()
+ return queryReuse(ctx, path, query, current, allowed, nil)
+}
+
+// queryReuse is like Query but also takes a map of module info that can be reused
+// if the validation criteria in Origin are met.
+func queryReuse(ctx context.Context, path, query, current string, allowed AllowedFunc, reuse map[module.Version]*modinfo.ModulePublic) (*modfetch.RevInfo, error) {
var info *modfetch.RevInfo
err := modfetch.TryProxies(func(proxy string) (err error) {
- info, err = queryProxy(ctx, proxy, path, query, current, allowed)
+ info, err = queryProxy(ctx, proxy, path, query, current, allowed, reuse)
return err
})
return info, err
}
+// checkReuse checks whether a revision of a given module or a version list
+// for a given module may be reused, according to the information in origin.
+func checkReuse(ctx context.Context, path string, old *codehost.Origin) error {
+ return modfetch.TryProxies(func(proxy string) error {
+ repo, err := lookupRepo(proxy, path)
+ if err != nil {
+ return err
+ }
+ return repo.CheckReuse(old)
+ })
+}
+
// AllowedFunc is used by Query and other functions to filter out unsuitable
// versions, for example, those listed in exclude directives in the main
// module's go.mod file.
@@ -106,7 +129,7 @@ func (queryDisabledError) Error() string {
return fmt.Sprintf("cannot query module due to -mod=%s\n\t(%s)", cfg.BuildMod, cfg.BuildModReason)
}
-func queryProxy(ctx context.Context, proxy, path, query, current string, allowed AllowedFunc) (*modfetch.RevInfo, error) {
+func queryProxy(ctx context.Context, proxy, path, query, current string, allowed AllowedFunc, reuse map[module.Version]*modinfo.ModulePublic) (*modfetch.RevInfo, error) {
ctx, span := trace.StartSpan(ctx, "modload.queryProxy "+path+" "+query)
defer span.Done()
@@ -137,6 +160,19 @@ func queryProxy(ctx context.Context, proxy, path, query, current string, allowed
return nil, err
}
+ if old := reuse[module.Version{Path: path, Version: query}]; old != nil {
+ if err := repo.CheckReuse(old.Origin); err == nil {
+ info := &modfetch.RevInfo{
+ Version: old.Version,
+ Origin: old.Origin,
+ }
+ if old.Time != nil {
+ info.Time = *old.Time
+ }
+ return info, nil
+ }
+ }
+
// Parse query to detect parse errors (and possibly handle query)
// before any network I/O.
qm, err := newQueryMatcher(path, query, current, allowed)
@@ -177,15 +213,23 @@ func queryProxy(ctx context.Context, proxy, path, query, current string, allowed
if err != nil {
return nil, err
}
+ revErr := &modfetch.RevInfo{Origin: versions.Origin} // RevInfo to return with error
+
releases, prereleases, err := qm.filterVersions(ctx, versions.List)
if err != nil {
- return nil, err
+ return revErr, err
}
lookup := func(v string) (*modfetch.RevInfo, error) {
rev, err := repo.Stat(v)
+ // Stat can return a non-nil rev and a non-nil err,
+ // in order to provide origin information to make the error cacheable.
+ if rev == nil && err != nil {
+ return revErr, err
+ }
+ rev.Origin = mergeOrigin(rev.Origin, versions.Origin)
if err != nil {
- return nil, err
+ return rev, err
}
if (query == "upgrade" || query == "patch") && module.IsPseudoVersion(current) && !rev.Time.IsZero() {
@@ -210,9 +254,14 @@ func queryProxy(ctx context.Context, proxy, path, query, current string, allowed
currentTime, err := module.PseudoVersionTime(current)
if err == nil && rev.Time.Before(currentTime) {
if err := allowed(ctx, module.Version{Path: path, Version: current}); errors.Is(err, ErrDisallowed) {
- return nil, err
+ return revErr, err
}
- return repo.Stat(current)
+ info, err := repo.Stat(current)
+ if info == nil && err != nil {
+ return revErr, err
+ }
+ info.Origin = mergeOrigin(info.Origin, versions.Origin)
+ return info, err
}
}
@@ -242,7 +291,7 @@ func queryProxy(ctx context.Context, proxy, path, query, current string, allowed
return lookup(latest.Version)
}
} else if !errors.Is(err, fs.ErrNotExist) {
- return nil, err
+ return revErr, err
}
}
@@ -254,7 +303,7 @@ func queryProxy(ctx context.Context, proxy, path, query, current string, allowed
return lookup(current)
}
- return nil, &NoMatchingVersionError{query: query, current: current}
+ return revErr, &NoMatchingVersionError{query: query, current: current}
}
// IsRevisionQuery returns true if vers is a version query that may refer to
@@ -663,7 +712,7 @@ func QueryPattern(ctx context.Context, pattern, query string, current func(strin
pathCurrent := current(path)
r.Mod.Path = path
- r.Rev, err = queryProxy(ctx, proxy, path, query, pathCurrent, allowed)
+ r.Rev, err = queryProxy(ctx, proxy, path, query, pathCurrent, allowed, nil)
if err != nil {
return r, err
}
@@ -991,6 +1040,7 @@ func versionHasGoMod(_ context.Context, m module.Version) (bool, error) {
// available versions, but cannot fetch specific source files.
type versionRepo interface {
ModulePath() string
+ CheckReuse(*codehost.Origin) error
Versions(prefix string) (*modfetch.Versions, error)
Stat(rev string) (*modfetch.RevInfo, error)
Latest() (*modfetch.RevInfo, error)
@@ -1024,6 +1074,9 @@ type emptyRepo struct {
var _ versionRepo = emptyRepo{}
func (er emptyRepo) ModulePath() string { return er.path }
+func (er emptyRepo) CheckReuse(old *codehost.Origin) error {
+ return fmt.Errorf("empty repo")
+}
func (er emptyRepo) Versions(prefix string) (*modfetch.Versions, error) {
return &modfetch.Versions{}, nil
}
@@ -1044,6 +1097,10 @@ var _ versionRepo = (*replacementRepo)(nil)
func (rr *replacementRepo) ModulePath() string { return rr.repo.ModulePath() }
+func (rr *replacementRepo) CheckReuse(old *codehost.Origin) error {
+ return fmt.Errorf("replacement repo")
+}
+
// Versions returns the versions from rr.repo augmented with any matching
// replacement versions.
func (rr *replacementRepo) Versions(prefix string) (*modfetch.Versions, error) {
diff --git a/src/cmd/go/testdata/script/reuse_git.txt b/src/cmd/go/testdata/script/reuse_git.txt
new file mode 100644
index 0000000000..7d8844d932
--- /dev/null
+++ b/src/cmd/go/testdata/script/reuse_git.txt
@@ -0,0 +1,371 @@
+[short] skip
+[!exec:git] skip
+[!net] skip
+
+env GO111MODULE=on
+env GOPROXY=direct
+env GOSUMDB=off
+
+# go mod download with the pseudo-version should invoke git but not have a TagSum or Ref.
+go mod download -x -json vcs-test.golang.org/git/hello.git@v0.0.0-20170922010558-fc3a09f3dc5c
+stderr 'git fetch'
+cp stdout hellopseudo.json
+! stdout '"(Query|TagPrefix|TagSum|Ref)"'
+stdout '"Version": "v0.0.0-20170922010558-fc3a09f3dc5c"'
+stdout '"VCS": "git"'
+stdout '"URL": "https://vcs-test.golang.org/git/hello"'
+stdout '"Hash": "fc3a09f3dc5cfe0d7a743ea18f1f5226e68b3777"'
+go clean -modcache
+
+# go mod download vcstest/hello should invoke git, print origin info
+go mod download -x -json vcs-test.golang.org/git/hello.git@latest
+stderr 'git fetch'
+cp stdout hello.json
+stdout '"Version": "v0.0.0-20170922010558-fc3a09f3dc5c"'
+stdout '"VCS": "git"'
+stdout '"URL": "https://vcs-test.golang.org/git/hello"'
+stdout '"Query": "latest"'
+! stdout '"TagPrefix"'
+stdout '"TagSum": "t1:47DEQpj8HBSa[+]/TImW[+]5JCeuQeRkm5NMpJWZG3hSuFU="'
+stdout '"Ref": "HEAD"'
+stdout '"Hash": "fc3a09f3dc5cfe0d7a743ea18f1f5226e68b3777"'
+
+# pseudo-version again should not invoke git fetch (it has the version from the @latest query)
+# but still be careful not to include a TagSum or a Ref, especially not Ref set to HEAD,
+# which is easy to do when reusing the cached version from the @latest query.
+go mod download -x -json vcs-test.golang.org/git/hello.git@v0.0.0-20170922010558-fc3a09f3dc5c
+! stderr 'git fetch'
+cp stdout hellopseudo2.json
+cmp hellopseudo.json hellopseudo2.json
+
+# go mod download vcstest/hello@hash needs to check TagSum to find pseudoversion base.
+go mod download -x -json vcs-test.golang.org/git/hello.git@fc3a09f3dc5c
+! stderr 'git fetch'
+cp stdout hellohash.json
+stdout '"Version": "v0.0.0-20170922010558-fc3a09f3dc5c"'
+stdout '"Query": "fc3a09f3dc5c"'
+stdout '"VCS": "git"'
+stdout '"URL": "https://vcs-test.golang.org/git/hello"'
+stdout '"TagSum": "t1:47DEQpj8HBSa[+]/TImW[+]5JCeuQeRkm5NMpJWZG3hSuFU="'
+stdout '"Hash": "fc3a09f3dc5cfe0d7a743ea18f1f5226e68b3777"'
+
+# go mod download vcstest/hello/v9 should fail, still print origin info
+! go mod download -x -json vcs-test.golang.org/git/hello.git/v9@latest
+cp stdout hellov9.json
+stdout '"Version": "latest"'
+stdout '"Error":.*no matching versions'
+! stdout '"TagPrefix"'
+stdout '"TagSum": "t1:47DEQpj8HBSa[+]/TImW[+]5JCeuQeRkm5NMpJWZG3hSuFU="'
+! stdout '"Ref":'
+! stdout '"Hash":'
+
+# go mod download vcstest/hello/sub/v9 should also fail, print origin info with TagPrefix
+! go mod download -x -json vcs-test.golang.org/git/hello.git/sub/v9@latest
+cp stdout hellosubv9.json
+stdout '"Version": "latest"'
+stdout '"Error":.*no matching versions'
+stdout '"TagPrefix": "sub/"'
+stdout '"TagSum": "t1:47DEQpj8HBSa[+]/TImW[+]5JCeuQeRkm5NMpJWZG3hSuFU="'
+! stdout '"Ref":'
+! stdout '"Hash":'
+
+# go mod download vcstest/tagtests should invoke git, print origin info
+go mod download -x -json vcs-test.golang.org/git/tagtests.git@latest
+stderr 'git fetch'
+cp stdout tagtests.json
+stdout '"Version": "v0.2.2"'
+stdout '"Query": "latest"'
+stdout '"VCS": "git"'
+stdout '"URL": "https://vcs-test.golang.org/git/tagtests"'
+! stdout '"TagPrefix"'
+stdout '"TagSum": "t1:Dp7yRKDuE8WjG0429PN9hYWjqhy2te7P9Oki/sMEOGo="'
+stdout '"Ref": "refs/tags/v0.2.2"'
+stdout '"Hash": "59356c8cd18c5fe9a598167d98a6843e52d57952"'
+
+# go mod download vcstest/tagtests@v0.2.2 should print origin info, no TagSum needed
+go mod download -x -json vcs-test.golang.org/git/tagtests.git@v0.2.2
+cp stdout tagtestsv022.json
+stdout '"Version": "v0.2.2"'
+! stdout '"Query":'
+stdout '"VCS": "git"'
+stdout '"URL": "https://vcs-test.golang.org/git/tagtests"'
+! stdout '"TagPrefix"'
+! stdout '"TagSum"'
+stdout '"Ref": "refs/tags/v0.2.2"'
+stdout '"Hash": "59356c8cd18c5fe9a598167d98a6843e52d57952"'
+
+# go mod download vcstest/tagtests@master needs a TagSum again
+go mod download -x -json vcs-test.golang.org/git/tagtests.git@master
+cp stdout tagtestsmaster.json
+stdout '"Version": "v0.2.3-0.20190509225625-c7818c24fa2f"'
+stdout '"Query": "master"'
+stdout '"VCS": "git"'
+stdout '"URL": "https://vcs-test.golang.org/git/tagtests"'
+! stdout '"TagPrefix"'
+stdout '"TagSum": "t1:Dp7yRKDuE8WjG0429PN9hYWjqhy2te7P9Oki/sMEOGo="'
+stdout '"Ref": "refs/heads/master"'
+stdout '"Hash": "c7818c24fa2f3f714c67d0a6d3e411c85a518d1f"'
+
+# go mod download vcstest/prefixtagtests should invoke git, print origin info
+go mod download -x -json vcs-test.golang.org/git/prefixtagtests.git/sub@latest
+stderr 'git fetch'
+cp stdout prefixtagtests.json
+stdout '"Version": "v0.0.10"'
+stdout '"Query": "latest"'
+stdout '"VCS": "git"'
+stdout '"URL": "https://vcs-test.golang.org/git/prefixtagtests"'
+stdout '"Subdir": "sub"'
+stdout '"TagPrefix": "sub/"'
+stdout '"TagSum": "t1:YGSbWkJ8dn9ORAr[+]BlKHFK/2ZhXLb9hVuYfTZ9D8C7g="'
+stdout '"Ref": "refs/tags/sub/v0.0.10"'
+stdout '"Hash": "2b7c4692e12c109263cab51b416fcc835ddd7eae"'
+
+# go mod download of a bunch of these should fail (some are invalid) but write good JSON for later
+! go mod download -json vcs-test.golang.org/git/hello.git@latest vcs-test.golang.org/git/hello.git/v9@latest vcs-test.golang.org/git/hello.git/sub/v9@latest vcs-test.golang.org/git/tagtests.git@latest vcs-test.golang.org/git/tagtests.git@v0.2.2 vcs-test.golang.org/git/tagtests.git@master
+cp stdout all.json
+
+# clean the module cache, make sure that makes go mod download re-run git fetch, clean again
+go clean -modcache
+go mod download -x -json vcs-test.golang.org/git/hello.git@latest
+stderr 'git fetch'
+go clean -modcache
+
+# reuse go mod download vcstest/hello result
+go mod download -reuse=hello.json -x -json vcs-test.golang.org/git/hello.git@latest
+! stderr 'git fetch'
+stdout '"Reuse": true'
+stdout '"Version": "v0.0.0-20170922010558-fc3a09f3dc5c"'
+stdout '"VCS": "git"'
+stdout '"URL": "https://vcs-test.golang.org/git/hello"'
+! stdout '"TagPrefix"'
+stdout '"TagSum": "t1:47DEQpj8HBSa[+]/TImW[+]5JCeuQeRkm5NMpJWZG3hSuFU="'
+stdout '"Ref": "HEAD"'
+stdout '"Hash": "fc3a09f3dc5cfe0d7a743ea18f1f5226e68b3777"'
+! stdout '"Dir"'
+! stdout '"Info"'
+! stdout '"GoMod"'
+! stdout '"Zip"'
+
+# reuse go mod download vcstest/hello pseudoversion result
+go mod download -reuse=hellopseudo.json -x -json vcs-test.golang.org/git/hello.git@v0.0.0-20170922010558-fc3a09f3dc5c
+! stderr 'git fetch'
+stdout '"Reuse": true'
+stdout '"Version": "v0.0.0-20170922010558-fc3a09f3dc5c"'
+stdout '"VCS": "git"'
+stdout '"URL": "https://vcs-test.golang.org/git/hello"'
+! stdout '"(Query|TagPrefix|TagSum|Ref)"'
+stdout '"Hash": "fc3a09f3dc5cfe0d7a743ea18f1f5226e68b3777"'
+! stdout '"(Dir|Info|GoMod|Zip)"'
+
+# reuse go mod download vcstest/hello@hash
+go mod download -reuse=hellohash.json -x -json vcs-test.golang.org/git/hello.git@fc3a09f3dc5c
+! stderr 'git fetch'
+stdout '"Reuse": true'
+stdout '"Query": "fc3a09f3dc5c"'
+stdout '"Version": "v0.0.0-20170922010558-fc3a09f3dc5c"'
+stdout '"VCS": "git"'
+stdout '"URL": "https://vcs-test.golang.org/git/hello"'
+! stdout '"(TagPrefix|Ref)"'
+stdout '"TagSum": "t1:47DEQpj8HBSa[+]/TImW[+]5JCeuQeRkm5NMpJWZG3hSuFU="'
+stdout '"Hash": "fc3a09f3dc5cfe0d7a743ea18f1f5226e68b3777"'
+! stdout '"(Dir|Info|GoMod|Zip)"'
+
+# reuse go mod download vcstest/hello/v9 error result
+! go mod download -reuse=hellov9.json -x -json vcs-test.golang.org/git/hello.git/v9@latest
+! stderr 'git fetch'
+stdout '"Reuse": true'
+stdout '"Error":.*no matching versions'
+! stdout '"TagPrefix"'
+stdout '"TagSum": "t1:47DEQpj8HBSa[+]/TImW[+]5JCeuQeRkm5NMpJWZG3hSuFU="'
+! stdout '"(Ref|Hash)":'
+! stdout '"(Dir|Info|GoMod|Zip)"'
+
+# reuse go mod download vcstest/hello/sub/v9 error result
+! go mod download -reuse=hellosubv9.json -x -json vcs-test.golang.org/git/hello.git/sub/v9@latest
+! stderr 'git fetch'
+stdout '"Reuse": true'
+stdout '"Error":.*no matching versions'
+stdout '"TagPrefix": "sub/"'
+stdout '"TagSum": "t1:47DEQpj8HBSa[+]/TImW[+]5JCeuQeRkm5NMpJWZG3hSuFU="'
+! stdout '"(Ref|Hash)":'
+! stdout '"(Dir|Info|GoMod|Zip)"'
+
+# reuse go mod download vcstest/tagtests result
+go mod download -reuse=tagtests.json -x -json vcs-test.golang.org/git/tagtests.git@latest
+! stderr 'git fetch'
+stdout '"Reuse": true'
+stdout '"Version": "v0.2.2"'
+stdout '"Query": "latest"'
+stdout '"VCS": "git"'
+stdout '"URL": "https://vcs-test.golang.org/git/tagtests"'
+! stdout '"TagPrefix"'
+stdout '"TagSum": "t1:Dp7yRKDuE8WjG0429PN9hYWjqhy2te7P9Oki/sMEOGo="'
+stdout '"Ref": "refs/tags/v0.2.2"'
+stdout '"Hash": "59356c8cd18c5fe9a598167d98a6843e52d57952"'
+! stdout '"(Dir|Info|GoMod|Zip)"'
+
+# reuse go mod download vcstest/tagtests@v0.2.2 result
+go mod download -reuse=tagtestsv022.json -x -json vcs-test.golang.org/git/tagtests.git@v0.2.2
+! stderr 'git fetch'
+stdout '"Reuse": true'
+stdout '"Version": "v0.2.2"'
+! stdout '"Query":'
+stdout '"VCS": "git"'
+stdout '"URL": "https://vcs-test.golang.org/git/tagtests"'
+! stdout '"TagPrefix"'
+! stdout '"TagSum"'
+stdout '"Ref": "refs/tags/v0.2.2"'
+stdout '"Hash": "59356c8cd18c5fe9a598167d98a6843e52d57952"'
+! stdout '"(Dir|Info|GoMod|Zip)"'
+
+# reuse go mod download vcstest/tagtests@master result
+go mod download -reuse=tagtestsmaster.json -x -json vcs-test.golang.org/git/tagtests.git@master
+! stderr 'git fetch'
+stdout '"Reuse": true'
+stdout '"Version": "v0.2.3-0.20190509225625-c7818c24fa2f"'
+stdout '"Query": "master"'
+stdout '"VCS": "git"'
+stdout '"URL": "https://vcs-test.golang.org/git/tagtests"'
+! stdout '"TagPrefix"'
+stdout '"TagSum": "t1:Dp7yRKDuE8WjG0429PN9hYWjqhy2te7P9Oki/sMEOGo="'
+stdout '"Ref": "refs/heads/master"'
+stdout '"Hash": "c7818c24fa2f3f714c67d0a6d3e411c85a518d1f"'
+! stdout '"(Dir|Info|GoMod|Zip)"'
+
+# reuse go mod download vcstest/tagtests@master result again with all.json
+go mod download -reuse=all.json -x -json vcs-test.golang.org/git/tagtests.git@master
+! stderr 'git fetch'
+stdout '"Reuse": true'
+stdout '"Version": "v0.2.3-0.20190509225625-c7818c24fa2f"'
+stdout '"Query": "master"'
+stdout '"VCS": "git"'
+stdout '"URL": "https://vcs-test.golang.org/git/tagtests"'
+! stdout '"TagPrefix"'
+stdout '"TagSum": "t1:Dp7yRKDuE8WjG0429PN9hYWjqhy2te7P9Oki/sMEOGo="'
+stdout '"Ref": "refs/heads/master"'
+stdout '"Hash": "c7818c24fa2f3f714c67d0a6d3e411c85a518d1f"'
+! stdout '"(Dir|Info|GoMod|Zip)"'
+
+# go mod download vcstest/prefixtagtests result with json
+go mod download -reuse=prefixtagtests.json -x -json vcs-test.golang.org/git/prefixtagtests.git/sub@latest
+! stderr 'git fetch'
+stdout '"Version": "v0.0.10"'
+stdout '"Query": "latest"'
+stdout '"VCS": "git"'
+stdout '"URL": "https://vcs-test.golang.org/git/prefixtagtests"'
+stdout '"Subdir": "sub"'
+stdout '"TagPrefix": "sub/"'
+stdout '"TagSum": "t1:YGSbWkJ8dn9ORAr[+]BlKHFK/2ZhXLb9hVuYfTZ9D8C7g="'
+stdout '"Ref": "refs/tags/sub/v0.0.10"'
+stdout '"Hash": "2b7c4692e12c109263cab51b416fcc835ddd7eae"'
+! stdout '"(Dir|Info|GoMod|Zip)"'
+
+# reuse the bulk results with all.json
+! go mod download -reuse=all.json -json vcs-test.golang.org/git/hello.git@latest vcs-test.golang.org/git/hello.git/v9@latest vcs-test.golang.org/git/hello.git/sub/v9@latest vcs-test.golang.org/git/tagtests.git@latest vcs-test.golang.org/git/tagtests.git@v0.2.2 vcs-test.golang.org/git/tagtests.git@master
+! stderr 'git fetch'
+stdout '"Reuse": true'
+! stdout '"(Dir|Info|GoMod|Zip)"'
+
+# reuse attempt with stale hash should reinvoke git, not report reuse
+go mod download -reuse=tagtestsv022badhash.json -x -json vcs-test.golang.org/git/tagtests.git@v0.2.2
+stderr 'git fetch'
+! stdout '"Reuse": true'
+stdout '"Version": "v0.2.2"'
+! stdout '"Query"'
+stdout '"VCS": "git"'
+stdout '"URL": "https://vcs-test.golang.org/git/tagtests"'
+! stdout '"(TagPrefix|TagSum)"'
+stdout '"Ref": "refs/tags/v0.2.2"'
+stdout '"Hash": "59356c8cd18c5fe9a598167d98a6843e52d57952"'
+stdout '"Dir"'
+stdout '"Info"'
+stdout '"GoMod"'
+stdout '"Zip"'
+
+# reuse with stale repo URL
+go mod download -reuse=tagtestsv022badurl.json -x -json vcs-test.golang.org/git/tagtests.git@v0.2.2
+! stdout '"Reuse": true'
+stdout '"URL": "https://vcs-test.golang.org/git/tagtests"'
+stdout '"Dir"'
+stdout '"Info"'
+stdout '"GoMod"'
+stdout '"Zip"'
+
+# reuse with stale VCS
+go mod download -reuse=tagtestsv022badvcs.json -x -json vcs-test.golang.org/git/tagtests.git@v0.2.2
+! stdout '"Reuse": true'
+stdout '"URL": "https://vcs-test.golang.org/git/tagtests"'
+
+# reuse with stale Dir
+go mod download -reuse=tagtestsv022baddir.json -x -json vcs-test.golang.org/git/tagtests.git@v0.2.2
+! stdout '"Reuse": true'
+stdout '"URL": "https://vcs-test.golang.org/git/tagtests"'
+
+# reuse with stale TagSum
+go mod download -reuse=tagtestsbadtagsum.json -x -json vcs-test.golang.org/git/tagtests.git@latest
+! stdout '"Reuse": true'
+stdout '"TagSum": "t1:Dp7yRKDuE8WjG0429PN9hYWjqhy2te7P9Oki/sMEOGo="'
+
+-- tagtestsv022badhash.json --
+{
+ "Path": "vcs-test.golang.org/git/tagtests.git",
+ "Version": "v0.2.2",
+ "Origin": {
+ "VCS": "git",
+ "URL": "https://vcs-test.golang.org/git/tagtests",
+ "Ref": "refs/tags/v0.2.2",
+ "Hash": "59356c8cd18c5fe9a598167d98a6843e52d57952XXX"
+ }
+}
+
+-- tagtestsbadtagsum.json --
+{
+ "Path": "vcs-test.golang.org/git/tagtests.git",
+ "Version": "v0.2.2",
+ "Query": "latest",
+ "Origin": {
+ "VCS": "git",
+ "URL": "https://vcs-test.golang.org/git/tagtests",
+ "TagSum": "t1:Dp7yRKDuE8WjG0429PN9hYWjqhy2te7P9Oki/sMEOGo=XXX",
+ "Ref": "refs/tags/v0.2.2",
+ "Hash": "59356c8cd18c5fe9a598167d98a6843e52d57952"
+ },
+ "Reuse": true
+}
+
+-- tagtestsv022badvcs.json --
+{
+ "Path": "vcs-test.golang.org/git/tagtests.git",
+ "Version": "v0.2.2",
+ "Origin": {
+ "VCS": "gitXXX",
+ "URL": "https://vcs-test.golang.org/git/tagtests",
+ "Ref": "refs/tags/v0.2.2",
+ "Hash": "59356c8cd18c5fe9a598167d98a6843e52d57952"
+ }
+}
+
+-- tagtestsv022baddir.json --
+{
+ "Path": "vcs-test.golang.org/git/tagtests.git",
+ "Version": "v0.2.2",
+ "Origin": {
+ "VCS": "git",
+ "URL": "https://vcs-test.golang.org/git/tagtests",
+ "Subdir": "subdir",
+ "Ref": "refs/tags/v0.2.2",
+ "Hash": "59356c8cd18c5fe9a598167d98a6843e52d57952"
+ }
+}
+
+-- tagtestsv022badurl.json --
+{
+ "Path": "vcs-test.golang.org/git/tagtests.git",
+ "Version": "v0.2.2",
+ "Origin": {
+ "VCS": "git",
+ "URL": "https://vcs-test.golang.org/git/tagtestsXXX",
+ "Ref": "refs/tags/v0.2.2",
+ "Hash": "59356c8cd18c5fe9a598167d98a6843e52d57952"
+ }
+}
From c1110910713766eb06d75ed48db3722d318a845e Mon Sep 17 00:00:00 2001
From: Russ Cox
Date: Fri, 1 Jul 2022 16:10:19 -0400
Subject: [PATCH 12/78] cmd/go: make module@nonexistentversion failures
reusable
CL 411398 added the -reuse flag for reusing cached JSON output
when the remote Git repository has not changed. One case that was
not yet cached is a lookup of a nonexistent version.
This CL adds caching of failed lookups of nonexistent versions,
by saving a checksum of all the heads and tags refs on the remote
server (we never consider other kinds of refs). If none of those have
changed, then we don't need to download the full server.
Fixes #53644.
Change-Id: I428bbc8ec8475bd7d03788934d643e1e2be3add0
Reviewed-on: https://go-review.googlesource.com/c/go/+/415678
Run-TryBot: Russ Cox
Reviewed-by: Bryan Mills
TryBot-Result: Gopher Robot
---
src/cmd/go/internal/modfetch/cache.go | 26 +++++---
.../go/internal/modfetch/codehost/codehost.go | 9 ++-
src/cmd/go/internal/modfetch/codehost/git.go | 48 ++++++++++++--
src/cmd/go/internal/modfetch/coderepo.go | 10 ++-
src/cmd/go/internal/modload/query.go | 2 +-
src/cmd/go/testdata/script/reuse_git.txt | 62 +++++++++++++++++--
6 files changed, 136 insertions(+), 21 deletions(-)
diff --git a/src/cmd/go/internal/modfetch/cache.go b/src/cmd/go/internal/modfetch/cache.go
index 7ebe208c12..c1ed18736c 100644
--- a/src/cmd/go/internal/modfetch/cache.go
+++ b/src/cmd/go/internal/modfetch/cache.go
@@ -253,11 +253,12 @@ func (r *cachingRepo) Stat(rev string) (*RevInfo, error) {
return cachedInfo{info, err}
}).(cachedInfo)
- if c.err != nil {
- return nil, c.err
+ info := c.info
+ if info != nil {
+ copy := *info
+ info = ©
}
- info := *c.info
- return &info, nil
+ return info, c.err
}
func (r *cachingRepo) Latest() (*RevInfo, error) {
@@ -277,11 +278,12 @@ func (r *cachingRepo) Latest() (*RevInfo, error) {
return cachedInfo{info, err}
}).(cachedInfo)
- if c.err != nil {
- return nil, c.err
+ info := c.info
+ if info != nil {
+ copy := *info
+ info = ©
}
- info := *c.info
- return &info, nil
+ return info, c.err
}
func (r *cachingRepo) GoMod(version string) ([]byte, error) {
@@ -330,15 +332,21 @@ func InfoFile(path, version string) (*RevInfo, string, error) {
}
var info *RevInfo
+ var err2info map[error]*RevInfo
err := TryProxies(func(proxy string) error {
i, err := Lookup(proxy, path).Stat(version)
if err == nil {
info = i
+ } else {
+ if err2info == nil {
+ err2info = make(map[error]*RevInfo)
+ }
+ err2info[err] = info
}
return err
})
if err != nil {
- return nil, "", err
+ return err2info[err], "", err
}
// Stat should have populated the disk cache for us.
diff --git a/src/cmd/go/internal/modfetch/codehost/codehost.go b/src/cmd/go/internal/modfetch/codehost/codehost.go
index 937ac6819a..8eaf254b44 100644
--- a/src/cmd/go/internal/modfetch/codehost/codehost.go
+++ b/src/cmd/go/internal/modfetch/codehost/codehost.go
@@ -110,12 +110,18 @@ type Origin struct {
// with a mutable meaning while Hash is a name with an immutable meaning.
Ref string `json:",omitempty"`
Hash string `json:",omitempty"`
+
+ // If RepoSum is non-empty, then the resolution of this module version
+ // failed due to the repo being available but the version not being present.
+ // This depends on the entire state of the repo, which RepoSum summarizes.
+ // For Git, this is a hash of all the refs and their hashes.
+ RepoSum string `json:",omitempty"`
}
// Checkable reports whether the Origin contains anything that can be checked.
// If not, the Origin is purely informational and should fail a CheckReuse call.
func (o *Origin) Checkable() bool {
- return o.TagSum != "" || o.Ref != "" || o.Hash != ""
+ return o.TagSum != "" || o.Ref != "" || o.Hash != "" || o.RepoSum != ""
}
// ClearCheckable clears the Origin enough to make Checkable return false.
@@ -124,6 +130,7 @@ func (o *Origin) ClearCheckable() {
o.TagPrefix = ""
o.Ref = ""
o.Hash = ""
+ o.RepoSum = ""
}
// A Tags describes the available tags in a code repository.
diff --git a/src/cmd/go/internal/modfetch/codehost/git.go b/src/cmd/go/internal/modfetch/codehost/git.go
index a225aaf1ed..35f77e870e 100644
--- a/src/cmd/go/internal/modfetch/codehost/git.go
+++ b/src/cmd/go/internal/modfetch/codehost/git.go
@@ -182,12 +182,12 @@ func (r *gitRepo) CheckReuse(old *Origin, subdir string) error {
return fmt.Errorf("origin moved from %v %q %q to %v %q %q", old.VCS, old.URL, old.Subdir, "git", r.remoteURL, subdir)
}
- // Note: Can have Hash with no Ref and no TagSum,
+ // Note: Can have Hash with no Ref and no TagSum and no RepoSum,
// meaning the Hash simply has to remain in the repo.
// In that case we assume it does in the absence of any real way to check.
// But if neither Hash nor TagSum is present, we have nothing to check,
// which we take to mean we didn't record enough information to be sure.
- if old.Hash == "" && old.TagSum == "" {
+ if old.Hash == "" && old.TagSum == "" && old.RepoSum == "" {
return fmt.Errorf("non-specific origin")
}
@@ -214,7 +214,11 @@ func (r *gitRepo) CheckReuse(old *Origin, subdir string) error {
return fmt.Errorf("tags changed")
}
}
-
+ if old.RepoSum != "" {
+ if r.repoSum(r.refs) != old.RepoSum {
+ return fmt.Errorf("refs changed")
+ }
+ }
return nil
}
@@ -307,6 +311,35 @@ func (r *gitRepo) Tags(prefix string) (*Tags, error) {
return tags, nil
}
+// repoSum returns a checksum of the entire repo state,
+// which can be checked (as Origin.RepoSum) to cache
+// the absence of a specific module version.
+// The caller must supply refs, the result of a successful r.loadRefs.
+func (r *gitRepo) repoSum(refs map[string]string) string {
+ var list []string
+ for ref := range refs {
+ list = append(list, ref)
+ }
+ sort.Strings(list)
+ h := sha256.New()
+ for _, ref := range list {
+ fmt.Fprintf(h, "%q %s\n", ref, refs[ref])
+ }
+ return "r1:" + base64.StdEncoding.EncodeToString(h.Sum(nil))
+}
+
+// unknownRevisionInfo returns a RevInfo containing an Origin containing a RepoSum of refs,
+// for use when returning an UnknownRevisionError.
+func (r *gitRepo) unknownRevisionInfo(refs map[string]string) *RevInfo {
+ return &RevInfo{
+ Origin: &Origin{
+ VCS: "git",
+ URL: r.remoteURL,
+ RepoSum: r.repoSum(refs),
+ },
+ }
+}
+
func (r *gitRepo) Latest() (*RevInfo, error) {
refs, err := r.loadRefs()
if err != nil {
@@ -418,7 +451,7 @@ func (r *gitRepo) stat(rev string) (info *RevInfo, err error) {
hash = rev
}
} else {
- return nil, &UnknownRevisionError{Rev: rev}
+ return r.unknownRevisionInfo(refs), &UnknownRevisionError{Rev: rev}
}
defer func() {
@@ -532,7 +565,12 @@ func (r *gitRepo) fetchRefsLocked() error {
func (r *gitRepo) statLocal(version, rev string) (*RevInfo, error) {
out, err := Run(r.dir, "git", "-c", "log.showsignature=false", "log", "--no-decorate", "-n1", "--format=format:%H %ct %D", rev, "--")
if err != nil {
- return nil, &UnknownRevisionError{Rev: rev}
+ // Return info with Origin.RepoSum if possible to allow caching of negative lookup.
+ var info *RevInfo
+ if refs, err := r.loadRefs(); err == nil {
+ info = r.unknownRevisionInfo(refs)
+ }
+ return info, &UnknownRevisionError{Rev: rev}
}
f := strings.Fields(string(out))
if len(f) < 2 {
diff --git a/src/cmd/go/internal/modfetch/coderepo.go b/src/cmd/go/internal/modfetch/coderepo.go
index 86e3ee9d1c..b934e362a4 100644
--- a/src/cmd/go/internal/modfetch/coderepo.go
+++ b/src/cmd/go/internal/modfetch/coderepo.go
@@ -297,7 +297,15 @@ func (r *codeRepo) Stat(rev string) (*RevInfo, error) {
codeRev := r.revToRev(rev)
info, err := r.code.Stat(codeRev)
if err != nil {
- return nil, &module.ModuleError{
+ // Note: info may be non-nil to supply Origin for caching error.
+ var revInfo *RevInfo
+ if info != nil {
+ revInfo = &RevInfo{
+ Origin: info.Origin,
+ Version: rev,
+ }
+ }
+ return revInfo, &module.ModuleError{
Path: r.modPath,
Err: &module.InvalidVersionError{
Version: rev,
diff --git a/src/cmd/go/internal/modload/query.go b/src/cmd/go/internal/modload/query.go
index 1d2f5d5e15..01df14fca4 100644
--- a/src/cmd/go/internal/modload/query.go
+++ b/src/cmd/go/internal/modload/query.go
@@ -197,7 +197,7 @@ func queryProxy(ctx context.Context, proxy, path, query, current string, allowed
}
}
if err != nil {
- return nil, queryErr
+ return info, queryErr
}
}
if err := allowed(ctx, module.Version{Path: path, Version: info.Version}); errors.Is(err, ErrDisallowed) {
diff --git a/src/cmd/go/testdata/script/reuse_git.txt b/src/cmd/go/testdata/script/reuse_git.txt
index 7d8844d932..a5a0c8a9a0 100644
--- a/src/cmd/go/testdata/script/reuse_git.txt
+++ b/src/cmd/go/testdata/script/reuse_git.txt
@@ -56,8 +56,7 @@ stdout '"Version": "latest"'
stdout '"Error":.*no matching versions'
! stdout '"TagPrefix"'
stdout '"TagSum": "t1:47DEQpj8HBSa[+]/TImW[+]5JCeuQeRkm5NMpJWZG3hSuFU="'
-! stdout '"Ref":'
-! stdout '"Hash":'
+! stdout '"(Ref|Hash|RepoSum)":'
# go mod download vcstest/hello/sub/v9 should also fail, print origin info with TagPrefix
! go mod download -x -json vcs-test.golang.org/git/hello.git/sub/v9@latest
@@ -66,8 +65,33 @@ stdout '"Version": "latest"'
stdout '"Error":.*no matching versions'
stdout '"TagPrefix": "sub/"'
stdout '"TagSum": "t1:47DEQpj8HBSa[+]/TImW[+]5JCeuQeRkm5NMpJWZG3hSuFU="'
-! stdout '"Ref":'
-! stdout '"Hash":'
+! stdout '"(Ref|Hash|RepoSum)":'
+
+# go mod download vcstest/hello@nonexist should fail, still print origin info
+! go mod download -x -json vcs-test.golang.org/git/hello.git@nonexist
+cp stdout hellononexist.json
+stdout '"Version": "nonexist"'
+stdout '"Error":.*unknown revision nonexist'
+stdout '"RepoSum": "r1:c0/9JCZ25lxoBiK3[+]3BhACU4giH49flcJmBynJ[+]Jvmc="'
+! stdout '"(TagPrefix|TagSum|Ref|Hash)"'
+
+# go mod download vcstest/hello@1234567890123456789012345678901234567890 should fail, still print origin info
+# (40 hex digits is assumed to be a full hash and is a slightly different code path from @nonexist)
+! go mod download -x -json vcs-test.golang.org/git/hello.git@1234567890123456789012345678901234567890
+cp stdout hellononhash.json
+stdout '"Version": "1234567890123456789012345678901234567890"'
+stdout '"Error":.*unknown revision 1234567890123456789012345678901234567890'
+stdout '"RepoSum": "r1:c0/9JCZ25lxoBiK3[+]3BhACU4giH49flcJmBynJ[+]Jvmc="'
+! stdout '"(TagPrefix|TagSum|Ref|Hash)"'
+
+# go mod download vcstest/hello@v0.0.0-20220101120101-123456789abc should fail, still print origin info
+# (non-existent pseudoversion)
+! go mod download -x -json vcs-test.golang.org/git/hello.git@v0.0.0-20220101120101-123456789abc
+cp stdout hellononpseudo.json
+stdout '"Version": "v0.0.0-20220101120101-123456789abc"'
+stdout '"Error":.*unknown revision 123456789abc'
+stdout '"RepoSum": "r1:c0/9JCZ25lxoBiK3[+]3BhACU4giH49flcJmBynJ[+]Jvmc="'
+! stdout '"(TagPrefix|TagSum|Ref|Hash)"'
# go mod download vcstest/tagtests should invoke git, print origin info
go mod download -x -json vcs-test.golang.org/git/tagtests.git@latest
@@ -190,6 +214,36 @@ stdout '"TagSum": "t1:47DEQpj8HBSa[+]/TImW[+]5JCeuQeRkm5NMpJWZG3hSuFU="'
! stdout '"(Ref|Hash)":'
! stdout '"(Dir|Info|GoMod|Zip)"'
+# reuse go mod download vcstest/hello@nonexist
+! go mod download -reuse=hellononexist.json -x -json vcs-test.golang.org/git/hello.git@nonexist
+! stderr 'git fetch'
+stdout '"Reuse": true'
+stdout '"Version": "nonexist"'
+stdout '"Error":.*unknown revision nonexist'
+stdout '"RepoSum": "r1:c0/9JCZ25lxoBiK3[+]3BhACU4giH49flcJmBynJ[+]Jvmc="'
+! stdout '"(TagPrefix|TagSum|Ref|Hash)"'
+! stdout '"(Dir|Info|GoMod|Zip)"'
+
+# reuse go mod download vcstest/hello@1234567890123456789012345678901234567890
+! go mod download -reuse=hellononhash.json -x -json vcs-test.golang.org/git/hello.git@1234567890123456789012345678901234567890
+! stderr 'git fetch'
+stdout '"Reuse": true'
+stdout '"Version": "1234567890123456789012345678901234567890"'
+stdout '"Error":.*unknown revision 1234567890123456789012345678901234567890'
+stdout '"RepoSum": "r1:c0/9JCZ25lxoBiK3[+]3BhACU4giH49flcJmBynJ[+]Jvmc="'
+! stdout '"(TagPrefix|TagSum|Ref|Hash)"'
+! stdout '"(Dir|Info|GoMod|Zip)"'
+
+# reuse go mod download vcstest/hello@v0.0.0-20220101120101-123456789abc
+! go mod download -reuse=hellononpseudo.json -x -json vcs-test.golang.org/git/hello.git@v0.0.0-20220101120101-123456789abc
+! stderr 'git fetch'
+stdout '"Reuse": true'
+stdout '"Version": "v0.0.0-20220101120101-123456789abc"'
+stdout '"Error":.*unknown revision 123456789abc'
+stdout '"RepoSum": "r1:c0/9JCZ25lxoBiK3[+]3BhACU4giH49flcJmBynJ[+]Jvmc="'
+! stdout '"(TagPrefix|TagSum|Ref|Hash)"'
+! stdout '"(Dir|Info|GoMod|Zip)"'
+
# reuse go mod download vcstest/tagtests result
go mod download -reuse=tagtests.json -x -json vcs-test.golang.org/git/tagtests.git@latest
! stderr 'git fetch'
From d602380f58e2a2ab4b262c7d69b78ff634cba5e8 Mon Sep 17 00:00:00 2001
From: Michael Pratt
Date: Tue, 28 Jun 2022 14:47:49 -0400
Subject: [PATCH 13/78] cmd/compile: drop "buildcfg" from no instrument
packages
Package buildcfg was added to this list by CL 403851, but package
buildcfg does not exist.
This was probably intended to refer to internal/buildcfg, but
internal/buildcfg is only used by the compiler so it is not clear why it
couldn't be instrumented.
For #44853.
Change-Id: Iad2517358be79c3eabf240376156bcff0c4bcefc
Reviewed-on: https://go-review.googlesource.com/c/go/+/414516
Reviewed-by: Bryan Mills
Reviewed-by: Cherry Mui
TryBot-Result: Gopher Robot
Run-TryBot: Michael Pratt
---
src/cmd/compile/internal/base/base.go | 1 -
1 file changed, 1 deletion(-)
diff --git a/src/cmd/compile/internal/base/base.go b/src/cmd/compile/internal/base/base.go
index 5e1493e275..39ce8e66f7 100644
--- a/src/cmd/compile/internal/base/base.go
+++ b/src/cmd/compile/internal/base/base.go
@@ -70,7 +70,6 @@ var NoInstrumentPkgs = []string{
"runtime/msan",
"runtime/asan",
"internal/cpu",
- "buildcfg",
}
// Don't insert racefuncenter/racefuncexit into the following packages.
From 2007599dc83aff17d8261338e8d2ab1f2c518a9b Mon Sep 17 00:00:00 2001
From: Ian Lance Taylor
Date: Mon, 4 Jul 2022 13:16:46 -0700
Subject: [PATCH 14/78] test: recognize new gofrontend error message
The new gofrontend message matches other gofrontend error messages,
so adjust the test to accept it.
For #27938
For #51237
Change-Id: I29b536f83a0cf22b1dbdae9abc2f5f6cf21d522d
Reviewed-on: https://go-review.googlesource.com/c/go/+/416014
Run-TryBot: Ian Lance Taylor
Reviewed-by: Ian Lance Taylor
TryBot-Result: Gopher Robot
Reviewed-by: Than McIntosh
---
test/fixedbugs/issue27938.go | 6 +++---
1 file changed, 3 insertions(+), 3 deletions(-)
diff --git a/test/fixedbugs/issue27938.go b/test/fixedbugs/issue27938.go
index 2589e1eff8..5392c65f1f 100644
--- a/test/fixedbugs/issue27938.go
+++ b/test/fixedbugs/issue27938.go
@@ -11,13 +11,13 @@
package p
type _ struct {
- F sync.Mutex // ERROR "undefined: sync|expected package"
+ F sync.Mutex // ERROR "undefined: sync|expected package|reference to undefined name"
}
type _ struct {
- sync.Mutex // ERROR "undefined: sync|expected package"
+ sync.Mutex // ERROR "undefined: sync|expected package|reference to undefined name"
}
type _ interface {
- sync.Mutex // ERROR "undefined: sync|expected package|expected signature or type name"
+ sync.Mutex // ERROR "undefined: sync|expected package|expected signature or type name|reference to undefined name"
}
From 4484c30f788835d8dda0afcefdb12e4b25b2c312 Mon Sep 17 00:00:00 2001
From: Cherry Mui
Date: Fri, 1 Jul 2022 12:56:39 -0400
Subject: [PATCH 15/78] misc/cgo/test: make TestSetgidStress cheaper
TestSetgidStress spawns 1000 threads, which can be expensive on
some platforms or slow builders. Run with 50 threads in short
mode instead.
This makes the failure less reproducible even with buggy code. But
one can manually stress test it (e.g. when a flaky failure appear
on the builder).
Fixes #53641.
Change-Id: I33b5ea5ecaa8c7a56f59c16f9171657ee295db47
Reviewed-on: https://go-review.googlesource.com/c/go/+/415677
Reviewed-by: Austin Clements
TryBot-Result: Gopher Robot
Run-TryBot: Cherry Mui
---
misc/cgo/test/setgid2_linux.go | 5 ++++-
1 file changed, 4 insertions(+), 1 deletion(-)
diff --git a/misc/cgo/test/setgid2_linux.go b/misc/cgo/test/setgid2_linux.go
index d239893f43..9069cff334 100644
--- a/misc/cgo/test/setgid2_linux.go
+++ b/misc/cgo/test/setgid2_linux.go
@@ -20,7 +20,10 @@ import (
)
func testSetgidStress(t *testing.T) {
- const N = 1000
+ var N = 1000
+ if testing.Short() {
+ N = 50
+ }
ch := make(chan int, N)
for i := 0; i < N; i++ {
go func() {
From f4755fc7333f524666f6ba3140cee0d180bce8b3 Mon Sep 17 00:00:00 2001
From: Ian Lance Taylor
Date: Sun, 3 Jul 2022 13:32:54 -0700
Subject: [PATCH 16/78] cmd/dist: use purego tag when building the bootstrap
binaries
This is in addition to the current math_big_pure_go tag.
Using purego ensures that we can build the cmd binaries with gccgo.
For #53662
Change-Id: Ib82f8bf10659b5f94935f2b427ae8b2da875cd3b
Reviewed-on: https://go-review.googlesource.com/c/go/+/415934
Reviewed-by: David Chase
Run-TryBot: Ian Lance Taylor
Reviewed-by: Ian Lance Taylor
Run-TryBot: Ian Lance Taylor
TryBot-Result: Gopher Robot
Auto-Submit: Ian Lance Taylor
---
src/cmd/dist/buildtool.go | 4 +++-
1 file changed, 3 insertions(+), 1 deletion(-)
diff --git a/src/cmd/dist/buildtool.go b/src/cmd/dist/buildtool.go
index 947da115e3..400c2e85b6 100644
--- a/src/cmd/dist/buildtool.go
+++ b/src/cmd/dist/buildtool.go
@@ -204,6 +204,8 @@ func bootstrapBuildTools() {
// https://groups.google.com/d/msg/golang-dev/Ss7mCKsvk8w/Gsq7VYI0AwAJ
// Use the math_big_pure_go build tag to disable the assembly in math/big
// which may contain unsupported instructions.
+ // Use the purego build tag to disable other assembly code,
+ // such as in cmd/internal/notsha256.
// Note that if we are using Go 1.10 or later as bootstrap, the -gcflags=-l
// only applies to the final cmd/go binary, but that's OK: if this is Go 1.10
// or later we don't need to disable inlining to work around bugs in the Go 1.4 compiler.
@@ -211,7 +213,7 @@ func bootstrapBuildTools() {
pathf("%s/bin/go", goroot_bootstrap),
"install",
"-gcflags=-l",
- "-tags=math_big_pure_go compiler_bootstrap",
+ "-tags=math_big_pure_go compiler_bootstrap purego",
}
if vflag > 0 {
cmd = append(cmd, "-v")
From 177306f6305b35bf6993c2d74baa7fb60cd3f5d4 Mon Sep 17 00:00:00 2001
From: Ian Lance Taylor
Date: Sun, 3 Jul 2022 13:34:47 -0700
Subject: [PATCH 17/78] cmd/internal/notsha256: add purego tag as needed
This permits building the package with gccgo, when using gccgo
as a bootstrap compiler.
Fixes #53662
Change-Id: Ic7ae9323ec5954e9306a32e1160e9aa1ed3aa202
Reviewed-on: https://go-review.googlesource.com/c/go/+/415935
TryBot-Result: Gopher Robot
Run-TryBot: Ian Lance Taylor
Auto-Submit: Ian Lance Taylor
Reviewed-by: Ian Lance Taylor
Reviewed-by: David Chase
---
src/cmd/internal/notsha256/sha256block_386.s | 3 +++
src/cmd/internal/notsha256/sha256block_amd64.go | 3 +++
src/cmd/internal/notsha256/sha256block_amd64.s | 3 +++
src/cmd/internal/notsha256/sha256block_decl.go | 3 ++-
src/cmd/internal/notsha256/sha256block_generic.go | 4 ++--
src/cmd/internal/notsha256/sha256block_ppc64x.s | 3 ++-
6 files changed, 15 insertions(+), 4 deletions(-)
diff --git a/src/cmd/internal/notsha256/sha256block_386.s b/src/cmd/internal/notsha256/sha256block_386.s
index 086a0ab25c..f2ba7d7a9b 100644
--- a/src/cmd/internal/notsha256/sha256block_386.s
+++ b/src/cmd/internal/notsha256/sha256block_386.s
@@ -2,6 +2,9 @@
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
+//go:build !purego
+// +build !purego
+
// SHA256 block routine. See sha256block.go for Go equivalent.
//
// The algorithm is detailed in FIPS 180-4:
diff --git a/src/cmd/internal/notsha256/sha256block_amd64.go b/src/cmd/internal/notsha256/sha256block_amd64.go
index 676c4f70d9..27b84a86b1 100644
--- a/src/cmd/internal/notsha256/sha256block_amd64.go
+++ b/src/cmd/internal/notsha256/sha256block_amd64.go
@@ -2,6 +2,9 @@
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
+//go:build !purego
+// +build !purego
+
package notsha256
var useAVX2 = false
diff --git a/src/cmd/internal/notsha256/sha256block_amd64.s b/src/cmd/internal/notsha256/sha256block_amd64.s
index b2ae7c5fc9..36ea74451d 100644
--- a/src/cmd/internal/notsha256/sha256block_amd64.s
+++ b/src/cmd/internal/notsha256/sha256block_amd64.s
@@ -2,6 +2,9 @@
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
+//go:build !purego
+// +build !purego
+
#include "textflag.h"
// SHA256 block routine. See sha256block.go for Go equivalent.
diff --git a/src/cmd/internal/notsha256/sha256block_decl.go b/src/cmd/internal/notsha256/sha256block_decl.go
index 5a822ee479..631f1a4a1b 100644
--- a/src/cmd/internal/notsha256/sha256block_decl.go
+++ b/src/cmd/internal/notsha256/sha256block_decl.go
@@ -2,7 +2,8 @@
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
-//go:build 386 || amd64 || ppc64le || ppc64
+//go:build !purego && (386 || amd64 || ppc64le || ppc64)
+// +build !purego
// +build 386 amd64 ppc64le ppc64
package notsha256
diff --git a/src/cmd/internal/notsha256/sha256block_generic.go b/src/cmd/internal/notsha256/sha256block_generic.go
index 20ae841383..2664722bc2 100644
--- a/src/cmd/internal/notsha256/sha256block_generic.go
+++ b/src/cmd/internal/notsha256/sha256block_generic.go
@@ -2,8 +2,8 @@
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
-//go:build !amd64 && !386 && !ppc64le && !ppc64
-// +build !amd64,!386,!ppc64le,!ppc64
+//go:build purego || (!amd64 && !386 && !ppc64le && !ppc64)
+// +build purego !amd64,!386,!ppc64le,!ppc64
package notsha256
diff --git a/src/cmd/internal/notsha256/sha256block_ppc64x.s b/src/cmd/internal/notsha256/sha256block_ppc64x.s
index 6e0f1d6133..e907d3b71b 100644
--- a/src/cmd/internal/notsha256/sha256block_ppc64x.s
+++ b/src/cmd/internal/notsha256/sha256block_ppc64x.s
@@ -8,7 +8,8 @@
// bootstrap toolchain.
//
-//go:build ppc64 || ppc64le
+//go:build !purego && (ppc64 || ppc64le)
+// +build !purego
// +build ppc64 ppc64le
// Based on CRYPTOGAMS code with the following comment:
From 53a4152d478d75ef4b71e428b9d69ed54144081f Mon Sep 17 00:00:00 2001
From: Ian Lance Taylor
Date: Fri, 24 Jun 2022 14:34:28 -0700
Subject: [PATCH 18/78] os/exec: clarify that Wait must be called
Fixes #52580
Change-Id: Ib2dd8a793b9c6fcb083abb3f7c346f6279adefc9
Reviewed-on: https://go-review.googlesource.com/c/go/+/414056
Run-TryBot: Ian Lance Taylor
TryBot-Result: Gopher Robot
Reviewed-by: Alan Donovan
Reviewed-by: Cherry Mui
---
src/os/exec/exec.go | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
diff --git a/src/os/exec/exec.go b/src/os/exec/exec.go
index f0dc7dab7d..57d18420bb 100644
--- a/src/os/exec/exec.go
+++ b/src/os/exec/exec.go
@@ -462,8 +462,8 @@ func lookExtensions(path, dir string) (string, error) {
//
// If Start returns successfully, the c.Process field will be set.
//
-// The Wait method will return the exit code and release associated resources
-// once the command exits.
+// After a successful call to Start the Wait method must be called in
+// order to release associated system resources.
func (c *Cmd) Start() error {
if c.Path == "" && c.Err == nil && c.lookPathErr == nil {
c.Err = errors.New("exec: no command")
From 2acd3646fc448b760e82fcace189adda94a1904a Mon Sep 17 00:00:00 2001
From: Keith Randall
Date: Sat, 2 Jul 2022 11:07:55 -0700
Subject: [PATCH 19/78] cmd/compile: rework induction variable detector
Induction variable detection is still not quite right. I've added
another failing test.
Redo the overflow/underflow detector so it is more obviously correct.
Update #53600
Fixes #53653
Fixes #53663
Change-Id: Id95228e282fdbf6bd80b26e1c41d62e935ba08ff
Reviewed-on: https://go-review.googlesource.com/c/go/+/415874
Run-TryBot: Keith Randall
TryBot-Result: Gopher Robot
Reviewed-by: Russ Cox
Reviewed-by: David Chase
---
src/cmd/compile/internal/ssa/loopbce.go | 350 ++++++++++++++----------
test/fixedbugs/issue53600.go | 11 +
test/fixedbugs/issue53600.out | 1 +
test/fixedbugs/issue53653.go | 42 +++
test/fixedbugs/issue53653.out | 8 +
test/loopbce.go | 65 ++++-
6 files changed, 324 insertions(+), 153 deletions(-)
create mode 100644 test/fixedbugs/issue53653.go
create mode 100644 test/fixedbugs/issue53653.out
diff --git a/src/cmd/compile/internal/ssa/loopbce.go b/src/cmd/compile/internal/ssa/loopbce.go
index a934cd2c7b..22fb5118ce 100644
--- a/src/cmd/compile/internal/ssa/loopbce.go
+++ b/src/cmd/compile/internal/ssa/loopbce.go
@@ -5,6 +5,7 @@
package ssa
import (
+ "cmd/compile/internal/base"
"fmt"
"math"
)
@@ -90,41 +91,42 @@ func findIndVar(f *Func) []indVar {
continue
}
- var flags indVarFlags
- var ind, max *Value // induction, and maximum
+ var ind *Value // induction variable
+ var init *Value // starting value
+ var limit *Value // ending value
- // Check thet the control if it either ind <= max or max >/>= ind.
+ // Check thet the control if it either ind <= limit or limit <= ind.
// TODO: Handle 32-bit comparisons.
// TODO: Handle unsigned comparisons?
c := b.Controls[0]
+ inclusive := false
switch c.Op {
case OpLeq64:
- flags |= indVarMaxInc
+ inclusive = true
fallthrough
case OpLess64:
- ind, max = c.Args[0], c.Args[1]
+ ind, limit = c.Args[0], c.Args[1]
default:
continue
}
// See if this is really an induction variable
less := true
- min, inc, nxt := parseIndVar(ind)
- if min == nil {
+ init, inc, nxt := parseIndVar(ind)
+ if init == nil {
// We failed to parse the induction variable. Before punting, we want to check
- // whether the control op was written with arguments in non-idiomatic order,
- // so that we believe being "max" (the upper bound) is actually the induction
- // variable itself. This would happen for code like:
- // for i := 0; len(n) > i; i++
- min, inc, nxt = parseIndVar(max)
- if min == nil {
+ // whether the control op was written with the induction variable on the RHS
+ // instead of the LHS. This happens for the downwards case, like:
+ // for i := len(n)-1; i >= 0; i--
+ init, inc, nxt = parseIndVar(limit)
+ if init == nil {
// No recognied induction variable on either operand
continue
}
// Ok, the arguments were reversed. Swap them, and remember that we're
// looking at a ind >/>= loop (so the induction must be decrementing).
- ind, max = max, ind
+ ind, limit = limit, ind
less = false
}
@@ -138,8 +140,8 @@ func findIndVar(f *Func) []indVar {
}
// Increment sign must match comparison direction.
- // When incrementing, the termination comparison must be ind <= max.
- // When decrementing, the termination comparison must be ind >/>= max.
+ // When incrementing, the termination comparison must be ind <= limit.
+ // When decrementing, the termination comparison must be ind >/>= limit.
// See issue 26116.
if step > 0 && !less {
continue
@@ -148,177 +150,229 @@ func findIndVar(f *Func) []indVar {
continue
}
- // If the increment is negative, swap min/max and their flags
- if step < 0 {
- min, max = max, min
- oldf := flags
- flags = indVarMaxInc
- if oldf&indVarMaxInc == 0 {
- flags |= indVarMinExc
- }
- step = -step
- }
-
- if flags&indVarMaxInc != 0 && max.Op == OpConst64 && max.AuxInt+step < max.AuxInt {
- // For a <= comparison, we need to make sure that a value equal to
- // max can be incremented without overflowing.
- // (For a < comparison, the %step check below ensures no overflow.)
- continue
- }
-
// Up to now we extracted the induction variable (ind),
// the increment delta (inc), the temporary sum (nxt),
- // the minimum value (min) and the maximum value (max).
+ // the initial value (init) and the limiting value (limit).
//
- // We also know that ind has the form (Phi min nxt) where
+ // We also know that ind has the form (Phi init nxt) where
// nxt is (Add inc nxt) which means: 1) inc dominates nxt
// and 2) there is a loop starting at inc and containing nxt.
//
// We need to prove that the induction variable is incremented
- // only when it's smaller than the maximum value.
+ // only when it's smaller than the limiting value.
// Two conditions must happen listed below to accept ind
// as an induction variable.
// First condition: loop entry has a single predecessor, which
// is the header block. This implies that b.Succs[0] is
- // reached iff ind < max.
+ // reached iff ind < limit.
if len(b.Succs[0].b.Preds) != 1 {
// b.Succs[1] must exit the loop.
continue
}
// Second condition: b.Succs[0] dominates nxt so that
- // nxt is computed when inc < max, meaning nxt <= max.
+ // nxt is computed when inc < limit.
if !sdom.IsAncestorEq(b.Succs[0].b, nxt.Block) {
// inc+ind can only be reached through the branch that enters the loop.
continue
}
- // We can only guarantee that the loop runs within limits of induction variable
- // if (one of)
- // (1) the increment is ±1
- // (2) the limits are constants
- // (3) loop is of the form k0 upto Known_not_negative-k inclusive, step <= k
- // (4) loop is of the form k0 upto Known_not_negative-k exclusive, step <= k+1
- // (5) loop is of the form Known_not_negative downto k0, minint+step < k0
- if step > 1 {
- ok := false
- if min.Op == OpConst64 && max.Op == OpConst64 {
- if max.AuxInt > min.AuxInt && max.AuxInt%step == min.AuxInt%step { // handle overflow
- ok = true
- }
- }
- // Handle induction variables of these forms.
- // KNN is known-not-negative.
- // SIGNED ARITHMETIC ONLY. (see switch on c above)
- // Possibilities for KNN are len and cap; perhaps we can infer others.
- // for i := 0; i <= KNN-k ; i += k
- // for i := 0; i < KNN-(k-1); i += k
- // Also handle decreasing.
-
- // "Proof" copied from https://go-review.googlesource.com/c/go/+/104041/10/src/cmd/compile/internal/ssa/loopbce.go#164
- //
- // In the case of
- // // PC is Positive Constant
- // L := len(A)-PC
- // for i := 0; i < L; i = i+PC
- //
- // we know:
- //
- // 0 + PC does not over/underflow.
- // len(A)-PC does not over/underflow
- // maximum value for L is MaxInt-PC
- // i < L <= MaxInt-PC means i + PC < MaxInt hence no overflow.
-
- // To match in SSA:
- // if (a) min.Op == OpConst64(k0)
- // and (b) k0 >= MININT + step
- // and (c) max.Op == OpSubtract(Op{StringLen,SliceLen,SliceCap}, k)
- // or (c) max.Op == OpAdd(Op{StringLen,SliceLen,SliceCap}, -k)
- // or (c) max.Op == Op{StringLen,SliceLen,SliceCap}
- // and (d) if upto loop, require indVarMaxInc && step <= k or !indVarMaxInc && step-1 <= k
-
- if min.Op == OpConst64 && min.AuxInt >= step+math.MinInt64 {
- knn := max
- k := int64(0)
- var kArg *Value
-
- switch max.Op {
- case OpSub64:
- knn = max.Args[0]
- kArg = max.Args[1]
-
- case OpAdd64:
- knn = max.Args[0]
- kArg = max.Args[1]
- if knn.Op == OpConst64 {
- knn, kArg = kArg, knn
- }
- }
- switch knn.Op {
- case OpSliceLen, OpStringLen, OpSliceCap:
- default:
- knn = nil
- }
-
- if kArg != nil && kArg.Op == OpConst64 {
- k = kArg.AuxInt
- if max.Op == OpAdd64 {
- k = -k
- }
- }
- if k >= 0 && knn != nil {
- if inc.AuxInt > 0 { // increasing iteration
- // The concern for the relation between step and k is to ensure that iv never exceeds knn
- // i.e., iv < knn-(K-1) ==> iv + K <= knn; iv <= knn-K ==> iv +K < knn
- if step <= k || flags&indVarMaxInc == 0 && step-1 == k {
- ok = true
+ // Check for overflow/underflow. We need to make sure that inc never causes
+ // the induction variable to wrap around.
+ // We use a function wrapper here for easy return true / return false / keep going logic.
+ // This function returns true if the increment will never overflow/underflow.
+ ok := func() bool {
+ if step > 0 {
+ if limit.Op == OpConst64 {
+ // Figure out the actual largest value.
+ v := limit.AuxInt
+ if !inclusive {
+ if v == math.MinInt64 {
+ return false // < minint is never satisfiable.
}
- } else { // decreasing iteration
- // Will be decrementing from max towards min; max is knn-k; will only attempt decrement if
- // knn-k >[=] min; underflow is only a concern if min-step is not smaller than min.
- // This all assumes signed integer arithmetic
- // This is already assured by the test above: min.AuxInt >= step+math.MinInt64
- ok = true
+ v--
}
+ if init.Op == OpConst64 {
+ // Use stride to compute a better lower limit.
+ if init.AuxInt > v {
+ return false
+ }
+ v = addU(init.AuxInt, diff(v, init.AuxInt)/uint64(step)*uint64(step))
+ }
+ // It is ok if we can't overflow when incrementing from the largest value.
+ return !addWillOverflow(v, step)
+ }
+ if step == 1 && !inclusive {
+ // Can't overflow because maxint is never a possible value.
+ return true
+ }
+ // If the limit is not a constant, check to see if it is a
+ // negative offset from a known non-negative value.
+ knn, k := findKNN(limit)
+ if knn == nil || k < 0 {
+ return false
+ }
+ // limit == (something nonnegative) - k. That subtraction can't underflow, so
+ // we can trust it.
+ if inclusive {
+ // ind <= knn - k cannot overflow if step is at most k
+ return step <= k
+ }
+ // ind < knn - k cannot overflow if step is at most k+1
+ return step <= k+1 && k != math.MaxInt64
+ } else { // step < 0
+ if limit.Op == OpConst64 {
+ // Figure out the actual smallest value.
+ v := limit.AuxInt
+ if !inclusive {
+ if v == math.MaxInt64 {
+ return false // > maxint is never satisfiable.
+ }
+ v++
+ }
+ if init.Op == OpConst64 {
+ // Use stride to compute a better lower limit.
+ if init.AuxInt < v {
+ return false
+ }
+ v = subU(init.AuxInt, diff(init.AuxInt, v)/uint64(-step)*uint64(-step))
+ }
+ // It is ok if we can't underflow when decrementing from the smallest value.
+ return !subWillUnderflow(v, -step)
+ }
+ if step == -1 && !inclusive {
+ // Can't underflow because minint is never a possible value.
+ return true
}
}
+ return false
- // TODO: other unrolling idioms
- // for i := 0; i < KNN - KNN % k ; i += k
- // for i := 0; i < KNN&^(k-1) ; i += k // k a power of 2
- // for i := 0; i < KNN&(-k) ; i += k // k a power of 2
+ }
- if !ok {
- continue
+ if ok() {
+ flags := indVarFlags(0)
+ var min, max *Value
+ if step > 0 {
+ min = init
+ max = limit
+ if inclusive {
+ flags |= indVarMaxInc
+ }
+ } else {
+ min = limit
+ max = init
+ flags |= indVarMaxInc
+ if !inclusive {
+ flags |= indVarMinExc
+ }
+ step = -step
}
+ if f.pass.debug >= 1 {
+ printIndVar(b, ind, min, max, step, flags)
+ }
+
+ iv = append(iv, indVar{
+ ind: ind,
+ min: min,
+ max: max,
+ entry: b.Succs[0].b,
+ flags: flags,
+ })
+ b.Logf("found induction variable %v (inc = %v, min = %v, max = %v)\n", ind, inc, min, max)
}
- if f.pass.debug >= 1 {
- printIndVar(b, ind, min, max, step, flags)
- }
-
- iv = append(iv, indVar{
- ind: ind,
- min: min,
- max: max,
- entry: b.Succs[0].b,
- flags: flags,
- })
- b.Logf("found induction variable %v (inc = %v, min = %v, max = %v)\n", ind, inc, min, max)
+ // TODO: other unrolling idioms
+ // for i := 0; i < KNN - KNN % k ; i += k
+ // for i := 0; i < KNN&^(k-1) ; i += k // k a power of 2
+ // for i := 0; i < KNN&(-k) ; i += k // k a power of 2
}
return iv
}
-func dropAdd64(v *Value) (*Value, int64) {
- if v.Op == OpAdd64 && v.Args[0].Op == OpConst64 {
- return v.Args[1], v.Args[0].AuxInt
+// addWillOverflow reports whether x+y would result in a value more than maxint.
+func addWillOverflow(x, y int64) bool {
+ return x+y < x
+}
+
+// subWillUnderflow reports whether x-y would result in a value less than minint.
+func subWillUnderflow(x, y int64) bool {
+ return x-y > x
+}
+
+// diff returns x-y as a uint64. Requires x>=y.
+func diff(x, y int64) uint64 {
+ if x < y {
+ base.Fatalf("diff %d - %d underflowed", x, y)
}
- if v.Op == OpAdd64 && v.Args[1].Op == OpConst64 {
- return v.Args[0], v.Args[1].AuxInt
+ return uint64(x - y)
+}
+
+// addU returns x+y. Requires that x+y does not overflow an int64.
+func addU(x int64, y uint64) int64 {
+ if y >= 1<<63 {
+ if x >= 0 {
+ base.Fatalf("addU overflowed %d + %d", x, y)
+ }
+ x += 1<<63 - 1
+ x += 1
+ y -= 1 << 63
}
- return v, 0
+ if addWillOverflow(x, int64(y)) {
+ base.Fatalf("addU overflowed %d + %d", x, y)
+ }
+ return x + int64(y)
+}
+
+// subU returns x-y. Requires that x-y does not underflow an int64.
+func subU(x int64, y uint64) int64 {
+ if y >= 1<<63 {
+ if x < 0 {
+ base.Fatalf("subU underflowed %d - %d", x, y)
+ }
+ x -= 1<<63 - 1
+ x -= 1
+ y -= 1 << 63
+ }
+ if subWillUnderflow(x, int64(y)) {
+ base.Fatalf("subU underflowed %d - %d", x, y)
+ }
+ return x - int64(y)
+}
+
+// if v is known to be x - c, where x is known to be nonnegative and c is a
+// constant, return x, c. Otherwise return nil, 0.
+func findKNN(v *Value) (*Value, int64) {
+ var x, y *Value
+ x = v
+ switch v.Op {
+ case OpSub64:
+ x = v.Args[0]
+ y = v.Args[1]
+
+ case OpAdd64:
+ x = v.Args[0]
+ y = v.Args[1]
+ if x.Op == OpConst64 {
+ x, y = y, x
+ }
+ }
+ switch x.Op {
+ case OpSliceLen, OpStringLen, OpSliceCap:
+ default:
+ return nil, 0
+ }
+ if y == nil {
+ return x, 0
+ }
+ if y.Op != OpConst64 {
+ return nil, 0
+ }
+ if v.Op == OpAdd64 {
+ return x, -y.AuxInt
+ }
+ return x, y.AuxInt
}
func printIndVar(b *Block, i, min, max *Value, inc int64, flags indVarFlags) {
diff --git a/test/fixedbugs/issue53600.go b/test/fixedbugs/issue53600.go
index fd3a9e5e47..ead40b57af 100644
--- a/test/fixedbugs/issue53600.go
+++ b/test/fixedbugs/issue53600.go
@@ -12,6 +12,7 @@ func main() {
f()
g()
h()
+ j(math.MinInt64)
}
func f() {
for i := int64(math.MaxInt64); i <= math.MaxInt64; i++ {
@@ -40,3 +41,13 @@ func h() {
println(i, i < 0)
}
}
+
+//go:noinline
+func j(i int64) {
+ for j := int64(math.MaxInt64); j <= i-1; j++ {
+ if j < 0 {
+ break
+ }
+ println(j)
+ }
+}
diff --git a/test/fixedbugs/issue53600.out b/test/fixedbugs/issue53600.out
index 5590c7dcfb..577b50fd2c 100644
--- a/test/fixedbugs/issue53600.out
+++ b/test/fixedbugs/issue53600.out
@@ -6,3 +6,4 @@ done
9223372036854775805 false
9223372036854775807 false
done
+9223372036854775807
diff --git a/test/fixedbugs/issue53653.go b/test/fixedbugs/issue53653.go
new file mode 100644
index 0000000000..555f7da528
--- /dev/null
+++ b/test/fixedbugs/issue53653.go
@@ -0,0 +1,42 @@
+// run
+
+// Copyright 2022 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package main
+
+import "math"
+
+func main() {
+ f()
+ g()
+ h()
+}
+func f() {
+ for i := int64(math.MinInt64); i >= math.MinInt64; i-- {
+ if i > 0 {
+ println("done")
+ return
+ }
+ println(i, i > 0)
+ }
+}
+func g() {
+ for i := int64(math.MinInt64) + 1; i >= math.MinInt64; i-- {
+ if i > 0 {
+ println("done")
+ return
+ }
+ println(i, i > 0)
+ }
+}
+func h() {
+ for i := int64(math.MinInt64) + 2; i >= math.MinInt64; i -= 2 {
+ if i > 0 {
+ println("done")
+ return
+ }
+ println(i, i > 0)
+ }
+}
diff --git a/test/fixedbugs/issue53653.out b/test/fixedbugs/issue53653.out
new file mode 100644
index 0000000000..f699392cf3
--- /dev/null
+++ b/test/fixedbugs/issue53653.out
@@ -0,0 +1,8 @@
+-9223372036854775808 false
+done
+-9223372036854775807 false
+-9223372036854775808 false
+done
+-9223372036854775806 false
+-9223372036854775808 false
+done
diff --git a/test/loopbce.go b/test/loopbce.go
index f0c9bd0f81..4ae9a6a630 100644
--- a/test/loopbce.go
+++ b/test/loopbce.go
@@ -3,6 +3,8 @@
package main
+import "math"
+
func f0a(a []int) int {
x := 0
for i := range a { // ERROR "Induction variable: limits \[0,\?\), increment 1$"
@@ -281,8 +283,8 @@ func d2(a [100]int) [100]int {
func d3(a [100]int) [100]int {
for i := 0; i <= 99; i++ { // ERROR "Induction variable: limits \[0,99\], increment 1$"
- for j := 0; j <= i-1; j++ { // ERROR "Induction variable: limits \[0,\?\], increment 1$"
- a[j] = 0 // ERROR "Proved IsInBounds$"
+ for j := 0; j <= i-1; j++ {
+ a[j] = 0
a[j+1] = 0 // ERROR "Proved IsInBounds$"
a[j+2] = 0
}
@@ -290,7 +292,61 @@ func d3(a [100]int) [100]int {
return a
}
-func nobce1() {
+func d4() {
+ for i := int64(math.MaxInt64 - 9); i < math.MaxInt64-2; i += 4 { // ERROR "Induction variable: limits \[9223372036854775798,9223372036854775805\), increment 4$"
+ useString("foo")
+ }
+ for i := int64(math.MaxInt64 - 8); i < math.MaxInt64-2; i += 4 { // ERROR "Induction variable: limits \[9223372036854775799,9223372036854775805\), increment 4$"
+ useString("foo")
+ }
+ for i := int64(math.MaxInt64 - 7); i < math.MaxInt64-2; i += 4 {
+ useString("foo")
+ }
+ for i := int64(math.MaxInt64 - 6); i < math.MaxInt64-2; i += 4 { // ERROR "Induction variable: limits \[9223372036854775801,9223372036854775805\), increment 4$"
+ useString("foo")
+ }
+ for i := int64(math.MaxInt64 - 9); i <= math.MaxInt64-2; i += 4 { // ERROR "Induction variable: limits \[9223372036854775798,9223372036854775805\], increment 4$"
+ useString("foo")
+ }
+ for i := int64(math.MaxInt64 - 8); i <= math.MaxInt64-2; i += 4 { // ERROR "Induction variable: limits \[9223372036854775799,9223372036854775805\], increment 4$"
+ useString("foo")
+ }
+ for i := int64(math.MaxInt64 - 7); i <= math.MaxInt64-2; i += 4 {
+ useString("foo")
+ }
+ for i := int64(math.MaxInt64 - 6); i <= math.MaxInt64-2; i += 4 {
+ useString("foo")
+ }
+}
+
+func d5() {
+ for i := int64(math.MinInt64 + 9); i > math.MinInt64+2; i -= 4 { // ERROR "Induction variable: limits \(-9223372036854775806,-9223372036854775799\], increment 4"
+ useString("foo")
+ }
+ for i := int64(math.MinInt64 + 8); i > math.MinInt64+2; i -= 4 { // ERROR "Induction variable: limits \(-9223372036854775806,-9223372036854775800\], increment 4"
+ useString("foo")
+ }
+ for i := int64(math.MinInt64 + 7); i > math.MinInt64+2; i -= 4 {
+ useString("foo")
+ }
+ for i := int64(math.MinInt64 + 6); i > math.MinInt64+2; i -= 4 { // ERROR "Induction variable: limits \(-9223372036854775806,-9223372036854775802\], increment 4"
+ useString("foo")
+ }
+ for i := int64(math.MinInt64 + 9); i >= math.MinInt64+2; i -= 4 { // ERROR "Induction variable: limits \[-9223372036854775806,-9223372036854775799\], increment 4"
+ useString("foo")
+ }
+ for i := int64(math.MinInt64 + 8); i >= math.MinInt64+2; i -= 4 { // ERROR "Induction variable: limits \[-9223372036854775806,-9223372036854775800\], increment 4"
+ useString("foo")
+ }
+ for i := int64(math.MinInt64 + 7); i >= math.MinInt64+2; i -= 4 {
+ useString("foo")
+ }
+ for i := int64(math.MinInt64 + 6); i >= math.MinInt64+2; i -= 4 {
+ useString("foo")
+ }
+}
+
+func bce1() {
// tests overflow of max-min
a := int64(9223372036854774057)
b := int64(-1547)
@@ -300,8 +356,7 @@ func nobce1() {
panic("invalid test: modulos should differ")
}
- for i := b; i < a; i += z {
- // No induction variable is possible because i will overflow a first iteration.
+ for i := b; i < a; i += z { // ERROR "Induction variable: limits \[-1547,9223372036854774057\), increment 1337"
useString("foobar")
}
}
From c391156f96357593fa18fccee305401e3f82a1a6 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Daniel=20Mart=C3=AD?=
Date: Tue, 5 Jul 2022 16:14:22 +0100
Subject: [PATCH 20/78] cmd/go: set up git identity for build_buildvcs_auto.txt
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit
Just like in other tests like get_dotfiles.txt or
version_buildvcs_git.txt. Without it, I get a failure on my machine:
fatal: empty ident name (for ) not allowed
Change-Id: I1c17c0d58c539b59154570b5438c7bd850bac5aa
Reviewed-on: https://go-review.googlesource.com/c/go/+/416095
Reviewed-by: Ian Lance Taylor
TryBot-Result: Gopher Robot
Run-TryBot: Daniel Martí
Reviewed-by: Dmitri Shuralyov
---
src/cmd/go/testdata/script/build_buildvcs_auto.txt | 4 ++++
1 file changed, 4 insertions(+)
diff --git a/src/cmd/go/testdata/script/build_buildvcs_auto.txt b/src/cmd/go/testdata/script/build_buildvcs_auto.txt
index 9eac568045..dd9eef5f82 100644
--- a/src/cmd/go/testdata/script/build_buildvcs_auto.txt
+++ b/src/cmd/go/testdata/script/build_buildvcs_auto.txt
@@ -6,11 +6,15 @@
cd sub
exec git init .
+exec git config user.name 'Nameless Gopher'
+exec git config user.email 'nobody@golang.org'
exec git add sub.go
exec git commit -m 'initial state'
cd ..
exec git init
+exec git config user.name 'Nameless Gopher'
+exec git config user.email 'nobody@golang.org'
exec git submodule add ./sub
exec git add go.mod example.go
exec git commit -m 'initial state'
From 1243ec9c177007879958443262fe4d25099c5ede Mon Sep 17 00:00:00 2001
From: Cuong Manh Le
Date: Wed, 29 Jun 2022 02:05:21 +0700
Subject: [PATCH 21/78] cmd/compile: only check implicit dots for method call
enabled by a type bound
Fixes #53419
Change-Id: Ibad64f5c4af2112deeb0a9ecc9c589b17594bd05
Reviewed-on: https://go-review.googlesource.com/c/go/+/414836
Reviewed-by: Matthew Dempsky
Reviewed-by: David Chase
Reviewed-by: Keith Randall
TryBot-Result: Gopher Robot
Run-TryBot: Cuong Manh Le
---
src/cmd/compile/internal/noder/stencil.go | 14 ++++++++------
test/run.go | 1 -
2 files changed, 8 insertions(+), 7 deletions(-)
diff --git a/src/cmd/compile/internal/noder/stencil.go b/src/cmd/compile/internal/noder/stencil.go
index cf2f0b38db..796a740528 100644
--- a/src/cmd/compile/internal/noder/stencil.go
+++ b/src/cmd/compile/internal/noder/stencil.go
@@ -1654,12 +1654,14 @@ func (g *genInst) getDictionarySym(gf *ir.Name, targs []*types.Type, isMeth bool
se := call.X.(*ir.SelectorExpr)
if se.X.Type().IsShape() {
// This is a method call enabled by a type bound.
-
- // We need this extra check for method expressions,
- // which don't add in the implicit XDOTs.
- tmpse := ir.NewSelectorExpr(src.NoXPos, ir.OXDOT, se.X, se.Sel)
- tmpse = typecheck.AddImplicitDots(tmpse)
- tparam := tmpse.X.Type()
+ tparam := se.X.Type()
+ if call.X.Op() == ir.ODOTMETH {
+ // We need this extra check for method expressions,
+ // which don't add in the implicit XDOTs.
+ tmpse := ir.NewSelectorExpr(src.NoXPos, ir.OXDOT, se.X, se.Sel)
+ tmpse = typecheck.AddImplicitDots(tmpse)
+ tparam = tmpse.X.Type()
+ }
if !tparam.IsShape() {
// The method expression is not
// really on a typeparam.
diff --git a/test/run.go b/test/run.go
index 8934e23b38..cb1622ccc9 100644
--- a/test/run.go
+++ b/test/run.go
@@ -1966,7 +1966,6 @@ var types2Failures32Bit = setOf(
var go118Failures = setOf(
"typeparam/nested.go", // 1.18 compiler doesn't support function-local types with generics
"typeparam/issue51521.go", // 1.18 compiler produces bad panic message and link error
- "typeparam/issue53419.go", // 1.18 compiler mishandles generic selector resolution
)
// In all of these cases, the 1.17 compiler reports reasonable errors, but either the
From eaf21256545ae04a35fa070763faa6eb2098591d Mon Sep 17 00:00:00 2001
From: Than McIntosh
Date: Wed, 6 Jul 2022 07:45:19 -0400
Subject: [PATCH 22/78] cmd/go: default to "exe" build mode for windows -race
This patch changes the default build mode from "pie" to "exe" when
building programs on windows with "-race" in effect. The Go command
already issues an error if users explicitly ask for -buildmode=pie in
combination with -race on windows, but wasn't revising the default
"pie" build mode if a specific buildmode was not requested.
Updates #53539.
Updates #35006.
Change-Id: I2f81a41a1d15a0b4f5ae943146175c5a1202cbe0
Reviewed-on: https://go-review.googlesource.com/c/go/+/416174
Reviewed-by: Alex Brainman
Reviewed-by: Cherry Mui
TryBot-Result: Gopher Robot
Run-TryBot: Than McIntosh
---
src/cmd/go/internal/work/init.go | 6 +++++-
1 file changed, 5 insertions(+), 1 deletion(-)
diff --git a/src/cmd/go/internal/work/init.go b/src/cmd/go/internal/work/init.go
index 5bf548db32..255ff3a0c5 100644
--- a/src/cmd/go/internal/work/init.go
+++ b/src/cmd/go/internal/work/init.go
@@ -211,7 +211,11 @@ func buildModeInit() {
codegenArg = "-shared"
ldBuildmode = "pie"
case "windows":
- ldBuildmode = "pie"
+ if cfg.BuildRace {
+ ldBuildmode = "exe"
+ } else {
+ ldBuildmode = "pie"
+ }
case "ios":
codegenArg = "-shared"
ldBuildmode = "pie"
From 0c7fcf6bd1fd8df2bfae3a482f1261886f6313c1 Mon Sep 17 00:00:00 2001
From: Than McIntosh
Date: Fri, 1 Jul 2022 08:39:12 -0400
Subject: [PATCH 23/78] cmd/link: explicitly disable PIE for windows/amd64
-race mode
Turn off PIE explicitly for windows/amd64 when -race is in effect,
since at the moment the race detector runtime doesn't seem to handle
PIE binaries correctly. Note that newer C compilers on windows
produce PIE binaries by default, so the Go linker needs to explicitly
turn off PIE when invoking the external linker in this case.
Updates #53539.
Change-Id: Ib990621f22cf61a5fa383584bab81d3dfd7552e8
Reviewed-on: https://go-review.googlesource.com/c/go/+/415676
Reviewed-by: Cherry Mui
TryBot-Result: Gopher Robot
Run-TryBot: Than McIntosh
Reviewed-by: Alex Brainman
---
src/cmd/link/internal/ld/lib.go | 13 +++++++++++++
1 file changed, 13 insertions(+)
diff --git a/src/cmd/link/internal/ld/lib.go b/src/cmd/link/internal/ld/lib.go
index 565ff9d634..18910ddb85 100644
--- a/src/cmd/link/internal/ld/lib.go
+++ b/src/cmd/link/internal/ld/lib.go
@@ -1426,10 +1426,23 @@ func (ctxt *Link) hostlink() {
argv = append(argv, "-Wl,-pagezero_size,4000000")
}
}
+ if *flagRace && ctxt.HeadType == objabi.Hwindows {
+ // Current windows/amd64 race detector tsan support
+ // library can't handle PIE mode (see #53539 for more details).
+ // For now, explicitly disable PIE (since some compilers
+ // default to it) if -race is in effect.
+ argv = addASLRargs(argv, false)
+ }
case BuildModePIE:
switch ctxt.HeadType {
case objabi.Hdarwin, objabi.Haix:
case objabi.Hwindows:
+ if *flagAslr && *flagRace {
+ // Current windows/amd64 race detector tsan support
+ // library can't handle PIE mode (see #53539 for more details).
+ // Disable alsr if -race in effect.
+ *flagAslr = false
+ }
argv = addASLRargs(argv, *flagAslr)
default:
// ELF.
From 8ac58de1857637f372a00ea16ab5497193b784a6 Mon Sep 17 00:00:00 2001
From: Aaron Gable
Date: Wed, 6 Jul 2022 16:59:03 -0700
Subject: [PATCH 24/78] crypto/x509: populate Number and AKI of parsed CRLs
The x509.RevocationList type has two fields which correspond to
extensions, rather than native fields, of the underlying ASN.1 CRL:
the .Number field corresponds to the crlNumber extension, and
the .AuthorityKeyId field corresponds to the authorityKeyIdentifier
extension.
The x509.CreateRevocationList() function uses these fields to populate
their respective extensions in the resulting CRL. However, the
x509.ParseRevocationList() function does not perform the reverse
operation: the fields retain their zero-values even after parsing a CRL
which contains the relevant extensions.
Add code which populates these fields when parsing their extensions.
Add assertions to the existing tests to confirm that the values are
populated appropriately.
Fixes #53726
Change-Id: Ie5b71081e53034e0b5b9ff3c122065c62f15cf23
Reviewed-on: https://go-review.googlesource.com/c/go/+/416354
Run-TryBot: Roland Shoemaker
Auto-Submit: Roland Shoemaker
Reviewed-by: Roland Shoemaker
TryBot-Result: Gopher Robot
Reviewed-by: Damien Neil
---
src/crypto/x509/parser.go | 17 +++++++++++++----
src/crypto/x509/x509.go | 7 +++++--
src/crypto/x509/x509_test.go | 13 +++++++++++++
3 files changed, 31 insertions(+), 6 deletions(-)
diff --git a/src/crypto/x509/parser.go b/src/crypto/x509/parser.go
index e0e8f6125f..cd87044d17 100644
--- a/src/crypto/x509/parser.go
+++ b/src/crypto/x509/parser.go
@@ -1008,22 +1008,22 @@ func ParseRevocationList(der []byte) (*RevocationList, error) {
// we can populate RevocationList.Raw, before unwrapping the
// SEQUENCE so it can be operated on
if !input.ReadASN1Element(&input, cryptobyte_asn1.SEQUENCE) {
- return nil, errors.New("x509: malformed certificate")
+ return nil, errors.New("x509: malformed crl")
}
rl.Raw = input
if !input.ReadASN1(&input, cryptobyte_asn1.SEQUENCE) {
- return nil, errors.New("x509: malformed certificate")
+ return nil, errors.New("x509: malformed crl")
}
var tbs cryptobyte.String
// do the same trick again as above to extract the raw
// bytes for Certificate.RawTBSCertificate
if !input.ReadASN1Element(&tbs, cryptobyte_asn1.SEQUENCE) {
- return nil, errors.New("x509: malformed tbs certificate")
+ return nil, errors.New("x509: malformed tbs crl")
}
rl.RawTBSRevocationList = tbs
if !tbs.ReadASN1(&tbs, cryptobyte_asn1.SEQUENCE) {
- return nil, errors.New("x509: malformed tbs certificate")
+ return nil, errors.New("x509: malformed tbs crl")
}
var version int
@@ -1148,6 +1148,15 @@ func ParseRevocationList(der []byte) (*RevocationList, error) {
if err != nil {
return nil, err
}
+ if ext.Id.Equal(oidExtensionAuthorityKeyId) {
+ rl.AuthorityKeyId = ext.Value
+ } else if ext.Id.Equal(oidExtensionCRLNumber) {
+ value := cryptobyte.String(ext.Value)
+ rl.Number = new(big.Int)
+ if !value.ReadASN1Integer(rl.Number) {
+ return nil, errors.New("x509: malformed crl number")
+ }
+ }
rl.Extensions = append(rl.Extensions, ext)
}
}
diff --git a/src/crypto/x509/x509.go b/src/crypto/x509/x509.go
index 87eb1f7720..7dcebfa5f1 100644
--- a/src/crypto/x509/x509.go
+++ b/src/crypto/x509/x509.go
@@ -2109,7 +2109,9 @@ type RevocationList struct {
// Issuer contains the DN of the issuing certificate.
Issuer pkix.Name
// AuthorityKeyId is used to identify the public key associated with the
- // issuing certificate.
+ // issuing certificate. It is populated from the authorityKeyIdentifier
+ // extension when parsing a CRL. It is ignored when creating a CRL; the
+ // extension is populated from the issuing certificate itself.
AuthorityKeyId []byte
Signature []byte
@@ -2125,7 +2127,8 @@ type RevocationList struct {
// Number is used to populate the X.509 v2 cRLNumber extension in the CRL,
// which should be a monotonically increasing sequence number for a given
- // CRL scope and CRL issuer.
+ // CRL scope and CRL issuer. It is also populated from the cRLNumber
+ // extension when parsing a CRL.
Number *big.Int
// ThisUpdate is used to populate the thisUpdate field in the CRL, which
diff --git a/src/crypto/x509/x509_test.go b/src/crypto/x509/x509_test.go
index 8ef6115df4..594ee1dceb 100644
--- a/src/crypto/x509/x509_test.go
+++ b/src/crypto/x509/x509_test.go
@@ -2681,6 +2681,19 @@ func TestCreateRevocationList(t *testing.T) {
t.Fatalf("Extensions mismatch: got %v; want %v.",
parsedCRL.Extensions[2:], tc.template.ExtraExtensions)
}
+
+ if tc.template.Number != nil && parsedCRL.Number == nil {
+ t.Fatalf("Generated CRL missing Number: got nil, want %s",
+ tc.template.Number.String())
+ }
+ if tc.template.Number != nil && tc.template.Number.Cmp(parsedCRL.Number) != 0 {
+ t.Fatalf("Generated CRL has wrong Number: got %s, want %s",
+ parsedCRL.Number.String(), tc.template.Number.String())
+ }
+ if !bytes.Equal(parsedCRL.AuthorityKeyId, expectedAKI) {
+ t.Fatalf("Generated CRL has wrong Number: got %x, want %x",
+ parsedCRL.AuthorityKeyId, expectedAKI)
+ }
})
}
}
From 486fc0177068277a51235c7794660b238e70d622 Mon Sep 17 00:00:00 2001
From: Aaron Gable
Date: Tue, 28 Jun 2022 15:28:21 -0700
Subject: [PATCH 25/78] crypto/x509: correctly parse CRL entry extensions
When checking to see if a CRL entry has any extensions, attempt to read
them from the individual revokedCertificate, rather than from the parent
TBSCertList.
Additionally, crlEntryExtensions is not an EXPLICIT field (c.f.
crlExtension and Certificate extensions), so do not perform an extra
layer of unwrapping when parsing the field.
The added test case fails without the accompanying changes.
Fixes #53592
Change-Id: Icc00e4c911f196aef77e3248117de64ddc5ea27f
Reviewed-on: https://go-review.googlesource.com/c/go/+/414877
Reviewed-by: Damien Neil
Reviewed-by: Roland Shoemaker
Run-TryBot: Roland Shoemaker
Auto-Submit: Roland Shoemaker
TryBot-Result: Gopher Robot
---
src/crypto/x509/parser.go | 5 +----
src/crypto/x509/x509_test.go | 28 ++++++++++++++++++++++++++++
2 files changed, 29 insertions(+), 4 deletions(-)
diff --git a/src/crypto/x509/parser.go b/src/crypto/x509/parser.go
index cd87044d17..a2d3d80964 100644
--- a/src/crypto/x509/parser.go
+++ b/src/crypto/x509/parser.go
@@ -1106,13 +1106,10 @@ func ParseRevocationList(der []byte) (*RevocationList, error) {
}
var extensions cryptobyte.String
var present bool
- if !tbs.ReadOptionalASN1(&extensions, &present, cryptobyte_asn1.SEQUENCE) {
+ if !certSeq.ReadOptionalASN1(&extensions, &present, cryptobyte_asn1.SEQUENCE) {
return nil, errors.New("x509: malformed extensions")
}
if present {
- if !extensions.ReadASN1(&extensions, cryptobyte_asn1.SEQUENCE) {
- return nil, errors.New("x509: malformed extensions")
- }
for !extensions.Empty() {
var extension cryptobyte.String
if !extensions.ReadASN1(&extension, cryptobyte_asn1.SEQUENCE) {
diff --git a/src/crypto/x509/x509_test.go b/src/crypto/x509/x509_test.go
index 594ee1dceb..cddad1e246 100644
--- a/src/crypto/x509/x509_test.go
+++ b/src/crypto/x509/x509_test.go
@@ -2524,6 +2524,34 @@ func TestCreateRevocationList(t *testing.T) {
NextUpdate: time.Time{}.Add(time.Hour * 48),
},
},
+ {
+ name: "valid, extra entry extension",
+ key: ec256Priv,
+ issuer: &Certificate{
+ KeyUsage: KeyUsageCRLSign,
+ Subject: pkix.Name{
+ CommonName: "testing",
+ },
+ SubjectKeyId: []byte{1, 2, 3},
+ },
+ template: &RevocationList{
+ RevokedCertificates: []pkix.RevokedCertificate{
+ {
+ SerialNumber: big.NewInt(2),
+ RevocationTime: time.Time{}.Add(time.Hour),
+ Extensions: []pkix.Extension{
+ {
+ Id: []int{2, 5, 29, 99},
+ Value: []byte{5, 0},
+ },
+ },
+ },
+ },
+ Number: big.NewInt(5),
+ ThisUpdate: time.Time{}.Add(time.Hour * 24),
+ NextUpdate: time.Time{}.Add(time.Hour * 48),
+ },
+ },
{
name: "valid, Ed25519 key",
key: ed25519Priv,
From c177d9d98a7bfb21346f6309c115d0a2bf3167e3 Mon Sep 17 00:00:00 2001
From: Roland Shoemaker
Date: Wed, 29 Jun 2022 11:30:47 -0700
Subject: [PATCH 26/78] crypto/x509: restrict CRL number to <=20 octets
Similar to certificate serial numbers, RFC 5280 restricts the length of
the CRL number field to no more than 20 octets. Enforce this in
CreateRevocationList.
Fixes #53543
Change-Id: If392ef6b0844db716ae9ee6ef317135fceab039c
Reviewed-on: https://go-review.googlesource.com/c/go/+/415134
Auto-Submit: Roland Shoemaker
Reviewed-by: Tatiana Bradley
Reviewed-by: Damien Neil
TryBot-Result: Gopher Robot
Run-TryBot: Roland Shoemaker
---
src/crypto/x509/x509.go | 4 ++++
src/crypto/x509/x509_test.go | 34 ++++++++++++++++++++++++++++++++++
2 files changed, 38 insertions(+)
diff --git a/src/crypto/x509/x509.go b/src/crypto/x509/x509.go
index 7dcebfa5f1..950f6d08c8 100644
--- a/src/crypto/x509/x509.go
+++ b/src/crypto/x509/x509.go
@@ -2196,6 +2196,10 @@ func CreateRevocationList(rand io.Reader, template *RevocationList, issuer *Cert
if err != nil {
return nil, err
}
+
+ if numBytes := template.Number.Bytes(); len(numBytes) > 20 || (len(numBytes) == 20 && numBytes[0]&0x80 != 0) {
+ return nil, errors.New("x509: CRL number exceeds 20 octets")
+ }
crlNum, err := asn1.Marshal(template.Number)
if err != nil {
return nil, err
diff --git a/src/crypto/x509/x509_test.go b/src/crypto/x509/x509_test.go
index cddad1e246..cba44f6f8c 100644
--- a/src/crypto/x509/x509_test.go
+++ b/src/crypto/x509/x509_test.go
@@ -2478,6 +2478,40 @@ func TestCreateRevocationList(t *testing.T) {
},
expectedError: "x509: template contains nil Number field",
},
+ {
+ name: "long Number",
+ key: ec256Priv,
+ issuer: &Certificate{
+ KeyUsage: KeyUsageCRLSign,
+ Subject: pkix.Name{
+ CommonName: "testing",
+ },
+ SubjectKeyId: []byte{1, 2, 3},
+ },
+ template: &RevocationList{
+ ThisUpdate: time.Time{}.Add(time.Hour * 24),
+ NextUpdate: time.Time{}.Add(time.Hour * 48),
+ Number: big.NewInt(0).SetBytes(append([]byte{1}, make([]byte, 20)...)),
+ },
+ expectedError: "x509: CRL number exceeds 20 octets",
+ },
+ {
+ name: "long Number (20 bytes, MSB set)",
+ key: ec256Priv,
+ issuer: &Certificate{
+ KeyUsage: KeyUsageCRLSign,
+ Subject: pkix.Name{
+ CommonName: "testing",
+ },
+ SubjectKeyId: []byte{1, 2, 3},
+ },
+ template: &RevocationList{
+ ThisUpdate: time.Time{}.Add(time.Hour * 24),
+ NextUpdate: time.Time{}.Add(time.Hour * 48),
+ Number: big.NewInt(0).SetBytes(append([]byte{255}, make([]byte, 19)...)),
+ },
+ expectedError: "x509: CRL number exceeds 20 octets",
+ },
{
name: "invalid signature algorithm",
key: ec256Priv,
From 1ebc983000ed411a1c06f6b8a61770be1392e707 Mon Sep 17 00:00:00 2001
From: Michael Anthony Knyszek
Date: Thu, 7 Jul 2022 20:01:21 +0000
Subject: [PATCH 27/78] runtime: overestimate the amount of allocated memory in
heapLive
CL 377516 made it so that memory metrics are truly monotonic, but also
updated how heapLive tracked allocated memory to also be monotonic.
The result is that cached spans with allocated memory aren't fully
accounted for by the GC, causing it to make a worse assumption (the
exact mechanism is at this time unknown), resulting in a memory
regression, especially for smaller heaps.
This change is a partial revert of CL 377516 that makes heapLive a
non-monotonic overestimate again, which appears to resolve the
regression.
For #53738.
Change-Id: I5c51067abc0b8e0a6b89dd8dbd4a0be2e8c0c1b2
Reviewed-on: https://go-review.googlesource.com/c/go/+/416417
Reviewed-by: Michael Pratt
Run-TryBot: Michael Knyszek
TryBot-Result: Gopher Robot
---
src/runtime/mcache.go | 36 ++++++++++++++++++++++++++++++------
1 file changed, 30 insertions(+), 6 deletions(-)
diff --git a/src/runtime/mcache.go b/src/runtime/mcache.go
index 7c785900db..1f484fb9b6 100644
--- a/src/runtime/mcache.go
+++ b/src/runtime/mcache.go
@@ -173,10 +173,6 @@ func (c *mcache) refill(spc spanClass) {
bytesAllocated := slotsUsed * int64(s.elemsize)
gcController.totalAlloc.Add(bytesAllocated)
- // Update heapLive and flush scanAlloc.
- gcController.update(bytesAllocated, int64(c.scanAlloc))
- c.scanAlloc = 0
-
// Clear the second allocCount just to be safe.
s.allocCountBeforeCache = 0
}
@@ -198,6 +194,23 @@ func (c *mcache) refill(spc spanClass) {
// Store the current alloc count for accounting later.
s.allocCountBeforeCache = s.allocCount
+ // Update heapLive and flush scanAlloc.
+ //
+ // We have not yet allocated anything new into the span, but we
+ // assume that all of its slots will get used, so this makes
+ // heapLive an overestimate.
+ //
+ // When the span gets uncached, we'll fix up this overestimate
+ // if necessary (see releaseAll).
+ //
+ // We pick an overestimate here because an underestimate leads
+ // the pacer to believe that it's in better shape than it is,
+ // which appears to lead to more memory used. See #53738 for
+ // more details.
+ usedBytes := uintptr(s.allocCount) * s.elemsize
+ gcController.update(int64(s.npages*pageSize)-int64(usedBytes), int64(c.scanAlloc))
+ c.scanAlloc = 0
+
c.alloc[spc] = s
}
@@ -247,6 +260,8 @@ func (c *mcache) releaseAll() {
scanAlloc := int64(c.scanAlloc)
c.scanAlloc = 0
+ sg := mheap_.sweepgen
+ dHeapLive := int64(0)
for i := range c.alloc {
s := c.alloc[i]
if s != &emptymspan {
@@ -262,6 +277,15 @@ func (c *mcache) releaseAll() {
// We assumed earlier that the full span gets allocated.
gcController.totalAlloc.Add(slotsUsed * int64(s.elemsize))
+ if s.sweepgen != sg+1 {
+ // refill conservatively counted unallocated slots in gcController.heapLive.
+ // Undo this.
+ //
+ // If this span was cached before sweep, then gcController.heapLive was totally
+ // recomputed since caching this span, so we don't do this for stale spans.
+ dHeapLive -= int64(uintptr(s.nelems)-uintptr(s.allocCount)) * int64(s.elemsize)
+ }
+
// Release the span to the mcentral.
mheap_.central[i].mcentral.uncacheSpan(s)
c.alloc[i] = &emptymspan
@@ -277,8 +301,8 @@ func (c *mcache) releaseAll() {
c.tinyAllocs = 0
memstats.heapStats.release()
- // Updated heapScan.
- gcController.update(0, scanAlloc)
+ // Update heapLive and heapScan.
+ gcController.update(dHeapLive, scanAlloc)
}
// prepareForSweep flushes c if the system has entered a new sweep phase
From 14abe8aa7324bdf0e09e1dfebfb3519cc30f4918 Mon Sep 17 00:00:00 2001
From: Cuong Manh Le
Date: Fri, 1 Jul 2022 16:33:54 +0700
Subject: [PATCH 28/78] cmd/compile: don't convert to interface{} for
un-comparable types in generic switch
Fixes #53635
Change-Id: I41f383be8870432fc0d29fa83687911ddd8217f1
Reviewed-on: https://go-review.googlesource.com/c/go/+/415634
TryBot-Result: Gopher Robot
Run-TryBot: Cuong Manh Le
Reviewed-by: Keith Randall
Reviewed-by: Keith Randall
Reviewed-by: Matthew Dempsky
---
src/cmd/compile/internal/noder/stencil.go | 3 +++
test/fixedbugs/issue53635.go | 31 +++++++++++++++++++++++
2 files changed, 34 insertions(+)
create mode 100644 test/fixedbugs/issue53635.go
diff --git a/src/cmd/compile/internal/noder/stencil.go b/src/cmd/compile/internal/noder/stencil.go
index 796a740528..1534a1fa49 100644
--- a/src/cmd/compile/internal/noder/stencil.go
+++ b/src/cmd/compile/internal/noder/stencil.go
@@ -1214,6 +1214,9 @@ func (subst *subster) node(n ir.Node) ir.Node {
if m.Tag != nil && m.Tag.Op() == ir.OTYPESW {
break // Nothing to do here for type switches.
}
+ if m.Tag != nil && !types.IsComparable(m.Tag.Type()) {
+ break // Nothing to do here for un-comparable types.
+ }
if m.Tag != nil && !m.Tag.Type().IsEmptyInterface() && m.Tag.Type().HasShape() {
// To implement a switch on a value that is or has a type parameter, we first convert
// that thing we're switching on to an interface{}.
diff --git a/test/fixedbugs/issue53635.go b/test/fixedbugs/issue53635.go
new file mode 100644
index 0000000000..bea5493805
--- /dev/null
+++ b/test/fixedbugs/issue53635.go
@@ -0,0 +1,31 @@
+// run
+
+// Copyright 2022 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package main
+
+func main() {
+ f[int]()
+}
+
+func f[T any]() {
+ switch []T(nil) {
+ case nil:
+ default:
+ panic("FAIL")
+ }
+
+ switch (func() T)(nil) {
+ case nil:
+ default:
+ panic("FAIL")
+ }
+
+ switch (map[int]T)(nil) {
+ case nil:
+ default:
+ panic("FAIL")
+ }
+}
From 180bcad33dcd3d59443fe8eda5ae7556b1b2945b Mon Sep 17 00:00:00 2001
From: Damien Neil
Date: Tue, 31 May 2022 14:47:33 -0700
Subject: [PATCH 29/78] net/http: wait for listeners to exit in Server.Close
and Shutdown
Avoid race conditions when a new connection is accepted just after
Server.Close or Server.Shutdown is called by waiting for the
listener goroutines to exit before proceeding to clean up active
connections.
No test because the mechanism required to trigger the race condition
reliably requires such tight coupling to the Server internals that
any test would be quite fragile in the face of reasonable refactorings.
Fixes #48642
Updates #33313, #36819
Change-Id: I109a93362680991bf298e0a95637595dcaa884af
Reviewed-on: https://go-review.googlesource.com/c/go/+/409537
TryBot-Result: Gopher Robot
Run-TryBot: Damien Neil
Reviewed-by: Brad Fitzpatrick
Reviewed-by: Bryan Mills
---
src/net/http/server.go | 22 +++++++++++++++-------
1 file changed, 15 insertions(+), 7 deletions(-)
diff --git a/src/net/http/server.go b/src/net/http/server.go
index bc3a4633da..87dd412984 100644
--- a/src/net/http/server.go
+++ b/src/net/http/server.go
@@ -2690,6 +2690,8 @@ type Server struct {
activeConn map[*conn]struct{}
doneChan chan struct{}
onShutdown []func()
+
+ listenerGroup sync.WaitGroup
}
func (s *Server) getDoneChan() <-chan struct{} {
@@ -2732,6 +2734,15 @@ func (srv *Server) Close() error {
defer srv.mu.Unlock()
srv.closeDoneChanLocked()
err := srv.closeListenersLocked()
+
+ // Unlock srv.mu while waiting for listenerGroup.
+ // The group Add and Done calls are made with srv.mu held,
+ // to avoid adding a new listener in the window between
+ // us setting inShutdown above and waiting here.
+ srv.mu.Unlock()
+ srv.listenerGroup.Wait()
+ srv.mu.Lock()
+
for c := range srv.activeConn {
c.rwc.Close()
delete(srv.activeConn, c)
@@ -2778,6 +2789,7 @@ func (srv *Server) Shutdown(ctx context.Context) error {
go f()
}
srv.mu.Unlock()
+ srv.listenerGroup.Wait()
pollIntervalBase := time.Millisecond
nextPollInterval := func() time.Duration {
@@ -2794,7 +2806,7 @@ func (srv *Server) Shutdown(ctx context.Context) error {
timer := time.NewTimer(nextPollInterval())
defer timer.Stop()
for {
- if srv.closeIdleConns() && srv.numListeners() == 0 {
+ if srv.closeIdleConns() {
return lnerr
}
select {
@@ -2817,12 +2829,6 @@ func (srv *Server) RegisterOnShutdown(f func()) {
srv.mu.Unlock()
}
-func (s *Server) numListeners() int {
- s.mu.Lock()
- defer s.mu.Unlock()
- return len(s.listeners)
-}
-
// closeIdleConns closes all idle connections and reports whether the
// server is quiescent.
func (s *Server) closeIdleConns() bool {
@@ -3157,8 +3163,10 @@ func (s *Server) trackListener(ln *net.Listener, add bool) bool {
return false
}
s.listeners[ln] = struct{}{}
+ s.listenerGroup.Add(1)
} else {
delete(s.listeners, ln)
+ s.listenerGroup.Done()
}
return true
}
From 5c1a13e7a47bc47c07057c0acf626e3fafe064c9 Mon Sep 17 00:00:00 2001
From: "Bryan C. Mills"
Date: Fri, 8 Jul 2022 10:02:14 -0400
Subject: [PATCH 30/78] cmd/go: avoid setting variables for '/' and ':' in
TestScript subprocess environments
Also simplify platform-dependent handling of the PATH variable,
to make it more like the existing platform-dependent handling for
HOME and TMPDIR.
Fixes #53671.
Change-Id: Ica2665d3f61988c66fb6982b9feb61ca48eced79
Reviewed-on: https://go-review.googlesource.com/c/go/+/416554
Reviewed-by: Ian Lance Taylor
Run-TryBot: Bryan Mills
TryBot-Result: Gopher Robot
---
src/cmd/go/go_test.go | 9 +++++++++
src/cmd/go/script_test.go | 19 ++++++-------------
src/cmd/go/testdata/script/README | 17 ++++++++++++-----
3 files changed, 27 insertions(+), 18 deletions(-)
diff --git a/src/cmd/go/go_test.go b/src/cmd/go/go_test.go
index b39a62f3e4..c100316f47 100644
--- a/src/cmd/go/go_test.go
+++ b/src/cmd/go/go_test.go
@@ -1363,6 +1363,15 @@ func tempEnvName() string {
}
}
+func pathEnvName() string {
+ switch runtime.GOOS {
+ case "plan9":
+ return "path"
+ default:
+ return "PATH"
+ }
+}
+
func TestDefaultGOPATH(t *testing.T) {
tg := testgo(t)
defer tg.cleanup()
diff --git a/src/cmd/go/script_test.go b/src/cmd/go/script_test.go
index 5e82929f19..809dfb452f 100644
--- a/src/cmd/go/script_test.go
+++ b/src/cmd/go/script_test.go
@@ -163,7 +163,7 @@ func (ts *testScript) setup() {
ts.cd = filepath.Join(ts.workdir, "gopath/src")
ts.env = []string{
"WORK=" + ts.workdir, // must be first for ts.abbrev
- "PATH=" + testBin + string(filepath.ListSeparator) + os.Getenv("PATH"),
+ pathEnvName() + "=" + testBin + string(filepath.ListSeparator) + os.Getenv(pathEnvName()),
homeEnvName() + "=/no-home",
"CCACHE_DISABLE=1", // ccache breaks with non-existent HOME
"GOARCH=" + runtime.GOARCH,
@@ -187,8 +187,6 @@ func (ts *testScript) setup() {
tempEnvName() + "=" + filepath.Join(ts.workdir, "tmp"),
"devnull=" + os.DevNull,
"goversion=" + goVersion(ts),
- ":=" + string(os.PathListSeparator),
- "/=" + string(os.PathSeparator),
"CMDGO_TEST_RUN_MAIN=true",
}
if testenv.Builder() != "" || os.Getenv("GIT_TRACE_CURL") == "1" {
@@ -203,10 +201,6 @@ func (ts *testScript) setup() {
ts.env = append(ts.env, "TESTGONETWORK=panic", "TESTGOVCS=panic")
}
- if runtime.GOOS == "plan9" {
- ts.env = append(ts.env, "path="+testBin+string(filepath.ListSeparator)+os.Getenv("path"))
- }
-
for _, key := range extraEnvKeys {
if val := os.Getenv(key); val != "" {
ts.env = append(ts.env, key+"="+val)
@@ -219,6 +213,10 @@ func (ts *testScript) setup() {
ts.envMap[kv[:i]] = kv[i+1:]
}
}
+ // Add entries for ${:} and ${/} to make it easier to write platform-independent
+ // environment variables.
+ ts.envMap["/"] = string(os.PathSeparator)
+ ts.envMap[":"] = string(os.PathListSeparator)
fmt.Fprintf(&ts.log, "# (%s)\n", time.Now().UTC().Format(time.RFC3339))
ts.mark = ts.log.Len()
@@ -1264,12 +1262,7 @@ func (ts *testScript) lookPath(command string) (string, error) {
}
}
- pathName := "PATH"
- if runtime.GOOS == "plan9" {
- pathName = "path"
- }
-
- for _, dir := range strings.Split(ts.envMap[pathName], string(filepath.ListSeparator)) {
+ for _, dir := range strings.Split(ts.envMap[pathEnvName()], string(filepath.ListSeparator)) {
if searchExt {
ents, err := os.ReadDir(dir)
if err != nil {
diff --git a/src/cmd/go/testdata/script/README b/src/cmd/go/testdata/script/README
index c575bff1a5..e52917684f 100644
--- a/src/cmd/go/testdata/script/README
+++ b/src/cmd/go/testdata/script/README
@@ -41,12 +41,19 @@ Scripts also have access to these other environment variables:
GODEBUG=
devnull=
goversion=
- :=
-The scripts' supporting files are unpacked relative to $GOPATH/src (aka $WORK/gopath/src)
-and then the script begins execution in that directory as well. Thus the example above runs
-in $WORK/gopath/src with GOPATH=$WORK/gopath and $WORK/gopath/src/hello.go
-containing the listed contents.
+On Plan 9, the variables $path and $home are set instead of $PATH and $HOME.
+On Windows, the variables $USERPROFILE and $TMP are set instead of
+$HOME and $TMPDIR.
+
+In addition, variables named ':' and '/' are expanded within script arguments
+(expanding to the value of os.PathListSeparator and os.PathSeparator
+respectively) but are not inherited in subprocess environments.
+
+The scripts' supporting files are unpacked relative to $GOPATH/src
+(aka $WORK/gopath/src) and then the script begins execution in that directory as
+well. Thus the example above runs in $WORK/gopath/src with GOPATH=$WORK/gopath
+and $WORK/gopath/src/hello.go containing the listed contents.
The lines at the top of the script are a sequence of commands to be executed
by a tiny script engine in ../../script_test.go (not the system shell).
From c1a4e0fe014568501b194eb8b04309f54eee6b4c Mon Sep 17 00:00:00 2001
From: Keith Randall
Date: Fri, 8 Jul 2022 14:52:23 -0700
Subject: [PATCH 31/78] cmd/compile: fix libfuzzer instrumentation line number
Set a reasonable starting line number before processing the body of
the function in the order pass.
We update base.Pos each time we process a node, but some of the
libfuzzer instrumentation is added before we process any node, so the
base.Pos used is junk.
Fixes #53688
Change-Id: I3654b805eabb8866a9a1574845ef4ff062797319
Reviewed-on: https://go-review.googlesource.com/c/go/+/416654
Reviewed-by: David Chase
Reviewed-by: Keith Randall
Run-TryBot: Keith Randall
TryBot-Result: Gopher Robot
---
src/cmd/compile/internal/walk/order.go | 8 +++++++-
1 file changed, 7 insertions(+), 1 deletion(-)
diff --git a/src/cmd/compile/internal/walk/order.go b/src/cmd/compile/internal/walk/order.go
index 8d1089dcc1..2d1e88238c 100644
--- a/src/cmd/compile/internal/walk/order.go
+++ b/src/cmd/compile/internal/walk/order.go
@@ -63,7 +63,7 @@ func order(fn *ir.Func) {
s := fmt.Sprintf("\nbefore order %v", fn.Sym())
ir.DumpList(s, fn.Body)
}
-
+ ir.SetPos(fn) // Set reasonable position for instrumenting code. See issue 53688.
orderBlock(&fn.Body, map[string][]*ir.Name{})
}
@@ -477,6 +477,12 @@ func (o *orderState) edge() {
// and then replaces the old slice in n with the new slice.
// free is a map that can be used to obtain temporary variables by type.
func orderBlock(n *ir.Nodes, free map[string][]*ir.Name) {
+ if len(*n) != 0 {
+ // Set reasonable position for instrumenting code. See issue 53688.
+ // It would be nice if ir.Nodes had a position (the opening {, probably),
+ // but it doesn't. So we use the first statement's position instead.
+ ir.SetPos((*n)[0])
+ }
var order orderState
order.free = free
mark := order.markTemp()
From 59ab6f351a370a27458755dc69f4a837e55a05a6 Mon Sep 17 00:00:00 2001
From: Mitar
Date: Sun, 10 Jul 2022 14:06:09 +0000
Subject: [PATCH 32/78] net/http: remove Content-Encoding in writeNotModified
Additional header to remove if set before calling http.ServeContent.
The API of ServeContent is that one should set Content-Encoding before calling it, if the content is encoded (e.g., compressed). But then, if content has not been modified, that header should be removed, according to RFC 7232 section 4.1.
Change-Id: If51b35b7811a4dbb19de2ddb73f40c5e68fcec7e
GitHub-Last-Rev: 53df6e73c44b63f351f7aeeb45cab82d706311eb
GitHub-Pull-Request: golang/go#50903
Reviewed-on: https://go-review.googlesource.com/c/go/+/381955
Run-TryBot: hopehook
Reviewed-by: Damien Neil
TryBot-Result: Gopher Robot
Reviewed-by: Benny Siegert
---
src/net/http/fs.go | 1 +
src/net/http/fs_test.go | 54 +++++++++++++++++++++++++++++++++++++++++
2 files changed, 55 insertions(+)
diff --git a/src/net/http/fs.go b/src/net/http/fs.go
index 7a1d5f4be5..4f144ebad2 100644
--- a/src/net/http/fs.go
+++ b/src/net/http/fs.go
@@ -541,6 +541,7 @@ func writeNotModified(w ResponseWriter) {
h := w.Header()
delete(h, "Content-Type")
delete(h, "Content-Length")
+ delete(h, "Content-Encoding")
if h.Get("Etag") != "" {
delete(h, "Last-Modified")
}
diff --git a/src/net/http/fs_test.go b/src/net/http/fs_test.go
index d627dfd4be..4be561cdfa 100644
--- a/src/net/http/fs_test.go
+++ b/src/net/http/fs_test.go
@@ -564,6 +564,60 @@ func testServeFileWithContentEncoding(t *testing.T, h2 bool) {
}
}
+// Tests that ServeFile does not generate representation metadata when
+// file has not been modified, as per RFC 7232 section 4.1.
+func TestServeFileNotModified_h1(t *testing.T) { testServeFileNotModified(t, h1Mode) }
+func TestServeFileNotModified_h2(t *testing.T) { testServeFileNotModified(t, h2Mode) }
+func testServeFileNotModified(t *testing.T, h2 bool) {
+ defer afterTest(t)
+ cst := newClientServerTest(t, h2, HandlerFunc(func(w ResponseWriter, r *Request) {
+ w.Header().Set("Content-Type", "application/json")
+ w.Header().Set("Content-Encoding", "foo")
+ w.Header().Set("Etag", `"123"`)
+ ServeFile(w, r, "testdata/file")
+
+ // Because the testdata is so small, it would fit in
+ // both the h1 and h2 Server's write buffers. For h1,
+ // sendfile is used, though, forcing a header flush at
+ // the io.Copy. http2 doesn't do a header flush so
+ // buffers all 11 bytes and then adds its own
+ // Content-Length. To prevent the Server's
+ // Content-Length and test ServeFile only, flush here.
+ w.(Flusher).Flush()
+ }))
+ defer cst.close()
+ req, err := NewRequest("GET", cst.ts.URL, nil)
+ if err != nil {
+ t.Fatal(err)
+ }
+ req.Header.Set("If-None-Match", `"123"`)
+ resp, err := cst.c.Do(req)
+ if err != nil {
+ t.Fatal(err)
+ }
+ b, err := io.ReadAll(resp.Body)
+ resp.Body.Close()
+ if err != nil {
+ t.Fatal("reading Body:", err)
+ }
+ if len(b) != 0 {
+ t.Errorf("non-empty body")
+ }
+ if g, e := resp.StatusCode, StatusNotModified; g != e {
+ t.Errorf("status mismatch: got %d, want %d", g, e)
+ }
+ // HTTP1 transport sets ContentLength to 0.
+ if g, e1, e2 := resp.ContentLength, int64(-1), int64(0); g != e1 && g != e2 {
+ t.Errorf("Content-Length mismatch: got %d, want %d or %d", g, e1, e2)
+ }
+ if resp.Header.Get("Content-Type") != "" {
+ t.Errorf("Content-Type present, but it should not be")
+ }
+ if resp.Header.Get("Content-Encoding") != "" {
+ t.Errorf("Content-Encoding present, but it should not be")
+ }
+}
+
func TestServeIndexHtml(t *testing.T) {
defer afterTest(t)
From f956941b0f5a5a841827bd3e84401d32916bb73e Mon Sep 17 00:00:00 2001
From: Russ Cox
Date: Tue, 5 Jul 2022 13:53:35 -0400
Subject: [PATCH 33/78] cmd/go: use package index for std in
load.loadPackageData
load.loadPackageData was only using an index for modules,
not for standard library packages. Other parts of the code were
using the index, so there was some benefit, but not as much
as you'd hope.
With the index disabled, the Script/work test takes 2.2s on my Mac.
With the index enabled before this CL, it took 2.0s.
With the index enabled after this CL, it takes 1.6s.
Before this CL, the Script/work test issued:
429 IsDir
19 IsDirWithGoFiles
7 Lstat
9072 Open
993 ReadDir
256 Stat
7 Walk
3 indexModule
24 openIndexModule
525 openIndexPackage
After this CL, it issued:
19 IsDirWithGoFiles
7 Lstat
60 Open
606 ReadDir
256 Stat
7 Walk
3 indexModule
24 openIndexModule
525 openIndexPackage
This speedup helps the Dragonfly builder, which has very slow
file I/O and is timing out since a recent indexing change.
Times for go test -run=Script/^work$ on the Dragonfly builder:
50s before indexing changes
31s full module indexing of std
46s per-package indexing of std
It cuts the time for go test -run=Script/^work$ from 44s to 20s.
For #53577.
Change-Id: I7189a77fc7fdf61de3ab3447efc4e84d1fc52c25
Reviewed-on: https://go-review.googlesource.com/c/go/+/416134
Reviewed-by: Bryan Mills
---
src/cmd/go/internal/fsys/fsys.go | 65 +++++++++++++++++++++++++++-
src/cmd/go/internal/load/pkg.go | 9 +++-
src/cmd/go/internal/modindex/read.go | 2 +
src/cmd/go/internal/modindex/scan.go | 2 +
src/cmd/go/testdata/script/index.txt | 6 +++
5 files changed, 82 insertions(+), 2 deletions(-)
create mode 100644 src/cmd/go/testdata/script/index.txt
diff --git a/src/cmd/go/internal/fsys/fsys.go b/src/cmd/go/internal/fsys/fsys.go
index 41d0bbfe66..d96a290de5 100644
--- a/src/cmd/go/internal/fsys/fsys.go
+++ b/src/cmd/go/internal/fsys/fsys.go
@@ -6,16 +6,65 @@ import (
"encoding/json"
"errors"
"fmt"
+ "internal/godebug"
"io/fs"
"io/ioutil"
+ "log"
"os"
+ pathpkg "path"
"path/filepath"
"runtime"
+ "runtime/debug"
"sort"
"strings"
+ "sync"
"time"
)
+// Trace emits a trace event for the operation and file path to the trace log,
+// but only when $GODEBUG contains gofsystrace=1.
+// The traces are appended to the file named by the $GODEBUG setting gofsystracelog, or else standard error.
+// For debugging, if the $GODEBUG setting gofsystracestack is non-empty, then trace events for paths
+// matching that glob pattern (using path.Match) will be followed by a full stack trace.
+func Trace(op, path string) {
+ if !doTrace {
+ return
+ }
+ traceMu.Lock()
+ defer traceMu.Unlock()
+ fmt.Fprintf(traceFile, "%d gofsystrace %s %s\n", os.Getpid(), op, path)
+ if traceStack != "" {
+ if match, _ := pathpkg.Match(traceStack, path); match {
+ traceFile.Write(debug.Stack())
+ }
+ }
+}
+
+var (
+ doTrace bool
+ traceStack string
+ traceFile *os.File
+ traceMu sync.Mutex
+)
+
+func init() {
+ if godebug.Get("gofsystrace") != "1" {
+ return
+ }
+ doTrace = true
+ traceStack = godebug.Get("gofsystracestack")
+ if f := godebug.Get("gofsystracelog"); f != "" {
+ // Note: No buffering on writes to this file, so no need to worry about closing it at exit.
+ var err error
+ traceFile, err = os.OpenFile(f, os.O_WRONLY|os.O_APPEND|os.O_CREATE, 0666)
+ if err != nil {
+ log.Fatal(err)
+ }
+ } else {
+ traceFile = os.Stderr
+ }
+}
+
// OverlayFile is the path to a text file in the OverlayJSON format.
// It is the value of the -overlay flag.
var OverlayFile string
@@ -86,6 +135,7 @@ func Init(wd string) error {
return nil
}
+ Trace("ReadFile", OverlayFile)
b, err := os.ReadFile(OverlayFile)
if err != nil {
return fmt.Errorf("reading overlay file: %v", err)
@@ -191,6 +241,7 @@ func initFromJSON(overlayJSON OverlayJSON) error {
// IsDir returns true if path is a directory on disk or in the
// overlay.
func IsDir(path string) (bool, error) {
+ Trace("IsDir", path)
path = canonicalize(path)
if _, ok := parentIsOverlayFile(path); ok {
@@ -260,6 +311,7 @@ func readDir(dir string) ([]fs.FileInfo, error) {
// ReadDir provides a slice of fs.FileInfo entries corresponding
// to the overlaid files in the directory.
func ReadDir(dir string) ([]fs.FileInfo, error) {
+ Trace("ReadDir", dir)
dir = canonicalize(dir)
if _, ok := parentIsOverlayFile(dir); ok {
return nil, &fs.PathError{Op: "ReadDir", Path: dir, Err: errNotDir}
@@ -327,11 +379,17 @@ func OverlayPath(path string) (string, bool) {
// Open opens the file at or overlaid on the given path.
func Open(path string) (*os.File, error) {
- return OpenFile(path, os.O_RDONLY, 0)
+ Trace("Open", path)
+ return openFile(path, os.O_RDONLY, 0)
}
// OpenFile opens the file at or overlaid on the given path with the flag and perm.
func OpenFile(path string, flag int, perm os.FileMode) (*os.File, error) {
+ Trace("OpenFile", path)
+ return openFile(path, flag, perm)
+}
+
+func openFile(path string, flag int, perm os.FileMode) (*os.File, error) {
cpath := canonicalize(path)
if node, ok := overlay[cpath]; ok {
// Opening a file in the overlay.
@@ -360,6 +418,7 @@ func OpenFile(path string, flag int, perm os.FileMode) (*os.File, error) {
// IsDirWithGoFiles reports whether dir is a directory containing Go files
// either on disk or in the overlay.
func IsDirWithGoFiles(dir string) (bool, error) {
+ Trace("IsDirWithGoFiles", dir)
fis, err := ReadDir(dir)
if os.IsNotExist(err) || errors.Is(err, errNotDir) {
return false, nil
@@ -436,6 +495,7 @@ func walk(path string, info fs.FileInfo, walkFn filepath.WalkFunc) error {
// Walk walks the file tree rooted at root, calling walkFn for each file or
// directory in the tree, including root.
func Walk(root string, walkFn filepath.WalkFunc) error {
+ Trace("Walk", root)
info, err := Lstat(root)
if err != nil {
err = walkFn(root, nil, err)
@@ -450,11 +510,13 @@ func Walk(root string, walkFn filepath.WalkFunc) error {
// lstat implements a version of os.Lstat that operates on the overlay filesystem.
func Lstat(path string) (fs.FileInfo, error) {
+ Trace("Lstat", path)
return overlayStat(path, os.Lstat, "lstat")
}
// Stat implements a version of os.Stat that operates on the overlay filesystem.
func Stat(path string) (fs.FileInfo, error) {
+ Trace("Stat", path)
return overlayStat(path, os.Stat, "stat")
}
@@ -528,6 +590,7 @@ func (f fakeDir) Sys() any { return nil }
// Glob is like filepath.Glob but uses the overlay file system.
func Glob(pattern string) (matches []string, err error) {
+ Trace("Glob", pattern)
// Check pattern is well-formed.
if _, err := filepath.Match(pattern, ""); err != nil {
return nil, err
diff --git a/src/cmd/go/internal/load/pkg.go b/src/cmd/go/internal/load/pkg.go
index fcb72b07b2..046f508545 100644
--- a/src/cmd/go/internal/load/pkg.go
+++ b/src/cmd/go/internal/load/pkg.go
@@ -877,7 +877,14 @@ func loadPackageData(ctx context.Context, path, parentPath, parentDir, parentRoo
if !cfg.ModulesEnabled {
buildMode = build.ImportComment
}
- if modroot := modload.PackageModRoot(ctx, r.path); modroot != "" {
+ modroot := modload.PackageModRoot(ctx, r.path)
+ if modroot == "" && str.HasPathPrefix(r.dir, cfg.GOROOTsrc) {
+ modroot = cfg.GOROOTsrc
+ if str.HasPathPrefix(r.dir, cfg.GOROOTsrc+string(filepath.Separator)+"cmd") {
+ modroot += string(filepath.Separator) + "cmd"
+ }
+ }
+ if modroot != "" {
if rp, err := modindex.GetPackage(modroot, r.dir); err == nil {
data.p, data.err = rp.Import(cfg.BuildContext, buildMode)
goto Happy
diff --git a/src/cmd/go/internal/modindex/read.go b/src/cmd/go/internal/modindex/read.go
index 7ee4669e67..436bbebb39 100644
--- a/src/cmd/go/internal/modindex/read.go
+++ b/src/cmd/go/internal/modindex/read.go
@@ -179,6 +179,7 @@ func openIndexModule(modroot string, ismodcache bool) (*Module, error) {
err error
}
r := mcache.Do(modroot, func() any {
+ fsys.Trace("openIndexModule", modroot)
id, err := moduleHash(modroot, ismodcache)
if err != nil {
return result{nil, err}
@@ -212,6 +213,7 @@ func openIndexPackage(modroot, pkgdir string) (*IndexPackage, error) {
err error
}
r := pcache.Do([2]string{modroot, pkgdir}, func() any {
+ fsys.Trace("openIndexPackage", pkgdir)
id, err := dirHash(modroot, pkgdir)
if err != nil {
return result{nil, err}
diff --git a/src/cmd/go/internal/modindex/scan.go b/src/cmd/go/internal/modindex/scan.go
index 1ba7c0cad1..d3f059bcfc 100644
--- a/src/cmd/go/internal/modindex/scan.go
+++ b/src/cmd/go/internal/modindex/scan.go
@@ -46,6 +46,7 @@ func moduleWalkErr(modroot string, path string, info fs.FileInfo, err error) err
// encoded representation. It returns ErrNotIndexed if the module can't
// be indexed because it contains symlinks.
func indexModule(modroot string) ([]byte, error) {
+ fsys.Trace("indexModule", modroot)
var packages []*rawPackage
err := fsys.Walk(modroot, func(path string, info fs.FileInfo, err error) error {
if err := moduleWalkErr(modroot, path, info, err); err != nil {
@@ -72,6 +73,7 @@ func indexModule(modroot string) ([]byte, error) {
// encoded representation. It returns ErrNotIndexed if the package can't
// be indexed.
func indexPackage(modroot, pkgdir string) []byte {
+ fsys.Trace("indexPackage", pkgdir)
p := importRaw(modroot, relPath(pkgdir, modroot))
return encodePackageBytes(p)
}
diff --git a/src/cmd/go/testdata/script/index.txt b/src/cmd/go/testdata/script/index.txt
new file mode 100644
index 0000000000..6a2d13c8b5
--- /dev/null
+++ b/src/cmd/go/testdata/script/index.txt
@@ -0,0 +1,6 @@
+# Check that standard library packages are cached.
+go list -json math # refresh cache
+env GODEBUG=gofsystrace=1,gofsystracelog=fsys.log
+go list -json math
+! grep math/abs.go fsys.log
+grep 'openIndexPackage .*[\\/]math$' fsys.log
From 398dcd1cf00a1536dad98cf87c16f8ad0c8913fc Mon Sep 17 00:00:00 2001
From: Dmitri Goutnik
Date: Sat, 9 Jul 2022 07:36:45 -0500
Subject: [PATCH 34/78] database/sql: make TestTxContextWaitNoDiscard test more
robust
Similar to CL 385934, rely on waiter trigger instead of the WAIT query
prefix and factor out the common test code.
Fixes #53222
Change-Id: I46efc85ca102b350bb4dbe8e514921e016870ffb
Reviewed-on: https://go-review.googlesource.com/c/go/+/416655
Reviewed-by: Michael Knyszek
TryBot-Result: Gopher Robot
Auto-Submit: Bryan Mills
Run-TryBot: Dmitri Goutnik
Reviewed-by: Bryan Mills
---
src/database/sql/sql_test.go | 45 ++++++++++++------------------------
1 file changed, 15 insertions(+), 30 deletions(-)
diff --git a/src/database/sql/sql_test.go b/src/database/sql/sql_test.go
index 6bc869fc86..8c58723c03 100644
--- a/src/database/sql/sql_test.go
+++ b/src/database/sql/sql_test.go
@@ -449,6 +449,16 @@ func TestQueryContextWait(t *testing.T) {
// TestTxContextWait tests the transaction behavior when the tx context is canceled
// during execution of the query.
func TestTxContextWait(t *testing.T) {
+ testContextWait(t, false)
+}
+
+// TestTxContextWaitNoDiscard is the same as TestTxContextWait, but should not discard
+// the final connection.
+func TestTxContextWaitNoDiscard(t *testing.T) {
+ testContextWait(t, true)
+}
+
+func testContextWait(t *testing.T, keepConnOnRollback bool) {
db := newTestDB(t, "people")
defer closeDB(t, db)
@@ -458,7 +468,7 @@ func TestTxContextWait(t *testing.T) {
if err != nil {
t.Fatal(err)
}
- tx.keepConnOnRollback = false
+ tx.keepConnOnRollback = keepConnOnRollback
tx.dc.ci.(*fakeConn).waiter = func(c context.Context) {
cancel()
@@ -472,36 +482,11 @@ func TestTxContextWait(t *testing.T) {
t.Fatalf("expected QueryContext to error with context canceled but returned %v", err)
}
- waitForFree(t, db, 0)
-}
-
-// TestTxContextWaitNoDiscard is the same as TestTxContextWait, but should not discard
-// the final connection.
-func TestTxContextWaitNoDiscard(t *testing.T) {
- db := newTestDB(t, "people")
- defer closeDB(t, db)
-
- ctx, cancel := context.WithTimeout(context.Background(), 15*time.Millisecond)
- defer cancel()
-
- tx, err := db.BeginTx(ctx, nil)
- if err != nil {
- // Guard against the context being canceled before BeginTx completes.
- if err == context.DeadlineExceeded {
- t.Skip("tx context canceled prior to first use")
- }
- t.Fatal(err)
+ if keepConnOnRollback {
+ waitForFree(t, db, 1)
+ } else {
+ waitForFree(t, db, 0)
}
-
- // This will trigger the *fakeConn.Prepare method which will take time
- // performing the query. The ctxDriverPrepare func will check the context
- // after this and close the rows and return an error.
- _, err = tx.QueryContext(ctx, "WAIT|1s|SELECT|people|age,name|")
- if err != context.DeadlineExceeded {
- t.Fatalf("expected QueryContext to error with context deadline exceeded but returned %v", err)
- }
-
- waitForFree(t, db, 1)
}
// TestUnsupportedOptions checks that the database fails when a driver that
From bf5898ef53d1693aa572da0da746c05e9a6f15c5 Mon Sep 17 00:00:00 2001
From: Sean Liao
Date: Sat, 9 Jul 2022 18:38:45 +0100
Subject: [PATCH 35/78] net/url: use EscapedPath for url.JoinPath
Fixes #53763
Change-Id: I08b53f159ebdce7907e8cc17316fd0c982363239
Reviewed-on: https://go-review.googlesource.com/c/go/+/416774
TryBot-Result: Gopher Robot
Reviewed-by: Damien Neil
Reviewed-by: Bryan Mills
Run-TryBot: Ian Lance Taylor
---
src/net/url/url.go | 2 +-
src/net/url/url_test.go | 10 ++++++++++
2 files changed, 11 insertions(+), 1 deletion(-)
diff --git a/src/net/url/url.go b/src/net/url/url.go
index db4d6385e3..e82ae6aeef 100644
--- a/src/net/url/url.go
+++ b/src/net/url/url.go
@@ -1193,7 +1193,7 @@ func (u *URL) UnmarshalBinary(text []byte) error {
func (u *URL) JoinPath(elem ...string) *URL {
url := *u
if len(elem) > 0 {
- elem = append([]string{u.Path}, elem...)
+ elem = append([]string{u.EscapedPath()}, elem...)
p := path.Join(elem...)
// path.Join will remove any trailing slashes.
// Preserve at least one.
diff --git a/src/net/url/url_test.go b/src/net/url/url_test.go
index 478cc34872..263eddffcf 100644
--- a/src/net/url/url_test.go
+++ b/src/net/url/url_test.go
@@ -2119,6 +2119,16 @@ func TestJoinPath(t *testing.T) {
elem: nil,
out: "https://go.googlesource.com/",
},
+ {
+ base: "https://go.googlesource.com/a%2fb",
+ elem: []string{"c"},
+ out: "https://go.googlesource.com/a%2fb/c",
+ },
+ {
+ base: "https://go.googlesource.com/a%2fb",
+ elem: []string{"c%2fd"},
+ out: "https://go.googlesource.com/a%2fb/c%2fd",
+ },
{
base: "/",
elem: nil,
From ad641e8521381886bc6274d78e986f2bb8ac561b Mon Sep 17 00:00:00 2001
From: "Bryan C. Mills"
Date: Mon, 11 Jul 2022 13:06:56 -0400
Subject: [PATCH 36/78] misc/cgo/testcarchive: don't rely on an erroneous
install target in tests
Non-main packages in module mode should not be installed to
GOPATH/pkg, but due to #37015 they were installed there anyway.
This change switches the 'go install' command in TestPIE to instead
use 'go build', and switches TestInstall and TestCachedInstall
(which appear to be explicitly testing 'go install') to explicitly
request GOPATH mode (which does have a well-defined install target).
For #37015.
Change-Id: Ifb24657d2781d1e35cf40078e8e3ebf56aab9cc8
Reviewed-on: https://go-review.googlesource.com/c/go/+/416954
TryBot-Result: Gopher Robot
Run-TryBot: Bryan Mills
Auto-Submit: Bryan Mills
Reviewed-by: Ian Lance Taylor
---
misc/cgo/testcarchive/carchive_test.go | 20 ++++++++++++--------
1 file changed, 12 insertions(+), 8 deletions(-)
diff --git a/misc/cgo/testcarchive/carchive_test.go b/misc/cgo/testcarchive/carchive_test.go
index d36b97b70e..c409c317dc 100644
--- a/misc/cgo/testcarchive/carchive_test.go
+++ b/misc/cgo/testcarchive/carchive_test.go
@@ -205,6 +205,7 @@ func genHeader(t *testing.T, header, dir string) {
func testInstall(t *testing.T, exe, libgoa, libgoh string, buildcmd ...string) {
t.Helper()
cmd := exec.Command(buildcmd[0], buildcmd[1:]...)
+ cmd.Env = append(cmd.Environ(), "GO111MODULE=off") // 'go install' only works in GOPATH mode
t.Log(buildcmd)
if out, err := cmd.CombinedOutput(); err != nil {
t.Logf("%s", out)
@@ -238,7 +239,7 @@ func testInstall(t *testing.T, exe, libgoa, libgoh string, buildcmd ...string) {
binArgs := append(cmdToRun(exe), "arg1", "arg2")
cmd = exec.Command(binArgs[0], binArgs[1:]...)
if runtime.Compiler == "gccgo" {
- cmd.Env = append(os.Environ(), "GCCGO=1")
+ cmd.Env = append(cmd.Environ(), "GCCGO=1")
}
if out, err := cmd.CombinedOutput(); err != nil {
t.Logf("%s", out)
@@ -822,9 +823,15 @@ func TestPIE(t *testing.T) {
t.Skipf("skipping PIE test on %s", GOOS)
}
+ libgoa := "libgo.a"
+ if runtime.Compiler == "gccgo" {
+ libgoa = "liblibgo.a"
+ }
+
if !testWork {
defer func() {
os.Remove("testp" + exeSuffix)
+ os.Remove(libgoa)
os.RemoveAll(filepath.Join(GOPATH, "pkg"))
}()
}
@@ -837,18 +844,13 @@ func TestPIE(t *testing.T) {
// be running this test in a GOROOT owned by root.)
genHeader(t, "p.h", "./p")
- cmd := exec.Command("go", "install", "-buildmode=c-archive", "./libgo")
+ cmd := exec.Command("go", "build", "-buildmode=c-archive", "./libgo")
if out, err := cmd.CombinedOutput(); err != nil {
t.Logf("%s", out)
t.Fatal(err)
}
- libgoa := "libgo.a"
- if runtime.Compiler == "gccgo" {
- libgoa = "liblibgo.a"
- }
-
- ccArgs := append(cc, "-fPIE", "-pie", "-o", "testp"+exeSuffix, "main.c", "main_unix.c", filepath.Join(libgodir, libgoa))
+ ccArgs := append(cc, "-fPIE", "-pie", "-o", "testp"+exeSuffix, "main.c", "main_unix.c", libgoa)
if runtime.Compiler == "gccgo" {
ccArgs = append(ccArgs, "-lgo")
}
@@ -1035,6 +1037,7 @@ func TestCachedInstall(t *testing.T) {
buildcmd := []string{"go", "install", "-buildmode=c-archive", "./libgo"}
cmd := exec.Command(buildcmd[0], buildcmd[1:]...)
+ cmd.Env = append(cmd.Environ(), "GO111MODULE=off") // 'go install' only works in GOPATH mode
t.Log(buildcmd)
if out, err := cmd.CombinedOutput(); err != nil {
t.Logf("%s", out)
@@ -1050,6 +1053,7 @@ func TestCachedInstall(t *testing.T) {
}
cmd = exec.Command(buildcmd[0], buildcmd[1:]...)
+ cmd.Env = append(cmd.Environ(), "GO111MODULE=off")
t.Log(buildcmd)
if out, err := cmd.CombinedOutput(); err != nil {
t.Logf("%s", out)
From b8bf820d5d4602f7e83ff89c0f8d0f2bd3a220d4 Mon Sep 17 00:00:00 2001
From: "Bryan C. Mills"
Date: Mon, 11 Jul 2022 13:32:40 -0400
Subject: [PATCH 37/78] cmd/nm: don't rely on an erroneous install target in
tests
Non-main packages in module mode should not be installed to
GOPATH/pkg, but due to #37015 they were installed there anyway.
This change switches the 'go install' command in testGoLib to instead
use 'go build -buildmode=archive' with an explicit output file.
For #37015.
Change-Id: I15781aa33d1b2adc6a4437a58622276f4e20b889
Reviewed-on: https://go-review.googlesource.com/c/go/+/416955
Reviewed-by: Ian Lance Taylor
TryBot-Result: Gopher Robot
Run-TryBot: Bryan Mills
Auto-Submit: Bryan Mills
---
src/cmd/nm/nm_test.go | 13 ++-----------
1 file changed, 2 insertions(+), 11 deletions(-)
diff --git a/src/cmd/nm/nm_test.go b/src/cmd/nm/nm_test.go
index 226c2c3bcd..4bc9bf9079 100644
--- a/src/cmd/nm/nm_test.go
+++ b/src/cmd/nm/nm_test.go
@@ -250,23 +250,14 @@ func testGoLib(t *testing.T, iscgo bool) {
t.Fatal(err)
}
- args := []string{"install", "mylib"}
- cmd := exec.Command(testenv.GoToolPath(t), args...)
+ cmd := exec.Command(testenv.GoToolPath(t), "build", "-buildmode=archive", "-o", "mylib.a", ".")
cmd.Dir = libpath
cmd.Env = append(os.Environ(), "GOPATH="+gopath)
out, err := cmd.CombinedOutput()
if err != nil {
t.Fatalf("building test lib failed: %s %s", err, out)
}
- pat := filepath.Join(gopath, "pkg", "*", "mylib.a")
- ms, err := filepath.Glob(pat)
- if err != nil {
- t.Fatal(err)
- }
- if len(ms) == 0 {
- t.Fatalf("cannot found paths for pattern %s", pat)
- }
- mylib := ms[0]
+ mylib := filepath.Join(libpath, "mylib.a")
out, err = exec.Command(testnmpath, mylib).CombinedOutput()
if err != nil {
From 7510e597def68cee77e8ba280fc0f04d3cfd2a22 Mon Sep 17 00:00:00 2001
From: Russ Cox
Date: Wed, 6 Jul 2022 09:49:32 -0400
Subject: [PATCH 38/78] cmd/go: make module index loading O(1)
For a large module, opening the index was populating tables with
entries for every package in the module. If we are only using a small
number of those packages, this is wasted work that can dwarf the
benefit from the index.
This CL changes the index reader to avoid loading all packages
at module index open time. It also refactors the code somewhat
for clarity.
It also removes some duplication by defining that a per-package
index is a per-module index containing a single package, rather
than having two different formats and two different decoders.
It also changes the string table to use uvarint-prefixed data
instead of having to scan for a NUL byte. This makes random access
to long strings more efficient - O(1) instead of O(n) - and can significantly
speed up the strings.Compare operation in the binary search looking
for a given package.
Also add a direct test of the indexing code.
For #53577.
Change-Id: I7428d28133e4e7fe2d2993fa014896cd15af48af
Reviewed-on: https://go-review.googlesource.com/c/go/+/416178
Reviewed-by: Bryan Mills
---
src/cmd/go/internal/modindex/index_test.go | 87 ++++
src/cmd/go/internal/modindex/read.go | 514 ++++++++++-----------
src/cmd/go/internal/modindex/write.go | 46 +-
src/cmd/go/internal/modload/search.go | 13 +-
4 files changed, 356 insertions(+), 304 deletions(-)
create mode 100644 src/cmd/go/internal/modindex/index_test.go
diff --git a/src/cmd/go/internal/modindex/index_test.go b/src/cmd/go/internal/modindex/index_test.go
new file mode 100644
index 0000000000..2c072f909d
--- /dev/null
+++ b/src/cmd/go/internal/modindex/index_test.go
@@ -0,0 +1,87 @@
+// Copyright 2022 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package modindex
+
+import (
+ "encoding/hex"
+ "encoding/json"
+ "go/build"
+ "internal/diff"
+ "path/filepath"
+ "reflect"
+ "runtime"
+ "testing"
+)
+
+func init() {
+ isTest = true
+ enabled = true // to allow GODEBUG=goindex=0 go test, when things are very broken
+}
+
+func TestIndex(t *testing.T) {
+ src := filepath.Join(runtime.GOROOT(), "src")
+ checkPkg := func(t *testing.T, m *Module, pkg string, data []byte) {
+ p := m.Package(pkg)
+ bp, err := p.Import(build.Default, build.ImportComment)
+ if err != nil {
+ t.Fatal(err)
+ }
+ bp1, err := build.Default.Import(pkg, filepath.Join(src, pkg), build.ImportComment)
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ if !reflect.DeepEqual(bp, bp1) {
+ t.Errorf("mismatch")
+ t.Logf("index:\n%s", hex.Dump(data))
+
+ js, err := json.MarshalIndent(bp, "", "\t")
+ if err != nil {
+ t.Fatal(err)
+ }
+ js1, err := json.MarshalIndent(bp1, "", "\t")
+ if err != nil {
+ t.Fatal(err)
+ }
+ t.Logf("diff:\n%s", diff.Diff("index", js, "correct", js1))
+ t.FailNow()
+ }
+ }
+
+ // Check packages in increasing complexity, one at a time.
+ pkgs := []string{
+ "crypto",
+ "encoding",
+ "unsafe",
+ "encoding/json",
+ "runtime",
+ "net",
+ }
+ var raws []*rawPackage
+ for _, pkg := range pkgs {
+ raw := importRaw(src, pkg)
+ raws = append(raws, raw)
+ t.Run(pkg, func(t *testing.T) {
+ data := encodeModuleBytes([]*rawPackage{raw})
+ m, err := fromBytes(src, data)
+ if err != nil {
+ t.Fatal(err)
+ }
+ checkPkg(t, m, pkg, data)
+ })
+ }
+
+ // Check that a multi-package index works too.
+ t.Run("all", func(t *testing.T) {
+ data := encodeModuleBytes(raws)
+ m, err := fromBytes(src, data)
+ if err != nil {
+ t.Fatal(err)
+ }
+ for _, pkg := range pkgs {
+ checkPkg(t, m, pkg, data)
+ }
+ })
+}
diff --git a/src/cmd/go/internal/modindex/read.go b/src/cmd/go/internal/modindex/read.go
index 436bbebb39..38ddfec70f 100644
--- a/src/cmd/go/internal/modindex/read.go
+++ b/src/cmd/go/internal/modindex/read.go
@@ -15,7 +15,6 @@ import (
"internal/godebug"
"internal/goroot"
"internal/unsafeheader"
- "math"
"path"
"path/filepath"
"runtime"
@@ -45,10 +44,9 @@ var enabled bool = godebug.Get("goindex") != "0"
// do the equivalent of build.Import of packages in the module and answer other
// questions based on the index file's data.
type Module struct {
- modroot string
- od offsetDecoder
- packages map[string]int // offsets of each package
- packagePaths []string // paths to package directories relative to modroot; these are the keys of packages
+ modroot string
+ d *decoder
+ n int // number of packages
}
// moduleHash returns an ActionID corresponding to the state of the module
@@ -236,110 +234,131 @@ func openIndexPackage(modroot, pkgdir string) (*IndexPackage, error) {
return r.pkg, r.err
}
+var errCorrupt = errors.New("corrupt index")
+
+// protect marks the start of a large section of code that accesses the index.
+// It should be used as:
+//
+// defer unprotect(protect, &err)
+//
+// It should not be used for trivial accesses which would be
+// dwarfed by the overhead of the defer.
+func protect() bool {
+ return debug.SetPanicOnFault(true)
+}
+
+var isTest = false
+
+// unprotect marks the end of a large section of code that accesses the index.
+// It should be used as:
+//
+// defer unprotect(protect, &err)
+//
+// end looks for panics due to errCorrupt or bad mmap accesses.
+// When it finds them, it adds explanatory text, consumes the panic, and sets *errp instead.
+// If errp is nil, end adds the explanatory text but then calls base.Fatalf.
+func unprotect(old bool, errp *error) {
+ // SetPanicOnFault's errors _may_ satisfy this interface. Even though it's not guaranteed
+ // that all its errors satisfy this interface, we'll only check for these errors so that
+ // we don't suppress panics that could have been produced from other sources.
+ type addrer interface {
+ Addr() uintptr
+ }
+
+ debug.SetPanicOnFault(old)
+
+ if e := recover(); e != nil {
+ if _, ok := e.(addrer); ok || e == errCorrupt {
+ // This panic was almost certainly caused by SetPanicOnFault or our panic(errCorrupt).
+ err := fmt.Errorf("error reading module index: %v", e)
+ if errp != nil {
+ *errp = err
+ return
+ }
+ if isTest {
+ panic(err)
+ }
+ base.Fatalf("%v", err)
+ }
+ // The panic was likely not caused by SetPanicOnFault.
+ panic(e)
+ }
+}
+
// fromBytes returns a *Module given the encoded representation.
-func fromBytes(moddir string, data []byte) (mi *Module, err error) {
+func fromBytes(moddir string, data []byte) (m *Module, err error) {
if !enabled {
panic("use of index")
}
- // SetPanicOnFault's errors _may_ satisfy this interface. Even though it's not guaranteed
- // that all its errors satisfy this interface, we'll only check for these errors so that
- // we don't suppress panics that could have been produced from other sources.
- type addrer interface {
- Addr() uintptr
+ defer unprotect(protect(), &err)
+
+ if !bytes.HasPrefix(data, []byte(indexVersion+"\n")) {
+ return nil, errCorrupt
}
- // set PanicOnFault to true so that we can catch errors on the initial reads of the slice,
- // in case it's mmapped (the common case).
- old := debug.SetPanicOnFault(true)
- defer func() {
- debug.SetPanicOnFault(old)
- if e := recover(); e != nil {
- if _, ok := e.(addrer); ok {
- // This panic was almost certainly caused by SetPanicOnFault.
- err = fmt.Errorf("error reading module index: %v", e)
- return
- }
- // The panic was likely not caused by SetPanicOnFault.
- panic(e)
- }
- }()
-
- gotVersion, unread, _ := bytes.Cut(data, []byte{'\n'})
- if string(gotVersion) != indexVersion {
- return nil, fmt.Errorf("bad index version string: %q", gotVersion)
+ const hdr = len(indexVersion + "\n")
+ d := &decoder{data: data}
+ str := d.intAt(hdr)
+ if str < hdr+8 || len(d.data) < str {
+ return nil, errCorrupt
}
- stringTableOffset, unread := binary.LittleEndian.Uint32(unread[:4]), unread[4:]
- st := newStringTable(data[stringTableOffset:])
- d := decoder{unread, st}
- numPackages := d.int()
-
- packagePaths := make([]string, numPackages)
- for i := range packagePaths {
- packagePaths[i] = d.string()
- }
- packageOffsets := make([]int, numPackages)
- for i := range packageOffsets {
- packageOffsets[i] = d.int()
- }
- packages := make(map[string]int, numPackages)
- for i := range packagePaths {
- packages[packagePaths[i]] = packageOffsets[i]
+ d.data, d.str = data[:str], d.data[str:]
+ // Check that string table looks valid.
+ // First string is empty string (length 0),
+ // and we leave a marker byte 0xFF at the end
+ // just to make sure that the file is not truncated.
+ if len(d.str) == 0 || d.str[0] != 0 || d.str[len(d.str)-1] != 0xFF {
+ return nil, errCorrupt
}
- return &Module{
+ n := d.intAt(hdr + 4)
+ if n < 0 || n > (len(d.data)-8)/8 {
+ return nil, errCorrupt
+ }
+
+ m = &Module{
moddir,
- offsetDecoder{data, st},
- packages,
- packagePaths,
- }, nil
+ d,
+ n,
+ }
+ return m, nil
}
// packageFromBytes returns a *IndexPackage given the encoded representation.
func packageFromBytes(modroot string, data []byte) (p *IndexPackage, err error) {
- if !enabled {
- panic("use of package index when not enabled")
+ m, err := fromBytes(modroot, data)
+ if err != nil {
+ return nil, err
}
-
- // SetPanicOnFault's errors _may_ satisfy this interface. Even though it's not guaranteed
- // that all its errors satisfy this interface, we'll only check for these errors so that
- // we don't suppress panics that could have been produced from other sources.
- type addrer interface {
- Addr() uintptr
+ if m.n != 1 {
+ return nil, fmt.Errorf("corrupt single-package index")
}
-
- // set PanicOnFault to true so that we can catch errors on the initial reads of the slice,
- // in case it's mmapped (the common case).
- old := debug.SetPanicOnFault(true)
- defer func() {
- debug.SetPanicOnFault(old)
- if e := recover(); e != nil {
- if _, ok := e.(addrer); ok {
- // This panic was almost certainly caused by SetPanicOnFault.
- err = fmt.Errorf("error reading module index: %v", e)
- return
- }
- // The panic was likely not caused by SetPanicOnFault.
- panic(e)
- }
- }()
-
- gotVersion, unread, _ := bytes.Cut(data, []byte{'\n'})
- if string(gotVersion) != indexVersion {
- return nil, fmt.Errorf("bad index version string: %q", gotVersion)
- }
- stringTableOffset, unread := binary.LittleEndian.Uint32(unread[:4]), unread[4:]
- st := newStringTable(data[stringTableOffset:])
- d := &decoder{unread, st}
- p = decodePackage(d, offsetDecoder{data, st})
- p.modroot = modroot
- return p, nil
+ return m.pkg(0), nil
}
-// Returns a list of directory paths, relative to the modroot, for
-// packages contained in the module index.
-func (mi *Module) Packages() []string {
- return mi.packagePaths
+// pkgDir returns the dir string of the i'th package in the index.
+func (m *Module) pkgDir(i int) string {
+ if i < 0 || i >= m.n {
+ panic(errCorrupt)
+ }
+ return m.d.stringAt(12 + 8 + 8*i)
+}
+
+// pkgOff returns the offset of the data for the i'th package in the index.
+func (m *Module) pkgOff(i int) int {
+ if i < 0 || i >= m.n {
+ panic(errCorrupt)
+ }
+ return m.d.intAt(12 + 8 + 8*i + 4)
+}
+
+// Walk calls f for each package in the index, passing the path to that package relative to the module root.
+func (m *Module) Walk(f func(path string)) {
+ defer unprotect(protect(), nil)
+ for i := 0; i < m.n; i++ {
+ f(m.pkgDir(i))
+ }
}
// relPath returns the path relative to the module's root.
@@ -349,11 +368,7 @@ func relPath(path, modroot string) string {
// Import is the equivalent of build.Import given the information in Module.
func (rp *IndexPackage) Import(bctxt build.Context, mode build.ImportMode) (p *build.Package, err error) {
- defer func() {
- if e := recover(); e != nil {
- err = fmt.Errorf("error reading module index: %v", e)
- }
- }()
+ defer unprotect(protect(), &err)
ctxt := (*Context)(&bctxt)
@@ -794,46 +809,44 @@ type IndexPackage struct {
var errCannotFindPackage = errors.New("cannot find package")
-// Package returns an IndexPackage constructed using the information in the Module.
-func (mi *Module) Package(path string) *IndexPackage {
- defer func() {
- if e := recover(); e != nil {
- base.Fatalf("error reading module index: %v", e)
- }
- }()
- offset, ok := mi.packages[path]
- if !ok {
- return &IndexPackage{error: fmt.Errorf("%w %q in:\n\t%s", errCannotFindPackage, path, filepath.Join(mi.modroot, path))}
- }
+// Package and returns finds the package with the given path (relative to the module root).
+// If the package does not exist, Package returns an IndexPackage that will return an
+// appropriate error from its methods.
+func (m *Module) Package(path string) *IndexPackage {
+ defer unprotect(protect(), nil)
- // TODO(matloob): do we want to lock on the module index?
- d := mi.od.decoderAt(offset)
- p := decodePackage(d, mi.od)
- p.modroot = mi.modroot
- return p
+ i, ok := sort.Find(m.n, func(i int) int {
+ return strings.Compare(path, m.pkgDir(i))
+ })
+ if !ok {
+ return &IndexPackage{error: fmt.Errorf("%w %q in:\n\t%s", errCannotFindPackage, path, filepath.Join(m.modroot, path))}
+ }
+ return m.pkg(i)
}
-func decodePackage(d *decoder, od offsetDecoder) *IndexPackage {
- rp := new(IndexPackage)
- if errstr := d.string(); errstr != "" {
- rp.error = errors.New(errstr)
+// pkgAt returns the i'th IndexPackage in m.
+func (m *Module) pkg(i int) *IndexPackage {
+ r := m.d.readAt(m.pkgOff(i))
+ p := new(IndexPackage)
+ if errstr := r.string(); errstr != "" {
+ p.error = errors.New(errstr)
}
- rp.dir = d.string()
- numSourceFiles := d.uint32()
- rp.sourceFiles = make([]*sourceFile, numSourceFiles)
- for i := uint32(0); i < numSourceFiles; i++ {
- offset := d.uint32()
- rp.sourceFiles[i] = &sourceFile{
- od: od.offsetDecoderAt(offset),
+ p.dir = r.string()
+ p.sourceFiles = make([]*sourceFile, r.int())
+ for i := range p.sourceFiles {
+ p.sourceFiles[i] = &sourceFile{
+ d: m.d,
+ pos: r.int(),
}
}
- return rp
+ p.modroot = m.modroot
+ return p
}
// sourceFile represents the information of a given source file in the module index.
type sourceFile struct {
- od offsetDecoder // od interprets all offsets relative to the start of the source file's data
-
+ d *decoder // encoding of this source file
+ pos int // start of sourceFile encoding in d
onceReadImports sync.Once
savedImports []rawImport // saved imports so that they're only read once
}
@@ -853,73 +866,67 @@ const (
)
func (sf *sourceFile) error() string {
- return sf.od.stringAt(sourceFileError)
+ return sf.d.stringAt(sf.pos + sourceFileError)
}
func (sf *sourceFile) parseError() string {
- return sf.od.stringAt(sourceFileParseError)
+ return sf.d.stringAt(sf.pos + sourceFileParseError)
}
func (sf *sourceFile) synopsis() string {
- return sf.od.stringAt(sourceFileSynopsis)
+ return sf.d.stringAt(sf.pos + sourceFileSynopsis)
}
func (sf *sourceFile) name() string {
- return sf.od.stringAt(sourceFileName)
+ return sf.d.stringAt(sf.pos + sourceFileName)
}
func (sf *sourceFile) pkgName() string {
- return sf.od.stringAt(sourceFilePkgName)
+ return sf.d.stringAt(sf.pos + sourceFilePkgName)
}
func (sf *sourceFile) ignoreFile() bool {
- return sf.od.boolAt(sourceFileIgnoreFile)
+ return sf.d.boolAt(sf.pos + sourceFileIgnoreFile)
}
func (sf *sourceFile) binaryOnly() bool {
- return sf.od.boolAt(sourceFileBinaryOnly)
+ return sf.d.boolAt(sf.pos + sourceFileBinaryOnly)
}
func (sf *sourceFile) cgoDirectives() string {
- return sf.od.stringAt(sourceFileCgoDirectives)
+ return sf.d.stringAt(sf.pos + sourceFileCgoDirectives)
}
func (sf *sourceFile) goBuildConstraint() string {
- return sf.od.stringAt(sourceFileGoBuildConstraint)
+ return sf.d.stringAt(sf.pos + sourceFileGoBuildConstraint)
}
func (sf *sourceFile) plusBuildConstraints() []string {
- d := sf.od.decoderAt(sourceFileNumPlusBuildConstraints)
- n := d.int()
+ pos := sf.pos + sourceFileNumPlusBuildConstraints
+ n := sf.d.intAt(pos)
+ pos += 4
ret := make([]string, n)
for i := 0; i < n; i++ {
- ret[i] = d.string()
+ ret[i] = sf.d.stringAt(pos)
+ pos += 4
}
return ret
}
-func importsOffset(numPlusBuildConstraints int) int {
- // 4 bytes per uin32, add one to advance past numPlusBuildConstraints itself
- return sourceFileNumPlusBuildConstraints + 4*(numPlusBuildConstraints+1)
-}
-
func (sf *sourceFile) importsOffset() int {
- numPlusBuildConstraints := sf.od.intAt(sourceFileNumPlusBuildConstraints)
- return importsOffset(numPlusBuildConstraints)
-}
-
-func embedsOffset(importsOffset, numImports int) int {
- // 4 bytes per uint32; 1 to advance past numImports itself, and 5 uint32s per import
- return importsOffset + 4*(1+(5*numImports))
+ pos := sf.pos + sourceFileNumPlusBuildConstraints
+ n := sf.d.intAt(pos)
+ // each build constraint is 1 uint32
+ return pos + 4 + n*4
}
func (sf *sourceFile) embedsOffset() int {
- importsOffset := sf.importsOffset()
- numImports := sf.od.intAt(importsOffset)
- return embedsOffset(importsOffset, numImports)
+ pos := sf.importsOffset()
+ n := sf.d.intAt(pos)
+ // each import is 5 uint32s (string + tokpos)
+ return pos + 4 + n*(4*5)
}
func (sf *sourceFile) imports() []rawImport {
sf.onceReadImports.Do(func() {
importsOffset := sf.importsOffset()
- d := sf.od.decoderAt(importsOffset)
- numImports := d.int()
+ r := sf.d.readAt(importsOffset)
+ numImports := r.int()
ret := make([]rawImport, numImports)
for i := 0; i < numImports; i++ {
- ret[i].path = d.string()
- ret[i].position = d.tokpos()
+ ret[i] = rawImport{r.string(), r.tokpos()}
}
sf.savedImports = ret
})
@@ -928,125 +935,15 @@ func (sf *sourceFile) imports() []rawImport {
func (sf *sourceFile) embeds() []embed {
embedsOffset := sf.embedsOffset()
- d := sf.od.decoderAt(embedsOffset)
- numEmbeds := d.int()
+ r := sf.d.readAt(embedsOffset)
+ numEmbeds := r.int()
ret := make([]embed, numEmbeds)
for i := range ret {
- pattern := d.string()
- pos := d.tokpos()
- ret[i] = embed{pattern, pos}
+ ret[i] = embed{r.string(), r.tokpos()}
}
return ret
}
-// A decoder reads from the current position of the file and advances its position as it
-// reads.
-type decoder struct {
- b []byte
- st *stringTable
-}
-
-func (d *decoder) uint32() uint32 {
- n := binary.LittleEndian.Uint32(d.b[:4])
- d.b = d.b[4:]
- return n
-}
-
-func (d *decoder) int() int {
- n := d.uint32()
- if int64(n) > math.MaxInt {
- base.Fatalf("go: attempting to read a uint32 from the index that overflows int")
- }
- return int(n)
-}
-
-func (d *decoder) tokpos() token.Position {
- file := d.string()
- offset := d.int()
- line := d.int()
- column := d.int()
- return token.Position{
- Filename: file,
- Offset: offset,
- Line: line,
- Column: column,
- }
-}
-
-func (d *decoder) string() string {
- return d.st.string(d.int())
-}
-
-// And offset decoder reads information offset from its position in the file.
-// It's either offset from the beginning of the index, or the beginning of a sourceFile's data.
-type offsetDecoder struct {
- b []byte
- st *stringTable
-}
-
-func (od *offsetDecoder) uint32At(offset int) uint32 {
- if offset > len(od.b) {
- base.Fatalf("go: trying to read from index file at offset higher than file length. This indicates a corrupt offset file in the cache.")
- }
- return binary.LittleEndian.Uint32(od.b[offset:])
-}
-
-func (od *offsetDecoder) intAt(offset int) int {
- n := od.uint32At(offset)
- if int64(n) > math.MaxInt {
- base.Fatalf("go: attempting to read a uint32 from the index that overflows int")
- }
- return int(n)
-}
-
-func (od *offsetDecoder) boolAt(offset int) bool {
- switch v := od.uint32At(offset); v {
- case 0:
- return false
- case 1:
- return true
- default:
- base.Fatalf("go: invalid bool value in index file encoding: %v", v)
- }
- panic("unreachable")
-}
-
-func (od *offsetDecoder) stringAt(offset int) string {
- return od.st.string(od.intAt(offset))
-}
-
-func (od *offsetDecoder) decoderAt(offset int) *decoder {
- return &decoder{od.b[offset:], od.st}
-}
-
-func (od *offsetDecoder) offsetDecoderAt(offset uint32) offsetDecoder {
- return offsetDecoder{od.b[offset:], od.st}
-}
-
-type stringTable struct {
- b []byte
-}
-
-func newStringTable(b []byte) *stringTable {
- return &stringTable{b: b}
-}
-
-func (st *stringTable) string(pos int) string {
- if pos == 0 {
- return ""
- }
-
- bb := st.b[pos:]
- i := bytes.IndexByte(bb, 0)
-
- if i == -1 {
- panic("reached end of string table trying to read string")
- }
- s := asString(bb[:i])
-
- return s
-}
-
func asString(b []byte) string {
p := (*unsafeheader.Slice)(unsafe.Pointer(&b)).Data
@@ -1057,3 +954,82 @@ func asString(b []byte) string {
return s
}
+
+// A decoder helps decode the index format.
+type decoder struct {
+ data []byte // data after header
+ str []byte // string table
+}
+
+// intAt returns the int at the given offset in d.data.
+func (d *decoder) intAt(off int) int {
+ if off < 0 || len(d.data)-off < 4 {
+ panic(errCorrupt)
+ }
+ i := binary.LittleEndian.Uint32(d.data[off : off+4])
+ if int32(i)>>31 != 0 {
+ panic(errCorrupt)
+ }
+ return int(i)
+}
+
+// boolAt returns the bool at the given offset in d.data.
+func (d *decoder) boolAt(off int) bool {
+ return d.intAt(off) != 0
+}
+
+// stringTableAt returns the string pointed at by the int at the given offset in d.data.
+func (d *decoder) stringAt(off int) string {
+ return d.stringTableAt(d.intAt(off))
+}
+
+// stringTableAt returns the string at the given offset in the string table d.str.
+func (d *decoder) stringTableAt(off int) string {
+ if off < 0 || off >= len(d.str) {
+ panic(errCorrupt)
+ }
+ s := d.str[off:]
+ v, n := binary.Uvarint(s)
+ if n <= 0 || v > uint64(len(s[n:])) {
+ panic(errCorrupt)
+ }
+ return asString(s[n : n+int(v)])
+}
+
+// A reader reads sequential fields from a section of the index format.
+type reader struct {
+ d *decoder
+ pos int
+}
+
+// readAt returns a reader starting at the given position in d.
+func (d *decoder) readAt(pos int) *reader {
+ return &reader{d, pos}
+}
+
+// int reads the next int.
+func (r *reader) int() int {
+ i := r.d.intAt(r.pos)
+ r.pos += 4
+ return i
+}
+
+// string reads the next string.
+func (r *reader) string() string {
+ return r.d.stringTableAt(r.int())
+}
+
+// bool reads the next bool.
+func (r *reader) bool() bool {
+ return r.int() != 0
+}
+
+// tokpos reads the next token.Position.
+func (r *reader) tokpos() token.Position {
+ return token.Position{
+ Filename: r.string(),
+ Offset: r.int(),
+ Line: r.int(),
+ Column: r.int(),
+ }
+}
diff --git a/src/cmd/go/internal/modindex/write.go b/src/cmd/go/internal/modindex/write.go
index 3408248bd9..7db1fb0870 100644
--- a/src/cmd/go/internal/modindex/write.go
+++ b/src/cmd/go/internal/modindex/write.go
@@ -1,54 +1,46 @@
+// Copyright 2022 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
package modindex
import (
"cmd/go/internal/base"
"encoding/binary"
"go/token"
- "math"
"sort"
- "strings"
)
-const indexVersion = "go index v0"
+const indexVersion = "go index v1" // 11 bytes (plus \n), to align uint32s in index
// encodeModuleBytes produces the encoded representation of the module index.
// encodeModuleBytes may modify the packages slice.
func encodeModuleBytes(packages []*rawPackage) []byte {
e := newEncoder()
- e.Bytes([]byte(indexVersion))
- e.Bytes([]byte{'\n'})
+ e.Bytes([]byte(indexVersion + "\n"))
stringTableOffsetPos := e.Pos() // fill this at the end
e.Uint32(0) // string table offset
- e.Int(len(packages))
sort.Slice(packages, func(i, j int) bool {
return packages[i].dir < packages[j].dir
})
+ e.Int(len(packages))
+ packagesPos := e.Pos()
for _, p := range packages {
e.String(p.dir)
- }
- packagesOffsetPos := e.Pos()
- for range packages {
e.Int(0)
}
for i, p := range packages {
- e.IntAt(e.Pos(), packagesOffsetPos+4*i)
+ e.IntAt(e.Pos(), packagesPos+8*i+4)
encodePackage(e, p)
}
e.IntAt(e.Pos(), stringTableOffsetPos)
e.Bytes(e.stringTable)
+ e.Bytes([]byte{0xFF}) // end of string table marker
return e.b
}
func encodePackageBytes(p *rawPackage) []byte {
- e := newEncoder()
- e.Bytes([]byte(indexVersion))
- e.Bytes([]byte{'\n'})
- stringTableOffsetPos := e.Pos() // fill this at the end
- e.Uint32(0) // string table offset
- encodePackage(e, p)
- e.IntAt(e.Pos(), stringTableOffsetPos)
- e.Bytes(e.stringTable)
- return e.b
+ return encodeModuleBytes([]*rawPackage{p})
}
func encodePackage(e *encoder, p *rawPackage) {
@@ -126,9 +118,6 @@ func (e *encoder) Bytes(b []byte) {
}
func (e *encoder) String(s string) {
- if strings.IndexByte(s, 0) >= 0 {
- base.Fatalf("go: attempting to encode a string containing a null byte")
- }
if n, ok := e.strings[s]; ok {
e.Int(n)
return
@@ -136,8 +125,8 @@ func (e *encoder) String(s string) {
pos := len(e.stringTable)
e.strings[s] = pos
e.Int(pos)
+ e.stringTable = binary.AppendUvarint(e.stringTable, uint64(len(s)))
e.stringTable = append(e.stringTable, []byte(s)...)
- e.stringTable = append(e.stringTable, 0)
}
func (e *encoder) Bool(b bool) {
@@ -152,17 +141,18 @@ func (e *encoder) Uint32(n uint32) {
e.b = binary.LittleEndian.AppendUint32(e.b, n)
}
-// Int encodes n. Note that all ints are written to the index as uint32s.
+// Int encodes n. Note that all ints are written to the index as uint32s,
+// and to avoid problems on 32-bit systems we require fitting into a 32-bit int.
func (e *encoder) Int(n int) {
- if n < 0 || int64(n) > math.MaxUint32 {
- base.Fatalf("go: attempting to write an int to the index that overflows uint32")
+ if n < 0 || int(int32(n)) != n {
+ base.Fatalf("go: attempting to write an int to the index that overflows int32")
}
e.Uint32(uint32(n))
}
func (e *encoder) IntAt(n int, at int) {
- if n < 0 || int64(n) > math.MaxUint32 {
- base.Fatalf("go: attempting to write an int to the index that overflows uint32")
+ if n < 0 || int(int32(n)) != n {
+ base.Fatalf("go: attempting to write an int to the index that overflows int32")
}
binary.LittleEndian.PutUint32(e.b[at:], uint32(n))
}
diff --git a/src/cmd/go/internal/modload/search.go b/src/cmd/go/internal/modload/search.go
index 856390a0f2..b2ac7f22b1 100644
--- a/src/cmd/go/internal/modload/search.go
+++ b/src/cmd/go/internal/modload/search.go
@@ -216,21 +216,20 @@ func matchPackages(ctx context.Context, m *search.Match, tags map[string]bool, f
// is the module's root directory on disk, index is the modindex.Module for the
// module, and importPathRoot is the module's path prefix.
func walkFromIndex(index *modindex.Module, importPathRoot string, isMatch, treeCanMatch func(string) bool, tags, have map[string]bool, addPkg func(string)) {
-loopPackages:
- for _, reldir := range index.Packages() {
+ index.Walk(func(reldir string) {
// Avoid .foo, _foo, and testdata subdirectory trees.
p := reldir
for {
elem, rest, found := strings.Cut(p, string(filepath.Separator))
if strings.HasPrefix(elem, ".") || strings.HasPrefix(elem, "_") || elem == "testdata" {
- continue loopPackages
+ return
}
if found && elem == "vendor" {
// Ignore this path if it contains the element "vendor" anywhere
// except for the last element (packages named vendor are allowed
// for historical reasons). Note that found is true when this
// isn't the last path element.
- continue loopPackages
+ return
}
if !found {
// Didn't find the separator, so we're considering the last element.
@@ -241,12 +240,12 @@ loopPackages:
// Don't use GOROOT/src.
if reldir == "" && importPathRoot == "" {
- continue
+ return
}
name := path.Join(importPathRoot, filepath.ToSlash(reldir))
if !treeCanMatch(name) {
- continue
+ return
}
if !have[name] {
@@ -257,7 +256,7 @@ loopPackages:
}
}
}
- }
+ })
}
// MatchInModule identifies the packages matching the given pattern within the
From b75ad09cae8918343000e304c66c5df27101829b Mon Sep 17 00:00:00 2001
From: Tobias Klauser
Date: Mon, 11 Jul 2022 19:10:50 +0200
Subject: [PATCH 39/78] cmd/trace: fix typo in web documentation
Change-Id: I950224c3f698fbdf2bcab6f847f4afddaa6e9c2d
Reviewed-on: https://go-review.googlesource.com/c/go/+/416974
TryBot-Result: Gopher Robot
Run-TryBot: Tobias Klauser
Run-TryBot: Alan Donovan
Reviewed-by: Ian Lance Taylor
Reviewed-by: Alan Donovan
---
src/cmd/trace/main.go | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
diff --git a/src/cmd/trace/main.go b/src/cmd/trace/main.go
index 11804d0b90..0e4d882c97 100644
--- a/src/cmd/trace/main.go
+++ b/src/cmd/trace/main.go
@@ -247,7 +247,7 @@ p { color: grey85; font-size:85%; }
because it made a system call or tried to acquire a mutex.
Directly underneath each bar, a smaller bar or more commonly a fine
- vertical line indicates an event occuring during its execution.
+ vertical line indicates an event occurring during its execution.
Some of these are related to garbage collection; most indicate that
a goroutine yielded its logical processor but then immediately resumed execution
on the same logical processor. Clicking on the event displays the stack trace
@@ -274,7 +274,7 @@ p { color: grey85; font-size:85%; }
function written in C.
- Above the event trace for the first logical processor are
+ Above the event trace for the first logical processor are
traces for various runtime-internal events.
The "GC" bar shows when the garbage collector is running, and in which stage.
From 846490110ab1117b5f7366e3a531d24d88800074 Mon Sep 17 00:00:00 2001
From: Than McIntosh
Date: Mon, 11 Jul 2022 15:20:13 -0400
Subject: [PATCH 40/78] runtime/race: update amd64 syso images to avoid sse4
Rebuild selected amd64 syso images with updated LLVM build rules that
avoid the use of SSE4, so as to ensure that the Go race detector
continues to work on older x86 cpus. No changes to the syso files for
openbsd/amd64 (upstream support has been removed in LLVM) or
netbsd/amd64 (work still in progress there).
Fixes #53743.
Change-Id: I738ae4d1e0528c6e06dd4ddb78e7039a30a51779
Reviewed-on: https://go-review.googlesource.com/c/go/+/416857
Reviewed-by: Cherry Mui
Run-TryBot: Than McIntosh
---
src/runtime/race/README | 6 +++---
src/runtime/race/race_darwin_amd64.syso | Bin 538536 -> 541464 bytes
src/runtime/race/race_freebsd_amd64.syso | Bin 710664 -> 712464 bytes
src/runtime/race/race_linux_amd64.syso | Bin 552768 -> 557744 bytes
4 files changed, 3 insertions(+), 3 deletions(-)
diff --git a/src/runtime/race/README b/src/runtime/race/README
index eb18ad600b..ad8f55fb73 100644
--- a/src/runtime/race/README
+++ b/src/runtime/race/README
@@ -4,9 +4,9 @@ the LLVM project (https://github.com/llvm/llvm-project/tree/main/compiler-rt).
To update the .syso files use golang.org/x/build/cmd/racebuild.
-race_darwin_amd64.syso built with LLVM 41cb504b7c4b18ac15830107431a0c1eec73a6b2 and Go 851ecea4cc99ab276109493477b2c7e30c253ea8.
-race_freebsd_amd64.syso built with LLVM 41cb504b7c4b18ac15830107431a0c1eec73a6b2 and Go 851ecea4cc99ab276109493477b2c7e30c253ea8.
-race_linux_amd64.syso built with LLVM 41cb504b7c4b18ac15830107431a0c1eec73a6b2 and Go 851ecea4cc99ab276109493477b2c7e30c253ea8.
+race_darwin_amd64.syso built with LLVM 127e59048cd3d8dbb80c14b3036918c114089529 and Go 59ab6f351a370a27458755dc69f4a837e55a05a6.
+race_freebsd_amd64.syso built with LLVM 127e59048cd3d8dbb80c14b3036918c114089529 and Go 59ab6f351a370a27458755dc69f4a837e55a05a6.
+race_linux_amd64.syso built with LLVM 127e59048cd3d8dbb80c14b3036918c114089529 and Go 59ab6f351a370a27458755dc69f4a837e55a05a6.
race_linux_ppc64le.syso built with LLVM 41cb504b7c4b18ac15830107431a0c1eec73a6b2 and Go 851ecea4cc99ab276109493477b2c7e30c253ea8.
race_netbsd_amd64.syso built with LLVM 41cb504b7c4b18ac15830107431a0c1eec73a6b2 and Go 851ecea4cc99ab276109493477b2c7e30c253ea8.
race_windows_amd64.syso built with LLVM 89f7ccea6f6488c443655880229c54db1f180153 and Go f62d3202bf9dbb3a00ad2a2c63ff4fa4188c5d3b.
diff --git a/src/runtime/race/race_darwin_amd64.syso b/src/runtime/race/race_darwin_amd64.syso
index dde17add91495ad4459c341c1558a3095911be9a..e5d848c883c2bfd36689489939bef4fd42ef4b46 100644
GIT binary patch
delta 109938
zcmcG%d3;RQ8$UjG8D_|sAR)vusX;~MFAEn{ow;=X?AJx<(f34Bf
z29}I}^m)nmuP)@n@$WSf)j->EeW2V)D~1Gm67^@qSmXMzT9I0@b}i2U<9RVszdkB3
ztd>}-eEq=2QcZE7?0GdTQtTkFcZD4mq*0>QGqgpFFxa!KMMrT)?e#}m)Dy*v<(|te
zXNeoj*T3HC6LGk`T-GS=;O{}a?T~_msO^O`HHtfC3>VF$qW0BAN4e-46?K9rYCj9)
z%tNqY51GZU9zphvXK+M}XH~=$aquHgy*6!yMV{C;cJaqDPhOjuqNU6e-L|bMs0;y|
z7np^t?P-%-%-N1oar!ebCzg4#+P)yZ62Csb?bQbQpHDLD+fRD7^>&GkOzUlN(p+;(exilACkU%*e>xT^6)H$|+
zhY2Gy6Y+0(!%f^
zM@~&*NTZ+I7-n*01w|ful7DVtNaRQ=L)K^f+))zG5Akg9wh`A3@m%zFZMe6OA_#(J
zPfg9vvF(h_vyc>xX6zzQM`j?r0tQ
zI@+b*BUk-8ceu_sJ6e0@?x>dARxG_A>}(yn4mHsqcz{1kgLsw(?d%YmTump;uLCoA
zdxJd>kJk76y0cm6ZFEh3Tj74aXSrUZU9CJtJF6w68JA3hq}Q%GhQIuYfAlJ|Z9E?2
z3A|A+^d`CszwHx#y|Y{|bayLHF)TdsO9XEFjoqz6E!9Hb^XruJ(0g}Bg%+R(^T%+-
zuY*FMx%`=@#8bMvS^+=d(s_GYd0yI6$1vHS>r@{9@jb0V|3J_0x1HkG8OL?n>}};K
zL+C1%Es;wyAW7a^$I!+frYDtj_xulNtRJ;;{aq(pc^*L$@0Sc1;%T_gV#x6$o*|x;
zeP(^xMAmwnO!geTSzl{x9qY#yuem=1nv3j9Xa_#;!m%<17mf9dIy6R{{i0{zq3%uRr?70T1@XorRFy8Q+so@s
zhdYR!>Mmv~gO@<%XOvCU{whj)A@mOsS9zbB>=<1hrCVD*Pu+_qPlq#|1OD`juQ72Bz(M0G!wm7Pyb|pB1Kue{(ct1LYqZfe=^rJ&K+T~m?Lt8)EW4!0$xnR%Ib3F{h{OGF`wHZ42k?vft
z@A)ob_bw8ONSb^
zYrvmf3=RCKnWJlf8vJNOj%u%<&M~4Az=trA!R0SI5M_Y3=@M;%>*^jp6=!EKM
z8;#QTvxJhZ*Ai=m$%eTAc^
zf%f*J3P+=gs_DMW(OjTOe%*c?y$sa)FSN(?E`|aBLYDyT4b2!|BWt&0)Gv;!E4C3Kri{xs~nYYb}`)XqootkE!FqD
zcC)jg+>h3uNNAxttjlS-V!H$s}7hf80(7@my$7d;ccU`cfo3E9nVfrpW=@z$ZzimVU|@Zq>7n~)s+x#jZTO(X6?rO21z-3T>@$OGYDo=
zpMXT2D~|CmudsqTbe(i|2%VcO@SCw@%nO3P_5~KqV7*S}JA~(gd(hx~6tQqfV|AYP
zPup4)(4fJ6K)k)+&}uWhx0f+t?lSVFQy6U?0yB#9tl~VAWh$8mO+n_T!%2o>v}RvREYaz6vY3^{N7QA@1)b3GO$Xz5fOv~x
zwKyEMMse^v3N}I#im^tHN#fx54)w$e;o};uzkZ8_oV!;-&4-f4+r3I^U0nLpUX^?$
z#7W|N^T_#cggIjPJazE5La`*annMzP5FFyh+3JBGgyn*M?Q~|Z!wfQgov=V};_TJ(a0`JeB;tSr|$(b_k_S)=!Z%&Oe+xDoV}qmN`(9))h7F&=j(Gr!X`2
z!^tev5fHD3`YK9sbp}Pz#!UVWvVRx+5(>M6H-n(yU)UuymBj2<)d_oruCmyBjk@Yr
z;akb@a46F{GYnem)Ypy(mt=jwV1_wYk;LP|vyJB1q{^D`>nlvO@Kvam`}Ph57a-Js#%VLl&LFeR8E+1>dTE~VnD#7KMEB|x=UZkS
zS$b9IU{`)&5{)Ld*y`^P=z`4Z+-}V)463~NH3A8s}nT@o+!2`JuH
zu}Ac>