diff --git a/api/next/42888.txt b/api/next/42888.txt index f9b8e1e475..279536f2ab 100644 --- a/api/next/42888.txt +++ b/api/next/42888.txt @@ -1 +1,2 @@ -pkg runtime/debug, func SetCrashOutput(*os.File) error #42888 +pkg runtime/debug, type CrashOptions struct #67182 +pkg runtime/debug, func SetCrashOutput(*os.File, CrashOptions) error #42888 diff --git a/api/next/66249.txt b/api/next/66249.txt new file mode 100644 index 0000000000..f9d7a1addc --- /dev/null +++ b/api/next/66249.txt @@ -0,0 +1,5 @@ +pkg crypto/x509, func ParseOID(string) (OID, error) #66249 +pkg crypto/x509, method (*OID) UnmarshalBinary([]uint8) error #66249 +pkg crypto/x509, method (*OID) UnmarshalText([]uint8) error #66249 +pkg crypto/x509, method (OID) MarshalBinary() ([]uint8, error) #66249 +pkg crypto/x509, method (OID) MarshalText() ([]uint8, error) #66249 diff --git a/api/next/66339.txt b/api/next/66339.txt new file mode 100644 index 0000000000..a6b45adcd2 --- /dev/null +++ b/api/next/66339.txt @@ -0,0 +1 @@ +pkg go/ast, func Preorder(Node) iter.Seq[Node] #66339 diff --git a/api/next/66405.txt b/api/next/66405.txt new file mode 100644 index 0000000000..0b39494f92 --- /dev/null +++ b/api/next/66405.txt @@ -0,0 +1 @@ +pkg net/http, type Request struct, Pattern string #66405 diff --git a/api/next/67143.txt b/api/next/67143.txt new file mode 100644 index 0000000000..f124034849 --- /dev/null +++ b/api/next/67143.txt @@ -0,0 +1,4 @@ +pkg go/types, method (*Alias) Origin() *Alias #67143 +pkg go/types, method (*Alias) SetTypeParams([]*TypeParam) #67143 +pkg go/types, method (*Alias) TypeArgs() *TypeList #67143 +pkg go/types, method (*Alias) TypeParams() *TypeParamList #67143 diff --git a/doc/README.md b/doc/README.md index c2b320711f..3d0fb86102 100644 --- a/doc/README.md +++ b/doc/README.md @@ -27,22 +27,39 @@ should have a corresponding file named `doc/next/6-stdlib/99-minor/net/http/1234 At a minimum, that file should contain either a full sentence or a TODO, ideally referring to a person with the responsibility to complete the note. +If your CL addresses an accepted proposal, mention the proposal issue number in +your release note in the form `/issue/NUMBER`. A link to the issue in the text +will have this form (see below). If you don't want to mention the issue in the +text, add it as a comment: +``` + +``` +If an accepted proposal is mentioned in a CL but not in the release notes, it will be +flagged as a TODO by the automated tooling. That is true even for proposals that add API. + Use the following forms in your markdown: [http.Request] # symbol documentation; auto-linked as in Go doc strings + [Request] # short form, for symbols in the package being documented [#12345](/issue/12345) # GitHub issues [CL 6789](/cl/6789) # Gerrit changelists ## For the release team -At the start of a release development cycle, the contents of `next` should be deleted -and replaced with those of `initial`. From the repo root: +The `relnote` tool, at `golang.org/x/build/cmd/relnote`, operates on the files +in `doc/next`. + +As a release cycle nears completion, run `relnote todo` to get a list of +unfinished release note work. + +To prepare the release notes for a release, run `relnote generate`. +That will merge the `.md` files in `next` into a single file. + +To begin the next release development cycle, delete the contents of `next` +and replace them with those of `initial`. From the repo root: > cd doc > rm -r next/* > cp -r initial/* next Then edit `next/1-intro.md` to refer to the next version. - -To prepare the release notes for a release, run `golang.org/x/build/cmd/relnote generate`. -That will merge the `.md` files in `next` into a single file. diff --git a/doc/godebug.md b/doc/godebug.md index 704ebe3a2f..4cbc85f941 100644 --- a/doc/godebug.md +++ b/doc/godebug.md @@ -88,14 +88,38 @@ Because this method of setting GODEBUG defaults was introduced only in Go 1.21, programs listing versions of Go earlier than Go 1.20 are configured to match Go 1.20, not the older version. -To override these defaults, a main package's source files +To override these defaults, starting in Go 1.23, the work module's `go.mod` +or the workspace's `go.work` can list one or more `godebug` lines: + + godebug ( + default=go1.21 + panicnil=1 + asynctimerchan=0 + ) + +The special key `default` indicates a Go version to take unspecified +settings from. This allows setting the GODEBUG defaults separately +from the Go language version in the module. +In this example, the program is asking for Go 1.21 semantics and +then asking for the old pre-Go 1.21 `panic(nil)` behavior and the +new Go 1.23 `asynctimerchan=0` behavior. + +Only the work module's `go.mod` is consulted for `godebug` directives. +Any directives in required dependency modules are ignored. +It is an error to list a `godebug` with an unrecognized setting. +(Toolchains older than Go 1.23 reject all `godebug` lines, since they do not +understand `godebug` at all.) + +The defaults from the `go` and `godebug` lines apply to all main +packages that are built. For more fine-grained control, +starting in Go 1.21, a main package's source files can include one or more `//go:debug` directives at the top of the file (preceding the `package` statement). -Continuing the `panicnil` example, if the module or workspace is updated -to say `go` `1.21`, the program can opt back into the old `panic(nil)` -behavior by including this directive: +The `godebug` lines in the previous example would be written: + //go:debug default=go1.21 //go:debug panicnil=1 + //go:debug asynctimerchan=0 Starting in Go 1.21, the Go toolchain treats a `//go:debug` directive with an unrecognized GODEBUG setting as an invalid program. @@ -175,7 +199,7 @@ Whether the type checker produces `Alias` types or not is controlled by the [`gotypesalias` setting](/pkg/go/types#Alias). For Go 1.22 it defaults to `gotypesalias=0`. For Go 1.23, `gotypesalias=1` will become the default. -This setting will be removed in a future release, Go 1.24 at the earliest. +This setting will be removed in a future release, Go 1.27 at the earliest. Go 1.22 changed the default minimum TLS version supported by both servers and clients to TLS 1.2. The default can be reverted to TLS 1.0 using the diff --git a/doc/next/3-tools.md b/doc/next/3-tools.md index c052f3b084..4112fb61ac 100644 --- a/doc/next/3-tools.md +++ b/doc/next/3-tools.md @@ -8,6 +8,10 @@ Distributions that install the `go` command to a location other than `$GOROOT/bin/go` should install a symlink instead of relocating or copying the `go` binary. +The new go env `-changed` flag causes the command to print only +those settings whose effective value differs from the default value +that would be obtained in an empty environment with no prior uses of the `-w` flag. + ### Vet {#vet} The `go vet` subcommand now includes the diff --git a/doc/next/5-toolchain.md b/doc/next/5-toolchain.md index ce763f1b93..7f547f686b 100644 --- a/doc/next/5-toolchain.md +++ b/doc/next/5-toolchain.md @@ -4,6 +4,11 @@ The build time overhead to building with [Profile Guided Optimization](/doc/pgo) Previously, large builds could see 100%+ build time increase from enabling PGO. In Go 1.23, overhead should be in the single digit percentages. + +The compiler in Go 1.23 can now overlap the stack frame slots of local variables +accessed in disjoint regions of a function, which reduces stack usage +for Go applications. + ## Assembler {#assembler} ## Linker {#linker} diff --git a/doc/next/6-stdlib/99-minor/crypto/x509/66249.md b/doc/next/6-stdlib/99-minor/crypto/x509/66249.md new file mode 100644 index 0000000000..d449e74d66 --- /dev/null +++ b/doc/next/6-stdlib/99-minor/crypto/x509/66249.md @@ -0,0 +1,3 @@ +The new [ParseOID] function parses a dot-encoded ASN.1 Object Identifier string. +The [OID] type now implements the [encoding.BinaryMarshaler], +[encoding.BinaryUnmarshaler], [encoding.TextMarshaler], [encoding.TextUnmarshaler] interfaces. diff --git a/doc/next/6-stdlib/99-minor/go/ast/66339.md b/doc/next/6-stdlib/99-minor/go/ast/66339.md new file mode 100644 index 0000000000..0eec51ecd6 --- /dev/null +++ b/doc/next/6-stdlib/99-minor/go/ast/66339.md @@ -0,0 +1,2 @@ +The new [Preorder] function returns a convenient iterator over all the +nodes of a syntax tree. diff --git a/doc/next/6-stdlib/99-minor/go/types/67143.md b/doc/next/6-stdlib/99-minor/go/types/67143.md new file mode 100644 index 0000000000..405c679378 --- /dev/null +++ b/doc/next/6-stdlib/99-minor/go/types/67143.md @@ -0,0 +1,2 @@ +The methods [Alias.Origin], [Alias.SetTypeParams], [Alias.TypeParams], +and [Alias.TypeArgs] have been added. They are needed for generic alias types. diff --git a/doc/next/6-stdlib/99-minor/net/http/64910.md b/doc/next/6-stdlib/99-minor/net/http/64910.md index 78b13fb711..28452ee932 100644 --- a/doc/next/6-stdlib/99-minor/net/http/64910.md +++ b/doc/next/6-stdlib/99-minor/net/http/64910.md @@ -1,2 +1,2 @@ -The patterns used by [ServeMux] allow multiple spaces matching -regexp '[ \t]+'. +The patterns used by [ServeMux] now allow one or more spaces or tabs after the method name. +Previously, only a single space was permitted. diff --git a/doc/next/6-stdlib/99-minor/net/http/66405.md b/doc/next/6-stdlib/99-minor/net/http/66405.md new file mode 100644 index 0000000000..c827b4b219 --- /dev/null +++ b/doc/next/6-stdlib/99-minor/net/http/66405.md @@ -0,0 +1,3 @@ +For inbound requests, the new [Request.Pattern] field contains the [ServeMux] +pattern (if any) that matched the request. This field is not set when +`GODEBUG=httpmuxgo121=1` is set. diff --git a/doc/next/6-stdlib/99-minor/runtime/debug/67182.md b/doc/next/6-stdlib/99-minor/runtime/debug/67182.md new file mode 100644 index 0000000000..d83864a3db --- /dev/null +++ b/doc/next/6-stdlib/99-minor/runtime/debug/67182.md @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/src/archive/zip/reader.go b/src/archive/zip/reader.go index ff6fedf632..60b34b76ee 100644 --- a/src/archive/zip/reader.go +++ b/src/archive/zip/reader.go @@ -699,9 +699,13 @@ func findSignatureInBlock(b []byte) int { if b[i] == 'P' && b[i+1] == 'K' && b[i+2] == 0x05 && b[i+3] == 0x06 { // n is length of comment n := int(b[i+directoryEndLen-2]) | int(b[i+directoryEndLen-1])<<8 - if n+directoryEndLen+i <= len(b) { - return i + if n+directoryEndLen+i > len(b) { + // Truncated comment. + // Some parsers (such as Info-ZIP) ignore the truncated comment + // rather than treating it as a hard error. + return -1 } + return i } } return -1 diff --git a/src/archive/zip/reader_test.go b/src/archive/zip/reader_test.go index 9f651da530..00e5ec3e05 100644 --- a/src/archive/zip/reader_test.go +++ b/src/archive/zip/reader_test.go @@ -570,6 +570,14 @@ var tests = []ZipTest{ }, }, }, + // Issue 66869: Don't skip over an EOCDR with a truncated comment. + // The test file sneakily hides a second EOCDR before the first one; + // previously we would extract one file ("file") from this archive, + // while most other tools would reject the file or extract a different one ("FILE"). + { + Name: "comment-truncated.zip", + Error: ErrFormat, + }, } func TestReader(t *testing.T) { diff --git a/src/archive/zip/testdata/comment-truncated.zip b/src/archive/zip/testdata/comment-truncated.zip new file mode 100644 index 0000000000..1bc19a8557 Binary files /dev/null and b/src/archive/zip/testdata/comment-truncated.zip differ diff --git a/src/cmd/addr2line/main.go b/src/cmd/addr2line/main.go index 6e005a8fac..e77785f156 100644 --- a/src/cmd/addr2line/main.go +++ b/src/cmd/addr2line/main.go @@ -28,6 +28,7 @@ import ( "strings" "cmd/internal/objfile" + "cmd/internal/telemetry" ) func printUsage(w *os.File) { @@ -45,6 +46,7 @@ func usage() { func main() { log.SetFlags(0) log.SetPrefix("addr2line: ") + telemetry.Start() // pprof expects this behavior when checking for addr2line if len(os.Args) > 1 && os.Args[1] == "--help" { @@ -54,6 +56,8 @@ func main() { flag.Usage = usage flag.Parse() + telemetry.Inc("addr2line/invocations") + telemetry.CountFlags("addr2line/flag:", *flag.CommandLine) if flag.NArg() != 1 { usage() } diff --git a/src/cmd/asm/internal/asm/testdata/ppc64.s b/src/cmd/asm/internal/asm/testdata/ppc64.s index 8627408f06..7e8c6f9cf2 100644 --- a/src/cmd/asm/internal/asm/testdata/ppc64.s +++ b/src/cmd/asm/internal/asm/testdata/ppc64.s @@ -260,13 +260,32 @@ TEXT asmtest(SB),DUPOK|NOSPLIT,$0 XORIS $15, R3, R4 // 6c64000f XOR $983040, R3, R4 // 6c64000f - // TODO: the order of CR operands don't match + // TODO: cleanup inconsistency of printing CMPx opcodes with explicit CR arguments. CMP R3, R4 // 7c232000 + CMP R3, R0 // 7c230000 + CMP R3, R0, CR1 // CMP R3,CR1,R0 // 7ca30000 CMPU R3, R4 // 7c232040 + CMPU R3, R0 // 7c230040 + CMPU R3, R0, CR2 // CMPU R3,CR2,R0 // 7d230040 CMPW R3, R4 // 7c032000 + CMPW R3, R0 // 7c030000 + CMPW R3, R0, CR3 // CMPW R3,CR3,R0 // 7d830000 CMPWU R3, R4 // 7c032040 - CMPB R3,R4,R4 // 7c6423f8 + CMPWU R3, R0 // 7c030040 + CMPWU R3, R0, CR4 // CMPWU R3,CR4,R0 // 7e030040 + CMP R3, $0 // 2c230000 + CMPU R3, $0 // 28230000 + CMPW R3, $0 // 2c030000 + CMPWU R3, $0 // 28030000 + CMP R3, $0, CR0 // CMP R3,CR0,$0 // 2c230000 + CMPU R3, $0, CR1 // CMPU R3,CR1,$0 // 28a30000 + CMPW R3, $0, CR2 // CMPW R3,CR2,$0 // 2d030000 + CMPW R3, $-32768, CR2 // CMPW R3,CR2,$-32768 // 2d038000 + CMPWU R3, $0, CR3 // CMPWU R3,CR3,$0 // 29830000 + CMPWU R3, $0x8008, CR3 // CMPWU R3,CR3,$32776 // 29838008 + CMPEQB R3,R4,CR6 // 7f0321c0 + CMPB R3,R4,R4 // 7c6423f8 ADD R3, R4 // 7c841a14 ADD R3, R4, R5 // 7ca41a14 diff --git a/src/cmd/asm/main.go b/src/cmd/asm/main.go index ba69195056..82a2fa80e0 100644 --- a/src/cmd/asm/main.go +++ b/src/cmd/asm/main.go @@ -20,16 +20,20 @@ import ( "cmd/internal/bio" "cmd/internal/obj" "cmd/internal/objabi" + "cmd/internal/telemetry" ) func main() { log.SetFlags(0) log.SetPrefix("asm: ") + telemetry.Start() buildcfg.Check() GOARCH := buildcfg.GOARCH flags.Parse() + telemetry.Inc("asm/invocations") + telemetry.CountFlags("asm/flag:", *flag.CommandLine) architecture := arch.Set(GOARCH, *flags.Shared || *flags.Dynlink) if architecture == nil { diff --git a/src/cmd/buildid/buildid.go b/src/cmd/buildid/buildid.go index 72ad80dbbb..7abc37283f 100644 --- a/src/cmd/buildid/buildid.go +++ b/src/cmd/buildid/buildid.go @@ -12,6 +12,7 @@ import ( "strings" "cmd/internal/buildid" + "cmd/internal/telemetry" ) func usage() { @@ -25,8 +26,11 @@ var wflag = flag.Bool("w", false, "write build ID") func main() { log.SetPrefix("buildid: ") log.SetFlags(0) + telemetry.Start() flag.Usage = usage flag.Parse() + telemetry.Inc("buildid/invocations") + telemetry.CountFlags("buildid/flag:", *flag.CommandLine) if flag.NArg() != 1 { usage() } diff --git a/src/cmd/cgo/doc.go b/src/cmd/cgo/doc.go index 1c5d4b060d..3d4789fafb 100644 --- a/src/cmd/cgo/doc.go +++ b/src/cmd/cgo/doc.go @@ -529,6 +529,9 @@ The following options are available when running cgo directly: Write out input file in Go syntax replacing C package names with real values. Used to generate files in the syscall package when bootstrapping a new target. + -ldflags flags + Flags to pass to the C linker. The cmd/go tool uses + this to pass in the flags in the CGO_LDFLAGS variable. -objdir directory Put all generated files in directory. -srcdir directory diff --git a/src/cmd/cgo/main.go b/src/cmd/cgo/main.go index a9095dee3d..c258985fee 100644 --- a/src/cmd/cgo/main.go +++ b/src/cmd/cgo/main.go @@ -28,6 +28,7 @@ import ( "cmd/internal/edit" "cmd/internal/notsha256" "cmd/internal/objabi" + "cmd/internal/telemetry" ) // A Package collects information about the package we're going to write. @@ -242,6 +243,8 @@ var objDir = flag.String("objdir", "", "object directory") var importPath = flag.String("importpath", "", "import path of package being built (for comments in generated files)") var exportHeader = flag.String("exportheader", "", "where to write export header if any exported functions") +var ldflags = flag.String("ldflags", "", "flags to pass to C linker") + var gccgo = flag.Bool("gccgo", false, "generate files for use with gccgo") var gccgoprefix = flag.String("gccgoprefix", "", "-fgo-prefix option used with gccgo") var gccgopkgpath = flag.String("gccgopkgpath", "", "-fgo-pkgpath option used with gccgo") @@ -255,8 +258,11 @@ var goarch, goos, gomips, gomips64 string var gccBaseCmd []string func main() { + telemetry.Start() objabi.AddVersionFlag() // -V objabi.Flagparse(usage) + telemetry.Inc("cgo/invocations") + telemetry.CountFlags("cgo/flag:", *flag.CommandLine) if *gccgoDefineCgoIncomplete { if !*gccgo { @@ -328,11 +334,11 @@ func main() { os.Exit(2) } - // Record CGO_LDFLAGS from the environment for external linking. - if ldflags := os.Getenv("CGO_LDFLAGS"); ldflags != "" { - args, err := splitQuoted(ldflags) + // Record linker flags for external linking. + if *ldflags != "" { + args, err := splitQuoted(*ldflags) if err != nil { - fatalf("bad CGO_LDFLAGS: %q (%s)", ldflags, err) + fatalf("bad -ldflags option: %q (%s)", *ldflags, err) } p.addToFlag("LDFLAGS", args) } diff --git a/src/cmd/compile/internal/base/debug.go b/src/cmd/compile/internal/base/debug.go index 672e3909e4..c1b62f27ca 100644 --- a/src/cmd/compile/internal/base/debug.go +++ b/src/cmd/compile/internal/base/debug.go @@ -16,6 +16,7 @@ var Debug DebugFlags // The -d option takes a comma-separated list of settings. // Each setting is name=value; for ints, name is short for name=1. type DebugFlags struct { + AlignHot int `help:"enable hot block alignment (currently requires -pgo)" concurrent:"ok"` Append int `help:"print information about append compilation"` Checkptr int `help:"instrument unsafe pointer conversions\n0: instrumentation disabled\n1: conversions involving unsafe.Pointer are instrumented\n2: conversions to unsafe.Pointer force heap allocation" concurrent:"ok"` Closure int `help:"print information about closure compilation"` diff --git a/src/cmd/compile/internal/base/flag.go b/src/cmd/compile/internal/base/flag.go index 0d3c7c2226..fe515aafbf 100644 --- a/src/cmd/compile/internal/base/flag.go +++ b/src/cmd/compile/internal/base/flag.go @@ -178,6 +178,7 @@ func ParseFlags() { Debug.ConcurrentOk = true Debug.MaxShapeLen = 500 + Debug.AlignHot = 1 Debug.InlFuncsWithClosures = 1 Debug.InlStaticInit = 1 Debug.PGOInline = 1 @@ -212,6 +213,8 @@ func ParseFlags() { Flag.CompilingRuntime = true } + Ctxt.Std = Flag.Std + // Three inputs govern loop iteration variable rewriting, hash, experiment, flag. // The loop variable rewriting is: // IF non-empty hash, then hash determines behavior (function+line match) (*) diff --git a/src/cmd/compile/internal/gc/compile.go b/src/cmd/compile/internal/gc/compile.go index 0f57f8ca82..159fd29c48 100644 --- a/src/cmd/compile/internal/gc/compile.go +++ b/src/cmd/compile/internal/gc/compile.go @@ -14,6 +14,7 @@ import ( "cmd/compile/internal/ir" "cmd/compile/internal/liveness" "cmd/compile/internal/objw" + "cmd/compile/internal/pgoir" "cmd/compile/internal/ssagen" "cmd/compile/internal/staticinit" "cmd/compile/internal/types" @@ -112,7 +113,7 @@ func prepareFunc(fn *ir.Func) { // compileFunctions compiles all functions in compilequeue. // It fans out nBackendWorkers to do the work // and waits for them to complete. -func compileFunctions() { +func compileFunctions(profile *pgoir.Profile) { if race.Enabled { // Randomize compilation order to try to shake out races. tmp := make([]*ir.Func, len(compilequeue)) @@ -179,7 +180,7 @@ func compileFunctions() { for _, fn := range fns { fn := fn queue(func(worker int) { - ssagen.Compile(fn, worker) + ssagen.Compile(fn, worker, profile) compile(fn.Closures) wg.Done() }) diff --git a/src/cmd/compile/internal/gc/main.go b/src/cmd/compile/internal/gc/main.go index 7ab64f4748..41f5e43ec6 100644 --- a/src/cmd/compile/internal/gc/main.go +++ b/src/cmd/compile/internal/gc/main.go @@ -303,7 +303,7 @@ func Main(archInit func(*ssagen.ArchInfo)) { // as late as possible to maximize how much work we can batch and // process concurrently. if len(compilequeue) != 0 { - compileFunctions() + compileFunctions(profile) continue } diff --git a/src/cmd/compile/internal/inline/inl.go b/src/cmd/compile/internal/inline/inl.go index 931f79552a..ed3d3d4eaf 100644 --- a/src/cmd/compile/internal/inline/inl.go +++ b/src/cmd/compile/internal/inline/inl.go @@ -61,6 +61,9 @@ var ( // TODO(prattmic): Make this non-global. candHotCalleeMap = make(map[*pgoir.IRNode]struct{}) + // Set of functions that contain hot call sites. + hasHotCall = make(map[*ir.Func]struct{}) + // List of all hot call sites. CallSiteInfo.Callee is always nil. // TODO(prattmic): Make this non-global. candHotEdgeMap = make(map[pgoir.CallSiteInfo]struct{}) @@ -78,6 +81,22 @@ var ( inlineHotMaxBudget int32 = 2000 ) +func IsPgoHotFunc(fn *ir.Func, profile *pgoir.Profile) bool { + if profile == nil { + return false + } + if n, ok := profile.WeightedCG.IRNodes[ir.LinkFuncName(fn)]; ok { + _, ok := candHotCalleeMap[n] + return ok + } + return false +} + +func HasPgoHotInline(fn *ir.Func) bool { + _, has := hasHotCall[fn] + return has +} + // PGOInlinePrologue records the hot callsites from ir-graph. func PGOInlinePrologue(p *pgoir.Profile) { if base.Debug.PGOInlineCDFThreshold != "" { @@ -228,14 +247,10 @@ func GarbageCollectUnreferencedHiddenClosures() { func inlineBudget(fn *ir.Func, profile *pgoir.Profile, relaxed bool, verbose bool) int32 { // Update the budget for profile-guided inlining. budget := int32(inlineMaxBudget) - if profile != nil { - if n, ok := profile.WeightedCG.IRNodes[ir.LinkFuncName(fn)]; ok { - if _, ok := candHotCalleeMap[n]; ok { - budget = inlineHotMaxBudget - if verbose { - fmt.Printf("hot-node enabled increased budget=%v for func=%v\n", budget, ir.PkgFuncName(fn)) - } - } + if IsPgoHotFunc(fn, profile) { + budget = inlineHotMaxBudget + if verbose { + fmt.Printf("hot-node enabled increased budget=%v for func=%v\n", budget, ir.PkgFuncName(fn)) } } if relaxed { @@ -580,7 +595,7 @@ opSwitch: // Check whether we'd actually inline this call. Set // log == false since we aren't actually doing inlining // yet. - if ok, _ := canInlineCallExpr(v.curFunc, n, callee, v.isBigFunc, false); ok { + if ok, _, _ := canInlineCallExpr(v.curFunc, n, callee, v.isBigFunc, false); ok { // mkinlcall would inline this call [1], so use // the cost of the inline body as the cost of // the call, as that is what will actually @@ -873,10 +888,11 @@ var InlineCall = func(callerfn *ir.Func, call *ir.CallExpr, fn *ir.Func, inlInde // inlineCostOK returns true if call n from caller to callee is cheap enough to // inline. bigCaller indicates that caller is a big function. // -// In addition to the "cost OK" boolean, it also returns the "max -// cost" limit used to make the decision (which may differ depending -// on func size), and the score assigned to this specific callsite. -func inlineCostOK(n *ir.CallExpr, caller, callee *ir.Func, bigCaller bool) (bool, int32, int32) { +// In addition to the "cost OK" boolean, it also returns +// - the "max cost" limit used to make the decision (which may differ depending on func size) +// - the score assigned to this specific callsite +// - whether the inlined function is "hot" according to PGO. +func inlineCostOK(n *ir.CallExpr, caller, callee *ir.Func, bigCaller bool) (bool, int32, int32, bool) { maxCost := int32(inlineMaxBudget) if bigCaller { // We use this to restrict inlining into very big functions. @@ -892,19 +908,21 @@ func inlineCostOK(n *ir.CallExpr, caller, callee *ir.Func, bigCaller bool) (bool } } + lineOffset := pgoir.NodeLineOffset(n, caller) + csi := pgoir.CallSiteInfo{LineOffset: lineOffset, Caller: caller} + _, hot := candHotEdgeMap[csi] + if metric <= maxCost { // Simple case. Function is already cheap enough. - return true, 0, metric + return true, 0, metric, hot } // We'll also allow inlining of hot functions below inlineHotMaxBudget, // but only in small functions. - lineOffset := pgoir.NodeLineOffset(n, caller) - csi := pgoir.CallSiteInfo{LineOffset: lineOffset, Caller: caller} - if _, ok := candHotEdgeMap[csi]; !ok { + if !hot { // Cold - return false, maxCost, metric + return false, maxCost, metric, false } // Hot @@ -913,49 +931,50 @@ func inlineCostOK(n *ir.CallExpr, caller, callee *ir.Func, bigCaller bool) (bool if base.Debug.PGODebug > 0 { fmt.Printf("hot-big check disallows inlining for call %s (cost %d) at %v in big function %s\n", ir.PkgFuncName(callee), callee.Inl.Cost, ir.Line(n), ir.PkgFuncName(caller)) } - return false, maxCost, metric + return false, maxCost, metric, false } if metric > inlineHotMaxBudget { - return false, inlineHotMaxBudget, metric + return false, inlineHotMaxBudget, metric, false } if !base.PGOHash.MatchPosWithInfo(n.Pos(), "inline", nil) { // De-selected by PGO Hash. - return false, maxCost, metric + return false, maxCost, metric, false } if base.Debug.PGODebug > 0 { fmt.Printf("hot-budget check allows inlining for call %s (cost %d) at %v in function %s\n", ir.PkgFuncName(callee), callee.Inl.Cost, ir.Line(n), ir.PkgFuncName(caller)) } - return true, 0, metric + return true, 0, metric, hot } // canInlineCallExpr returns true if the call n from caller to callee -// can be inlined, plus the score computed for the call expr in -// question. bigCaller indicates that caller is a big function. log +// can be inlined, plus the score computed for the call expr in question, +// and whether the callee is hot according to PGO. +// bigCaller indicates that caller is a big function. log // indicates that the 'cannot inline' reason should be logged. // // Preconditions: CanInline(callee) has already been called. -func canInlineCallExpr(callerfn *ir.Func, n *ir.CallExpr, callee *ir.Func, bigCaller bool, log bool) (bool, int32) { +func canInlineCallExpr(callerfn *ir.Func, n *ir.CallExpr, callee *ir.Func, bigCaller bool, log bool) (bool, int32, bool) { if callee.Inl == nil { // callee is never inlinable. if log && logopt.Enabled() { logopt.LogOpt(n.Pos(), "cannotInlineCall", "inline", ir.FuncName(callerfn), fmt.Sprintf("%s cannot be inlined", ir.PkgFuncName(callee))) } - return false, 0 + return false, 0, false } - ok, maxCost, callSiteScore := inlineCostOK(n, callerfn, callee, bigCaller) + ok, maxCost, callSiteScore, hot := inlineCostOK(n, callerfn, callee, bigCaller) if !ok { // callee cost too high for this call site. if log && logopt.Enabled() { logopt.LogOpt(n.Pos(), "cannotInlineCall", "inline", ir.FuncName(callerfn), fmt.Sprintf("cost %d of %s exceeds max caller cost %d", callee.Inl.Cost, ir.PkgFuncName(callee), maxCost)) } - return false, 0 + return false, 0, false } if callee == callerfn { @@ -963,7 +982,7 @@ func canInlineCallExpr(callerfn *ir.Func, n *ir.CallExpr, callee *ir.Func, bigCa if log && logopt.Enabled() { logopt.LogOpt(n.Pos(), "cannotInlineCall", "inline", fmt.Sprintf("recursive call to %s", ir.FuncName(callerfn))) } - return false, 0 + return false, 0, false } if base.Flag.Cfg.Instrumenting && types.IsNoInstrumentPkg(callee.Sym().Pkg) { @@ -977,7 +996,7 @@ func canInlineCallExpr(callerfn *ir.Func, n *ir.CallExpr, callee *ir.Func, bigCa logopt.LogOpt(n.Pos(), "cannotInlineCall", "inline", ir.FuncName(callerfn), fmt.Sprintf("call to runtime function %s in instrumented build", ir.PkgFuncName(callee))) } - return false, 0 + return false, 0, false } if base.Flag.Race && types.IsNoRacePkg(callee.Sym().Pkg) { @@ -985,7 +1004,7 @@ func canInlineCallExpr(callerfn *ir.Func, n *ir.CallExpr, callee *ir.Func, bigCa logopt.LogOpt(n.Pos(), "cannotInlineCall", "inline", ir.FuncName(callerfn), fmt.Sprintf(`call to into "no-race" package function %s in race build`, ir.PkgFuncName(callee))) } - return false, 0 + return false, 0, false } // Check if we've already inlined this function at this particular @@ -1008,11 +1027,11 @@ func canInlineCallExpr(callerfn *ir.Func, n *ir.CallExpr, callee *ir.Func, bigCa fmt.Sprintf("repeated recursive cycle to %s", ir.PkgFuncName(callee))) } } - return false, 0 + return false, 0, false } } - return true, callSiteScore + return true, callSiteScore, hot } // mkinlcall returns an OINLCALL node that can replace OCALLFUNC n, or @@ -1023,10 +1042,13 @@ func canInlineCallExpr(callerfn *ir.Func, n *ir.CallExpr, callee *ir.Func, bigCa // // n.Left = mkinlcall(n.Left, fn, isddd) func mkinlcall(callerfn *ir.Func, n *ir.CallExpr, fn *ir.Func, bigCaller bool) *ir.InlinedCallExpr { - ok, score := canInlineCallExpr(callerfn, n, fn, bigCaller, true) + ok, score, hot := canInlineCallExpr(callerfn, n, fn, bigCaller, true) if !ok { return nil } + if hot { + hasHotCall[callerfn] = struct{}{} + } typecheck.AssertFixedCall(n) parent := base.Ctxt.PosTable.Pos(n.Pos()).Base().InliningIndex() diff --git a/src/cmd/compile/internal/liveness/plive.go b/src/cmd/compile/internal/liveness/plive.go index dd48d10bc5..1a36035f46 100644 --- a/src/cmd/compile/internal/liveness/plive.go +++ b/src/cmd/compile/internal/liveness/plive.go @@ -1551,6 +1551,7 @@ func WriteFuncMap(fn *ir.Func, abiInfo *abi.ABIParamResultInfo) { nbitmap = 2 } lsym := base.Ctxt.Lookup(fn.LSym.Name + ".args_stackmap") + lsym.Set(obj.AttrLinkname, true) // allow args_stackmap referenced from assembly off := objw.Uint32(lsym, 0, uint32(nbitmap)) off = objw.Uint32(lsym, off, uint32(bv.N)) off = objw.BitVec(lsym, off, bv) diff --git a/src/cmd/compile/internal/noder/irgen.go b/src/cmd/compile/internal/noder/irgen.go index 34201545b5..281f619f6f 100644 --- a/src/cmd/compile/internal/noder/irgen.go +++ b/src/cmd/compile/internal/noder/irgen.go @@ -49,9 +49,7 @@ func checkFiles(m posMap, noders []*noder) (*types2.Package, *types2.Info) { IgnoreBranchErrors: true, // parser already checked via syntax.CheckBranches mode Importer: &importer, Sizes: types2.SizesFor("gc", buildcfg.GOARCH), - // Currently, the compiler panics when using Alias types. - // TODO(gri) set to true once this is fixed (issue #66873) - EnableAlias: false, + EnableAlias: true, } if base.Flag.ErrorURL { conf.ErrorURL = " [go.dev/e/%s]" diff --git a/src/cmd/compile/internal/noder/reader.go b/src/cmd/compile/internal/noder/reader.go index abd07ebb62..a7feadaf6e 100644 --- a/src/cmd/compile/internal/noder/reader.go +++ b/src/cmd/compile/internal/noder/reader.go @@ -427,7 +427,9 @@ func (pr *pkgReader) typIdx(info typeInfo, dict *readerDict, wrapped bool) *type r.dict = dict typ := r.doTyp() - assert(typ != nil) + if typ == nil { + base.Fatalf("doTyp returned nil for info=%v", info) + } // For recursive type declarations involving interfaces and aliases, // above r.doTyp() call may have already set pr.typs[idx], so just @@ -741,7 +743,26 @@ func (pr *pkgReader) objIdxMayFail(idx pkgbits.Index, implicits, explicits []*ty case pkgbits.ObjAlias: name := do(ir.OTYPE, false) - setType(name, r.typ()) + + // Clumsy dance: the r.typ() call here might recursively find this + // type alias name, before we've set its type (#66873). So we + // temporarily clear sym.Def and then restore it later, if still + // unset. + hack := sym.Def == name + if hack { + sym.Def = nil + } + typ := r.typ() + if hack { + if sym.Def != nil { + name = sym.Def.(*ir.Name) + assert(name.Type() == typ) + return name, nil + } + sym.Def = name + } + + setType(name, typ) name.SetAlias(true) return name, nil diff --git a/src/cmd/compile/internal/noder/writer.go b/src/cmd/compile/internal/noder/writer.go index 453b08dbf9..13706f9dd2 100644 --- a/src/cmd/compile/internal/noder/writer.go +++ b/src/cmd/compile/internal/noder/writer.go @@ -12,6 +12,7 @@ import ( "internal/buildcfg" "internal/pkgbits" "os" + "strings" "cmd/compile/internal/base" "cmd/compile/internal/ir" @@ -488,6 +489,18 @@ func (w *writer) typInfo(info typeInfo) { // typIdx also reports whether typ is a derived type; that is, whether // its identity depends on type parameters. func (pw *pkgWriter) typIdx(typ types2.Type, dict *writerDict) typeInfo { + // Strip non-global aliases, because they only appear in inline + // bodies anyway. Otherwise, they can cause types.Sym collisions + // (e.g., "main.C" for both of the local type aliases in + // test/fixedbugs/issue50190.go). + for { + if alias, ok := typ.(*types2.Alias); ok && !isGlobal(alias.Obj()) { + typ = alias.Rhs() + } else { + break + } + } + if idx, ok := pw.typsIdx[typ]; ok { return typeInfo{idx: idx, derived: false} } @@ -2598,6 +2611,10 @@ func (pw *pkgWriter) collectDecls(noders []*noder) { pw.errorf(l.pos, "//go:linkname only allowed in Go files that import \"unsafe\"") continue } + if strings.Contains(l.remote, "[") && strings.Contains(l.remote, "]") { + pw.errorf(l.pos, "//go:linkname reference of an instantiation is not allowed") + continue + } switch obj := pw.curpkg.Scope().Lookup(l.local).(type) { case *types2.Func, *types2.Var: diff --git a/src/cmd/compile/internal/ppc64/ssa.go b/src/cmd/compile/internal/ppc64/ssa.go index db420b7cb4..d4974ba77e 100644 --- a/src/cmd/compile/internal/ppc64/ssa.go +++ b/src/cmd/compile/internal/ppc64/ssa.go @@ -2004,8 +2004,8 @@ func ssaGenBlock(s *ssagen.State, b, next *ssa.Block) { p := s.Prog(ppc64.ACMP) p.From.Type = obj.TYPE_REG p.From.Reg = ppc64.REG_R3 - p.To.Type = obj.TYPE_REG - p.To.Reg = ppc64.REG_R0 + p.To.Type = obj.TYPE_CONST + p.To.Offset = 0 p = s.Prog(ppc64.ABNE) p.To.Type = obj.TYPE_BRANCH diff --git a/src/cmd/compile/internal/ssa/_gen/PPC64.rules b/src/cmd/compile/internal/ssa/_gen/PPC64.rules index f0cb23ba9f..6e07aa2ec7 100644 --- a/src/cmd/compile/internal/ssa/_gen/PPC64.rules +++ b/src/cmd/compile/internal/ssa/_gen/PPC64.rules @@ -645,6 +645,19 @@ (MOVBreg (MOVBZreg x)) => (MOVBreg x) (MOVBZreg (MOVBreg x)) => (MOVBZreg x) +// Catch any remaining rotate+shift cases +(MOVBZreg (SRWconst x [s])) && mergePPC64AndSrwi(0xFF,s) != 0 => (RLWINM [mergePPC64AndSrwi(0xFF,s)] x) +(MOVBZreg (RLWINM [r] y)) && mergePPC64AndRlwinm(0xFF,r) != 0 => (RLWINM [mergePPC64AndRlwinm(0xFF,r)] y) +(MOVHZreg (RLWINM [r] y)) && mergePPC64AndRlwinm(0xFFFF,r) != 0 => (RLWINM [mergePPC64AndRlwinm(0xFFFF,r)] y) +(MOVWZreg (RLWINM [r] y)) && mergePPC64AndRlwinm(0xFFFFFFFF,r) != 0 => (RLWINM [mergePPC64AndRlwinm(0xFFFFFFFF,r)] y) +(Select0 (ANDCCconst [m] (RLWINM [r] y))) && mergePPC64AndRlwinm(uint32(m),r) != 0 => (RLWINM [mergePPC64AndRlwinm(uint32(m),r)] y) +(SLDconst [s] (RLWINM [r] y)) && mergePPC64SldiRlwinm(s,r) != 0 => (RLWINM [mergePPC64SldiRlwinm(s,r)] y) +(RLWINM [r] (MOVHZreg u)) && mergePPC64RlwinmAnd(r,0xFFFF) != 0 => (RLWINM [mergePPC64RlwinmAnd(r,0xFFFF)] u) +(RLWINM [r] (Select0 (ANDCCconst [a] u))) && mergePPC64RlwinmAnd(r,uint32(a)) != 0 => (RLWINM [mergePPC64RlwinmAnd(r,uint32(a))] u) +// SLWconst is a special case of RLWNM which always zero-extends the result. +(SLWconst [s] (MOVWZreg w)) => (SLWconst [s] w) +(MOVWZreg w:(SLWconst u)) => w + // H - there are more combinations than these (MOVHZreg y:(MOV(H|B)Zreg _)) => y // repeat diff --git a/src/cmd/compile/internal/ssa/block.go b/src/cmd/compile/internal/ssa/block.go index 26af10b59c..02733eaf16 100644 --- a/src/cmd/compile/internal/ssa/block.go +++ b/src/cmd/compile/internal/ssa/block.go @@ -31,6 +31,9 @@ type Block struct { // After flagalloc, records whether flags are live at the end of the block. FlagsLiveAtEnd bool + // A block that would be good to align (according to the optimizer's guesses) + Hotness Hotness + // Subsequent blocks, if any. The number and order depend on the block kind. Succs []Edge @@ -112,7 +115,7 @@ func (e Edge) String() string { } // BlockKind is the kind of SSA block. -type BlockKind int16 +type BlockKind uint8 // short form print func (b *Block) String() string { @@ -426,3 +429,17 @@ const ( BranchUnknown = BranchPrediction(0) BranchLikely = BranchPrediction(+1) ) + +type Hotness int8 // Could use negative numbers for specifically non-hot blocks, but don't, yet. +const ( + // These values are arranged in what seems to be order of increasing alignment importance. + // Currently only a few are relevant. Implicitly, they are all in a loop. + HotNotFlowIn Hotness = 1 << iota // This block is only reached by branches + HotInitial // In the block order, the first one for a given loop. Not necessarily topological header. + HotPgo // By PGO-based heuristics, this block occurs in a hot loop + + HotNot = 0 + HotInitialNotFlowIn = HotInitial | HotNotFlowIn // typically first block of a rotated loop, loop is entered with a branch (not to this block). No PGO + HotPgoInitial = HotPgo | HotInitial // special case; single block loop, initial block is header block has a flow-in entry, but PGO says it is hot + HotPgoInitialNotFLowIn = HotPgo | HotInitial | HotNotFlowIn // PGO says it is hot, and the loop is rotated so flow enters loop with a branch +) diff --git a/src/cmd/compile/internal/ssa/fmahash_test.go b/src/cmd/compile/internal/ssa/fmahash_test.go index dfa1aa1147..c563d5b8d9 100644 --- a/src/cmd/compile/internal/ssa/fmahash_test.go +++ b/src/cmd/compile/internal/ssa/fmahash_test.go @@ -41,7 +41,7 @@ func TestFmaHash(t *testing.T) { t.Logf("%v", cmd.Env) b, e := cmd.CombinedOutput() if e != nil { - t.Error(e) + t.Errorf("build failed: %v\n%s", e, b) } s := string(b) // Looking for "GOFMAHASH triggered main.main:24" re := "fmahash(0?) triggered .*fma.go:29:..;.*fma.go:18:.." diff --git a/src/cmd/compile/internal/ssa/func.go b/src/cmd/compile/internal/ssa/func.go index 38b459a2ff..2bb34a41cb 100644 --- a/src/cmd/compile/internal/ssa/func.go +++ b/src/cmd/compile/internal/ssa/func.go @@ -45,6 +45,7 @@ type Func struct { laidout bool // Blocks are ordered NoSplit bool // true if function is marked as nosplit. Used by schedule check pass. dumpFileSeq uint8 // the sequence numbers of dump file. (%s_%02d__%s.dump", funcname, dumpFileSeq, phaseName) + IsPgoHot bool // when register allocation is done, maps value ids to locations RegAlloc []Location diff --git a/src/cmd/compile/internal/ssa/looprotate.go b/src/cmd/compile/internal/ssa/looprotate.go index 844a8f7124..f32125576f 100644 --- a/src/cmd/compile/internal/ssa/looprotate.go +++ b/src/cmd/compile/internal/ssa/looprotate.go @@ -56,9 +56,20 @@ func loopRotate(f *Func) { } p = e.b } - if p == nil || p == b { + if p == nil { continue } + p.Hotness |= HotInitial + if f.IsPgoHot { + p.Hotness |= HotPgo + } + // blocks will be arranged so that p is ordered first, if it isn't already. + if p == b { // p is header, already first (and also, only block in the loop) + continue + } + p.Hotness |= HotNotFlowIn + + // the loop header b follows p after[p.ID] = []*Block{b} for { nextIdx := idToIdx[b.ID] + 1 diff --git a/src/cmd/compile/internal/ssa/rewrite.go b/src/cmd/compile/internal/ssa/rewrite.go index 4e4d99af0b..f9566368c0 100644 --- a/src/cmd/compile/internal/ssa/rewrite.go +++ b/src/cmd/compile/internal/ssa/rewrite.go @@ -1653,6 +1653,56 @@ func mergePPC64ClrlsldiRlwinm(sld int32, rlw int64) int64 { return encodePPC64RotateMask(r_3, int64(mask_3), 32) } +// Test if RLWINM feeding into an ANDconst can be merged. Return the encoded RLWINM constant, +// or 0 if they cannot be merged. +func mergePPC64AndRlwinm(mask uint32, rlw int64) int64 { + r, _, _, mask_rlw := DecodePPC64RotateMask(rlw) + mask_out := (mask_rlw & uint64(mask)) + + // Verify the result is still a valid bitmask of <= 32 bits. + if !isPPC64WordRotateMask(int64(mask_out)) { + return 0 + } + return encodePPC64RotateMask(r, int64(mask_out), 32) +} + +// Test if AND feeding into an ANDconst can be merged. Return the encoded RLWINM constant, +// or 0 if they cannot be merged. +func mergePPC64RlwinmAnd(rlw int64, mask uint32) int64 { + r, _, _, mask_rlw := DecodePPC64RotateMask(rlw) + + // Rotate the input mask, combine with the rlwnm mask, and test if it is still a valid rlwinm mask. + r_mask := bits.RotateLeft32(mask, int(r)) + + mask_out := (mask_rlw & uint64(r_mask)) + + // Verify the result is still a valid bitmask of <= 32 bits. + if !isPPC64WordRotateMask(int64(mask_out)) { + return 0 + } + return encodePPC64RotateMask(r, int64(mask_out), 32) +} + +// Test if RLWINM feeding into SRDconst can be merged. Return the encoded RLIWNM constant, +// or 0 if they cannot be merged. +func mergePPC64SldiRlwinm(sldi, rlw int64) int64 { + r_1, mb, me, mask_1 := DecodePPC64RotateMask(rlw) + if mb > me || mb < sldi { + // Wrapping masks cannot be merged as the upper 32 bits are effectively undefined in this case. + // Likewise, if mb is less than the shift amount, it cannot be merged. + return 0 + } + // combine the masks, and adjust for the final left shift. + mask_3 := mask_1 << sldi + r_3 := (r_1 + sldi) & 31 // This can wrap. + + // Verify the result is still a valid bitmask of <= 32 bits. + if uint64(uint32(mask_3)) != mask_3 { + return 0 + } + return encodePPC64RotateMask(r_3, int64(mask_3), 32) +} + // Compute the encoded RLWINM constant from combining (SLDconst [sld] (SRWconst [srw] x)), // or return 0 if they cannot be combined. func mergePPC64SldiSrw(sld, srw int64) int64 { diff --git a/src/cmd/compile/internal/ssa/rewritePPC64.go b/src/cmd/compile/internal/ssa/rewritePPC64.go index 266ac14c38..cef2f21e50 100644 --- a/src/cmd/compile/internal/ssa/rewritePPC64.go +++ b/src/cmd/compile/internal/ssa/rewritePPC64.go @@ -611,6 +611,8 @@ func rewriteValuePPC64(v *Value) bool { return rewriteValuePPC64_OpPPC64ORN(v) case OpPPC64ORconst: return rewriteValuePPC64_OpPPC64ORconst(v) + case OpPPC64RLWINM: + return rewriteValuePPC64_OpPPC64RLWINM(v) case OpPPC64ROTL: return rewriteValuePPC64_OpPPC64ROTL(v) case OpPPC64ROTLW: @@ -6765,6 +6767,40 @@ func rewriteValuePPC64_OpPPC64MOVBZreg(v *Value) bool { v.AddArg(x) return true } + // match: (MOVBZreg (SRWconst x [s])) + // cond: mergePPC64AndSrwi(0xFF,s) != 0 + // result: (RLWINM [mergePPC64AndSrwi(0xFF,s)] x) + for { + if v_0.Op != OpPPC64SRWconst { + break + } + s := auxIntToInt64(v_0.AuxInt) + x := v_0.Args[0] + if !(mergePPC64AndSrwi(0xFF, s) != 0) { + break + } + v.reset(OpPPC64RLWINM) + v.AuxInt = int64ToAuxInt(mergePPC64AndSrwi(0xFF, s)) + v.AddArg(x) + return true + } + // match: (MOVBZreg (RLWINM [r] y)) + // cond: mergePPC64AndRlwinm(0xFF,r) != 0 + // result: (RLWINM [mergePPC64AndRlwinm(0xFF,r)] y) + for { + if v_0.Op != OpPPC64RLWINM { + break + } + r := auxIntToInt64(v_0.AuxInt) + y := v_0.Args[0] + if !(mergePPC64AndRlwinm(0xFF, r) != 0) { + break + } + v.reset(OpPPC64RLWINM) + v.AuxInt = int64ToAuxInt(mergePPC64AndRlwinm(0xFF, r)) + v.AddArg(y) + return true + } // match: (MOVBZreg (OR x (MOVWZreg y))) // result: (MOVBZreg (OR x y)) for { @@ -8610,6 +8646,23 @@ func rewriteValuePPC64_OpPPC64MOVHZreg(v *Value) bool { v.AddArg(x) return true } + // match: (MOVHZreg (RLWINM [r] y)) + // cond: mergePPC64AndRlwinm(0xFFFF,r) != 0 + // result: (RLWINM [mergePPC64AndRlwinm(0xFFFF,r)] y) + for { + if v_0.Op != OpPPC64RLWINM { + break + } + r := auxIntToInt64(v_0.AuxInt) + y := v_0.Args[0] + if !(mergePPC64AndRlwinm(0xFFFF, r) != 0) { + break + } + v.reset(OpPPC64RLWINM) + v.AuxInt = int64ToAuxInt(mergePPC64AndRlwinm(0xFFFF, r)) + v.AddArg(y) + return true + } // match: (MOVHZreg y:(MOVHZreg _)) // result: y for { @@ -9960,6 +10013,33 @@ func rewriteValuePPC64_OpPPC64MOVWZreg(v *Value) bool { v.AddArg(x) return true } + // match: (MOVWZreg (RLWINM [r] y)) + // cond: mergePPC64AndRlwinm(0xFFFFFFFF,r) != 0 + // result: (RLWINM [mergePPC64AndRlwinm(0xFFFFFFFF,r)] y) + for { + if v_0.Op != OpPPC64RLWINM { + break + } + r := auxIntToInt64(v_0.AuxInt) + y := v_0.Args[0] + if !(mergePPC64AndRlwinm(0xFFFFFFFF, r) != 0) { + break + } + v.reset(OpPPC64RLWINM) + v.AuxInt = int64ToAuxInt(mergePPC64AndRlwinm(0xFFFFFFFF, r)) + v.AddArg(y) + return true + } + // match: (MOVWZreg w:(SLWconst u)) + // result: w + for { + w := v_0 + if w.Op != OpPPC64SLWconst { + break + } + v.copyOf(w) + return true + } // match: (MOVWZreg y:(MOVWZreg _)) // result: y for { @@ -11346,6 +11426,49 @@ func rewriteValuePPC64_OpPPC64ORconst(v *Value) bool { } return false } +func rewriteValuePPC64_OpPPC64RLWINM(v *Value) bool { + v_0 := v.Args[0] + // match: (RLWINM [r] (MOVHZreg u)) + // cond: mergePPC64RlwinmAnd(r,0xFFFF) != 0 + // result: (RLWINM [mergePPC64RlwinmAnd(r,0xFFFF)] u) + for { + r := auxIntToInt64(v.AuxInt) + if v_0.Op != OpPPC64MOVHZreg { + break + } + u := v_0.Args[0] + if !(mergePPC64RlwinmAnd(r, 0xFFFF) != 0) { + break + } + v.reset(OpPPC64RLWINM) + v.AuxInt = int64ToAuxInt(mergePPC64RlwinmAnd(r, 0xFFFF)) + v.AddArg(u) + return true + } + // match: (RLWINM [r] (Select0 (ANDCCconst [a] u))) + // cond: mergePPC64RlwinmAnd(r,uint32(a)) != 0 + // result: (RLWINM [mergePPC64RlwinmAnd(r,uint32(a))] u) + for { + r := auxIntToInt64(v.AuxInt) + if v_0.Op != OpSelect0 { + break + } + v_0_0 := v_0.Args[0] + if v_0_0.Op != OpPPC64ANDCCconst { + break + } + a := auxIntToInt64(v_0_0.AuxInt) + u := v_0_0.Args[0] + if !(mergePPC64RlwinmAnd(r, uint32(a)) != 0) { + break + } + v.reset(OpPPC64RLWINM) + v.AuxInt = int64ToAuxInt(mergePPC64RlwinmAnd(r, uint32(a))) + v.AddArg(u) + return true + } + return false +} func rewriteValuePPC64_OpPPC64ROTL(v *Value) bool { v_1 := v.Args[1] v_0 := v.Args[0] @@ -12061,6 +12184,24 @@ func rewriteValuePPC64_OpPPC64SLDconst(v *Value) bool { v.AddArg(x) return true } + // match: (SLDconst [s] (RLWINM [r] y)) + // cond: mergePPC64SldiRlwinm(s,r) != 0 + // result: (RLWINM [mergePPC64SldiRlwinm(s,r)] y) + for { + s := auxIntToInt64(v.AuxInt) + if v_0.Op != OpPPC64RLWINM { + break + } + r := auxIntToInt64(v_0.AuxInt) + y := v_0.Args[0] + if !(mergePPC64SldiRlwinm(s, r) != 0) { + break + } + v.reset(OpPPC64RLWINM) + v.AuxInt = int64ToAuxInt(mergePPC64SldiRlwinm(s, r)) + v.AddArg(y) + return true + } // match: (SLDconst [c] z:(MOVBZreg x)) // cond: c < 8 && z.Uses == 1 // result: (CLRLSLDI [newPPC64ShiftAuxInt(c,56,63,64)] x) @@ -12206,6 +12347,19 @@ func rewriteValuePPC64_OpPPC64SLW(v *Value) bool { } func rewriteValuePPC64_OpPPC64SLWconst(v *Value) bool { v_0 := v.Args[0] + // match: (SLWconst [s] (MOVWZreg w)) + // result: (SLWconst [s] w) + for { + s := auxIntToInt64(v.AuxInt) + if v_0.Op != OpPPC64MOVWZreg { + break + } + w := v_0.Args[0] + v.reset(OpPPC64SLWconst) + v.AuxInt = int64ToAuxInt(s) + v.AddArg(w) + return true + } // match: (SLWconst [c] z:(MOVBZreg x)) // cond: z.Uses == 1 && c < 8 // result: (CLRLSLWI [newPPC64ShiftAuxInt(c,24,31,32)] x) @@ -14665,6 +14819,28 @@ func rewriteValuePPC64_OpSelect0(v *Value) bool { v.AddArg(v0) return true } + // match: (Select0 (ANDCCconst [m] (RLWINM [r] y))) + // cond: mergePPC64AndRlwinm(uint32(m),r) != 0 + // result: (RLWINM [mergePPC64AndRlwinm(uint32(m),r)] y) + for { + if v_0.Op != OpPPC64ANDCCconst { + break + } + m := auxIntToInt64(v_0.AuxInt) + v_0_0 := v_0.Args[0] + if v_0_0.Op != OpPPC64RLWINM { + break + } + r := auxIntToInt64(v_0_0.AuxInt) + y := v_0_0.Args[0] + if !(mergePPC64AndRlwinm(uint32(m), r) != 0) { + break + } + v.reset(OpPPC64RLWINM) + v.AuxInt = int64ToAuxInt(mergePPC64AndRlwinm(uint32(m), r)) + v.AddArg(y) + return true + } // match: (Select0 (ANDCCconst [1] z:(SRADconst [63] x))) // cond: z.Uses == 1 // result: (SRDconst [63] x) diff --git a/src/cmd/compile/internal/ssagen/abi.go b/src/cmd/compile/internal/ssagen/abi.go index 5c4a8aff69..d5ae3b1793 100644 --- a/src/cmd/compile/internal/ssagen/abi.go +++ b/src/cmd/compile/internal/ssagen/abi.go @@ -148,6 +148,11 @@ func (s *SymABIs) GenABIWrappers() { // offsets to dispatch arguments, which currently using ABI0 // frame layout. Pin it to ABI0. fn.ABI = obj.ABI0 + // Propagate linkname attribute, which was set on the ABIInternal + // symbol. + if sym.Linksym().IsLinkname() { + sym.LinksymABI(fn.ABI).Set(obj.AttrLinkname, true) + } } // If cgo-exported, add the definition ABI to the cgo diff --git a/src/cmd/compile/internal/ssagen/nowb.go b/src/cmd/compile/internal/ssagen/nowb.go index b8756eea61..8e776695e3 100644 --- a/src/cmd/compile/internal/ssagen/nowb.go +++ b/src/cmd/compile/internal/ssagen/nowb.go @@ -174,6 +174,14 @@ func (c *nowritebarrierrecChecker) check() { fmt.Fprintf(&err, "\n\t%v: called by %v", base.FmtPos(call.lineno), call.target.Nname) call = funcs[call.target] } + // Seeing this error in a failed CI run? It indicates that + // a function in the runtime package marked nowritebarrierrec + // (the outermost stack element) was found, by a static + // reachability analysis over the fully lowered optimized code, + // to call a function (fn) that involves a write barrier. + // + // Even if the call path is infeasable, + // you will need to reorganize the code to avoid it. base.ErrorfAt(fn.WBPos, 0, "write barrier prohibited by caller; %v%s", fn.Nname, err.String()) continue } diff --git a/src/cmd/compile/internal/ssagen/pgen.go b/src/cmd/compile/internal/ssagen/pgen.go index 5b57c8a825..e666c22a7d 100644 --- a/src/cmd/compile/internal/ssagen/pgen.go +++ b/src/cmd/compile/internal/ssagen/pgen.go @@ -12,9 +12,11 @@ import ( "sync" "cmd/compile/internal/base" + "cmd/compile/internal/inline" "cmd/compile/internal/ir" "cmd/compile/internal/liveness" "cmd/compile/internal/objw" + "cmd/compile/internal/pgoir" "cmd/compile/internal/ssa" "cmd/compile/internal/types" "cmd/internal/obj" @@ -296,8 +298,8 @@ const maxStackSize = 1 << 30 // uses it to generate a plist, // and flushes that plist to machine code. // worker indicates which of the backend workers is doing the processing. -func Compile(fn *ir.Func, worker int) { - f := buildssa(fn, worker) +func Compile(fn *ir.Func, worker int, profile *pgoir.Profile) { + f := buildssa(fn, worker, inline.IsPgoHotFunc(fn, profile) || inline.HasPgoHotInline(fn)) // Note: check arg size to fix issue 25507. if f.Frontend().(*ssafn).stksize >= maxStackSize || f.OwnAux.ArgWidth() >= maxStackSize { largeStackFramesMu.Lock() diff --git a/src/cmd/compile/internal/ssagen/ssa.go b/src/cmd/compile/internal/ssagen/ssa.go index 9e384fe016..9b23935df7 100644 --- a/src/cmd/compile/internal/ssagen/ssa.go +++ b/src/cmd/compile/internal/ssagen/ssa.go @@ -291,7 +291,7 @@ func (s *state) emitOpenDeferInfo() { // buildssa builds an SSA function for fn. // worker indicates which of the backend workers is doing the processing. -func buildssa(fn *ir.Func, worker int) *ssa.Func { +func buildssa(fn *ir.Func, worker int, isPgoHot bool) *ssa.Func { name := ir.FuncName(fn) abiSelf := abiForFunc(fn, ssaConfig.ABI0, ssaConfig.ABI1) @@ -373,6 +373,7 @@ func buildssa(fn *ir.Func, worker int) *ssa.Func { // Allocate starting block s.f.Entry = s.f.NewBlock(ssa.BlockPlain) s.f.Entry.Pos = fn.Pos() + s.f.IsPgoHot = isPgoHot if printssa { ssaDF := ssaDumpFile @@ -7302,12 +7303,47 @@ func genssa(f *ssa.Func, pp *objw.Progs) { var argLiveIdx int = -1 // argument liveness info index + // These control cache line alignment; if the required portion of + // a cache line is not available, then pad to obtain cache line + // alignment. Not implemented on all architectures, may not be + // useful on all architectures. + var hotAlign, hotRequire int64 + + if base.Debug.AlignHot > 0 { + switch base.Ctxt.Arch.Name { + // enable this on a case-by-case basis, with benchmarking. + // currently shown: + // good for amd64 + // not helpful for Apple Silicon + // + case "amd64", "386": + // Align to 64 if 31 or fewer bytes remain in a cache line + // benchmarks a little better than always aligning, and also + // adds slightly less to the (PGO-compiled) binary size. + hotAlign = 64 + hotRequire = 31 + } + } + // Emit basic blocks for i, b := range f.Blocks { - s.bstart[b.ID] = s.pp.Next + s.lineRunStart = nil s.SetPos(s.pp.Pos.WithNotStmt()) // It needs a non-empty Pos, but cannot be a statement boundary (yet). + if hotAlign > 0 && b.Hotness&ssa.HotPgoInitial == ssa.HotPgoInitial { + // So far this has only been shown profitable for PGO-hot loop headers. + // The Hotness values allows distinctions betwen initial blocks that are "hot" or not, and "flow-in" or not. + // Currently only the initial blocks of loops are tagged in this way; + // there are no blocks tagged "pgo-hot" that are not also tagged "initial". + // TODO more heuristics, more architectures. + p := s.pp.Prog(obj.APCALIGNMAX) + p.From.SetConst(hotAlign) + p.To.SetConst(hotRequire) + } + + s.bstart[b.ID] = s.pp.Next + if idx, ok := argLiveBlockMap[b.ID]; ok && idx != argLiveIdx { argLiveIdx = idx p := s.pp.Prog(obj.APCDATA) @@ -7466,7 +7502,8 @@ func genssa(f *ssa.Func, pp *objw.Progs) { // going to emit anyway, and use those instructions instead of the // inline marks. for p := s.pp.Text; p != nil; p = p.Link { - if p.As == obj.ANOP || p.As == obj.AFUNCDATA || p.As == obj.APCDATA || p.As == obj.ATEXT || p.As == obj.APCALIGN || Arch.LinkArch.Family == sys.Wasm { + if p.As == obj.ANOP || p.As == obj.AFUNCDATA || p.As == obj.APCDATA || p.As == obj.ATEXT || + p.As == obj.APCALIGN || p.As == obj.APCALIGNMAX || Arch.LinkArch.Family == sys.Wasm { // Don't use 0-sized instructions as inline marks, because we need // to identify inline mark instructions by pc offset. // (Some of these instructions are sometimes zero-sized, sometimes not. diff --git a/src/cmd/compile/internal/syntax/type.go b/src/cmd/compile/internal/syntax/type.go index 53132a442d..0be7e250ee 100644 --- a/src/cmd/compile/internal/syntax/type.go +++ b/src/cmd/compile/internal/syntax/type.go @@ -10,9 +10,12 @@ import "go/constant" // All types implement the Type interface. // (This type originally lived in types2. We moved it here // so we could depend on it from other packages without -// introducing a circularity.) +// introducing an import cycle.) type Type interface { // Underlying returns the underlying type of a type. + // Underlying types are never Named, TypeParam, or Alias types. + // + // See https://go.dev/ref/spec#Underlying_types. Underlying() Type // String returns a string representation of a type. diff --git a/src/cmd/compile/internal/types2/alias.go b/src/cmd/compile/internal/types2/alias.go index 9b7a13f81e..68475c54a4 100644 --- a/src/cmd/compile/internal/types2/alias.go +++ b/src/cmd/compile/internal/types2/alias.go @@ -14,7 +14,9 @@ import "fmt" // which points directly to the actual (aliased) type. type Alias struct { obj *TypeName // corresponding declared alias object + orig *Alias // original, uninstantiated alias tparams *TypeParamList // type parameters, or nil + targs *TypeList // type arguments, or nil fromRHS Type // RHS of type alias declaration; may be an alias actual Type // actual (aliased) type; never an alias } @@ -28,9 +30,34 @@ func NewAlias(obj *TypeName, rhs Type) *Alias { return alias } -func (a *Alias) Obj() *TypeName { return a.obj } +func (a *Alias) Obj() *TypeName { return a.obj } +func (a *Alias) String() string { return TypeString(a, nil) } + +// Underlying returns the [underlying type] of the alias type a, which is the +// underlying type of the aliased type. Underlying types are never Named, +// TypeParam, or Alias types. +// +// [underlying type]: https://go.dev/ref/spec#Underlying_types. func (a *Alias) Underlying() Type { return unalias(a).Underlying() } -func (a *Alias) String() string { return TypeString(a, nil) } + +// Origin returns the generic Alias type of which a is an instance. +// If a is not an instance of a generic alias, Origin returns a. +func (a *Alias) Origin() *Alias { return a.orig } + +// TypeParams returns the type parameters of the alias type a, or nil. +// A generic Alias and its instances have the same type parameters. +func (a *Alias) TypeParams() *TypeParamList { return a.tparams } + +// SetTypeParams sets the type parameters of the alias type a. +// The alias a must not have type arguments. +func (a *Alias) SetTypeParams(tparams []*TypeParam) { + assert(a.targs == nil) + a.tparams = bindTParams(tparams) +} + +// TypeArgs returns the type arguments used to instantiate the Alias type. +// If a is not an instance of a generic alias, the result is nil. +func (a *Alias) TypeArgs() *TypeList { return a.targs } // Rhs returns the type R on the right-hand side of an alias // declaration "type A = R", which may be another alias. @@ -82,7 +109,10 @@ func asNamed(t Type) *Named { // rhs must not be nil. func (check *Checker) newAlias(obj *TypeName, rhs Type) *Alias { assert(rhs != nil) - a := &Alias{obj, nil, rhs, nil} + a := new(Alias) + a.obj = obj + a.orig = a + a.fromRHS = rhs if obj.typ == nil { obj.typ = a } diff --git a/src/cmd/compile/internal/types2/instantiate.go b/src/cmd/compile/internal/types2/instantiate.go index a25cb141ec..5630d06bc9 100644 --- a/src/cmd/compile/internal/types2/instantiate.go +++ b/src/cmd/compile/internal/types2/instantiate.go @@ -14,6 +14,12 @@ import ( . "internal/types/errors" ) +// A genericType implements access to its type parameters. +type genericType interface { + Type + TypeParams() *TypeParamList +} + // Instantiate instantiates the type orig with the given type arguments targs. // orig must be a *Named or a *Signature type. If there is no error, the // resulting Type is an instantiated type of the same kind (either a *Named or @@ -41,17 +47,15 @@ import ( // count is incorrect; for *Named types, a panic may occur later inside the // *Named API. func Instantiate(ctxt *Context, orig Type, targs []Type, validate bool) (Type, error) { + assert(len(targs) > 0) if ctxt == nil { ctxt = NewContext() } + orig_ := orig.(genericType) // signature of Instantiate must not change for backward-compatibility + if validate { - var tparams []*TypeParam - switch t := orig.(type) { - case *Named: - tparams = t.TypeParams().list() - case *Signature: - tparams = t.TypeParams().list() - } + tparams := orig_.TypeParams().list() + assert(len(tparams) > 0) if len(targs) != len(tparams) { return nil, fmt.Errorf("got %d type arguments but %s has %d type parameters", len(targs), orig, len(tparams)) } @@ -60,7 +64,7 @@ func Instantiate(ctxt *Context, orig Type, targs []Type, validate bool) (Type, e } } - inst := (*Checker)(nil).instance(nopos, orig, targs, nil, ctxt) + inst := (*Checker)(nil).instance(nopos, orig_, targs, nil, ctxt) return inst, nil } @@ -75,7 +79,7 @@ func Instantiate(ctxt *Context, orig Type, targs []Type, validate bool) (Type, e // must be non-nil. // // For Named types the resulting instance may be unexpanded. -func (check *Checker) instance(pos syntax.Pos, orig Type, targs []Type, expanding *Named, ctxt *Context) (res Type) { +func (check *Checker) instance(pos syntax.Pos, orig genericType, targs []Type, expanding *Named, ctxt *Context) (res Type) { // The order of the contexts below matters: we always prefer instances in the // expanding instance context in order to preserve reference cycles. // diff --git a/src/cmd/compile/internal/types2/named.go b/src/cmd/compile/internal/types2/named.go index aa7ab00c33..1859b27aa4 100644 --- a/src/cmd/compile/internal/types2/named.go +++ b/src/cmd/compile/internal/types2/named.go @@ -485,9 +485,17 @@ func (t *Named) methodIndex(name string, foldCase bool) int { return -1 } -// TODO(gri) Investigate if Unalias can be moved to where underlying is set. -func (t *Named) Underlying() Type { return Unalias(t.resolve().underlying) } -func (t *Named) String() string { return TypeString(t, nil) } +// Underlying returns the [underlying type] of the named type t, resolving all +// forwarding declarations. Underlying types are never Named, TypeParam, or +// Alias types. +// +// [underlying type]: https://go.dev/ref/spec#Underlying_types. +func (t *Named) Underlying() Type { + // TODO(gri) Investigate if Unalias can be moved to where underlying is set. + return Unalias(t.resolve().underlying) +} + +func (t *Named) String() string { return TypeString(t, nil) } // ---------------------------------------------------------------------------- // Implementation diff --git a/src/cmd/compile/internal/types2/signature.go b/src/cmd/compile/internal/types2/signature.go index bb4d32b016..7a5a2c155f 100644 --- a/src/cmd/compile/internal/types2/signature.go +++ b/src/cmd/compile/internal/types2/signature.go @@ -73,9 +73,6 @@ func (s *Signature) Recv() *Var { return s.recv } // TypeParams returns the type parameters of signature s, or nil. func (s *Signature) TypeParams() *TypeParamList { return s.tparams } -// SetTypeParams sets the type parameters of signature s. -func (s *Signature) SetTypeParams(tparams []*TypeParam) { s.tparams = bindTParams(tparams) } - // RecvTypeParams returns the receiver type parameters of signature s, or nil. func (s *Signature) RecvTypeParams() *TypeParamList { return s.rparams } diff --git a/src/cmd/compile/internal/types2/stmt.go b/src/cmd/compile/internal/types2/stmt.go index 1984777008..655d072171 100644 --- a/src/cmd/compile/internal/types2/stmt.go +++ b/src/cmd/compile/internal/types2/stmt.go @@ -898,7 +898,7 @@ func (check *Checker) rangeStmt(inner stmtContext, s *syntax.ForStmt, rclause *s lhs := [2]Expr{sKey, sValue} // sKey, sValue may be nil rhs := [2]Type{key, val} // key, val may be nil - constIntRange := x.mode == constant_ && isInteger(x.typ) + rangeOverInt := isInteger(x.typ) if isDef { // short variable declaration @@ -933,14 +933,15 @@ func (check *Checker) rangeStmt(inner stmtContext, s *syntax.ForStmt, rclause *s continue } - // initialize lhs variable - if constIntRange { + if rangeOverInt { + assert(i == 0) // at most one iteration variable (rhs[1] == nil for rangeOverInt) check.initVar(obj, &x, "range clause") } else { - x.mode = value - x.expr = lhs // we don't have a better rhs expression to use here - x.typ = typ - check.initVar(obj, &x, "assignment") // error is on variable, use "assignment" not "range clause" + var y operand + y.mode = value + y.expr = lhs // we don't have a better rhs expression to use here + y.typ = typ + check.initVar(obj, &y, "assignment") // error is on variable, use "assignment" not "range clause" } assert(obj.typ != nil) } @@ -967,21 +968,30 @@ func (check *Checker) rangeStmt(inner stmtContext, s *syntax.ForStmt, rclause *s continue } - if constIntRange { + if rangeOverInt { + assert(i == 0) // at most one iteration variable (rhs[1] == nil for rangeOverInt) check.assignVar(lhs, nil, &x, "range clause") + // If the assignment succeeded, if x was untyped before, it now + // has a type inferred via the assignment. It must be an integer. + // (go.dev/issues/67027) + if x.mode != invalid && !isInteger(x.typ) { + check.softErrorf(lhs, InvalidRangeExpr, "cannot use iteration variable of type %s", x.typ) + } } else { - x.mode = value - x.expr = lhs // we don't have a better rhs expression to use here - x.typ = typ - check.assignVar(lhs, nil, &x, "assignment") // error is on variable, use "assignment" not "range clause" + var y operand + y.mode = value + y.expr = lhs // we don't have a better rhs expression to use here + y.typ = typ + check.assignVar(lhs, nil, &y, "assignment") // error is on variable, use "assignment" not "range clause" } } - } else if constIntRange { + } else if rangeOverInt { // If we don't have any iteration variables, we still need to // check that a (possibly untyped) integer range expression x // is valid. // We do this by checking the assignment _ = x. This ensures - // that an untyped x can be converted to a value of type int. + // that an untyped x can be converted to a value of its default + // type (rune or int). check.assignment(&x, nil, "range clause") } diff --git a/src/cmd/compile/internal/types2/typeparam.go b/src/cmd/compile/internal/types2/typeparam.go index 5c6030b3fb..9ad064906f 100644 --- a/src/cmd/compile/internal/types2/typeparam.go +++ b/src/cmd/compile/internal/types2/typeparam.go @@ -86,6 +86,10 @@ func (t *TypeParam) SetConstraint(bound Type) { t.iface() } +// Underlying returns the [underlying type] of the type parameter t, which is +// the underlying type of its constraint. This type is always an interface. +// +// [underlying type]: https://go.dev/ref/spec#Underlying_types. func (t *TypeParam) Underlying() Type { return t.iface() } diff --git a/src/cmd/covdata/covdata.go b/src/cmd/covdata/covdata.go index 549efea20a..b280203f0c 100644 --- a/src/cmd/covdata/covdata.go +++ b/src/cmd/covdata/covdata.go @@ -7,6 +7,7 @@ package main import ( "cmd/internal/cov" "cmd/internal/pkgpattern" + "cmd/internal/telemetry" "flag" "fmt" "os" @@ -108,6 +109,8 @@ const ( ) func main() { + telemetry.Start() + // First argument should be mode/subcommand. if len(os.Args) < 2 { usage("missing command selector") @@ -143,6 +146,8 @@ func main() { op.Usage("") } flag.Parse() + telemetry.Inc("covdata/invocations") + telemetry.CountFlags("covdata/flag:", *flag.CommandLine) // Mode-independent flag setup dbgtrace(1, "starting mode-independent setup") diff --git a/src/cmd/cover/cover.go b/src/cmd/cover/cover.go index d4e529bcde..912f7cafb5 100644 --- a/src/cmd/cover/cover.go +++ b/src/cmd/cover/cover.go @@ -26,6 +26,7 @@ import ( "cmd/internal/edit" "cmd/internal/objabi" + "cmd/internal/telemetry" ) const usageMessage = "" + @@ -86,9 +87,13 @@ const ( ) func main() { + telemetry.Start() + objabi.AddVersionFlag() flag.Usage = usage objabi.Flagparse(usage) + telemetry.Inc("cover/invocations") + telemetry.CountFlags("cover/flag:", *flag.CommandLine) // Usage information when no arguments. if flag.NFlag() == 0 && flag.NArg() == 0 { diff --git a/src/cmd/distpack/pack.go b/src/cmd/distpack/pack.go index cf507edb4d..0faab5c0b8 100644 --- a/src/cmd/distpack/pack.go +++ b/src/cmd/distpack/pack.go @@ -44,6 +44,8 @@ import ( "runtime" "strings" "time" + + "cmd/internal/telemetry" ) func usage() { @@ -67,8 +69,11 @@ var ( func main() { log.SetPrefix("distpack: ") log.SetFlags(0) + telemetry.Start() flag.Usage = usage flag.Parse() + telemetry.Inc("distpack/invocations") + telemetry.CountFlags("distpack/flag:", *flag.CommandLine) if flag.NArg() != 0 { usage() } diff --git a/src/cmd/doc/main.go b/src/cmd/doc/main.go index 273d7febbc..d02bf65c40 100644 --- a/src/cmd/doc/main.go +++ b/src/cmd/doc/main.go @@ -54,6 +54,8 @@ import ( "path" "path/filepath" "strings" + + "cmd/internal/telemetry" ) var ( @@ -85,6 +87,7 @@ func usage() { func main() { log.SetFlags(0) log.SetPrefix("doc: ") + telemetry.Start() dirsInit() err := do(os.Stdout, flag.CommandLine, os.Args[1:]) if err != nil { @@ -105,6 +108,8 @@ func do(writer io.Writer, flagSet *flag.FlagSet, args []string) (err error) { flagSet.BoolVar(&showSrc, "src", false, "show source code for symbol") flagSet.BoolVar(&short, "short", false, "one-line representation for each symbol") flagSet.Parse(args) + telemetry.Inc("doc/invocations") + telemetry.CountFlags("doc/flag:", *flag.CommandLine) if chdir != "" { if err := os.Chdir(chdir); err != nil { return err diff --git a/src/cmd/fix/main.go b/src/cmd/fix/main.go index db67b4ba07..b0aabae889 100644 --- a/src/cmd/fix/main.go +++ b/src/cmd/fix/main.go @@ -21,6 +21,8 @@ import ( "path/filepath" "sort" "strings" + + "cmd/internal/telemetry" ) var ( @@ -63,8 +65,11 @@ func usage() { } func main() { + telemetry.Start() flag.Usage = usage flag.Parse() + telemetry.Inc("fix/invocations") + telemetry.CountFlags("fix/flag:", *flag.CommandLine) if !version.IsValid(*goVersion) { report(fmt.Errorf("invalid -go=%s", *goVersion)) diff --git a/src/cmd/go.mod b/src/cmd/go.mod index 6c5b5c46ea..05d9a1b62b 100644 --- a/src/cmd/go.mod +++ b/src/cmd/go.mod @@ -6,10 +6,10 @@ require ( github.com/google/pprof v0.0.0-20240207164012-fb44976bdcd5 golang.org/x/arch v0.7.0 golang.org/x/build v0.0.0-20240222153247-cf4ed81bb19f - golang.org/x/mod v0.17.1-0.20240507203540-6686f416970d + golang.org/x/mod v0.17.1-0.20240514174713-c0bdc7bd01c9 golang.org/x/sync v0.7.0 golang.org/x/sys v0.20.0 - golang.org/x/telemetry v0.0.0-20240510223629-51e8b5d718eb + golang.org/x/telemetry v0.0.0-20240515213752-9ff3ad9b3e68 golang.org/x/term v0.18.0 golang.org/x/tools v0.20.1-0.20240429173604-74c9cfe4d22f ) diff --git a/src/cmd/go.sum b/src/cmd/go.sum index 3e0b99dc67..41c50d7a2b 100644 --- a/src/cmd/go.sum +++ b/src/cmd/go.sum @@ -26,14 +26,14 @@ golang.org/x/arch v0.7.0 h1:pskyeJh/3AmoQ8CPE95vxHLqp1G1GfGNXTmcl9NEKTc= golang.org/x/arch v0.7.0/go.mod h1:FEVrYAQjsQXMVJ1nsMoVVXPZg6p2JE2mx8psSWTDQys= golang.org/x/build v0.0.0-20240222153247-cf4ed81bb19f h1:XQ2eu0I26WsNCKQkRehp+5mwjjChw94trD9LT8LLSq0= golang.org/x/build v0.0.0-20240222153247-cf4ed81bb19f/go.mod h1:HTqTCkubWT8epEK9hDWWGkoOOB7LGSrU1qvWZCSwO50= -golang.org/x/mod v0.17.1-0.20240507203540-6686f416970d h1:QS9b5Jvh12iuDV+eYRspS3+7Fn6wOTYI6EAHdeGEsmY= -golang.org/x/mod v0.17.1-0.20240507203540-6686f416970d/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= +golang.org/x/mod v0.17.1-0.20240514174713-c0bdc7bd01c9 h1:EfMABMgrJ8+hRjLvhUzJkLKgFv3lYAglGXczg5ggNyk= +golang.org/x/mod v0.17.1-0.20240514174713-c0bdc7bd01c9/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= golang.org/x/sync v0.7.0 h1:YsImfSBoP9QPYL0xyKJPq0gcaJdG3rInoqxTWbfQu9M= golang.org/x/sync v0.7.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= golang.org/x/sys v0.20.0 h1:Od9JTbYCk261bKm4M/mw7AklTlFYIa0bIp9BgSm1S8Y= golang.org/x/sys v0.20.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= -golang.org/x/telemetry v0.0.0-20240510223629-51e8b5d718eb h1:UTGVF0T+nFaQu6f7USlW8TktAybpMdEjJcF5HyX4dxo= -golang.org/x/telemetry v0.0.0-20240510223629-51e8b5d718eb/go.mod h1:pRgIJT+bRLFKnoM1ldnzKoxTIn14Yxz928LQRYYgIN0= +golang.org/x/telemetry v0.0.0-20240515213752-9ff3ad9b3e68 h1:UpbHwFpoVYf6i5cMzwsNuPGNsZzfJXFr8R4uUv2HVgk= +golang.org/x/telemetry v0.0.0-20240515213752-9ff3ad9b3e68/go.mod h1:pRgIJT+bRLFKnoM1ldnzKoxTIn14Yxz928LQRYYgIN0= golang.org/x/term v0.18.0 h1:FcHjZXDMxI8mM3nwhX9HlKop4C0YQvCVCdwYl2wOtE8= golang.org/x/term v0.18.0/go.mod h1:ILwASektA3OnRv7amZ1xhE/KTR+u50pbXfZ03+6Nx58= golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ= diff --git a/src/cmd/go/alldocs.go b/src/cmd/go/alldocs.go index 7800c72af3..52fb4c25a8 100644 --- a/src/cmd/go/alldocs.go +++ b/src/cmd/go/alldocs.go @@ -27,6 +27,7 @@ // mod module maintenance // work workspace maintenance // run compile and run Go program +// telemetry manage telemetry data and settings // test test packages // tool run specified go tool // version print Go version @@ -456,7 +457,7 @@ // // Usage: // -// go env [-json] [-u] [-w] [var ...] +// go env [-json] [-changed] [-u] [-w] [var ...] // // Env prints Go environment information. // @@ -476,6 +477,10 @@ // form NAME=VALUE and changes the default settings // of the named environment variables to the given values. // +// The -changed flag prints only those settings whose effective +// value differs from the default value that would be obtained in +// an empty environment with no prior uses of the -w flag. +// // For more about environment variables, see 'go help environment'. // // # Update packages to use new APIs @@ -1201,6 +1206,12 @@ // // The -module flag changes the module's path (the go.mod file's module line). // +// The -godebug=key=value flag adds a godebug key=value line, +// replacing any existing godebug lines with the given key. +// +// The -dropgodebug=key flag drops any existing godebug lines +// with the given key. +// // The -require=path@version and -droprequire=path flags // add and drop a requirement on the given module path and version. // Note that -require overrides any existing requirements on path. @@ -1209,6 +1220,14 @@ // which make other go.mod adjustments as needed to satisfy // constraints imposed by other modules. // +// The -go=version flag sets the expected Go language version. +// This flag is mainly for tools that understand Go version dependencies. +// Users should prefer 'go get go@version'. +// +// The -toolchain=version flag sets the Go toolchain to use. +// This flag is mainly for tools that understand Go version dependencies. +// Users should prefer 'go get toolchain@version'. +// // The -exclude=path@version and -dropexclude=path@version flags // add and drop an exclusion for the given module path and version. // Note that -exclude=path@version is a no-op if that exclusion already exists. @@ -1230,13 +1249,9 @@ // like "v1.2.3" or a closed interval like "[v1.1.0,v1.1.9]". Note that // -retract=version is a no-op if that retraction already exists. // -// The -require, -droprequire, -exclude, -dropexclude, -replace, -// -dropreplace, -retract, and -dropretract editing flags may be repeated, -// and the changes are applied in the order given. -// -// The -go=version flag sets the expected Go language version. -// -// The -toolchain=name flag sets the Go toolchain to use. +// The -godebug, -dropgodebug, -require, -droprequire, -exclude, -dropexclude, +// -replace, -dropreplace, -retract, and -dropretract editing flags may be +// repeated, and the changes are applied in the order given. // // The -print flag prints the final go.mod in its text format instead of // writing it back to go.mod. @@ -1253,6 +1268,7 @@ // Module ModPath // Go string // Toolchain string +// Godebug []Godebug // Require []Require // Exclude []Module // Replace []Replace @@ -1264,9 +1280,14 @@ // Deprecated string // } // +// type Godebug struct { +// Key string +// Value string +// } +// // type Require struct { -// Path string -// Version string +// Path string +// Version string // Indirect bool // } // @@ -1530,6 +1551,12 @@ // rewrite the go.mod file. The only time this flag is needed is if no other // flags are specified, as in 'go work edit -fmt'. // +// The -godebug=key=value flag adds a godebug key=value line, +// replacing any existing godebug lines with the given key. +// +// The -dropgodebug=key flag drops any existing godebug lines +// with the given key. +// // The -use=path and -dropuse=path flags // add and drop a use directive from the go.work file's set of module directories. // @@ -1561,10 +1588,16 @@ // type GoWork struct { // Go string // Toolchain string +// Godebug []Godebug // Use []Use // Replace []Replace // } // +// type Godebug struct { +// Key string +// Value string +// } +// // type Use struct { // DiskPath string // ModulePath string @@ -1722,6 +1755,38 @@ // // See also: go build. // +// # Manage telemetry data and settings +// +// Usage: +// +// go telemetry [off|local|on] +// +// Telemetry is used to manage Go telemetry data and settings. +// +// Telemetry can be in one of three modes: off, local, or on. +// +// When telemetry is in local mode, counter data is written to the local file +// system, but will not be uploaded to remote servers. +// +// When telemetry is off, local counter data is neither collected nor uploaded. +// +// When telemetry is on, telemetry data is written to the local file system +// and periodically sent to https://telemetry.go.dev/. Uploaded data is used to +// help improve the Go toolchain and related tools, and it will be published as +// part of a public dataset. +// +// For more details, see https://telemetry.go.dev/privacy. +// This data is collected in accordance with the Google Privacy Policy +// (https://policies.google.com/privacy). +// +// To view the current telemetry mode, run "go telemetry". +// To disable telemetry uploading, but keep local data collection, run +// "go telemetry local". +// To enable both collection and uploading, run “go telemetry on”. +// To disable both collection and uploading, run "go telemetry off". +// +// See https://go.dev/doc/telemetry for more information on telemetry. +// // # Test packages // // Usage: diff --git a/src/cmd/go/go_test.go b/src/cmd/go/go_test.go index 5e5d539033..3a3383b271 100644 --- a/src/cmd/go/go_test.go +++ b/src/cmd/go/go_test.go @@ -196,7 +196,7 @@ func TestMain(m *testing.M) { defer removeAll(testTmpDir) } - testGOCACHE = cache.DefaultDir() + testGOCACHE, _ = cache.DefaultDir() if testenv.HasGoBuild() { testBin = filepath.Join(testTmpDir, "testbin") if err := os.Mkdir(testBin, 0777); err != nil { @@ -1058,7 +1058,7 @@ func TestGoListDeps(t *testing.T) { if runtime.Compiler != "gccgo" { // Check the list is in dependency order. tg.run("list", "-deps", "math") - want := "internal/cpu\nunsafe\nmath/bits\nmath\n" + want := "unsafe\ninternal/cpu\nmath/bits\nmath\n" out := tg.stdout.String() if !strings.Contains(out, "internal/cpu") { // Some systems don't use internal/cpu. diff --git a/src/cmd/go/internal/bug/bug.go b/src/cmd/go/internal/bug/bug.go index ed1813605e..d3f9065d3d 100644 --- a/src/cmd/go/internal/bug/bug.go +++ b/src/cmd/go/internal/bug/bug.go @@ -106,7 +106,7 @@ func printGoEnv(w io.Writer) { env := envcmd.MkEnv() env = append(env, envcmd.ExtraEnvVars()...) env = append(env, envcmd.ExtraEnvVarsCostly()...) - envcmd.PrintEnv(w, env) + envcmd.PrintEnv(w, env, false) } func printGoDetails(w io.Writer) { diff --git a/src/cmd/go/internal/cache/default.go b/src/cmd/go/internal/cache/default.go index b5650eac66..5430d9651e 100644 --- a/src/cmd/go/internal/cache/default.go +++ b/src/cmd/go/internal/cache/default.go @@ -39,7 +39,7 @@ See golang.org to learn more about Go. // initDefaultCache does the work of finding the default cache // the first time Default is called. func initDefaultCache() { - dir := DefaultDir() + dir, _ := DefaultDir() if dir == "off" { if defaultDirErr != nil { base.Fatalf("build cache is required, but could not be located: %v", defaultDirErr) @@ -67,14 +67,16 @@ func initDefaultCache() { } var ( - defaultDirOnce sync.Once - defaultDir string - defaultDirErr error + defaultDirOnce sync.Once + defaultDir string + defaultDirChanged bool // effective value differs from $GOCACHE + defaultDirErr error ) // DefaultDir returns the effective GOCACHE setting. -// It returns "off" if the cache is disabled. -func DefaultDir() string { +// It returns "off" if the cache is disabled, +// and reports whether the effective value differs from GOCACHE. +func DefaultDir() (string, bool) { // Save the result of the first call to DefaultDir for later use in // initDefaultCache. cmd/go/main.go explicitly sets GOCACHE so that // subprocesses will inherit it, but that means initDefaultCache can't @@ -82,10 +84,11 @@ func DefaultDir() string { defaultDirOnce.Do(func() { defaultDir = cfg.Getenv("GOCACHE") - if filepath.IsAbs(defaultDir) || defaultDir == "off" { - return - } if defaultDir != "" { + defaultDirChanged = true + if filepath.IsAbs(defaultDir) || defaultDir == "off" { + return + } defaultDir = "off" defaultDirErr = fmt.Errorf("GOCACHE is not an absolute path") return @@ -95,11 +98,12 @@ func DefaultDir() string { dir, err := os.UserCacheDir() if err != nil { defaultDir = "off" + defaultDirChanged = true defaultDirErr = fmt.Errorf("GOCACHE is not defined and %v", err) return } defaultDir = filepath.Join(dir, "go-build") }) - return defaultDir + return defaultDir, defaultDirChanged } diff --git a/src/cmd/go/internal/cfg/cfg.go b/src/cmd/go/internal/cfg/cfg.go index afb595a0c6..002d0006ed 100644 --- a/src/cmd/go/internal/cfg/cfg.go +++ b/src/cmd/go/internal/cfg/cfg.go @@ -101,7 +101,9 @@ var ( // GoPathError is set when GOPATH is not set. it contains an // explanation why GOPATH is unset. - GoPathError string + GoPathError string + GOPATHChanged bool + CGOChanged bool ) func defaultContext() build.Context { @@ -111,7 +113,7 @@ func defaultContext() build.Context { // Override defaults computed in go/build with defaults // from go environment configuration file, if known. - ctxt.GOPATH = envOr("GOPATH", gopath(ctxt)) + ctxt.GOPATH, GOPATHChanged = EnvOrAndChanged("GOPATH", gopath(ctxt)) ctxt.GOOS = Goos ctxt.GOARCH = Goarch @@ -125,14 +127,16 @@ func defaultContext() build.Context { ctxt.ToolTags = save // The go/build rule for whether cgo is enabled is: - // 1. If $CGO_ENABLED is set, respect it. - // 2. Otherwise, if this is a cross-compile, disable cgo. - // 3. Otherwise, use built-in default for GOOS/GOARCH. + // 1. If $CGO_ENABLED is set, respect it. + // 2. Otherwise, if this is a cross-compile, disable cgo. + // 3. Otherwise, use built-in default for GOOS/GOARCH. + // // Recreate that logic here with the new GOOS/GOARCH setting. - if v := Getenv("CGO_ENABLED"); v == "0" || v == "1" { - ctxt.CgoEnabled = v[0] == '1' - } else if ctxt.GOOS != runtime.GOOS || ctxt.GOARCH != runtime.GOARCH { - ctxt.CgoEnabled = false + // We need to run steps 2 and 3 to determine what the default value + // of CgoEnabled would be for computing CGOChanged. + defaultCgoEnabled := ctxt.CgoEnabled + if ctxt.GOOS != runtime.GOOS || ctxt.GOARCH != runtime.GOARCH { + defaultCgoEnabled = false } else { // Use built-in default cgo setting for GOOS/GOARCH. // Note that ctxt.GOOS/GOARCH are derived from the preference list @@ -159,11 +163,16 @@ func defaultContext() build.Context { if os.Getenv("CC") == "" { cc := DefaultCC(ctxt.GOOS, ctxt.GOARCH) if _, err := LookPath(cc); err != nil { - ctxt.CgoEnabled = false + defaultCgoEnabled = false } } } } + ctxt.CgoEnabled = defaultCgoEnabled + if v := Getenv("CGO_ENABLED"); v == "0" || v == "1" { + ctxt.CgoEnabled = v[0] == '1' + } + CGOChanged = ctxt.CgoEnabled != defaultCgoEnabled ctxt.OpenFile = func(path string) (io.ReadCloser, error) { return fsys.Open(path) @@ -262,8 +271,9 @@ func init() { // An EnvVar is an environment variable Name=Value. type EnvVar struct { - Name string - Value string + Name string + Value string + Changed bool // effective Value differs from default } // OrigEnv is the original environment of the program at startup. @@ -279,27 +289,28 @@ var envCache struct { m map[string]string } -// EnvFile returns the name of the Go environment configuration file. -func EnvFile() (string, error) { +// EnvFile returns the name of the Go environment configuration file, +// and reports whether the effective value differs from the default. +func EnvFile() (string, bool, error) { if file := os.Getenv("GOENV"); file != "" { if file == "off" { - return "", fmt.Errorf("GOENV=off") + return "", false, fmt.Errorf("GOENV=off") } - return file, nil + return file, true, nil } dir, err := os.UserConfigDir() if err != nil { - return "", err + return "", false, err } if dir == "" { - return "", fmt.Errorf("missing user-config dir") + return "", false, fmt.Errorf("missing user-config dir") } - return filepath.Join(dir, "go/env"), nil + return filepath.Join(dir, "go/env"), false, nil } func initEnvCache() { envCache.m = make(map[string]string) - if file, _ := EnvFile(); file != "" { + if file, _, _ := EnvFile(); file != "" { readEnvFile(file, "user") } goroot := findGOROOT(envCache.m["GOROOT"]) @@ -397,57 +408,67 @@ var ( GOROOTpkg string GOROOTsrc string - GOBIN = Getenv("GOBIN") - GOMODCACHE = envOr("GOMODCACHE", gopathDir("pkg/mod")) + GOBIN = Getenv("GOBIN") + GOMODCACHE, GOMODCACHEChanged = EnvOrAndChanged("GOMODCACHE", gopathDir("pkg/mod")) // Used in envcmd.MkEnv and build ID computations. - GOARM = envOr("GOARM", fmt.Sprint(buildcfg.GOARM)) - GOARM64 = envOr("GOARM64", fmt.Sprint(buildcfg.GOARM64)) - GO386 = envOr("GO386", buildcfg.GO386) - GOAMD64 = envOr("GOAMD64", fmt.Sprintf("%s%d", "v", buildcfg.GOAMD64)) - GOMIPS = envOr("GOMIPS", buildcfg.GOMIPS) - GOMIPS64 = envOr("GOMIPS64", buildcfg.GOMIPS64) - GOPPC64 = envOr("GOPPC64", fmt.Sprintf("%s%d", "power", buildcfg.GOPPC64)) - GORISCV64 = envOr("GORISCV64", fmt.Sprintf("rva%du64", buildcfg.GORISCV64)) - GOWASM = envOr("GOWASM", fmt.Sprint(buildcfg.GOWASM)) + GOARM64, goARM64Changed = EnvOrAndChanged("GOARM64", fmt.Sprint(buildcfg.GOARM64)) + GOARM, goARMChanged = EnvOrAndChanged("GOARM", fmt.Sprint(buildcfg.GOARM)) + GO386, go386Changed = EnvOrAndChanged("GO386", buildcfg.GO386) + GOAMD64, goAMD64Changed = EnvOrAndChanged("GOAMD64", fmt.Sprintf("%s%d", "v", buildcfg.GOAMD64)) + GOMIPS, goMIPSChanged = EnvOrAndChanged("GOMIPS", buildcfg.GOMIPS) + GOMIPS64, goMIPS64Changed = EnvOrAndChanged("GOMIPS64", buildcfg.GOMIPS64) + GOPPC64, goPPC64Changed = EnvOrAndChanged("GOPPC64", fmt.Sprintf("%s%d", "power", buildcfg.GOPPC64)) + GORISCV64, goRISCV64Changed = EnvOrAndChanged("GORISCV64", fmt.Sprintf("rva%du64", buildcfg.GORISCV64)) + GOWASM, goWASMChanged = EnvOrAndChanged("GOWASM", fmt.Sprint(buildcfg.GOWASM)) - GOPROXY = envOr("GOPROXY", "") - GOSUMDB = envOr("GOSUMDB", "") - GOPRIVATE = Getenv("GOPRIVATE") - GONOPROXY = envOr("GONOPROXY", GOPRIVATE) - GONOSUMDB = envOr("GONOSUMDB", GOPRIVATE) - GOINSECURE = Getenv("GOINSECURE") - GOVCS = Getenv("GOVCS") + GOPROXY, GOPROXYChanged = EnvOrAndChanged("GOPROXY", "") + GOSUMDB, GOSUMDBChanged = EnvOrAndChanged("GOSUMDB", "") + GOPRIVATE = Getenv("GOPRIVATE") + GONOPROXY, GONOPROXYChanged = EnvOrAndChanged("GONOPROXY", GOPRIVATE) + GONOSUMDB, GONOSUMDBChanged = EnvOrAndChanged("GONOSUMDB", GOPRIVATE) + GOINSECURE = Getenv("GOINSECURE") + GOVCS = Getenv("GOVCS") ) +// EnvOrAndChanged returns the environment variable value +// and reports whether it differs from the default value. +func EnvOrAndChanged(name, def string) (string, bool) { + val := Getenv(name) + if val != "" { + return val, val != def + } + return def, false +} + var SumdbDir = gopathDir("pkg/sumdb") // GetArchEnv returns the name and setting of the // GOARCH-specific architecture environment variable. // If the current architecture has no GOARCH-specific variable, // GetArchEnv returns empty key and value. -func GetArchEnv() (key, val string) { +func GetArchEnv() (key, val string, changed bool) { switch Goarch { case "arm": - return "GOARM", GOARM + return "GOARM", GOARM, goARMChanged case "arm64": - return "GOARM64", GOARM64 + return "GOARM64", GOARM64, goARM64Changed case "386": - return "GO386", GO386 + return "GO386", GO386, go386Changed case "amd64": - return "GOAMD64", GOAMD64 + return "GOAMD64", GOAMD64, goAMD64Changed case "mips", "mipsle": - return "GOMIPS", GOMIPS + return "GOMIPS", GOMIPS, goMIPSChanged case "mips64", "mips64le": - return "GOMIPS64", GOMIPS64 + return "GOMIPS64", GOMIPS64, goMIPS64Changed case "ppc64", "ppc64le": - return "GOPPC64", GOPPC64 + return "GOPPC64", GOPPC64, goPPC64Changed case "riscv64": - return "GORISCV64", GORISCV64 + return "GORISCV64", GORISCV64, goRISCV64Changed case "wasm": - return "GOWASM", GOWASM + return "GOWASM", GOWASM, goWASMChanged } - return "", "" + return "", "", false } // envOr returns Getenv(key) if set, or else def. @@ -565,6 +586,7 @@ func gopathDir(rel string) string { return filepath.Join(list[0], rel) } +// Keep consistent with go/build.defaultGOPATH. func gopath(ctxt build.Context) string { if len(ctxt.GOPATH) > 0 { return ctxt.GOPATH diff --git a/src/cmd/go/internal/clean/clean.go b/src/cmd/go/internal/clean/clean.go index b021b784da..de2ef9dcb9 100644 --- a/src/cmd/go/internal/clean/clean.go +++ b/src/cmd/go/internal/clean/clean.go @@ -153,7 +153,7 @@ func runClean(ctx context.Context, cmd *base.Command, args []string) { sh := work.NewShell("", fmt.Print) if cleanCache { - dir := cache.DefaultDir() + dir, _ := cache.DefaultDir() if dir != "off" { // Remove the cache subdirectories but not the top cache directory. // The top cache directory may have been created with special permissions @@ -180,7 +180,7 @@ func runClean(ctx context.Context, cmd *base.Command, args []string) { // Instead of walking through the entire cache looking for test results, // we write a file to the cache indicating that all test results from before // right now are to be ignored. - dir := cache.DefaultDir() + dir, _ := cache.DefaultDir() if dir != "off" { f, err := lockedfile.Edit(filepath.Join(dir, "testexpire.txt")) if err == nil { diff --git a/src/cmd/go/internal/envcmd/env.go b/src/cmd/go/internal/envcmd/env.go index bff3fe5d55..b25010a29a 100644 --- a/src/cmd/go/internal/envcmd/env.go +++ b/src/cmd/go/internal/envcmd/env.go @@ -29,10 +29,11 @@ import ( "cmd/go/internal/modload" "cmd/go/internal/work" "cmd/internal/quoted" + "cmd/internal/telemetry" ) var CmdEnv = &base.Command{ - UsageLine: "go env [-json] [-u] [-w] [var ...]", + UsageLine: "go env [-json] [-changed] [-u] [-w] [var ...]", Short: "print Go environment information", Long: ` Env prints Go environment information. @@ -53,6 +54,10 @@ The -w flag requires one or more arguments of the form NAME=VALUE and changes the default settings of the named environment variables to the given values. +The -changed flag prints only those settings whose effective +value differs from the default value that would be obtained in +an empty environment with no prior uses of the -w flag. + For more about environment variables, see 'go help environment'. `, } @@ -64,19 +69,20 @@ func init() { } var ( - envJson = CmdEnv.Flag.Bool("json", false, "") - envU = CmdEnv.Flag.Bool("u", false, "") - envW = CmdEnv.Flag.Bool("w", false, "") + envJson = CmdEnv.Flag.Bool("json", false, "") + envU = CmdEnv.Flag.Bool("u", false, "") + envW = CmdEnv.Flag.Bool("w", false, "") + envChanged = CmdEnv.Flag.Bool("changed", false, "") ) func MkEnv() []cfg.EnvVar { - envFile, _ := cfg.EnvFile() + envFile, envFileChanged, _ := cfg.EnvFile() env := []cfg.EnvVar{ {Name: "GO111MODULE", Value: cfg.Getenv("GO111MODULE")}, - {Name: "GOARCH", Value: cfg.Goarch}, + {Name: "GOARCH", Value: cfg.Goarch, Changed: cfg.Goarch != runtime.GOARCH}, {Name: "GOBIN", Value: cfg.GOBIN}, - {Name: "GOCACHE", Value: cache.DefaultDir()}, - {Name: "GOENV", Value: envFile}, + {Name: "GOCACHE"}, + {Name: "GOENV", Value: envFile, Changed: envFileChanged}, {Name: "GOEXE", Value: cfg.ExeSuffix}, // List the raw value of GOEXPERIMENT, not the cleaned one. @@ -90,63 +96,84 @@ func MkEnv() []cfg.EnvVar { {Name: "GOHOSTARCH", Value: runtime.GOARCH}, {Name: "GOHOSTOS", Value: runtime.GOOS}, {Name: "GOINSECURE", Value: cfg.GOINSECURE}, - {Name: "GOMODCACHE", Value: cfg.GOMODCACHE}, - {Name: "GONOPROXY", Value: cfg.GONOPROXY}, - {Name: "GONOSUMDB", Value: cfg.GONOSUMDB}, - {Name: "GOOS", Value: cfg.Goos}, - {Name: "GOPATH", Value: cfg.BuildContext.GOPATH}, + {Name: "GOMODCACHE", Value: cfg.GOMODCACHE, Changed: cfg.GOMODCACHEChanged}, + {Name: "GONOPROXY", Value: cfg.GONOPROXY, Changed: cfg.GONOPROXYChanged}, + {Name: "GONOSUMDB", Value: cfg.GONOSUMDB, Changed: cfg.GONOSUMDBChanged}, + {Name: "GOOS", Value: cfg.Goos, Changed: cfg.Goos != runtime.GOOS}, + {Name: "GOPATH", Value: cfg.BuildContext.GOPATH, Changed: cfg.GOPATHChanged}, {Name: "GOPRIVATE", Value: cfg.GOPRIVATE}, - {Name: "GOPROXY", Value: cfg.GOPROXY}, + {Name: "GOPROXY", Value: cfg.GOPROXY, Changed: cfg.GOPROXYChanged}, {Name: "GOROOT", Value: cfg.GOROOT}, - {Name: "GOSUMDB", Value: cfg.GOSUMDB}, + {Name: "GOSUMDB", Value: cfg.GOSUMDB, Changed: cfg.GOSUMDBChanged}, {Name: "GOTMPDIR", Value: cfg.Getenv("GOTMPDIR")}, {Name: "GOTOOLCHAIN", Value: cfg.Getenv("GOTOOLCHAIN")}, {Name: "GOTOOLDIR", Value: build.ToolDir}, {Name: "GOVCS", Value: cfg.GOVCS}, {Name: "GOVERSION", Value: runtime.Version()}, {Name: "GODEBUG", Value: os.Getenv("GODEBUG")}, + {Name: "GOTELEMETRY", Value: telemetry.Mode()}, + {Name: "GOTELEMETRYDIR", Value: telemetry.Dir()}, + } + + for i := range env { + switch env[i].Name { + case "GO111MODULE": + if env[i].Value != "on" && env[i].Value != "" { + env[i].Changed = true + } + case "GOBIN", "GOEXPERIMENT", "GOFLAGS", "GOINSECURE", "GOPRIVATE", "GOTMPDIR", "GOVCS": + if env[i].Value != "" { + env[i].Changed = true + } + case "GOCACHE": + env[i].Value, env[i].Changed = cache.DefaultDir() + case "GOTOOLCHAIN": + if env[i].Value != "auto" { + env[i].Changed = true + } + case "GODEBUG": + env[i].Value = os.Getenv("GODEBUG") + env[i].Changed = env[i].Value != "" + } } if work.GccgoBin != "" { - env = append(env, cfg.EnvVar{Name: "GCCGO", Value: work.GccgoBin}) + env = append(env, cfg.EnvVar{Name: "GCCGO", Value: work.GccgoBin, Changed: true}) } else { env = append(env, cfg.EnvVar{Name: "GCCGO", Value: work.GccgoName}) } - key, val := cfg.GetArchEnv() - if key != "" { - env = append(env, cfg.EnvVar{Name: key, Value: val}) + goarch, val, changed := cfg.GetArchEnv() + if goarch != "" { + env = append(env, cfg.EnvVar{Name: goarch, Value: val, Changed: changed}) } cc := cfg.Getenv("CC") + ccChanged := true if cc == "" { + ccChanged = false cc = cfg.DefaultCC(cfg.Goos, cfg.Goarch) } cxx := cfg.Getenv("CXX") + cxxChanged := true if cxx == "" { + cxxChanged = false cxx = cfg.DefaultCXX(cfg.Goos, cfg.Goarch) } - env = append(env, cfg.EnvVar{Name: "AR", Value: envOr("AR", "ar")}) - env = append(env, cfg.EnvVar{Name: "CC", Value: cc}) - env = append(env, cfg.EnvVar{Name: "CXX", Value: cxx}) + ar, arChanged := cfg.EnvOrAndChanged("AR", "ar") + env = append(env, cfg.EnvVar{Name: "AR", Value: ar, Changed: arChanged}) + env = append(env, cfg.EnvVar{Name: "CC", Value: cc, Changed: ccChanged}) + env = append(env, cfg.EnvVar{Name: "CXX", Value: cxx, Changed: cxxChanged}) if cfg.BuildContext.CgoEnabled { - env = append(env, cfg.EnvVar{Name: "CGO_ENABLED", Value: "1"}) + env = append(env, cfg.EnvVar{Name: "CGO_ENABLED", Value: "1", Changed: cfg.CGOChanged}) } else { - env = append(env, cfg.EnvVar{Name: "CGO_ENABLED", Value: "0"}) + env = append(env, cfg.EnvVar{Name: "CGO_ENABLED", Value: "0", Changed: cfg.CGOChanged}) } return env } -func envOr(name, def string) string { - val := cfg.Getenv(name) - if val != "" { - return val - } - return def -} - func findEnv(env []cfg.EnvVar, name string) string { for _, e := range env { if e.Name == name { @@ -206,7 +233,7 @@ func ExtraEnvVarsCostly() []cfg.EnvVar { return q } - return []cfg.EnvVar{ + ret := []cfg.EnvVar{ // Note: Update the switch in runEnv below when adding to this list. {Name: "CGO_CFLAGS", Value: join(cflags)}, {Name: "CGO_CPPFLAGS", Value: join(cppflags)}, @@ -216,6 +243,21 @@ func ExtraEnvVarsCostly() []cfg.EnvVar { {Name: "PKG_CONFIG", Value: b.PkgconfigCmd()}, {Name: "GOGCCFLAGS", Value: join(cmd[3:])}, } + + for i := range ret { + ev := &ret[i] + switch ev.Name { + case "GOGCCFLAGS": // GOGCCFLAGS cannot be modified + case "CGO_CPPFLAGS": + ev.Changed = ev.Value != "" + case "PKG_CONFIG": + ev.Changed = ev.Value != cfg.DefaultPkgConfig + case "CGO_CXXFLAGS", "CGO_CFLAGS", "CGO_FFLAGS", "GGO_LDFLAGS": + ev.Changed = ev.Value != work.DefaultCFlags + } + } + + return ret } // argKey returns the KEY part of the arg KEY=VAL, or else arg itself. @@ -297,27 +339,43 @@ func runEnv(ctx context.Context, cmd *base.Command, args []string) { } if len(args) > 0 { - if *envJson { + // Show only the named vars. + if !*envChanged { + if *envJson { + var es []cfg.EnvVar + for _, name := range args { + e := cfg.EnvVar{Name: name, Value: findEnv(env, name)} + es = append(es, e) + } + env = es + } else { + // Print just the values, without names. + for _, name := range args { + fmt.Printf("%s\n", findEnv(env, name)) + } + return + } + } else { + // Show only the changed, named vars. var es []cfg.EnvVar for _, name := range args { - e := cfg.EnvVar{Name: name, Value: findEnv(env, name)} - es = append(es, e) - } - printEnvAsJSON(es) - } else { - for _, name := range args { - fmt.Printf("%s\n", findEnv(env, name)) + for _, e := range env { + if e.Name == name { + es = append(es, e) + break + } + } } + env = es } - return } + // print if *envJson { - printEnvAsJSON(env) - return + printEnvAsJSON(env, *envChanged) + } else { + PrintEnv(os.Stdout, env, *envChanged) } - - PrintEnv(os.Stdout, env) } func runEnvW(args []string) { @@ -423,12 +481,15 @@ func checkBuildConfig(add map[string]string, del map[string]bool) error { } // PrintEnv prints the environment variables to w. -func PrintEnv(w io.Writer, env []cfg.EnvVar) { +func PrintEnv(w io.Writer, env []cfg.EnvVar, onlyChanged bool) { for _, e := range env { if e.Name != "TERM" { if runtime.GOOS != "plan9" && bytes.Contains([]byte(e.Value), []byte{0}) { base.Fatalf("go: internal error: encountered null byte in environment variable %s on non-plan9 platform", e.Name) } + if onlyChanged && !e.Changed { + continue + } switch runtime.GOOS { default: fmt.Fprintf(w, "%s=%s\n", e.Name, shellQuote(e.Value)) @@ -503,12 +564,15 @@ func batchEscape(s string) string { return b.String() } -func printEnvAsJSON(env []cfg.EnvVar) { +func printEnvAsJSON(env []cfg.EnvVar, onlyChanged bool) { m := make(map[string]string) for _, e := range env { if e.Name == "TERM" { continue } + if onlyChanged && !e.Changed { + continue + } m[e.Name] = e.Value } enc := json.NewEncoder(os.Stdout) @@ -591,7 +655,7 @@ func checkEnvWrite(key, val string) error { } func readEnvFileLines(mustExist bool) []string { - file, err := cfg.EnvFile() + file, _, err := cfg.EnvFile() if file == "" { if mustExist { base.Fatalf("go: cannot find go env config: %v", err) @@ -655,7 +719,7 @@ func updateEnvFile(add map[string]string, del map[string]bool) { } } - file, err := cfg.EnvFile() + file, _, err := cfg.EnvFile() if file == "" { base.Fatalf("go: cannot find go env config: %v", err) } diff --git a/src/cmd/go/internal/envcmd/env_test.go b/src/cmd/go/internal/envcmd/env_test.go index 7419cf3fc2..2f6470b871 100644 --- a/src/cmd/go/internal/envcmd/env_test.go +++ b/src/cmd/go/internal/envcmd/env_test.go @@ -56,7 +56,7 @@ func FuzzPrintEnvEscape(f *testing.F) { if runtime.GOOS == "windows" { b.WriteString("@echo off\n") } - PrintEnv(&b, []cfg.EnvVar{{Name: "var", Value: s}}) + PrintEnv(&b, []cfg.EnvVar{{Name: "var", Value: s}}, false) var want string if runtime.GOOS == "windows" { fmt.Fprintf(&b, "echo \"%%var%%\"\n") diff --git a/src/cmd/go/internal/load/godebug.go b/src/cmd/go/internal/load/godebug.go index c79245e5cd..4bb734ce64 100644 --- a/src/cmd/go/internal/load/godebug.go +++ b/src/cmd/go/internal/load/godebug.go @@ -5,7 +5,6 @@ package load import ( - "cmd/go/internal/modload" "errors" "fmt" "go/build" @@ -13,6 +12,9 @@ import ( "sort" "strconv" "strings" + + "cmd/go/internal/gover" + "cmd/go/internal/modload" ) var ErrNotGoDebug = errors.New("not //go:debug line") @@ -32,25 +34,10 @@ func ParseGoDebug(text string) (key, value string, err error) { if !ok { return "", "", fmt.Errorf("missing key=value") } - if strings.ContainsAny(k, " \t") { - return "", "", fmt.Errorf("key contains space") + if err := modload.CheckGodebug("//go:debug setting", k, v); err != nil { + return "", "", err } - if strings.ContainsAny(v, " \t") { - return "", "", fmt.Errorf("value contains space") - } - if strings.ContainsAny(k, ",") { - return "", "", fmt.Errorf("key contains comma") - } - if strings.ContainsAny(v, ",") { - return "", "", fmt.Errorf("value contains comma") - } - - for _, info := range godebugs.All { - if k == info.Name { - return k, v, nil - } - } - return "", "", fmt.Errorf("unknown //go:debug setting %q", k) + return k, v, nil } // defaultGODEBUG returns the default GODEBUG setting for the main package p. @@ -64,14 +51,21 @@ func defaultGODEBUG(p *Package, directives, testDirectives, xtestDirectives []bu if modload.RootMode == modload.NoRoot && p.Module != nil { // This is go install pkg@version or go run pkg@version. // Use the Go version from the package. - // If there isn't one, then + // If there isn't one, then assume Go 1.20, + // the last version before GODEBUGs were introduced. goVersion = p.Module.GoVersion if goVersion == "" { goVersion = "1.20" } } - m := godebugForGoVersion(goVersion) + var m map[string]string + for _, g := range modload.MainModules.Godebugs() { + if m == nil { + m = make(map[string]string) + } + m[g.Key] = g.Value + } for _, list := range [][]build.Directive{p.Internal.Build.Directives, directives, testDirectives, xtestDirectives} { for _, d := range list { k, v, err := ParseGoDebug(d.Text) @@ -84,6 +78,23 @@ func defaultGODEBUG(p *Package, directives, testDirectives, xtestDirectives []bu m[k] = v } } + if v, ok := m["default"]; ok { + delete(m, "default") + v = strings.TrimPrefix(v, "go") + if gover.IsValid(v) { + goVersion = v + } + } + + defaults := godebugForGoVersion(goVersion) + if defaults != nil { + // Apply m on top of defaults. + for k, v := range m { + defaults[k] = v + } + m = defaults + } + var keys []string for k := range m { keys = append(keys, k) diff --git a/src/cmd/go/internal/load/pkg.go b/src/cmd/go/internal/load/pkg.go index f241e93ee8..4bf2f381dd 100644 --- a/src/cmd/go/internal/load/pkg.go +++ b/src/cmd/go/internal/load/pkg.go @@ -604,51 +604,6 @@ func (sp *ImportStack) shorterThan(t []string) bool { // we return the same pointer each time. var packageCache = map[string]*Package{} -// ClearPackageCache clears the in-memory package cache and the preload caches. -// It is only for use by GOPATH-based "go get". -// TODO(jayconrod): When GOPATH-based "go get" is removed, delete this function. -func ClearPackageCache() { - clear(packageCache) - resolvedImportCache.Clear() - packageDataCache.Clear() -} - -// ClearPackageCachePartial clears packages with the given import paths from the -// in-memory package cache and the preload caches. It is only for use by -// GOPATH-based "go get". -// TODO(jayconrod): When GOPATH-based "go get" is removed, delete this function. -func ClearPackageCachePartial(args []string) { - shouldDelete := make(map[string]bool) - for _, arg := range args { - shouldDelete[arg] = true - if p := packageCache[arg]; p != nil { - delete(packageCache, arg) - } - } - resolvedImportCache.DeleteIf(func(key importSpec) bool { - return shouldDelete[key.path] - }) - packageDataCache.DeleteIf(func(key string) bool { - return shouldDelete[key] - }) -} - -// ReloadPackageNoFlags is like LoadImport but makes sure -// not to use the package cache. -// It is only for use by GOPATH-based "go get". -// TODO(rsc): When GOPATH-based "go get" is removed, delete this function. -func ReloadPackageNoFlags(arg string, stk *ImportStack) *Package { - p := packageCache[arg] - if p != nil { - delete(packageCache, arg) - resolvedImportCache.DeleteIf(func(key importSpec) bool { - return key.path == p.ImportPath - }) - packageDataCache.Delete(p.ImportPath) - } - return LoadPackage(context.TODO(), PackageOpts{}, arg, base.Cwd(), stk, nil, 0) -} - // dirToImportPath returns the pseudo-import path we use for a package // outside the Go path. It begins with _/ and then contains the full path // to the directory. If the package lives in c:\home\gopher\my\pkg then @@ -2437,7 +2392,7 @@ func (p *Package) setBuildInfo(ctx context.Context, autoVCS bool) { appendSetting("GOEXPERIMENT", cfg.RawGOEXPERIMENT) } appendSetting("GOOS", cfg.BuildContext.GOOS) - if key, val := cfg.GetArchEnv(); key != "" && val != "" { + if key, val, _ := cfg.GetArchEnv(); key != "" && val != "" { appendSetting(key, val) } diff --git a/src/cmd/go/internal/modcmd/edit.go b/src/cmd/go/internal/modcmd/edit.go index db131b0881..9b0c768ba2 100644 --- a/src/cmd/go/internal/modcmd/edit.go +++ b/src/cmd/go/internal/modcmd/edit.go @@ -44,6 +44,12 @@ flags are specified, as in 'go mod edit -fmt'. The -module flag changes the module's path (the go.mod file's module line). +The -godebug=key=value flag adds a godebug key=value line, +replacing any existing godebug lines with the given key. + +The -dropgodebug=key flag drops any existing godebug lines +with the given key. + The -require=path@version and -droprequire=path flags add and drop a requirement on the given module path and version. Note that -require overrides any existing requirements on path. @@ -52,6 +58,14 @@ Users should prefer 'go get path@version' or 'go get path@none', which make other go.mod adjustments as needed to satisfy constraints imposed by other modules. +The -go=version flag sets the expected Go language version. +This flag is mainly for tools that understand Go version dependencies. +Users should prefer 'go get go@version'. + +The -toolchain=version flag sets the Go toolchain to use. +This flag is mainly for tools that understand Go version dependencies. +Users should prefer 'go get toolchain@version'. + The -exclude=path@version and -dropexclude=path@version flags add and drop an exclusion for the given module path and version. Note that -exclude=path@version is a no-op if that exclusion already exists. @@ -73,13 +87,9 @@ retraction on the given version. The version may be a single version like "v1.2.3" or a closed interval like "[v1.1.0,v1.1.9]". Note that -retract=version is a no-op if that retraction already exists. -The -require, -droprequire, -exclude, -dropexclude, -replace, --dropreplace, -retract, and -dropretract editing flags may be repeated, -and the changes are applied in the order given. - -The -go=version flag sets the expected Go language version. - -The -toolchain=name flag sets the Go toolchain to use. +The -godebug, -dropgodebug, -require, -droprequire, -exclude, -dropexclude, +-replace, -dropreplace, -retract, and -dropretract editing flags may be +repeated, and the changes are applied in the order given. The -print flag prints the final go.mod in its text format instead of writing it back to go.mod. @@ -96,6 +106,7 @@ writing it back to go.mod. The JSON output corresponds to these Go types: Module ModPath Go string Toolchain string + Godebug []Godebug Require []Require Exclude []Module Replace []Replace @@ -107,9 +118,14 @@ writing it back to go.mod. The JSON output corresponds to these Go types: Deprecated string } + type Godebug struct { + Key string + Value string + } + type Require struct { - Path string - Version string + Path string + Version string Indirect bool } @@ -155,12 +171,14 @@ func (f flagFunc) Set(s string) error { f(s); return nil } func init() { cmdEdit.Run = runEdit // break init cycle + cmdEdit.Flag.Var(flagFunc(flagGodebug), "godebug", "") + cmdEdit.Flag.Var(flagFunc(flagDropGodebug), "dropgodebug", "") cmdEdit.Flag.Var(flagFunc(flagRequire), "require", "") cmdEdit.Flag.Var(flagFunc(flagDropRequire), "droprequire", "") cmdEdit.Flag.Var(flagFunc(flagExclude), "exclude", "") - cmdEdit.Flag.Var(flagFunc(flagDropReplace), "dropreplace", "") - cmdEdit.Flag.Var(flagFunc(flagReplace), "replace", "") cmdEdit.Flag.Var(flagFunc(flagDropExclude), "dropexclude", "") + cmdEdit.Flag.Var(flagFunc(flagReplace), "replace", "") + cmdEdit.Flag.Var(flagFunc(flagDropReplace), "dropreplace", "") cmdEdit.Flag.Var(flagFunc(flagRetract), "retract", "") cmdEdit.Flag.Var(flagFunc(flagDropRetract), "dropretract", "") @@ -369,6 +387,28 @@ func allowedVersionArg(arg string) bool { return !modfile.MustQuote(arg) } +// flagGodebug implements the -godebug flag. +func flagGodebug(arg string) { + key, value, ok := strings.Cut(arg, "=") + if !ok || strings.ContainsAny(arg, "\"`',") { + base.Fatalf("go: -godebug=%s: need key=value", arg) + } + edits = append(edits, func(f *modfile.File) { + if err := f.AddGodebug(key, value); err != nil { + base.Fatalf("go: -godebug=%s: %v", arg, err) + } + }) +} + +// flagDropGodebug implements the -dropgodebug flag. +func flagDropGodebug(arg string) { + edits = append(edits, func(f *modfile.File) { + if err := f.DropGodebug(arg); err != nil { + base.Fatalf("go: -dropgodebug=%s: %v", arg, err) + } + }) +} + // flagRequire implements the -require flag. func flagRequire(arg string) { path, version := parsePathVersion("require", arg) diff --git a/src/cmd/go/internal/modindex/read.go b/src/cmd/go/internal/modindex/read.go index bda3fb4338..9d8c48f2b0 100644 --- a/src/cmd/go/internal/modindex/read.go +++ b/src/cmd/go/internal/modindex/read.go @@ -147,7 +147,8 @@ func GetPackage(modroot, pkgdir string) (*IndexPackage, error) { // using the index, for instance because the index is disabled, or the package // is not in a module. func GetModule(modroot string) (*Module, error) { - if !enabled || cache.DefaultDir() == "off" { + dir, _ := cache.DefaultDir() + if !enabled || dir == "off" { return nil, errDisabled } if modroot == "" { diff --git a/src/cmd/go/internal/modload/buildlist.go b/src/cmd/go/internal/modload/buildlist.go index 9c11bd4d13..e7f0da1b69 100644 --- a/src/cmd/go/internal/modload/buildlist.go +++ b/src/cmd/go/internal/modload/buildlist.go @@ -795,13 +795,13 @@ func updateRoots(ctx context.Context, direct map[string]bool, rs *Requirements, case pruned: return updatePrunedRoots(ctx, direct, rs, pkgs, add, rootsImported) case workspace: - return updateWorkspaceRoots(ctx, rs, add) + return updateWorkspaceRoots(ctx, direct, rs, add) default: panic(fmt.Sprintf("unsupported pruning mode: %v", rs.pruning)) } } -func updateWorkspaceRoots(ctx context.Context, rs *Requirements, add []module.Version) (*Requirements, error) { +func updateWorkspaceRoots(ctx context.Context, direct map[string]bool, rs *Requirements, add []module.Version) (*Requirements, error) { if len(add) != 0 { // add should be empty in workspace mode because workspace mode implies // -mod=readonly, which in turn implies no new requirements. The code path @@ -812,7 +812,7 @@ func updateWorkspaceRoots(ctx context.Context, rs *Requirements, add []module.Ve // return an error. panic("add is not empty") } - return rs, nil + return newRequirements(workspace, rs.rootModules, direct), nil } // tidyPrunedRoots returns a minimal set of root requirements that maintains the diff --git a/src/cmd/go/internal/modload/init.go b/src/cmd/go/internal/modload/init.go index 1de9901563..2d82ea4187 100644 --- a/src/cmd/go/internal/modload/init.go +++ b/src/cmd/go/internal/modload/init.go @@ -10,6 +10,7 @@ import ( "encoding/json" "errors" "fmt" + "internal/godebugs" "internal/lazyregexp" "io" "os" @@ -241,6 +242,27 @@ func (mms *MainModuleSet) GoVersion() string { return gover.DefaultGoModVersion } +// Godebugs returns the godebug lines set on the single module, in module mode, +// or on the go.work file in workspace mode. +// The caller must not modify the result. +func (mms *MainModuleSet) Godebugs() []*modfile.Godebug { + if inWorkspaceMode() { + if mms.workFile != nil { + return mms.workFile.Godebug + } + return nil + } + if mms != nil && len(mms.versions) == 1 { + f := mms.ModFile(mms.mustGetSingleMainModule()) + if f == nil { + // Special case: we are outside a module, like 'go run x.go'. + return nil + } + return f.Godebug + } + return nil +} + // Toolchain returns the toolchain set on the single module, in module mode, // or the go.work file in workspace mode. func (mms *MainModuleSet) Toolchain() string { @@ -675,6 +697,12 @@ func loadWorkFile(path string) (workFile *modfile.WorkFile, modRoots []string, e modRoots = append(modRoots, modRoot) } + for _, g := range wf.Godebug { + if err := CheckGodebug("godebug", g.Key, g.Value); err != nil { + return nil, nil, err + } + } + return wf, modRoots, nil } @@ -914,6 +942,19 @@ func loadModFile(ctx context.Context, opts *PackageOpts) (*Requirements, error) } } + if !inWorkspaceMode() { + ok := true + for _, g := range f.Godebug { + if err := CheckGodebug("godebug", g.Key, g.Value); err != nil { + errs = append(errs, fmt.Errorf("%s: %v", base.ShortPath(filepath.Dir(gomod)), err)) + ok = false + } + } + if !ok { + continue + } + } + modFiles = append(modFiles, f) mainModule := f.Module.Mod mainModules = append(mainModules, mainModule) @@ -1257,6 +1298,7 @@ func makeMainModules(ms []module.Version, rootDirs []string, modFiles []*modfile } } } + return mainModules } @@ -1276,6 +1318,7 @@ func requirementsFromModFiles(ctx context.Context, workFile *modfile.WorkFile, m toolchain = workFile.Toolchain.Name } roots = appendGoAndToolchainRoots(roots, goVersion, toolchain, direct) + direct = directRequirements(modFiles) } else { pruning = pruningForGoVersion(MainModules.GoVersion()) if len(modFiles) != 1 { @@ -1297,6 +1340,18 @@ const ( withToolchainRoot = true ) +func directRequirements(modFiles []*modfile.File) map[string]bool { + direct := make(map[string]bool) + for _, modFile := range modFiles { + for _, r := range modFile.Require { + if !r.Indirect { + direct[r.Mod.Path] = true + } + } + } + return direct +} + func rootsFromModFile(m module.Version, modFile *modfile.File, addToolchainRoot addToolchainRoot) (roots []module.Version, direct map[string]bool) { direct = make(map[string]bool) padding := 2 // Add padding for the toolchain and go version, added upon return. @@ -2054,3 +2109,33 @@ func suggestGopkgIn(path string) string { } return url + ".v" + m } + +func CheckGodebug(verb, k, v string) error { + if strings.ContainsAny(k, " \t") { + return fmt.Errorf("key contains space") + } + if strings.ContainsAny(v, " \t") { + return fmt.Errorf("value contains space") + } + if strings.ContainsAny(k, ",") { + return fmt.Errorf("key contains comma") + } + if strings.ContainsAny(v, ",") { + return fmt.Errorf("value contains comma") + } + if k == "default" { + if !strings.HasPrefix(v, "go") || !gover.IsValid(v[len("go"):]) { + return fmt.Errorf("value for default= must be goVERSION") + } + if gover.Compare(v[len("go"):], gover.Local()) > 0 { + return fmt.Errorf("default=%s too new (toolchain is go%s)", v, gover.Local()) + } + return nil + } + for _, info := range godebugs.All { + if k == info.Name { + return nil + } + } + return fmt.Errorf("unknown %s %q", verb, k) +} diff --git a/src/cmd/go/internal/modload/load.go b/src/cmd/go/internal/modload/load.go index 408c109f5b..4e2eb63be2 100644 --- a/src/cmd/go/internal/modload/load.go +++ b/src/cmd/go/internal/modload/load.go @@ -1375,10 +1375,7 @@ func (ld *loader) updateRequirements(ctx context.Context) (changed bool, err err Module: dep.mod, } } - continue - } - - if pkg.err == nil && cfg.BuildMod != "mod" { + } else if pkg.err == nil && cfg.BuildMod != "mod" { if v, ok := rs.rootSelected(dep.mod.Path); !ok || v != dep.mod.Version { // dep.mod is not an explicit dependency, but needs to be. // Because we are not in "mod" mode, we will not be able to update it. diff --git a/src/cmd/go/internal/par/work.go b/src/cmd/go/internal/par/work.go index 3f1e69adfe..881b51be19 100644 --- a/src/cmd/go/internal/par/work.go +++ b/src/cmd/go/internal/par/work.go @@ -180,41 +180,3 @@ func (c *Cache[K, V]) Get(key K) (V, bool) { } return e.result, true } - -// Clear removes all entries in the cache. -// -// Concurrent calls to Get may return old values. Concurrent calls to Do -// may return old values or store results in entries that have been deleted. -// -// TODO(jayconrod): Delete this after the package cache clearing functions -// in internal/load have been removed. -func (c *Cache[K, V]) Clear() { - c.m.Clear() -} - -// Delete removes an entry from the map. It is safe to call Delete for an -// entry that does not exist. Delete will return quickly, even if the result -// for a key is still being computed; the computation will finish, but the -// result won't be accessible through the cache. -// -// TODO(jayconrod): Delete this after the package cache clearing functions -// in internal/load have been removed. -func (c *Cache[K, V]) Delete(key K) { - c.m.Delete(key) -} - -// DeleteIf calls pred for each key in the map. If pred returns true for a key, -// DeleteIf removes the corresponding entry. If the result for a key is -// still being computed, DeleteIf will remove the entry without waiting for -// the computation to finish. The result won't be accessible through the cache. -// -// TODO(jayconrod): Delete this after the package cache clearing functions -// in internal/load have been removed. -func (c *Cache[K, V]) DeleteIf(pred func(key K) bool) { - c.m.Range(func(key, _ any) bool { - if key := key.(K); pred(key) { - c.Delete(key) - } - return true - }) -} diff --git a/src/cmd/go/internal/script/cmds_other.go b/src/cmd/go/internal/script/cmds_nonunix.go similarity index 75% rename from src/cmd/go/internal/script/cmds_other.go rename to src/cmd/go/internal/script/cmds_nonunix.go index 847b225ae6..07bf284fa9 100644 --- a/src/cmd/go/internal/script/cmds_other.go +++ b/src/cmd/go/internal/script/cmds_nonunix.go @@ -2,10 +2,11 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. -//go:build !(unix || windows) +//go:build !unix package script func isETXTBSY(err error) bool { + // syscall.ETXTBSY is only meaningful on Unix platforms. return false } diff --git a/src/cmd/go/internal/script/cmds_posix.go b/src/cmd/go/internal/script/cmds_unix.go similarity index 91% rename from src/cmd/go/internal/script/cmds_posix.go rename to src/cmd/go/internal/script/cmds_unix.go index 2525f6e752..94a16b5e9a 100644 --- a/src/cmd/go/internal/script/cmds_posix.go +++ b/src/cmd/go/internal/script/cmds_unix.go @@ -2,7 +2,7 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. -//go:build unix || windows +//go:build unix package script diff --git a/src/cmd/go/internal/telemetrycmd/telemetry.go b/src/cmd/go/internal/telemetrycmd/telemetry.go new file mode 100644 index 0000000000..5542a02162 --- /dev/null +++ b/src/cmd/go/internal/telemetrycmd/telemetry.go @@ -0,0 +1,92 @@ +// Copyright 2024 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Package telemetrycmd implements the "go telemetry" command. +package telemetrycmd + +import ( + "context" + "fmt" + "os" + + "cmd/go/internal/base" + "cmd/internal/telemetry" +) + +var CmdTelemetry = &base.Command{ + UsageLine: "go telemetry [off|local|on]", + Short: "manage telemetry data and settings", + Long: `Telemetry is used to manage Go telemetry data and settings. + +Telemetry can be in one of three modes: off, local, or on. + +When telemetry is in local mode, counter data is written to the local file +system, but will not be uploaded to remote servers. + +When telemetry is off, local counter data is neither collected nor uploaded. + +When telemetry is on, telemetry data is written to the local file system +and periodically sent to https://telemetry.go.dev/. Uploaded data is used to +help improve the Go toolchain and related tools, and it will be published as +part of a public dataset. + +For more details, see https://telemetry.go.dev/privacy. +This data is collected in accordance with the Google Privacy Policy +(https://policies.google.com/privacy). + +To view the current telemetry mode, run "go telemetry". +To disable telemetry uploading, but keep local data collection, run +"go telemetry local". +To enable both collection and uploading, run “go telemetry on”. +To disable both collection and uploading, run "go telemetry off". + +See https://go.dev/doc/telemetry for more information on telemetry. +`, + Run: runTelemetry, +} + +func init() { + base.AddChdirFlag(&CmdTelemetry.Flag) +} + +func runTelemetry(ctx context.Context, cmd *base.Command, args []string) { + if len(args) == 0 { + fmt.Println(telemetry.Mode()) + return + } + + if len(args) != 1 { + cmd.Usage() + } + + mode := args[0] + if mode != "local" && mode != "off" && mode != "on" { + cmd.Usage() + } + if old := telemetry.Mode(); old == mode { + return + } + + if err := telemetry.SetMode(mode); err != nil { + base.Fatalf("go: failed to set the telemetry mode to %s: %v", mode, err) + } + if mode == "on" { + fmt.Fprintln(os.Stderr, telemetryOnMessage()) + } +} + +func telemetryOnMessage() string { + return `Telemetry uploading is now enabled and data will be periodically sent to +https://telemetry.go.dev/. Uploaded data is used to help improve the Go +toolchain and related tools, and it will be published as part of a public +dataset. + +For more details, see https://telemetry.go.dev/privacy. +This data is collected in accordance with the Google Privacy Policy +(https://policies.google.com/privacy). + +To disable telemetry uploading, but keep local data collection, run +“go telemetry local”. +To disable both collection and uploading, run “go telemetry off“.` +} diff --git a/src/cmd/go/internal/test/test.go b/src/cmd/go/internal/test/test.go index ac9d2721f5..a13070a91e 100644 --- a/src/cmd/go/internal/test/test.go +++ b/src/cmd/go/internal/test/test.go @@ -837,7 +837,7 @@ func runTest(ctx context.Context, cmd *base.Command, args []string) { // Read testcache expiration time, if present. // (We implement go clean -testcache by writing an expiration date // instead of searching out and deleting test result cache entries.) - if dir := cache.DefaultDir(); dir != "off" { + if dir, _ := cache.DefaultDir(); dir != "off" { if data, _ := lockedfile.Read(filepath.Join(dir, "testexpire.txt")); len(data) > 0 && data[len(data)-1] == '\n' { if t, err := strconv.ParseInt(string(data[:len(data)-1]), 10, 64); err == nil { testCacheExpire = time.Unix(0, t) diff --git a/src/cmd/go/internal/work/exec.go b/src/cmd/go/internal/work/exec.go index a3d1533899..90c61a9c30 100644 --- a/src/cmd/go/internal/work/exec.go +++ b/src/cmd/go/internal/work/exec.go @@ -46,7 +46,7 @@ import ( "cmd/internal/sys" ) -const defaultCFlags = "-O2 -g" +const DefaultCFlags = "-O2 -g" // actionList returns the list of actions in the dag rooted at root // as visited in a depth-first post-order traversal. @@ -337,7 +337,7 @@ func (b *Builder) buildActionID(a *Action) cache.ActionID { } // GOARM, GOMIPS, etc. - key, val := cfg.GetArchEnv() + key, val, _ := cfg.GetArchEnv() fmt.Fprintf(h, "%s=%s\n", key, val) if cfg.CleanGOEXPERIMENT != "" { @@ -1140,13 +1140,15 @@ type vetConfig struct { NonGoFiles []string // absolute paths to package non-Go files IgnoredFiles []string // absolute paths to ignored source files - ImportMap map[string]string // map import path in source code to package path - PackageFile map[string]string // map package path to .a file with export data - Standard map[string]bool // map package path to whether it's in the standard library - PackageVetx map[string]string // map package path to vetx data from earlier vet run - VetxOnly bool // only compute vetx data; don't report detected problems - VetxOutput string // write vetx data to this output file - GoVersion string // Go version for package + ModulePath string // module path (may be "" on module error) + ModuleVersion string // module version (may be "" on main module or module error) + ImportMap map[string]string // map import path in source code to package path + PackageFile map[string]string // map package path to .a file with export data + Standard map[string]bool // map package path to whether it's in the standard library + PackageVetx map[string]string // map package path to vetx data from earlier vet run + VetxOnly bool // only compute vetx data; don't report detected problems + VetxOutput string // write vetx data to this output file + GoVersion string // Go version for package SucceedOnTypecheckFailure bool // awful hack; see #18395 and below } @@ -1187,6 +1189,11 @@ func buildVetConfig(a *Action, srcfiles []string) { v = gover.DefaultGoModVersion } vcfg.GoVersion = "go" + v + + if a.Package.Module.Error == nil { + vcfg.ModulePath = a.Package.Module.Path + vcfg.ModuleVersion = a.Package.Module.Version + } } a.vetCfg = vcfg for i, raw := range a.Package.Internal.RawImports { @@ -1412,7 +1419,7 @@ func (b *Builder) printLinkerConfig(h io.Writer, p *load.Package) { } // GOARM, GOMIPS, etc. - key, val := cfg.GetArchEnv() + key, val, _ := cfg.GetArchEnv() fmt.Fprintf(h, "%s=%s\n", key, val) if cfg.CleanGOEXPERIMENT != "" { @@ -2453,13 +2460,13 @@ func (b *Builder) gccSupportsFlag(compiler []string, flag string) bool { cmdArgs := str.StringList(compiler, flag) if strings.HasPrefix(flag, "-Wl,") /* linker flag */ { - ldflags, err := buildFlags("LDFLAGS", defaultCFlags, nil, checkLinkerFlags) + ldflags, err := buildFlags("LDFLAGS", DefaultCFlags, nil, checkLinkerFlags) if err != nil { return false } cmdArgs = append(cmdArgs, ldflags...) } else { /* compiler flag, add "-c" */ - cflags, err := buildFlags("CFLAGS", defaultCFlags, nil, checkCompilerFlags) + cflags, err := buildFlags("CFLAGS", DefaultCFlags, nil, checkCompilerFlags) if err != nil { return false } @@ -2700,16 +2707,16 @@ func (b *Builder) CFlags(p *load.Package) (cppflags, cflags, cxxflags, fflags, l if cppflags, err = buildFlags("CPPFLAGS", "", p.CgoCPPFLAGS, checkCompilerFlags); err != nil { return } - if cflags, err = buildFlags("CFLAGS", defaultCFlags, p.CgoCFLAGS, checkCompilerFlags); err != nil { + if cflags, err = buildFlags("CFLAGS", DefaultCFlags, p.CgoCFLAGS, checkCompilerFlags); err != nil { return } - if cxxflags, err = buildFlags("CXXFLAGS", defaultCFlags, p.CgoCXXFLAGS, checkCompilerFlags); err != nil { + if cxxflags, err = buildFlags("CXXFLAGS", DefaultCFlags, p.CgoCXXFLAGS, checkCompilerFlags); err != nil { return } - if fflags, err = buildFlags("FFLAGS", defaultCFlags, p.CgoFFLAGS, checkCompilerFlags); err != nil { + if fflags, err = buildFlags("FFLAGS", DefaultCFlags, p.CgoFFLAGS, checkCompilerFlags); err != nil { return } - if ldflags, err = buildFlags("LDFLAGS", defaultCFlags, p.CgoLDFLAGS, checkLinkerFlags); err != nil { + if ldflags, err = buildFlags("LDFLAGS", DefaultCFlags, p.CgoLDFLAGS, checkLinkerFlags); err != nil { return } @@ -2812,7 +2819,10 @@ func (b *Builder) cgo(a *Action, cgoExe, objdir string, pcCFLAGS, pcLDFLAGS, cgo cgoflags = append(cgoflags, "-import_syscall=false") } - // Update $CGO_LDFLAGS with p.CgoLDFLAGS. + // cgoLDFLAGS, which includes p.CgoLDFLAGS, can be very long. + // Pass it to cgo on the command line, so that we use a + // response file if necessary. + // // These flags are recorded in the generated _cgo_gotypes.go file // using //go:cgo_ldflag directives, the compiler records them in the // object file for the package, and then the Go linker passes them @@ -2820,12 +2830,16 @@ func (b *Builder) cgo(a *Action, cgoExe, objdir string, pcCFLAGS, pcLDFLAGS, cgo // consists of the original $CGO_LDFLAGS (unchecked) and all the // flags put together from source code (checked). cgoenv := b.cCompilerEnv() + var ldflagsOption []string if len(cgoLDFLAGS) > 0 { flags := make([]string, len(cgoLDFLAGS)) for i, f := range cgoLDFLAGS { flags[i] = strconv.Quote(f) } - cgoenv = append(cgoenv, "CGO_LDFLAGS="+strings.Join(flags, " ")) + ldflagsOption = []string{"-ldflags=" + strings.Join(flags, " ")} + + // Remove CGO_LDFLAGS from the environment. + cgoenv = append(cgoenv, "CGO_LDFLAGS=") } if cfg.BuildToolchainName == "gccgo" { @@ -2863,7 +2877,7 @@ func (b *Builder) cgo(a *Action, cgoExe, objdir string, pcCFLAGS, pcLDFLAGS, cgo cgoflags = append(cgoflags, "-trimpath", strings.Join(trimpath, ";")) } - if err := sh.run(p.Dir, p.ImportPath, cgoenv, cfg.BuildToolexec, cgoExe, "-objdir", objdir, "-importpath", p.ImportPath, cgoflags, "--", cgoCPPFLAGS, cgoCFLAGS, cgofiles); err != nil { + if err := sh.run(p.Dir, p.ImportPath, cgoenv, cfg.BuildToolexec, cgoExe, "-objdir", objdir, "-importpath", p.ImportPath, cgoflags, ldflagsOption, "--", cgoCPPFLAGS, cgoCFLAGS, cgofiles); err != nil { return nil, nil, err } outGo = append(outGo, gofiles...) diff --git a/src/cmd/go/internal/work/shell.go b/src/cmd/go/internal/work/shell.go index 60817d9c3b..1fac8e3a45 100644 --- a/src/cmd/go/internal/work/shell.go +++ b/src/cmd/go/internal/work/shell.go @@ -114,7 +114,8 @@ func (sh *Shell) moveOrCopyFile(dst, src string, perm fs.FileMode, force bool) e // Otherwise fall back to standard copy. // If the source is in the build cache, we need to copy it. - if strings.HasPrefix(src, cache.DefaultDir()) { + dir, _ := cache.DefaultDir() + if strings.HasPrefix(src, dir) { return sh.CopyFile(dst, src, perm, force) } diff --git a/src/cmd/go/internal/workcmd/edit.go b/src/cmd/go/internal/workcmd/edit.go index 8d975b0b3d..18730436ca 100644 --- a/src/cmd/go/internal/workcmd/edit.go +++ b/src/cmd/go/internal/workcmd/edit.go @@ -38,6 +38,12 @@ This reformatting is also implied by any other modifications that use or rewrite the go.mod file. The only time this flag is needed is if no other flags are specified, as in 'go work edit -fmt'. +The -godebug=key=value flag adds a godebug key=value line, +replacing any existing godebug lines with the given key. + +The -dropgodebug=key flag drops any existing godebug lines +with the given key. + The -use=path and -dropuse=path flags add and drop a use directive from the go.work file's set of module directories. @@ -69,10 +75,16 @@ writing it back to go.mod. The JSON output corresponds to these Go types: type GoWork struct { Go string Toolchain string + Godebug []Godebug Use []Use Replace []Replace } + type Godebug struct { + Key string + Value string + } + type Use struct { DiskPath string ModulePath string @@ -110,6 +122,8 @@ func (f flagFunc) Set(s string) error { f(s); return nil } func init() { cmdEdit.Run = runEditwork // break init cycle + cmdEdit.Flag.Var(flagFunc(flagEditworkGodebug), "godebug", "") + cmdEdit.Flag.Var(flagFunc(flagEditworkDropGodebug), "dropgodebug", "") cmdEdit.Flag.Var(flagFunc(flagEditworkUse), "use", "") cmdEdit.Flag.Var(flagFunc(flagEditworkDropUse), "dropuse", "") cmdEdit.Flag.Var(flagFunc(flagEditworkReplace), "replace", "") @@ -206,6 +220,28 @@ func runEditwork(ctx context.Context, cmd *base.Command, args []string) { modload.WriteWorkFile(gowork, workFile) } +// flagEditworkGodebug implements the -godebug flag. +func flagEditworkGodebug(arg string) { + key, value, ok := strings.Cut(arg, "=") + if !ok || strings.ContainsAny(arg, "\"`',") { + base.Fatalf("go: -godebug=%s: need key=value", arg) + } + workedits = append(workedits, func(f *modfile.WorkFile) { + if err := f.AddGodebug(key, value); err != nil { + base.Fatalf("go: -godebug=%s: %v", arg, err) + } + }) +} + +// flagEditworkDropGodebug implements the -dropgodebug flag. +func flagEditworkDropGodebug(arg string) { + workedits = append(workedits, func(f *modfile.WorkFile) { + if err := f.DropGodebug(arg); err != nil { + base.Fatalf("go: -dropgodebug=%s: %v", arg, err) + } + }) +} + // flagEditworkUse implements the -use flag. func flagEditworkUse(arg string) { workedits = append(workedits, func(f *modfile.WorkFile) { diff --git a/src/cmd/go/main.go b/src/cmd/go/main.go index 72656dd903..03395b87f9 100644 --- a/src/cmd/go/main.go +++ b/src/cmd/go/main.go @@ -34,6 +34,7 @@ import ( "cmd/go/internal/modget" "cmd/go/internal/modload" "cmd/go/internal/run" + "cmd/go/internal/telemetrycmd" "cmd/go/internal/test" "cmd/go/internal/tool" "cmd/go/internal/toolchain" @@ -61,6 +62,7 @@ func init() { modcmd.CmdMod, workcmd.CmdWork, run.CmdRun, + telemetrycmd.CmdTelemetry, test.CmdTest, tool.CmdTool, version.CmdVersion, @@ -92,10 +94,11 @@ var counterErrorsGOPATHEntryRelative = telemetry.NewCounter("go/errors:gopath-en func main() { log.SetFlags(0) - telemetry.StartWithUpload() // Open the telemetry counter file so counters can be written to it. + telemetry.Start() // Open the telemetry counter file so counters can be written to it. handleChdirFlag() toolchain.Select() + telemetry.StartWithUpload() // Run the upload process. Opening the counter file is idempotent. flag.Usage = base.Usage flag.Parse() telemetry.Inc("go/invocations") diff --git a/src/cmd/go/testdata/script/cgo_long_cmd.txt b/src/cmd/go/testdata/script/cgo_long_cmd.txt new file mode 100644 index 0000000000..36b9133715 --- /dev/null +++ b/src/cmd/go/testdata/script/cgo_long_cmd.txt @@ -0,0 +1,46 @@ +# Issue #66456 + +[!cgo] skip +[GOOS:windows] skip +[GOOS:plan9] skip + +# Generate a file with a very long #cgo LDFLAGS line. +# This used to cause "go build" to fail with "argument list too long". +go generate + +# Build with the generated file. +go build + +-- go.mod -- +module cgolongcmd + +go 1.22 +-- generate.go -- +//go:build ignore + +package main + +import ( + "fmt" + "log" + "os" + "bytes" +) + +func main() { + var buf bytes.Buffer + buf.WriteString("package p\n") + buf.WriteString("// #cgo LDFLAGS:") + for i := range 10000 { + fmt.Fprintf(&buf, " -Wl,-rpath,/nonexistentpath/%d", i) + } + buf.WriteString("\n") + buf.WriteString(`import "C"`+"\n") + if err := os.WriteFile("generated.go", buf.Bytes(), 0o644); err != nil { + log.Fatal(err) + } +} +-- gen.go -- +package p + +//go:generate go run generate.go diff --git a/src/cmd/go/testdata/script/env_changed.txt b/src/cmd/go/testdata/script/env_changed.txt new file mode 100644 index 0000000000..7b7b154dae --- /dev/null +++ b/src/cmd/go/testdata/script/env_changed.txt @@ -0,0 +1,45 @@ +# Test query for non-defaults in the env + +env GOTOOLCHAIN=local +env GOSUMDB=nodefault +env GOPROXY=nodefault +env GO111MODULE=auto +env CGO_CFLAGS=nodefault +env CGO_CPPFLAGS=nodefault + +go env -changed +# linux output like GOTOOLCHAIN='local' +# windows output like GOTOOLCHAIN=local +stdout 'GOTOOLCHAIN=''?local''?' +stdout 'GOSUMDB=''?nodefault''?' +stdout 'GOPROXY=''?nodefault''?' +stdout 'GO111MODULE=''?auto''?' +stdout 'CGO_CFLAGS=''?nodefault''?' +stdout 'CGO_CPPFLAGS=''?nodefault''?' + +go env -changed -json +stdout '"GOTOOLCHAIN": "local"' +stdout '"GOSUMDB": "nodefault"' +stdout '"GOPROXY": "nodefault"' +stdout '"GO111MODULE": "auto"' +stdout '"CGO_CFLAGS": "nodefault"' +stdout '"CGO_CPPFLAGS": "nodefault"' + +[GOOS:windows] env GOOS=linux +[!GOOS:windows] env GOOS=windows +[GOARCH:amd64] env GOARCH=arm64 +[!GOARCH:amd64] env GOARCH=amd64 + +go env -changed GOOS +[GOOS:windows] stdout 'set GOOS=linux' +[!GOOS:windows] stdout 'GOOS=''windows''' +go env -changed GOARCH +[GOARCH:amd64] stdout 'set GOARCH=arm64|GOARCH=''arm64''' +[!GOARCH:amd64] stdout 'set GOARCH=amd64|GOARCH=''amd64''' + +go env -changed -json GOOS +[GOOS:windows] stdout '"GOOS": "linux"' +[!GOOS:windows] stdout '"GOOS": "windows"' +go env -changed -json GOARCH +[GOARCH:amd64] stdout '"GOARCH": "arm64"' +[!GOARCH:amd64] stdout '"GOARCH": "amd64"' diff --git a/src/cmd/go/testdata/script/godebug_default.txt b/src/cmd/go/testdata/script/godebug_default.txt index 5bb8cac4bb..fecdcdb6b5 100644 --- a/src/cmd/go/testdata/script/godebug_default.txt +++ b/src/cmd/go/testdata/script/godebug_default.txt @@ -45,6 +45,35 @@ cp go.mod.21 go.mod stderr 'go: module . listed in go.work file requires go >= 1.21' rm go.work +# Go 1.21 go.mod with godebug default=go1.20 +rm go.work +cp go.mod.21 go.mod +go mod edit -godebug default=go1.20 -godebug asynctimerchan=0 +go list -f '{{.Module.GoVersion}} {{.DefaultGODEBUG}}' +stdout panicnil=1 +stdout asynctimerchan=0 + +# Go 1.21 go.work with godebug default=go1.20 +cp go.work.21 go.work +go list -f '{{.Module.GoVersion}} {{.DefaultGODEBUG}}' +! stdout panicnil # go.work wins +stdout asynctimerchan=1 # go.work wins +go work edit -godebug default=go1.20 -godebug asynctimerchan=0 +go list -f '{{.Module.GoVersion}} {{.DefaultGODEBUG}}' +stdout panicnil=1 +stdout asynctimerchan=0 +rm go.work + +# Go 1.21 go.mod with //go:debug default=go1.20 in program +cp go.mod.21 go.mod +go list -tags godebug -f '{{.Module.GoVersion}} {{.DefaultGODEBUG}}' +stdout panicnil=1 +stdout asynctimerchan=0 + +# Invalid //go:debug line should be diagnosed at build. +! go build -tags godebugbad +stderr 'invalid //go:debug: value contains space' + [short] skip # Programs in Go 1.21 work module should trigger run-time error. @@ -105,6 +134,19 @@ func main() { panic(nil) } +-- godebug.go -- +//go:build godebug +//go:debug default=go1.20 +//go:debug asynctimerchan=0 + +package main + +-- godebugbad.go -- +//go:build godebugbad +//go:debug default=go1.20 asynctimerchan=0 + +package main + -- q/go.mod -- go 1.20 module q diff --git a/src/cmd/go/testdata/script/mod_edit.txt b/src/cmd/go/testdata/script/mod_edit.txt index 2d09b06c61..49ff464fa2 100644 --- a/src/cmd/go/testdata/script/mod_edit.txt +++ b/src/cmd/go/testdata/script/mod_edit.txt @@ -87,6 +87,16 @@ go mod init foo go mod edit -module local-only -require=other-local@v1.0.0 -replace other-local@v1.0.0=./other cmpenv go.mod go.mod.edit +# go mod edit -godebug +cd $WORK/g +cp go.mod.start go.mod +go mod edit -godebug key=value +cmpenv go.mod go.mod.edit +go mod edit -dropgodebug key2 +cmpenv go.mod go.mod.edit +go mod edit -dropgodebug key +cmpenv go.mod go.mod.start + -- x.go -- package x @@ -338,3 +348,13 @@ module m "Replace": null, "Retract": null } +-- $WORK/g/go.mod.start -- +module g + +go 1.10 +-- $WORK/g/go.mod.edit -- +module g + +go 1.10 + +godebug key=value diff --git a/src/cmd/go/testdata/script/mod_list_direct_work.txt b/src/cmd/go/testdata/script/mod_list_direct_work.txt new file mode 100644 index 0000000000..eeede6dad1 --- /dev/null +++ b/src/cmd/go/testdata/script/mod_list_direct_work.txt @@ -0,0 +1,76 @@ +# Test that ModuleDirect.Public is correctly set on go list output. +# This is a regression test for issue #66789. + +# In this test, the workspace contains modules example.com/a and +# example.com/b. Module example.com/a has a direct requirement +# on rsc.io/sampler, and an indirect requirement on golang.org/x/text +# through rsc.io/isampler. Module example.com/b has a direct +# requirement on example.com/c which is incorrectly marked as indirect +# in module example.com/b's go.mod file. + +# Check that go list -m processes the indirect annotations in the +# go.mod file. +go list -f '{{.Path}} {{.Indirect}}' -m all +stdout 'example.com/a false' +stdout 'example.com/b false' +stdout 'rsc.io/sampler false' +stdout 'golang.org/x/text true' +stdout 'example.com/c true' # Uses the information in go.mod without checking imports. + +# Check that 'go list all' correctly populates "indirect" module annotation. +go list -f '{{.ImportPath}} {{with .Module}}{{.Indirect}}{{end}}' all +stdout 'example.com/a false' +stdout 'example.com/b false' +stdout 'rsc.io/sampler false' +stdout 'golang.org/x/text/language true' +stdout 'example.com/c false' + +-- go.work -- +go 1.23 + +use ./a +use ./b +-- a/go.mod -- +module example.com/a + +go 1.23 + +require rsc.io/sampler v1.2.1 + +require golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c // indirect +-- a/a.go -- +package a + +import "rsc.io/sampler" + +func A() string { + return sampler.Hello() +} +-- b/go.mod -- +module example.com/b + +go 1.23 + +// The indrect comment below is inaccurate. Its purpose +// is to test that it is corrected when enough packages +// are loaded to correct it. + +require example.com/c v1.0.0 // indirect + +replace example.com/c => ../c +-- b/b.go -- +package b + +import "example.com/c" + +func B() { + c.C() +} +-- c/go.mod -- +module example.com/c + +go 1.23 +-- c/c.go -- +package c + +func C() {} \ No newline at end of file diff --git a/src/cmd/go/testdata/script/telemetry.txt b/src/cmd/go/testdata/script/telemetry.txt new file mode 100644 index 0000000000..e9aa0f1085 --- /dev/null +++ b/src/cmd/go/testdata/script/telemetry.txt @@ -0,0 +1,51 @@ +# Tests for the telemetry subcommand, + +# The script test framework sets TEST_TELEMETRY_DIR (overriding the +# default telemetry dir location) and then checks that at least one +# counter has been written per script tests. +# Run go before unsetting TEST_TELEMETRY_DIR to make the tests happy. +# We want to unset it so the environment we're testing is as close +# to a user's environment. +go help telemetry +env TEST_TELEMETRY_DIR= + +# Set userconfig dir, which is determined by os.UserConfigDir. +# The telemetry dir is determined using that. +mkdir $WORK/userconfig +env AppData=$WORK\userconfig # windows +[GOOS:windows] env userconfig=$AppData +env HOME=$WORK/userconfig # darwin,unix,ios +[GOOS:darwin] env userconfig=$HOME'/Library/Application Support' +[GOOS:ios] env userconfig=$HOME'/Library/Application Support' +[!GOOS:windows] [!GOOS:darwin] [!GOOS:ios] [!GOOS:plan9] env userconfig=$HOME/.config +env home=$WORK/userconfig # plan9 +[GOOS:plan9] env userconfig=$home/lib + +go telemetry +stdout 'local' + +go telemetry off +go telemetry +stdout 'off' +go env GOTELEMETRY +stdout 'off' + +go telemetry local +go telemetry +stdout 'local' +go env GOTELEMETRY +stdout 'local' + +go telemetry on +go telemetry +stdout 'on' +go env GOTELEMETRY +stdout 'on' + +go env +stdout 'GOTELEMETRY=''?on''?' +stdout 'GOTELEMETRYDIR=''?'$userconfig'[\\/]go[\\/]telemetry''?' +! go env -w GOTELEMETRY=off +stderr '^go: unknown go command variable GOTELEMETRY$' +! go env -w GOTELEMETRYDIR=foo +stderr '^go: unknown go command variable GOTELEMETRYDIR$' \ No newline at end of file diff --git a/src/cmd/go/testdata/script/work_edit.txt b/src/cmd/go/testdata/script/work_edit.txt index c67696dd6e..021346653f 100644 --- a/src/cmd/go/testdata/script/work_edit.txt +++ b/src/cmd/go/testdata/script/work_edit.txt @@ -34,9 +34,20 @@ cmp stdout go.work.want_print go work edit -json -go 1.19 -use b -dropuse c -replace 'x.1@v1.4.0 = ../z' -dropreplace x.1 -dropreplace x.1@v1.3.0 cmp stdout go.work.want_json +# go work edit -godebug +cd $WORK/g +cp go.work.start go.work +go work edit -godebug key=value +cmpenv go.work go.work.edit +go work edit -dropgodebug key2 +cmpenv go.work go.work.edit +go work edit -dropgodebug key +cmpenv go.work go.work.start + +# go work edit -print -fmt env GOWORK=$GOPATH/src/unformatted go work edit -print -fmt -cmp stdout formatted +cmp stdout $GOPATH/src/formatted -- m/go.mod -- module m @@ -164,3 +175,13 @@ replace ( x.1 v1.3.0 => y.1 v1.4.0 x.1 v1.4.0 => ../z ) +-- $WORK/g/go.work.start -- +use g + +go 1.10 +-- $WORK/g/go.work.edit -- +use g + +go 1.10 + +godebug key=value diff --git a/src/cmd/gofmt/gofmt.go b/src/cmd/gofmt/gofmt.go index 341c79ab8e..03f7bef89c 100644 --- a/src/cmd/gofmt/gofmt.go +++ b/src/cmd/gofmt/gofmt.go @@ -25,6 +25,8 @@ import ( "strconv" "strings" + "cmd/internal/telemetry" + "golang.org/x/sync/semaphore" ) @@ -372,8 +374,11 @@ func main() { } func gofmtMain(s *sequencer) { + telemetry.Start() flag.Usage = usage flag.Parse() + telemetry.Inc("gofmt/invocations") + telemetry.CountFlags("gofmt/flag:", *flag.CommandLine) if *cpuprofile != "" { fdSem <- true diff --git a/src/cmd/internal/goobj/objfile.go b/src/cmd/internal/goobj/objfile.go index fb87b04412..56ce76ad09 100644 --- a/src/cmd/internal/goobj/objfile.go +++ b/src/cmd/internal/goobj/objfile.go @@ -284,6 +284,7 @@ const ( _ // was ObjFlagNeedNameExpansion ObjFlagFromAssembly // object is from asm src, not go ObjFlagUnlinkable // unlinkable package (linker will emit an error) + ObjFlagStd // standard library package ) // Sym.Flag @@ -304,6 +305,7 @@ const ( SymFlagDict SymFlagPkgInit SymFlagLinkname + SymFlagABIWrapper ) // Returns the length of the name of the symbol. @@ -336,6 +338,7 @@ func (s *Sym) IsItab() bool { return s.Flag2()&SymFlagItab != 0 } func (s *Sym) IsDict() bool { return s.Flag2()&SymFlagDict != 0 } func (s *Sym) IsPkgInit() bool { return s.Flag2()&SymFlagPkgInit != 0 } func (s *Sym) IsLinkname() bool { return s.Flag2()&SymFlagLinkname != 0 } +func (s *Sym) ABIWrapper() bool { return s.Flag2()&SymFlagABIWrapper != 0 } func (s *Sym) SetName(x string, w *Writer) { binary.LittleEndian.PutUint32(s[:], uint32(len(x))) @@ -882,3 +885,4 @@ func (r *Reader) Flags() uint32 { func (r *Reader) Shared() bool { return r.Flags()&ObjFlagShared != 0 } func (r *Reader) FromAssembly() bool { return r.Flags()&ObjFlagFromAssembly != 0 } func (r *Reader) Unlinkable() bool { return r.Flags()&ObjFlagUnlinkable != 0 } +func (r *Reader) Std() bool { return r.Flags()&ObjFlagStd != 0 } diff --git a/src/cmd/internal/obj/arm64/asm7.go b/src/cmd/internal/obj/arm64/asm7.go index 03f0fb06da..c6601cb49e 100644 --- a/src/cmd/internal/obj/arm64/asm7.go +++ b/src/cmd/internal/obj/arm64/asm7.go @@ -889,9 +889,10 @@ var optab = []Optab{ {obj.ANOP, C_LCON, C_NONE, C_NONE, C_NONE, C_NONE, 0, 0, 0, 0, 0}, // nop variants, see #40689 {obj.ANOP, C_ZREG, C_NONE, C_NONE, C_NONE, C_NONE, 0, 0, 0, 0, 0}, {obj.ANOP, C_VREG, C_NONE, C_NONE, C_NONE, C_NONE, 0, 0, 0, 0, 0}, - {obj.ADUFFZERO, C_NONE, C_NONE, C_NONE, C_SBRA, C_NONE, 5, 4, 0, 0, 0}, // same as AB/ABL - {obj.ADUFFCOPY, C_NONE, C_NONE, C_NONE, C_SBRA, C_NONE, 5, 4, 0, 0, 0}, // same as AB/ABL - {obj.APCALIGN, C_LCON, C_NONE, C_NONE, C_NONE, C_NONE, 0, 0, 0, 0, 0}, // align code + {obj.ADUFFZERO, C_NONE, C_NONE, C_NONE, C_SBRA, C_NONE, 5, 4, 0, 0, 0}, // same as AB/ABL + {obj.ADUFFCOPY, C_NONE, C_NONE, C_NONE, C_SBRA, C_NONE, 5, 4, 0, 0, 0}, // same as AB/ABL + {obj.APCALIGN, C_LCON, C_NONE, C_NONE, C_NONE, C_NONE, 0, 0, 0, 0, 0}, // align code + {obj.APCALIGNMAX, C_LCON, C_NONE, C_NONE, C_LCON, C_NONE, 0, 0, 0, 0, 0}, // align code, conditional } // Valid pstate field values, and value to use in instruction. @@ -1109,13 +1110,8 @@ func span7(ctxt *obj.Link, cursym *obj.LSym, newprog obj.ProgAlloc) { m = o.size(c.ctxt, p) if m == 0 { switch p.As { - case obj.APCALIGN: - alignedValue := p.From.Offset - m = pcAlignPadLength(ctxt, pc, alignedValue) - // Update the current text symbol alignment value. - if int32(alignedValue) > cursym.Func().Align { - cursym.Func().Align = int32(alignedValue) - } + case obj.APCALIGN, obj.APCALIGNMAX: + m = obj.AlignmentPadding(int32(pc), p, ctxt, cursym) break case obj.ANOP, obj.AFUNCDATA, obj.APCDATA: continue @@ -1181,9 +1177,8 @@ func span7(ctxt *obj.Link, cursym *obj.LSym, newprog obj.ProgAlloc) { if m == 0 { switch p.As { - case obj.APCALIGN: - alignedValue := p.From.Offset - m = pcAlignPadLength(ctxt, pc, alignedValue) + case obj.APCALIGN, obj.APCALIGNMAX: + m = obj.AlignmentPaddingLength(int32(pc), p, ctxt) break case obj.ANOP, obj.AFUNCDATA, obj.APCDATA: continue @@ -1214,9 +1209,8 @@ func span7(ctxt *obj.Link, cursym *obj.LSym, newprog obj.ProgAlloc) { if sz > 4*len(out) { log.Fatalf("out array in span7 is too small, need at least %d for %v", sz/4, p) } - if p.As == obj.APCALIGN { - alignedValue := p.From.Offset - v := pcAlignPadLength(c.ctxt, p.Pc, alignedValue) + if p.As == obj.APCALIGN || p.As == obj.APCALIGNMAX { + v := obj.AlignmentPaddingLength(int32(p.Pc), p, c.ctxt) for i = 0; i < int(v/4); i++ { // emit ANOOP instruction by the padding size c.ctxt.Arch.ByteOrder.PutUint32(bp, OP_NOOP) @@ -3316,6 +3310,7 @@ func buildop(ctxt *obj.Link) { obj.AUNDEF, obj.AFUNCDATA, obj.APCALIGN, + obj.APCALIGNMAX, obj.APCDATA, obj.ADUFFZERO, obj.ADUFFCOPY: diff --git a/src/cmd/internal/obj/link.go b/src/cmd/internal/obj/link.go index dac6e209f1..647a459d59 100644 --- a/src/cmd/internal/obj/link.go +++ b/src/cmd/internal/obj/link.go @@ -416,6 +416,7 @@ const ( AJMP ANOP APCALIGN + APCALIGNMAX // currently x86, amd64 and arm64 APCDATA ARET AGETCALLERPC @@ -1047,6 +1048,7 @@ type Link struct { InParallel bool // parallel backend phase in effect UseBASEntries bool // use Base Address Selection Entries in location lists and PC ranges IsAsm bool // is the source assembly language, which may contain surprising idioms (e.g., call tables) + Std bool // is standard library package // state for writing objects Text []*LSym @@ -1058,6 +1060,10 @@ type Link struct { // to Data. constSyms []*LSym + // Windows SEH symbols are also data symbols that can be created + // concurrently. + SEHSyms []*LSym + // pkgIdx maps package path to index. The index is used for // symbol reference in the object file. pkgIdx map[string]int32 diff --git a/src/cmd/internal/obj/objfile.go b/src/cmd/internal/obj/objfile.go index ecc583ce4f..2ed98cb577 100644 --- a/src/cmd/internal/obj/objfile.go +++ b/src/cmd/internal/obj/objfile.go @@ -57,6 +57,9 @@ func WriteObjFile(ctxt *Link, b *bio.Writer) { if ctxt.IsAsm { flags |= goobj.ObjFlagFromAssembly } + if ctxt.Std { + flags |= goobj.ObjFlagStd + } h := goobj.Header{ Magic: goobj.Magic, Fingerprint: ctxt.Fingerprint, @@ -309,6 +312,7 @@ func (w *writer) StringTable() { const cutoff = int64(2e9) // 2 GB (or so; looks better in errors than 2^31) func (w *writer) Sym(s *LSym) { + name := s.Name abi := uint16(s.ABI()) if s.Static() { abi = goobj.SymABIstatic @@ -348,10 +352,15 @@ func (w *writer) Sym(s *LSym) { if s.IsPkgInit() { flag2 |= goobj.SymFlagPkgInit } - if s.IsLinkname() || w.ctxt.IsAsm { // assembly reference is treated the same as linkname + if s.IsLinkname() || (w.ctxt.IsAsm && name != "") || name == "main.main" { + // Assembly reference is treated the same as linkname, + // but not for unnamed (aux) symbols. + // The runtime linknames main.main. flag2 |= goobj.SymFlagLinkname } - name := s.Name + if s.ABIWrapper() { + flag2 |= goobj.SymFlagABIWrapper + } if strings.HasPrefix(name, "gofile..") { name = filepath.ToSlash(name) } @@ -788,14 +797,18 @@ func genFuncInfoSyms(ctxt *Link) { fn.FuncInfoSym = isym b.Reset() - auxsyms := []*LSym{fn.dwarfRangesSym, fn.dwarfLocSym, fn.dwarfDebugLinesSym, fn.dwarfInfoSym, fn.WasmImportSym, fn.sehUnwindInfoSym} + auxsyms := []*LSym{fn.dwarfRangesSym, fn.dwarfLocSym, fn.dwarfDebugLinesSym, fn.dwarfInfoSym, fn.WasmImportSym} for _, s := range auxsyms { if s == nil || s.Size == 0 { continue } + if s.OnList() { + panic("a symbol is added to defs multiple times") + } s.PkgIdx = goobj.PkgIdxSelf s.SymIdx = symidx s.Set(AttrIndexed, true) + s.Set(AttrOnList, true) symidx++ infosyms = append(infosyms, s) } @@ -839,6 +852,9 @@ func (ctxt *Link) writeSymDebugNamed(s *LSym, name string) { ver := "" if ctxt.Debugasm > 1 { ver = fmt.Sprintf("<%d>", s.ABI()) + if ctxt.Debugasm > 2 { + ver += fmt.Sprintf("", s.PkgIdx, s.SymIdx) + } } fmt.Fprintf(ctxt.Bso, "%s%s ", name, ver) if s.Type != 0 { diff --git a/src/cmd/internal/obj/ppc64/asm9.go b/src/cmd/internal/obj/ppc64/asm9.go index d9b7c2eed3..74f1772e3d 100644 --- a/src/cmd/internal/obj/ppc64/asm9.go +++ b/src/cmd/internal/obj/ppc64/asm9.go @@ -472,12 +472,12 @@ var optabBase = []Optab{ {as: ACMP, a1: C_REG, a6: C_REG, type_: 70, size: 4}, {as: ACMP, a1: C_REG, a2: C_CREG, a6: C_REG, type_: 70, size: 4}, - {as: ACMP, a1: C_REG, a6: C_S16CON, type_: 71, size: 4}, - {as: ACMP, a1: C_REG, a2: C_CREG, a6: C_S16CON, type_: 71, size: 4}, + {as: ACMP, a1: C_REG, a6: C_S16CON, type_: 70, size: 4}, + {as: ACMP, a1: C_REG, a2: C_CREG, a6: C_S16CON, type_: 70, size: 4}, {as: ACMPU, a1: C_REG, a6: C_REG, type_: 70, size: 4}, {as: ACMPU, a1: C_REG, a2: C_CREG, a6: C_REG, type_: 70, size: 4}, - {as: ACMPU, a1: C_REG, a6: C_U16CON, type_: 71, size: 4}, - {as: ACMPU, a1: C_REG, a2: C_CREG, a6: C_U16CON, type_: 71, size: 4}, + {as: ACMPU, a1: C_REG, a6: C_U16CON, type_: 70, size: 4}, + {as: ACMPU, a1: C_REG, a2: C_CREG, a6: C_U16CON, type_: 70, size: 4}, {as: AFCMPO, a1: C_FREG, a6: C_FREG, type_: 70, size: 4}, {as: AFCMPO, a1: C_FREG, a2: C_CREG, a6: C_FREG, type_: 70, size: 4}, {as: ATW, a1: C_32CON, a2: C_REG, a6: C_REG, type_: 60, size: 4}, @@ -3449,23 +3449,13 @@ func asmout(c *ctxt9, p *obj.Prog, o *Optab, out *[5]uint32) { o1 = AOP_RRR(OP_MTCRF, uint32(p.From.Reg), 0, 0) | uint32(v)<<12 - case 70: /* [f]cmp r,r,cr*/ - var r int - if p.Reg == 0 { - r = 0 + case 70: /* cmp* r,r,cr or cmp*i r,i,cr or fcmp f,f,cr or cmpeqb r,r */ + r := uint32(p.Reg&7) << 2 + if p.To.Type == obj.TYPE_CONST { + o1 = AOP_IRR(c.opirr(p.As), r, uint32(p.From.Reg), uint32(uint16(p.To.Offset))) } else { - r = (int(p.Reg) & 7) << 2 + o1 = AOP_RRR(c.oprrr(p.As), r, uint32(p.From.Reg), uint32(p.To.Reg)) } - o1 = AOP_RRR(c.oprrr(p.As), uint32(r), uint32(p.From.Reg), uint32(p.To.Reg)) - - case 71: /* cmp[l] r,i,cr*/ - var r int - if p.Reg == 0 { - r = 0 - } else { - r = (int(p.Reg) & 7) << 2 - } - o1 = AOP_RRR(c.opirr(p.As), uint32(r), uint32(p.From.Reg), 0) | uint32(c.regoff(&p.To))&0xffff case 72: /* slbmte (Rb+Rs -> slb[Rb]) -> Rs, Rb */ o1 = AOP_RRR(c.oprrr(p.As), uint32(p.From.Reg), 0, uint32(p.To.Reg)) diff --git a/src/cmd/internal/obj/sym.go b/src/cmd/internal/obj/sym.go index f27d4ef4fc..22153050f2 100644 --- a/src/cmd/internal/obj/sym.go +++ b/src/cmd/internal/obj/sym.go @@ -245,6 +245,13 @@ func (ctxt *Link) NumberSyms() { ctxt.Data = append(ctxt.Data, ctxt.constSyms...) ctxt.constSyms = nil + // So are SEH symbols. + sort.Slice(ctxt.SEHSyms, func(i, j int) bool { + return ctxt.SEHSyms[i].Name < ctxt.SEHSyms[j].Name + }) + ctxt.Data = append(ctxt.Data, ctxt.SEHSyms...) + ctxt.SEHSyms = nil + ctxt.pkgIdx = make(map[string]int32) ctxt.defs = []*LSym{} ctxt.hashed64defs = []*LSym{} diff --git a/src/cmd/internal/obj/util.go b/src/cmd/internal/obj/util.go index 3a071c21d4..0512d78ca0 100644 --- a/src/cmd/internal/obj/util.go +++ b/src/cmd/internal/obj/util.go @@ -6,6 +6,7 @@ package obj import ( "bytes" + "cmd/internal/objabi" "fmt" "internal/abi" "internal/buildcfg" @@ -642,6 +643,7 @@ var Anames = []string{ "JMP", "NOP", "PCALIGN", + "PCALIGNMAX", "PCDATA", "RET", "GETCALLERPC", @@ -667,3 +669,62 @@ func abiDecorate(a *Addr, abiDetail bool) string { } return fmt.Sprintf("<%s>", a.Sym.ABI()) } + +// AlignmentPadding bytes to add to align code as requested. +// Alignment is restricted to powers of 2 between 8 and 2048 inclusive. +// +// pc_: current offset in function, in bytes +// p: a PCALIGN or PCALIGNMAX prog +// ctxt: the context, for current function +// cursym: current function being assembled +// returns number of bytes of padding needed, +// updates minimum alignment for the function. +func AlignmentPadding(pc int32, p *Prog, ctxt *Link, cursym *LSym) int { + v := AlignmentPaddingLength(pc, p, ctxt) + requireAlignment(p.From.Offset, ctxt, cursym) + return v +} + +// AlignmentPaddingLength is the number of bytes to add to align code as requested. +// Alignment is restricted to powers of 2 between 8 and 2048 inclusive. +// This only computes the length and does not update the (missing parameter) +// current function's own required alignment. +// +// pc: current offset in function, in bytes +// p: a PCALIGN or PCALIGNMAX prog +// ctxt: the context, for current function +// returns number of bytes of padding needed, +func AlignmentPaddingLength(pc int32, p *Prog, ctxt *Link) int { + a := p.From.Offset + if !((a&(a-1) == 0) && 8 <= a && a <= 2048) { + ctxt.Diag("alignment value of an instruction must be a power of two and in the range [8, 2048], got %d\n", a) + return 0 + } + pc64 := int64(pc) + lob := pc64 & (a - 1) // Low Order Bits -- if not zero, then not aligned + if p.As == APCALIGN { + if lob != 0 { + return int(a - lob) + } + return 0 + } + // emit as many as s bytes of padding to obtain alignment + s := p.To.Offset + if s < 0 || s >= a { + ctxt.Diag("PCALIGNMAX 'amount' %d must be non-negative and smaller than the aligment %d\n", s, a) + return 0 + } + if s >= a-lob { + return int(a - lob) + } + return 0 +} + +// requireAlignment ensures that the function is aligned enough to support +// the required code alignment +func requireAlignment(a int64, ctxt *Link, cursym *LSym) { + // TODO remove explicit knowledge about AIX. + if ctxt.Headtype != objabi.Haix && cursym.Func().Align < int32(a) { + cursym.Func().Align = int32(a) + } +} diff --git a/src/cmd/internal/obj/x86/asm6.go b/src/cmd/internal/obj/x86/asm6.go index bdd75b4ef8..dc38069edc 100644 --- a/src/cmd/internal/obj/x86/asm6.go +++ b/src/cmd/internal/obj/x86/asm6.go @@ -2036,29 +2036,21 @@ type nopPad struct { n int32 // Size of the pad } -// Padding bytes to add to align code as requested. -// Alignment is restricted to powers of 2 between 8 and 2048 inclusive. +// requireAlignment ensures that the function alignment is at +// least as high as a, which should be a power of two +// and between 8 and 2048, inclusive. // -// pc: current offset in function, in bytes -// a: requested alignment, in bytes -// cursym: current function being assembled -// returns number of bytes of padding needed -func addpad(pc, a int64, ctxt *obj.Link, cursym *obj.LSym) int { +// the boolean result indicates whether the alignment meets those constraints +func requireAlignment(a int64, ctxt *obj.Link, cursym *obj.LSym) bool { if !((a&(a-1) == 0) && 8 <= a && a <= 2048) { ctxt.Diag("alignment value of an instruction must be a power of two and in the range [8, 2048], got %d\n", a) - return 0 + return false } - // By default function alignment is 32 bytes for amd64 if cursym.Func().Align < int32(a) { cursym.Func().Align = int32(a) } - - if pc&(a-1) != 0 { - return int(a - (pc & (a - 1))) - } - - return 0 + return true } func span6(ctxt *obj.Link, s *obj.LSym, newprog obj.ProgAlloc) { @@ -2144,17 +2136,17 @@ func span6(ctxt *obj.Link, s *obj.LSym, newprog obj.ProgAlloc) { c0 := c c = pjc.padJump(ctxt, s, p, c) - if p.As == obj.APCALIGN { - aln := p.From.Offset - v := addpad(int64(c), aln, ctxt, s) + if p.As == obj.APCALIGN || p.As == obj.APCALIGNMAX { + v := obj.AlignmentPadding(c, p, ctxt, s) if v > 0 { s.Grow(int64(c) + int64(v)) fillnop(s.P[c:], int(v)) } - + p.Pc = int64(c) c += int32(v) pPrev = p continue + } if maxLoopPad > 0 && p.Back&branchLoopHead != 0 && c&(loopAlign-1) != 0 { diff --git a/src/cmd/internal/obj/x86/seh.go b/src/cmd/internal/obj/x86/seh.go index 71cdd36642..11963e53f9 100644 --- a/src/cmd/internal/obj/x86/seh.go +++ b/src/cmd/internal/obj/x86/seh.go @@ -151,6 +151,7 @@ func populateSeh(ctxt *obj.Link, s *obj.LSym) (sehsym *obj.LSym) { s.Type = objabi.SSEHUNWINDINFO s.Set(obj.AttrDuplicateOK, true) s.Set(obj.AttrLocal, true) + s.Set(obj.AttrContentAddressable, true) if exceptionHandler != nil { r := obj.Addrel(s) r.Off = int32(len(buf.data) - 4) @@ -158,8 +159,6 @@ func populateSeh(ctxt *obj.Link, s *obj.LSym) (sehsym *obj.LSym) { r.Sym = exceptionHandler r.Type = objabi.R_PEIMAGEOFF } - // Note: AttrContentAddressable cannot be set here, - // because the content-addressable-handling code - // does not know about aux symbols. + ctxt.SEHSyms = append(ctxt.SEHSyms, s) }) } diff --git a/src/cmd/internal/objabi/pkgspecial.go b/src/cmd/internal/objabi/pkgspecial.go index 8ca9c7416d..867d92d357 100644 --- a/src/cmd/internal/objabi/pkgspecial.go +++ b/src/cmd/internal/objabi/pkgspecial.go @@ -50,6 +50,7 @@ var runtimePkgs = []string{ "internal/abi", "internal/bytealg", + "internal/byteorder", "internal/chacha8rand", "internal/coverage/rtcov", "internal/cpu", diff --git a/src/cmd/internal/telemetry/telemetry.go b/src/cmd/internal/telemetry/telemetry.go index 2420a07708..221b6a007d 100644 --- a/src/cmd/internal/telemetry/telemetry.go +++ b/src/cmd/internal/telemetry/telemetry.go @@ -74,3 +74,38 @@ func CountFlagValue(prefix string, flagSet flag.FlagSet, flagName string) { } }) } + +// Mode returns the current telemetry mode. +// +// The telemetry mode is a global value that controls both the local collection +// and uploading of telemetry data. Possible mode values are: +// - "on": both collection and uploading is enabled +// - "local": collection is enabled, but uploading is disabled +// - "off": both collection and uploading are disabled +// +// When mode is "on", or "local", telemetry data is written to the local file +// system and may be inspected with the [gotelemetry] command. +// +// If an error occurs while reading the telemetry mode from the file system, +// Mode returns the default value "local". +// +// [gotelemetry]: https://pkg.go.dev/golang.org/x/telemetry/cmd/gotelemetry +func Mode() string { + return telemetry.Mode() +} + +// SetMode sets the global telemetry mode to the given value. +// +// See the documentation of [Mode] for a description of the supported mode +// values. +// +// An error is returned if the provided mode value is invalid, or if an error +// occurs while persisting the mode value to the file system. +func SetMode(mode string) error { + return telemetry.SetMode(mode) +} + +// Dir returns the telemetry directory. +func Dir() string { + return telemetry.Dir() +} diff --git a/src/cmd/internal/telemetry/telemetry_bootstrap.go b/src/cmd/internal/telemetry/telemetry_bootstrap.go index 01549b6970..1740bdb701 100644 --- a/src/cmd/internal/telemetry/telemetry_bootstrap.go +++ b/src/cmd/internal/telemetry/telemetry_bootstrap.go @@ -19,3 +19,6 @@ func NewCounter(name string) dummyCounter { retu func NewStackCounter(name string, depth int) dummyCounter { return dummyCounter{} } func CountFlags(name string, flagSet flag.FlagSet) {} func CountFlagValue(prefix string, flagSet flag.FlagSet, flagName string) {} +func Mode() string { return "" } +func SetMode(mode string) error { return nil } +func Dir() string { return "" } diff --git a/src/cmd/link/internal/ld/lib.go b/src/cmd/link/internal/ld/lib.go index 823c395273..11df3a466d 100644 --- a/src/cmd/link/internal/ld/lib.go +++ b/src/cmd/link/internal/ld/lib.go @@ -47,6 +47,7 @@ import ( "sort" "strings" "sync" + "time" "cmd/internal/bio" "cmd/internal/goobj" @@ -517,6 +518,9 @@ func (ctxt *Link) findLibPath(libname string) string { func (ctxt *Link) loadlib() { var flags uint32 + if *flagCheckLinkname { + flags |= loader.FlagCheckLinkname + } switch *FlagStrictDups { case 0: // nothing to do @@ -1268,6 +1272,22 @@ func hostlinksetup(ctxt *Link) { } } +// cleanTimeStamps resets the timestamps for the specified list of +// existing files to the Unix epoch (1970-01-01 00:00:00 +0000 UTC). +// We take this step in order to help preserve reproducible builds; +// this seems to be primarily needed for external linking on on Darwin +// with later versions of xcode, which (unfortunately) seem to want to +// incorporate object file times into the final output file's build +// ID. See issue 64947 for the unpleasant details. +func cleanTimeStamps(files []string) { + epocht := time.Unix(0, 0) + for _, f := range files { + if err := os.Chtimes(f, epocht, epocht); err != nil { + Exitf("cannot chtimes %s: %v", f, err) + } + } +} + // hostobjCopy creates a copy of the object files in hostobj in a // temporary directory. func (ctxt *Link) hostobjCopy() (paths []string) { @@ -1360,9 +1380,14 @@ func (ctxt *Link) archive() { if ctxt.HeadType == objabi.Haix { argv = append(argv, "-X64") } + godotopath := filepath.Join(*flagTmpdir, "go.o") + cleanTimeStamps([]string{godotopath}) + hostObjCopyPaths := ctxt.hostobjCopy() + cleanTimeStamps(hostObjCopyPaths) + argv = append(argv, *flagOutfile) - argv = append(argv, filepath.Join(*flagTmpdir, "go.o")) - argv = append(argv, ctxt.hostobjCopy()...) + argv = append(argv, godotopath) + argv = append(argv, hostObjCopyPaths...) if ctxt.Debugvlog != 0 { ctxt.Logf("archive: %s\n", strings.Join(argv, " ")) @@ -1733,8 +1758,13 @@ func (ctxt *Link) hostlink() { argv = append(argv, compressDWARF) } - argv = append(argv, filepath.Join(*flagTmpdir, "go.o")) - argv = append(argv, ctxt.hostobjCopy()...) + hostObjCopyPaths := ctxt.hostobjCopy() + cleanTimeStamps(hostObjCopyPaths) + godotopath := filepath.Join(*flagTmpdir, "go.o") + cleanTimeStamps([]string{godotopath}) + + argv = append(argv, godotopath) + argv = append(argv, hostObjCopyPaths...) if ctxt.HeadType == objabi.Haix { // We want to have C files after Go files to remove // trampolines csects made by ld. diff --git a/src/cmd/link/internal/ld/main.go b/src/cmd/link/internal/ld/main.go index 8a67ccfb32..e6608fd791 100644 --- a/src/cmd/link/internal/ld/main.go +++ b/src/cmd/link/internal/ld/main.go @@ -96,6 +96,7 @@ var ( FlagS = flag.Bool("s", false, "disable symbol table") flag8 bool // use 64-bit addresses in symbol table flagInterpreter = flag.String("I", "", "use `linker` as ELF dynamic linker") + flagCheckLinkname = flag.Bool("checklinkname", false, "check linkname symbol references") FlagDebugTramp = flag.Int("debugtramp", 0, "debug trampolines") FlagDebugTextSize = flag.Int("debugtextsize", 0, "debug text section max size") flagDebugNosplit = flag.Bool("debugnosplit", false, "dump nosplit call graph") diff --git a/src/cmd/link/internal/loader/loader.go b/src/cmd/link/internal/loader/loader.go index 53ebb53a75..0a76c1fb0c 100644 --- a/src/cmd/link/internal/loader/loader.go +++ b/src/cmd/link/internal/loader/loader.go @@ -292,6 +292,7 @@ type extSymPayload struct { const ( // Loader.flags FlagStrictDups = 1 << iota + FlagCheckLinkname ) func NewLoader(flags uint32, reporter *ErrorReporter) *Loader { @@ -421,14 +422,6 @@ func (st *loadState) addSym(name string, ver int, r *oReader, li uint32, kind in } // Non-package (named) symbol. - if osym.IsLinkname() && r.DataSize(li) == 0 { - // This is a linknamed "var" "reference" (var x T with no data and //go:linkname x). - // Check if a linkname reference is allowed. - // Only check references (pull), not definitions (push, with non-zero size), - // so push is always allowed. - // Linkname is always a non-package reference. - checkLinkname(r.unit.Lib.Pkg, name) - } // Check if it already exists. oldi, existed := l.symsByName[ver][name] if !existed { @@ -2154,6 +2147,14 @@ type loadState struct { l *Loader hashed64Syms map[uint64]symAndSize // short hashed (content-addressable) symbols, keyed by content hash hashedSyms map[goobj.HashType]symAndSize // hashed (content-addressable) symbols, keyed by content hash + + linknameVarRefs []linknameVarRef // linknamed var refererces +} + +type linknameVarRef struct { + pkg string // package of reference (not definition) + name string + sym Sym } // Preload symbols of given kind from an object. @@ -2188,6 +2189,19 @@ func (st *loadState) preloadSyms(r *oReader, kind int) { } gi := st.addSym(name, v, r, i, kind, osym) r.syms[i] = gi + if kind == nonPkgDef && osym.IsLinkname() && r.DataSize(i) == 0 && strings.Contains(name, ".") { + // This is a linknamed "var" "reference" (var x T with no data and //go:linkname x). + // We want to check if a linkname reference is allowed. Here we haven't loaded all + // symbol definitions, so we don't yet know all the push linknames. So we add to a + // list and check later after all symbol defs are loaded. Linknamed vars are rare, + // so this list won't be long. + // Only check references (pull), not definitions (push, with non-zero size), + // so push is always allowed. + // This use of linkname is usually for referencing C symbols, so allow symbols + // with no "." in its name (not a regular Go symbol). + // Linkname is always a non-package reference. + st.linknameVarRefs = append(st.linknameVarRefs, linknameVarRef{r.unit.Lib.Pkg, name, gi}) + } if osym.Local() { l.SetAttrLocal(gi, true) } @@ -2237,6 +2251,9 @@ func (l *Loader) LoadSyms(arch *sys.Arch) { st.preloadSyms(r, hashedDef) st.preloadSyms(r, nonPkgDef) } + for _, vr := range st.linknameVarRefs { + l.checkLinkname(vr.pkg, vr.name, vr.sym) + } l.nhashedsyms = len(st.hashed64Syms) + len(st.hashedSyms) for _, r := range l.objs[goObjStart:] { loadObjRefs(l, r, arch) @@ -2252,15 +2269,15 @@ func loadObjRefs(l *Loader, r *oReader, arch *sys.Arch) { osym := r.Sym(ndef + i) name := osym.Name(r.Reader) v := abiToVer(osym.ABI(), r.version) + gi := l.LookupOrCreateSym(name, v) + r.syms[ndef+i] = gi if osym.IsLinkname() { // Check if a linkname reference is allowed. // Only check references (pull), not definitions (push), // so push is always allowed. // Linkname is always a non-package reference. - checkLinkname(r.unit.Lib.Pkg, name) + l.checkLinkname(r.unit.Lib.Pkg, name, gi) } - r.syms[ndef+i] = l.LookupOrCreateSym(name, v) - gi := r.syms[ndef+i] if osym.Local() { l.SetAttrLocal(gi, true) } @@ -2307,30 +2324,27 @@ func abiToVer(abi uint16, localSymVersion int) int { // A list of blocked linknames. Some linknames are allowed only // in specific packages. This maps symbol names to allowed packages. -// If a name is not in this map, and not with a blocked prefix (see -// blockedLinknamePrefixes), it is allowed everywhere. -// If a name is in this map, it is allowed only in listed packages. +// If a name is not in this map, it is allowed iff the definition +// has a linkname (push). +// If a name is in this map, it is allowed only in listed packages, +// even if it has a linknamed definition. var blockedLinknames = map[string][]string{ // coroutines - "runtime.coroexit": nil, - "runtime.corostart": nil, "runtime.coroswitch": {"iter"}, "runtime.newcoro": {"iter"}, // weak references "internal/weak.runtime_registerWeakPointer": {"internal/weak"}, "internal/weak.runtime_makeStrongFromWeak": {"internal/weak"}, - "runtime.getOrAddWeakHandle": nil, } -// A list of blocked linkname prefixes (packages). -var blockedLinknamePrefixes = []string{ - "internal/weak.", - "internal/concurrent.", -} +// check if a linkname reference to symbol s from pkg is allowed +func (l *Loader) checkLinkname(pkg, name string, s Sym) { + if l.flags&FlagCheckLinkname == 0 { + return + } -func checkLinkname(pkg, name string) { error := func() { - log.Fatalf("linkname or assembly reference of %s is not allowed in package %s", name, pkg) + log.Fatalf("%s: invalid reference to %s", pkg, name) } pkgs, ok := blockedLinknames[name] if ok { @@ -2341,11 +2355,26 @@ func checkLinkname(pkg, name string) { } error() } - for _, p := range blockedLinknamePrefixes { - if strings.HasPrefix(name, p) { - error() - } + r, li := l.toLocal(s) + if r == l.extReader { // referencing external symbol is okay + return } + if !r.Std() { // For now, only check for symbols defined in std + return + } + if r.unit.Lib.Pkg == pkg { // assembly reference from same package + return + } + osym := r.Sym(li) + if osym.IsLinkname() || osym.ABIWrapper() { + // Allow if the def has a linkname (push). + // ABI wrapper usually wraps an assembly symbol, a linknamed symbol, + // or an external symbol, or provide access of a Go symbol to assembly. + // For now, allow ABI wrappers. + // TODO: check the wrapped symbol? + return + } + error() } // TopLevelSym tests a symbol (by name and kind) to determine whether diff --git a/src/cmd/link/link_test.go b/src/cmd/link/link_test.go index 3abec64c5d..1ce484fe61 100644 --- a/src/cmd/link/link_test.go +++ b/src/cmd/link/link_test.go @@ -1416,7 +1416,7 @@ func TestRandLayout(t *testing.T) { } } -func TestBlockedLinkname(t *testing.T) { +func TestCheckLinkname(t *testing.T) { // Test that code containing blocked linknames does not build. testenv.MustHaveGoBuild(t) t.Parallel() @@ -1433,10 +1433,13 @@ func TestBlockedLinkname(t *testing.T) { {"push.go", true}, // pull linkname of blocked symbol is not ok {"coro.go", false}, - {"weak.go", false}, {"coro_var.go", false}, // assembly reference is not ok {"coro_asm", false}, + // pull-only linkname is not ok + {"coro2.go", false}, + // legacy bad linkname is ok, for now + {"fastrand.go", true}, } for _, test := range tests { test := test @@ -1444,7 +1447,7 @@ func TestBlockedLinkname(t *testing.T) { t.Parallel() src := filepath.Join("testdata", "linkname", test.src) exe := filepath.Join(tmpdir, test.src+".exe") - cmd := testenv.Command(t, testenv.GoToolPath(t), "build", "-o", exe, src) + cmd := testenv.Command(t, testenv.GoToolPath(t), "build", "-ldflags=-checklinkname=1", "-o", exe, src) out, err := cmd.CombinedOutput() if test.ok && err != nil { t.Errorf("build failed unexpectedly: %v:\n%s", err, out) diff --git a/src/cmd/link/testdata/linkname/coro2.go b/src/cmd/link/testdata/linkname/coro2.go new file mode 100644 index 0000000000..ae47147670 --- /dev/null +++ b/src/cmd/link/testdata/linkname/coro2.go @@ -0,0 +1,17 @@ +// Copyright 2024 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Linkname corostart is not allowed, as it doesn't have +// a linknamed definition. + +package main + +import _ "unsafe" + +//go:linkname corostart runtime.corostart +func corostart() + +func main() { + corostart() +} diff --git a/src/cmd/link/testdata/linkname/fastrand.go b/src/cmd/link/testdata/linkname/fastrand.go new file mode 100644 index 0000000000..ce51e2a7f3 --- /dev/null +++ b/src/cmd/link/testdata/linkname/fastrand.go @@ -0,0 +1,18 @@ +// Copyright 2024 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Linkname fastrand is allowed _for now_, as it has a +// linknamed definition, for legacy reason. +// NOTE: this may not be allowed in the future. Don't do this! + +package main + +import _ "unsafe" + +//go:linkname fastrand runtime.fastrand +func fastrand() uint32 + +func main() { + println(fastrand()) +} diff --git a/src/cmd/link/testdata/linkname/weak.go b/src/cmd/link/testdata/linkname/weak.go deleted file mode 100644 index 2bf0fbcbab..0000000000 --- a/src/cmd/link/testdata/linkname/weak.go +++ /dev/null @@ -1,22 +0,0 @@ -// Copyright 2024 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -// Linkname generic functions in internal/weak is not -// allowed; legitimate instantiation is ok. - -package main - -import ( - "unique" - "unsafe" -) - -//go:linkname weakMake internal/weak.Make[string] -func weakMake(string) unsafe.Pointer - -func main() { - h := unique.Make("xxx") - println(h.Value()) - weakMake("xxx") -} diff --git a/src/cmd/nm/nm.go b/src/cmd/nm/nm.go index 78fa60014b..62cf155362 100644 --- a/src/cmd/nm/nm.go +++ b/src/cmd/nm/nm.go @@ -13,6 +13,7 @@ import ( "sort" "cmd/internal/objfile" + "cmd/internal/telemetry" ) const helpText = `usage: go tool nm [options] file... @@ -67,8 +68,11 @@ func (nflag) String() string { func main() { log.SetFlags(0) + telemetry.Start() flag.Usage = usage flag.Parse() + telemetry.Inc("nm/invocations") + telemetry.CountFlags("nm/flag:", *flag.CommandLine) switch *sortOrder { case "address", "name", "none", "size": diff --git a/src/cmd/objdump/main.go b/src/cmd/objdump/main.go index 6605f8a60c..bd1762636d 100644 --- a/src/cmd/objdump/main.go +++ b/src/cmd/objdump/main.go @@ -41,6 +41,7 @@ import ( "strings" "cmd/internal/objfile" + "cmd/internal/telemetry" ) var printCode = flag.Bool("S", false, "print Go code alongside assembly") @@ -57,9 +58,12 @@ func usage() { func main() { log.SetFlags(0) log.SetPrefix("objdump: ") + telemetry.Start() flag.Usage = usage flag.Parse() + telemetry.Inc("objdump/invocations") + telemetry.CountFlags("objdump/flag:", *flag.CommandLine) if flag.NArg() != 1 && flag.NArg() != 3 { usage() } diff --git a/src/cmd/pack/pack.go b/src/cmd/pack/pack.go index 412ea36d60..6d7eaf7e5b 100644 --- a/src/cmd/pack/pack.go +++ b/src/cmd/pack/pack.go @@ -6,6 +6,7 @@ package main import ( "cmd/internal/archive" + "cmd/internal/telemetry" "fmt" "io" "io/fs" @@ -30,6 +31,7 @@ func usage() { func main() { log.SetFlags(0) log.SetPrefix("pack: ") + telemetry.Start() // need "pack op archive" at least. if len(os.Args) < 3 { log.Print("not enough arguments") @@ -37,6 +39,8 @@ func main() { usage() } setOp(os.Args[1]) + telemetry.Inc("pack/invocations") + telemetry.Inc("pack/op:" + string(op)) var ar *Archive switch op { case 'p': diff --git a/src/cmd/pprof/pprof.go b/src/cmd/pprof/pprof.go index 24d6ee04a0..69d3201cdb 100644 --- a/src/cmd/pprof/pprof.go +++ b/src/cmd/pprof/pprof.go @@ -12,6 +12,7 @@ package main import ( "crypto/tls" "debug/dwarf" + "flag" "fmt" "io" "net/http" @@ -24,18 +25,23 @@ import ( "time" "cmd/internal/objfile" + "cmd/internal/telemetry" "github.com/google/pprof/driver" "github.com/google/pprof/profile" ) func main() { + telemetry.Start() + telemetry.Inc("pprof/invocations") options := &driver.Options{ Fetch: new(fetcher), Obj: new(objTool), UI: newUI(), } - if err := driver.PProf(options); err != nil { + err := driver.PProf(options) + telemetry.CountFlags("pprof/flag:", *flag.CommandLine) // pprof will use the flag package as its default + if err != nil { fmt.Fprintf(os.Stderr, "%v\n", err) os.Exit(2) } diff --git a/src/cmd/preprofile/main.go b/src/cmd/preprofile/main.go index f29b5279e2..78063c1463 100644 --- a/src/cmd/preprofile/main.go +++ b/src/cmd/preprofile/main.go @@ -18,6 +18,7 @@ import ( "bufio" "cmd/internal/objabi" "cmd/internal/pgo" + "cmd/internal/telemetry" "flag" "fmt" "log" @@ -31,8 +32,8 @@ func usage() { } var ( - output = flag.String("o", "", "output file path") - input = flag.String("i", "", "input pprof file path") + output = flag.String("o", "", "output file path") + input = flag.String("i", "", "input pprof file path") ) func preprocess(profileFile string, outputFile string) error { @@ -72,9 +73,12 @@ func main() { log.SetFlags(0) log.SetPrefix("preprofile: ") + telemetry.Start() flag.Usage = usage flag.Parse() + telemetry.Inc("preprofile/invocations") + telemetry.CountFlags("preprofile/flag:", *flag.CommandLine) if *input == "" { log.Print("Input pprof path required (-i)") usage() diff --git a/src/cmd/test2json/main.go b/src/cmd/test2json/main.go index 09d5fcec79..36e7cf90b5 100644 --- a/src/cmd/test2json/main.go +++ b/src/cmd/test2json/main.go @@ -96,6 +96,7 @@ import ( "os/exec" "os/signal" + "cmd/internal/telemetry" "cmd/internal/test2json" ) @@ -115,8 +116,12 @@ func ignoreSignals() { } func main() { + telemetry.Start() + flag.Usage = usage flag.Parse() + telemetry.Inc("test2json/invocations") + telemetry.CountFlags("test2json/flag:", *flag.CommandLine) var mode test2json.Mode if *flagT { diff --git a/src/cmd/trace/main.go b/src/cmd/trace/main.go index 5f0d6f612b..2c0b15623d 100644 --- a/src/cmd/trace/main.go +++ b/src/cmd/trace/main.go @@ -7,6 +7,7 @@ package main import ( "bufio" "cmd/internal/browser" + "cmd/internal/telemetry" cmdv2 "cmd/trace/v2" "flag" "fmt" @@ -65,11 +66,14 @@ var ( ) func main() { + telemetry.Start() flag.Usage = func() { fmt.Fprint(os.Stderr, usageMessage) os.Exit(2) } flag.Parse() + telemetry.Inc("trace/invocations") + telemetry.CountFlags("trace/flag:", *flag.CommandLine) // Go 1.7 traces embed symbol info and does not require the binary. // But we optionally accept binary as first arg for Go 1.5 traces. diff --git a/src/cmd/vendor/golang.org/x/mod/modfile/rule.go b/src/cmd/vendor/golang.org/x/mod/modfile/rule.go index 0e7b7e2679..66dcaf9803 100644 --- a/src/cmd/vendor/golang.org/x/mod/modfile/rule.go +++ b/src/cmd/vendor/golang.org/x/mod/modfile/rule.go @@ -38,6 +38,7 @@ type File struct { Module *Module Go *Go Toolchain *Toolchain + Godebug []*Godebug Require []*Require Exclude []*Exclude Replace []*Replace @@ -65,6 +66,13 @@ type Toolchain struct { Syntax *Line } +// A Godebug is a single godebug key=value statement. +type Godebug struct { + Key string + Value string + Syntax *Line +} + // An Exclude is a single exclude statement. type Exclude struct { Mod module.Version @@ -289,7 +297,7 @@ func parseToFile(file string, data []byte, fix VersionFixer, strict bool) (parse }) } continue - case "module", "require", "exclude", "replace", "retract": + case "module", "godebug", "require", "exclude", "replace", "retract": for _, l := range x.Line { f.add(&errs, x, l, x.Token[0], l.Token, fix, strict) } @@ -308,7 +316,9 @@ var laxGoVersionRE = lazyregexp.New(`^v?(([1-9][0-9]*)\.(0|[1-9][0-9]*))([^0-9]. // Toolchains must be named beginning with `go1`, // like "go1.20.3" or "go1.20.3-gccgo". As a special case, "default" is also permitted. -// TODO(samthanawalla): Replace regex with https://pkg.go.dev/go/version#IsValid in 1.23+ +// Note that this regexp is a much looser condition than go/version.IsValid, +// for forward compatibility. +// (This code has to be work to identify new toolchains even if we tweak the syntax in the future.) var ToolchainRE = lazyregexp.New(`^default$|^go1($|\.)`) func (f *File) add(errs *ErrorList, block *LineBlock, line *Line, verb string, args []string, fix VersionFixer, strict bool) { @@ -384,7 +394,7 @@ func (f *File) add(errs *ErrorList, block *LineBlock, line *Line, verb string, a if len(args) != 1 { errorf("toolchain directive expects exactly one argument") return - } else if strict && !ToolchainRE.MatchString(args[0]) { + } else if !ToolchainRE.MatchString(args[0]) { errorf("invalid toolchain version '%s': must match format go1.23.0 or default", args[0]) return } @@ -412,6 +422,22 @@ func (f *File) add(errs *ErrorList, block *LineBlock, line *Line, verb string, a } f.Module.Mod = module.Version{Path: s} + case "godebug": + if len(args) != 1 || strings.ContainsAny(args[0], "\"`',") { + errorf("usage: godebug key=value") + return + } + key, value, ok := strings.Cut(args[0], "=") + if !ok { + errorf("usage: godebug key=value") + return + } + f.Godebug = append(f.Godebug, &Godebug{ + Key: key, + Value: value, + Syntax: line, + }) + case "require", "exclude": if len(args) != 2 { errorf("usage: %s module/path v1.2.3", verb) @@ -654,6 +680,22 @@ func (f *WorkFile) add(errs *ErrorList, line *Line, verb string, args []string, f.Toolchain = &Toolchain{Syntax: line} f.Toolchain.Name = args[0] + case "godebug": + if len(args) != 1 || strings.ContainsAny(args[0], "\"`',") { + errorf("usage: godebug key=value") + return + } + key, value, ok := strings.Cut(args[0], "=") + if !ok { + errorf("usage: godebug key=value") + return + } + f.Godebug = append(f.Godebug, &Godebug{ + Key: key, + Value: value, + Syntax: line, + }) + case "use": if len(args) != 1 { errorf("usage: %s local/dir", verb) @@ -929,6 +971,15 @@ func (f *File) Format() ([]byte, error) { // Cleanup cleans out all the cleared entries. func (f *File) Cleanup() { w := 0 + for _, g := range f.Godebug { + if g.Key != "" { + f.Godebug[w] = g + w++ + } + } + f.Godebug = f.Godebug[:w] + + w = 0 for _, r := range f.Require { if r.Mod.Path != "" { f.Require[w] = r @@ -1027,6 +1078,45 @@ func (f *File) AddToolchainStmt(name string) error { return nil } +// AddGodebug sets the first godebug line for key to value, +// preserving any existing comments for that line and removing all +// other godebug lines for key. +// +// If no line currently exists for key, AddGodebug adds a new line +// at the end of the last godebug block. +func (f *File) AddGodebug(key, value string) error { + need := true + for _, g := range f.Godebug { + if g.Key == key { + if need { + g.Value = value + f.Syntax.updateLine(g.Syntax, "godebug", key+"="+value) + need = false + } else { + g.Syntax.markRemoved() + *g = Godebug{} + } + } + } + + if need { + f.addNewGodebug(key, value) + } + return nil +} + +// addNewGodebug adds a new godebug key=value line at the end +// of the last godebug block, regardless of any existing godebug lines for key. +func (f *File) addNewGodebug(key, value string) { + line := f.Syntax.addLine(nil, "godebug", key+"="+value) + g := &Godebug{ + Key: key, + Value: value, + Syntax: line, + } + f.Godebug = append(f.Godebug, g) +} + // AddRequire sets the first require line for path to version vers, // preserving any existing comments for that line and removing all // other lines for path. @@ -1334,6 +1424,16 @@ func (f *File) SetRequireSeparateIndirect(req []*Require) { f.SortBlocks() } +func (f *File) DropGodebug(key string) error { + for _, g := range f.Godebug { + if g.Key == key { + g.Syntax.markRemoved() + *g = Godebug{} + } + } + return nil +} + func (f *File) DropRequire(path string) error { for _, r := range f.Require { if r.Mod.Path == path { diff --git a/src/cmd/vendor/golang.org/x/mod/modfile/work.go b/src/cmd/vendor/golang.org/x/mod/modfile/work.go index d7b99376eb..8f54897cf7 100644 --- a/src/cmd/vendor/golang.org/x/mod/modfile/work.go +++ b/src/cmd/vendor/golang.org/x/mod/modfile/work.go @@ -14,6 +14,7 @@ import ( type WorkFile struct { Go *Go Toolchain *Toolchain + Godebug []*Godebug Use []*Use Replace []*Replace @@ -68,7 +69,7 @@ func ParseWork(file string, data []byte, fix VersionFixer) (*WorkFile, error) { Err: fmt.Errorf("unknown block type: %s", strings.Join(x.Token, " ")), }) continue - case "use", "replace": + case "godebug", "use", "replace": for _, l := range x.Line { f.add(&errs, l, x.Token[0], l.Token, fix) } @@ -184,6 +185,55 @@ func (f *WorkFile) DropToolchainStmt() { } } +// AddGodebug sets the first godebug line for key to value, +// preserving any existing comments for that line and removing all +// other godebug lines for key. +// +// If no line currently exists for key, AddGodebug adds a new line +// at the end of the last godebug block. +func (f *WorkFile) AddGodebug(key, value string) error { + need := true + for _, g := range f.Godebug { + if g.Key == key { + if need { + g.Value = value + f.Syntax.updateLine(g.Syntax, "godebug", key+"="+value) + need = false + } else { + g.Syntax.markRemoved() + *g = Godebug{} + } + } + } + + if need { + f.addNewGodebug(key, value) + } + return nil +} + +// addNewGodebug adds a new godebug key=value line at the end +// of the last godebug block, regardless of any existing godebug lines for key. +func (f *WorkFile) addNewGodebug(key, value string) { + line := f.Syntax.addLine(nil, "godebug", key+"="+value) + g := &Godebug{ + Key: key, + Value: value, + Syntax: line, + } + f.Godebug = append(f.Godebug, g) +} + +func (f *WorkFile) DropGodebug(key string) error { + for _, g := range f.Godebug { + if g.Key == key { + g.Syntax.markRemoved() + *g = Godebug{} + } + } + return nil +} + func (f *WorkFile) AddUse(diskPath, modulePath string) error { need := true for _, d := range f.Use { diff --git a/src/cmd/vendor/golang.org/x/telemetry/internal/counter/file.go b/src/cmd/vendor/golang.org/x/telemetry/internal/counter/file.go index 43297f9b74..a0a1bd4cfb 100644 --- a/src/cmd/vendor/golang.org/x/telemetry/internal/counter/file.go +++ b/src/cmd/vendor/golang.org/x/telemetry/internal/counter/file.go @@ -14,13 +14,11 @@ import ( "path/filepath" "runtime" "runtime/debug" - "strings" "sync" "sync/atomic" "time" "unsafe" - "golang.org/x/mod/module" "golang.org/x/telemetry/internal/mmap" "golang.org/x/telemetry/internal/telemetry" ) @@ -135,10 +133,10 @@ func (f *file) init(begin, end time.Time) { return } - goVers, progPkgPath, prog, progVers := programInfo(info) + goVers, progPath, progVers := telemetry.ProgramInfo(info) f.meta = fmt.Sprintf("TimeBegin: %s\nTimeEnd: %s\nProgram: %s\nVersion: %s\nGoVersion: %s\nGOOS: %s\nGOARCH: %s\n\n", begin.Format(time.RFC3339), end.Format(time.RFC3339), - progPkgPath, progVers, goVers, runtime.GOOS, runtime.GOARCH) + progPath, progVers, goVers, runtime.GOOS, runtime.GOARCH) if len(f.meta) > maxMetaLen { // should be impossible for our use f.err = fmt.Errorf("metadata too long") return @@ -146,28 +144,10 @@ func (f *file) init(begin, end time.Time) { if progVers != "" { progVers = "@" + progVers } - prefix := fmt.Sprintf("%s%s-%s-%s-%s-", prog, progVers, goVers, runtime.GOOS, runtime.GOARCH) + prefix := fmt.Sprintf("%s%s-%s-%s-%s-", path.Base(progPath), progVers, goVers, runtime.GOOS, runtime.GOARCH) f.namePrefix = filepath.Join(dir, prefix) } -func programInfo(info *debug.BuildInfo) (goVers, progPkgPath, prog, progVers string) { - goVers = info.GoVersion - if strings.Contains(goVers, "devel") || strings.Contains(goVers, "-") { - goVers = "devel" - } - progPkgPath = info.Path - if progPkgPath == "" { - progPkgPath = strings.TrimSuffix(filepath.Base(os.Args[0]), ".exe") - } - prog = path.Base(progPkgPath) - progVers = info.Main.Version - if strings.Contains(progVers, "devel") || module.IsPseudoVersion(progVers) { - // we don't want to track pseudo versions, but may want to track prereleases. - progVers = "devel" - } - return goVers, progPkgPath, prog, progVers -} - // filename returns the name of the file to use for f, // given the current time now. // It also returns the time when that name will no longer be valid diff --git a/src/cmd/vendor/golang.org/x/telemetry/internal/crashmonitor/crash_go123.go b/src/cmd/vendor/golang.org/x/telemetry/internal/crashmonitor/crash_go123.go index 2e0c1b32e3..8f00c4856b 100644 --- a/src/cmd/vendor/golang.org/x/telemetry/internal/crashmonitor/crash_go123.go +++ b/src/cmd/vendor/golang.org/x/telemetry/internal/crashmonitor/crash_go123.go @@ -7,8 +7,11 @@ package crashmonitor -import "runtime/debug" +import ( + "os" + "runtime/debug" +) func init() { - setCrashOutput = debug.SetCrashOutput + setCrashOutput = func(f *os.File) error { return debug.SetCrashOutput(f, debug.CrashOptions{}) } } diff --git a/src/cmd/vendor/golang.org/x/telemetry/internal/telemetry/proginfo.go b/src/cmd/vendor/golang.org/x/telemetry/internal/telemetry/proginfo.go new file mode 100644 index 0000000000..20be9664fc --- /dev/null +++ b/src/cmd/vendor/golang.org/x/telemetry/internal/telemetry/proginfo.go @@ -0,0 +1,53 @@ +// Copyright 2024 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package telemetry + +import ( + "os" + "path/filepath" + "runtime/debug" + "strings" + + "golang.org/x/mod/module" +) + +// IsToolchainProgram reports whether a program with the given path is a Go +// toolchain program. +func IsToolchainProgram(progPath string) bool { + return strings.HasPrefix(progPath, "cmd/") +} + +// ProgramInfo extracts the go version, program package path, and program +// version to use for counter files. +// +// For programs in the Go toolchain, the program version will be the same as +// the Go version, and will typically be of the form "go1.2.3", not a semantic +// version of the form "v1.2.3". Go versions may also include spaces and +// special characters. +func ProgramInfo(info *debug.BuildInfo) (goVers, progPath, progVers string) { + goVers = info.GoVersion + if strings.Contains(goVers, "devel") || strings.Contains(goVers, "-") { + goVers = "devel" + } + + progPath = info.Path + if progPath == "" { + progPath = strings.TrimSuffix(filepath.Base(os.Args[0]), ".exe") + } + + // Main module version information is not populated for the cmd module, but + // we can re-use the Go version here. + if IsToolchainProgram(progPath) { + progVers = goVers + } else { + progVers = info.Main.Version + if strings.Contains(progVers, "devel") || module.IsPseudoVersion(progVers) { + // We don't want to track pseudo versions, but may want to track prereleases. + progVers = "devel" + } + } + + return goVers, progPath, progVers +} diff --git a/src/cmd/vendor/golang.org/x/telemetry/internal/upload/date.go b/src/cmd/vendor/golang.org/x/telemetry/internal/upload/date.go index 26e65bd9bf..22e4e8aa30 100644 --- a/src/cmd/vendor/golang.org/x/telemetry/internal/upload/date.go +++ b/src/cmd/vendor/golang.org/x/telemetry/internal/upload/date.go @@ -18,7 +18,7 @@ import ( var distantPast = 21 * 24 * time.Hour // reports that are too old (21 days) are not uploaded -func (u *Uploader) tooOld(date string, uploadStartTime time.Time) bool { +func (u *uploader) tooOld(date string, uploadStartTime time.Time) bool { t, err := time.Parse("2006-01-02", date) if err != nil { u.logger.Printf("tooOld: %v", err) @@ -31,7 +31,7 @@ func (u *Uploader) tooOld(date string, uploadStartTime time.Time) bool { // counterDateSpan parses the counter file named fname and returns the (begin, // end) span recorded in its metadata, or an error if this data could not be // extracted. -func (u *Uploader) counterDateSpan(fname string) (begin, end time.Time, _ error) { +func (u *uploader) counterDateSpan(fname string) (begin, end time.Time, _ error) { parsed, err := u.parseCountFile(fname) if err != nil { return time.Time{}, time.Time{}, err @@ -61,7 +61,7 @@ type parsedCache struct { m map[string]*counter.File } -func (u *Uploader) parseCountFile(fname string) (*counter.File, error) { +func (u *uploader) parseCountFile(fname string) (*counter.File, error) { u.cache.mu.Lock() defer u.cache.mu.Unlock() if u.cache.m == nil { diff --git a/src/cmd/vendor/golang.org/x/telemetry/internal/upload/findwork.go b/src/cmd/vendor/golang.org/x/telemetry/internal/upload/findwork.go index 6bd559a841..f1490be3a5 100644 --- a/src/cmd/vendor/golang.org/x/telemetry/internal/upload/findwork.go +++ b/src/cmd/vendor/golang.org/x/telemetry/internal/upload/findwork.go @@ -22,7 +22,7 @@ type work struct { // find all the files that look like counter files or reports // that need to be uploaded. (There may be unexpected leftover files // and uploading is supposed to be idempotent.) -func (u *Uploader) findWork() work { +func (u *uploader) findWork() work { localdir, uploaddir := u.dir.LocalDir(), u.dir.UploadDir() var ans work fis, err := os.ReadDir(localdir) diff --git a/src/cmd/vendor/golang.org/x/telemetry/internal/upload/reports.go b/src/cmd/vendor/golang.org/x/telemetry/internal/upload/reports.go index bb95971932..41757897ea 100644 --- a/src/cmd/vendor/golang.org/x/telemetry/internal/upload/reports.go +++ b/src/cmd/vendor/golang.org/x/telemetry/internal/upload/reports.go @@ -21,7 +21,7 @@ import ( ) // reports generates reports from inactive count files -func (u *Uploader) reports(todo *work) ([]string, error) { +func (u *uploader) reports(todo *work) ([]string, error) { if mode, _ := u.dir.Mode(); mode == "off" { return nil, nil // no reports } @@ -104,7 +104,7 @@ func notNeeded(date string, todo work) bool { return false } -func (u *Uploader) deleteFiles(files []string) { +func (u *uploader) deleteFiles(files []string) { for _, f := range files { if err := os.Remove(f); err != nil { // this could be a race condition. @@ -117,7 +117,7 @@ func (u *Uploader) deleteFiles(files []string) { // createReport for all the count files for the same date. // returns the absolute path name of the file containing the report -func (u *Uploader) createReport(start time.Time, expiryDate string, countFiles []string, lastWeek string) (string, error) { +func (u *uploader) createReport(start time.Time, expiryDate string, countFiles []string, lastWeek string) (string, error) { uploadOK := true mode, asof := u.dir.Mode() if mode != "on" { @@ -147,6 +147,7 @@ func (u *Uploader) createReport(start time.Time, expiryDate string, countFiles [ } var succeeded bool for _, f := range countFiles { + fok := false x, err := u.parseCountFile(f) if err != nil { u.logger.Printf("Unparseable count file %s: %v", filepath.Base(f), err) @@ -162,12 +163,14 @@ func (u *Uploader) createReport(start time.Time, expiryDate string, countFiles [ prog.Counters[k] += int64(v) } succeeded = true + fok = true + } + if !fok { + u.logger.Printf("no counters found in %s", f) } } if !succeeded { - // TODO(rfindley): this isn't right: a count file is not unparseable just - // because it has no counters - return "", fmt.Errorf("all %d count files for %s were unparseable", len(countFiles), expiryDate) + return "", fmt.Errorf("none of the %d count files for %s contained counters", len(countFiles), expiryDate) } // 1. generate the local report localContents, err := json.MarshalIndent(report, "", " ") diff --git a/src/cmd/vendor/golang.org/x/telemetry/internal/upload/run.go b/src/cmd/vendor/golang.org/x/telemetry/internal/upload/run.go index 2fb9fa670a..eba13b1a57 100644 --- a/src/cmd/vendor/golang.org/x/telemetry/internal/upload/run.go +++ b/src/cmd/vendor/golang.org/x/telemetry/internal/upload/run.go @@ -30,9 +30,24 @@ type RunConfig struct { StartTime time.Time // if set, overrides the upload start time } -// Uploader encapsulates a single upload operation, carrying parameters and +// Run generates and uploads reports, as allowed by the mode file. +func Run(config RunConfig) error { + defer func() { + if err := recover(); err != nil { + log.Printf("upload recover: %v", err) + } + }() + uploader, err := newUploader(config) + if err != nil { + return err + } + defer uploader.Close() + return uploader.Run() +} + +// uploader encapsulates a single upload operation, carrying parameters and // shared state. -type Uploader struct { +type uploader struct { // config is used to select counters to upload. config *telemetry.UploadConfig // configVersion string // version of the config @@ -47,11 +62,11 @@ type Uploader struct { logger *log.Logger } -// NewUploader creates a new uploader to use for running the upload for the +// newUploader creates a new uploader to use for running the upload for the // given config. // -// Uploaders should only be used for one call to [Run]. -func NewUploader(rcfg RunConfig) (*Uploader, error) { +// Uploaders should only be used for one call to [uploader.Run]. +func newUploader(rcfg RunConfig) (*uploader, error) { // Determine the upload directory. var dir telemetry.Dir if rcfg.TelemetryDir != "" { @@ -108,7 +123,7 @@ func NewUploader(rcfg RunConfig) (*Uploader, error) { startTime = rcfg.StartTime } - return &Uploader{ + return &uploader{ config: config, configVersion: configVersion, dir: dir, @@ -121,7 +136,7 @@ func NewUploader(rcfg RunConfig) (*Uploader, error) { } // Close cleans up any resources associated with the uploader. -func (u *Uploader) Close() error { +func (u *uploader) Close() error { if u.logFile == nil { return nil } @@ -129,7 +144,7 @@ func (u *Uploader) Close() error { } // Run generates and uploads reports -func (u *Uploader) Run() error { +func (u *uploader) Run() error { if telemetry.DisabledOnPlatform { return nil } @@ -174,9 +189,13 @@ func debugLogFile(debugDir string) (*os.File, error) { } prog := path.Base(progPkgPath) progVers := info.Main.Version - fname := filepath.Join(debugDir, fmt.Sprintf("%s-%s-%s-%4d%02d%02d-%d.log", - prog, progVers, goVers, year, month, day, os.Getpid())) - fname = strings.ReplaceAll(fname, " ", "") + if progVers == "(devel)" { // avoid special characters in created file names + progVers = "devel" + } + logBase := strings.ReplaceAll( + fmt.Sprintf("%s-%s-%s-%4d%02d%02d-%d.log", prog, progVers, goVers, year, month, day, os.Getpid()), + " ", "") + fname := filepath.Join(debugDir, logBase) if _, err := os.Stat(fname); err == nil { // This process previously called upload.Run return nil, nil diff --git a/src/cmd/vendor/golang.org/x/telemetry/internal/upload/upload.go b/src/cmd/vendor/golang.org/x/telemetry/internal/upload/upload.go index bec2230837..0e75d09c01 100644 --- a/src/cmd/vendor/golang.org/x/telemetry/internal/upload/upload.go +++ b/src/cmd/vendor/golang.org/x/telemetry/internal/upload/upload.go @@ -22,7 +22,7 @@ var ( // uploadReportDate returns the date component of the upload file name, or "" if the // date was unmatched. -func (u *Uploader) uploadReportDate(fname string) time.Time { +func (u *uploader) uploadReportDate(fname string) time.Time { match := dateRE.FindStringSubmatch(fname) if match == nil || len(match) < 2 { u.logger.Printf("malformed report name: missing date: %q", filepath.Base(fname)) @@ -36,7 +36,7 @@ func (u *Uploader) uploadReportDate(fname string) time.Time { return d } -func (u *Uploader) uploadReport(fname string) { +func (u *uploader) uploadReport(fname string) { thisInstant := u.startTime // TODO(rfindley): use uploadReportDate here, once we've done a gopls release. @@ -60,7 +60,7 @@ func (u *Uploader) uploadReport(fname string) { } // try to upload the report, 'true' if successful -func (u *Uploader) uploadReportContents(fname string, buf []byte) bool { +func (u *uploader) uploadReportContents(fname string, buf []byte) bool { b := bytes.NewReader(buf) fdate := strings.TrimSuffix(filepath.Base(fname), ".json") fdate = fdate[len(fdate)-len("2006-01-02"):] diff --git a/src/cmd/vendor/golang.org/x/telemetry/start.go b/src/cmd/vendor/golang.org/x/telemetry/start.go index 2b6b15be5c..414c9fc5c9 100644 --- a/src/cmd/vendor/golang.org/x/telemetry/start.go +++ b/src/cmd/vendor/golang.org/x/telemetry/start.go @@ -17,7 +17,7 @@ import ( "golang.org/x/telemetry/counter" "golang.org/x/telemetry/internal/crashmonitor" "golang.org/x/telemetry/internal/telemetry" - "golang.org/x/telemetry/upload" + "golang.org/x/telemetry/internal/upload" ) // Config controls the behavior of [Start]. diff --git a/src/cmd/vendor/golang.org/x/telemetry/upload/upload.go b/src/cmd/vendor/golang.org/x/telemetry/upload/upload.go deleted file mode 100644 index 0e2fb455d8..0000000000 --- a/src/cmd/vendor/golang.org/x/telemetry/upload/upload.go +++ /dev/null @@ -1,33 +0,0 @@ -// Copyright 2023 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -package upload - -import ( - "log" - - "golang.org/x/telemetry/internal/upload" -) - -// TODO(rfindley): remove, in favor of all callers using Start. - -// A RunConfig controls the behavior of Run. -// The zero value RunConfig is the default behavior; fields may be set to -// override various reporting and uploading choices. -type RunConfig = upload.RunConfig - -// Run generates and uploads reports, as allowed by the mode file. -func Run(config RunConfig) error { - defer func() { - if err := recover(); err != nil { - log.Printf("upload recover: %v", err) - } - }() - uploader, err := upload.NewUploader(config) - if err != nil { - return err - } - defer uploader.Close() - return uploader.Run() -} diff --git a/src/cmd/vendor/modules.txt b/src/cmd/vendor/modules.txt index 44a0b69008..5c691b1fe6 100644 --- a/src/cmd/vendor/modules.txt +++ b/src/cmd/vendor/modules.txt @@ -25,7 +25,7 @@ golang.org/x/arch/x86/x86asm # golang.org/x/build v0.0.0-20240222153247-cf4ed81bb19f ## explicit; go 1.21 golang.org/x/build/relnote -# golang.org/x/mod v0.17.1-0.20240507203540-6686f416970d +# golang.org/x/mod v0.17.1-0.20240514174713-c0bdc7bd01c9 ## explicit; go 1.18 golang.org/x/mod/internal/lazyregexp golang.org/x/mod/modfile @@ -45,7 +45,7 @@ golang.org/x/sync/semaphore golang.org/x/sys/plan9 golang.org/x/sys/unix golang.org/x/sys/windows -# golang.org/x/telemetry v0.0.0-20240510223629-51e8b5d718eb +# golang.org/x/telemetry v0.0.0-20240515213752-9ff3ad9b3e68 ## explicit; go 1.20 golang.org/x/telemetry golang.org/x/telemetry/counter @@ -57,7 +57,6 @@ golang.org/x/telemetry/internal/crashmonitor golang.org/x/telemetry/internal/mmap golang.org/x/telemetry/internal/telemetry golang.org/x/telemetry/internal/upload -golang.org/x/telemetry/upload # golang.org/x/term v0.18.0 ## explicit; go 1.18 golang.org/x/term diff --git a/src/cmd/vet/main.go b/src/cmd/vet/main.go index 7b9a700635..eff82dcc71 100644 --- a/src/cmd/vet/main.go +++ b/src/cmd/vet/main.go @@ -6,6 +6,8 @@ package main import ( "cmd/internal/objabi" + "cmd/internal/telemetry" + "flag" "golang.org/x/tools/go/analysis/unitchecker" @@ -45,8 +47,10 @@ import ( ) func main() { + telemetry.Start() objabi.AddVersionFlag() + telemetry.Inc("vet/invocations") unitchecker.Main( appends.Analyzer, asmdecl.Analyzer, @@ -82,4 +86,8 @@ func main() { unsafeptr.Analyzer, unusedresult.Analyzer, ) + + // It's possible that unitchecker will exit early. In + // those cases the flags won't be counted. + telemetry.CountFlags("vet/flag:", *flag.CommandLine) } diff --git a/src/crypto/aes/block.go b/src/crypto/aes/block.go index 53308ae92e..618eb7752a 100644 --- a/src/crypto/aes/block.go +++ b/src/crypto/aes/block.go @@ -36,17 +36,15 @@ package aes -import ( - "encoding/binary" -) +import "internal/byteorder" // Encrypt one block from src into dst, using the expanded key xk. func encryptBlockGo(xk []uint32, dst, src []byte) { _ = src[15] // early bounds check - s0 := binary.BigEndian.Uint32(src[0:4]) - s1 := binary.BigEndian.Uint32(src[4:8]) - s2 := binary.BigEndian.Uint32(src[8:12]) - s3 := binary.BigEndian.Uint32(src[12:16]) + s0 := byteorder.BeUint32(src[0:4]) + s1 := byteorder.BeUint32(src[4:8]) + s2 := byteorder.BeUint32(src[8:12]) + s3 := byteorder.BeUint32(src[12:16]) // First round just XORs input with key. s0 ^= xk[0] @@ -80,19 +78,19 @@ func encryptBlockGo(xk []uint32, dst, src []byte) { s3 ^= xk[k+3] _ = dst[15] // early bounds check - binary.BigEndian.PutUint32(dst[0:4], s0) - binary.BigEndian.PutUint32(dst[4:8], s1) - binary.BigEndian.PutUint32(dst[8:12], s2) - binary.BigEndian.PutUint32(dst[12:16], s3) + byteorder.BePutUint32(dst[0:4], s0) + byteorder.BePutUint32(dst[4:8], s1) + byteorder.BePutUint32(dst[8:12], s2) + byteorder.BePutUint32(dst[12:16], s3) } // Decrypt one block from src into dst, using the expanded key xk. func decryptBlockGo(xk []uint32, dst, src []byte) { _ = src[15] // early bounds check - s0 := binary.BigEndian.Uint32(src[0:4]) - s1 := binary.BigEndian.Uint32(src[4:8]) - s2 := binary.BigEndian.Uint32(src[8:12]) - s3 := binary.BigEndian.Uint32(src[12:16]) + s0 := byteorder.BeUint32(src[0:4]) + s1 := byteorder.BeUint32(src[4:8]) + s2 := byteorder.BeUint32(src[8:12]) + s3 := byteorder.BeUint32(src[12:16]) // First round just XORs input with key. s0 ^= xk[0] @@ -126,10 +124,10 @@ func decryptBlockGo(xk []uint32, dst, src []byte) { s3 ^= xk[k+3] _ = dst[15] // early bounds check - binary.BigEndian.PutUint32(dst[0:4], s0) - binary.BigEndian.PutUint32(dst[4:8], s1) - binary.BigEndian.PutUint32(dst[8:12], s2) - binary.BigEndian.PutUint32(dst[12:16], s3) + byteorder.BePutUint32(dst[0:4], s0) + byteorder.BePutUint32(dst[4:8], s1) + byteorder.BePutUint32(dst[8:12], s2) + byteorder.BePutUint32(dst[12:16], s3) } // Apply sbox0 to each byte in w. @@ -150,7 +148,7 @@ func expandKeyGo(key []byte, enc, dec []uint32) { var i int nk := len(key) / 4 for i = 0; i < nk; i++ { - enc[i] = binary.BigEndian.Uint32(key[4*i:]) + enc[i] = byteorder.BeUint32(key[4*i:]) } for ; i < len(enc); i++ { t := enc[i-1] diff --git a/src/crypto/aes/ctr_s390x.go b/src/crypto/aes/ctr_s390x.go index e5249d1842..56b82d5885 100644 --- a/src/crypto/aes/ctr_s390x.go +++ b/src/crypto/aes/ctr_s390x.go @@ -9,7 +9,7 @@ package aes import ( "crypto/cipher" "crypto/internal/alias" - "encoding/binary" + "internal/byteorder" ) // Assert that aesCipherAsm implements the ctrAble interface. @@ -41,8 +41,8 @@ func (c *aesCipherAsm) NewCTR(iv []byte) cipher.Stream { } var ac aesctr ac.block = c - ac.ctr[0] = binary.BigEndian.Uint64(iv[0:]) // high bits - ac.ctr[1] = binary.BigEndian.Uint64(iv[8:]) // low bits + ac.ctr[0] = byteorder.BeUint64(iv[0:]) // high bits + ac.ctr[1] = byteorder.BeUint64(iv[8:]) // low bits ac.buffer = ac.storage[:0] return &ac } @@ -52,8 +52,8 @@ func (c *aesctr) refill() { c.buffer = c.storage[:streamBufferSize] c0, c1 := c.ctr[0], c.ctr[1] for i := 0; i < streamBufferSize; i += 16 { - binary.BigEndian.PutUint64(c.buffer[i+0:], c0) - binary.BigEndian.PutUint64(c.buffer[i+8:], c1) + byteorder.BePutUint64(c.buffer[i+0:], c0) + byteorder.BePutUint64(c.buffer[i+8:], c1) // Increment in big endian: c0 is high, c1 is low. c1++ diff --git a/src/crypto/aes/gcm_ppc64x.go b/src/crypto/aes/gcm_ppc64x.go index 3e6e9ab4c3..f1e85129a8 100644 --- a/src/crypto/aes/gcm_ppc64x.go +++ b/src/crypto/aes/gcm_ppc64x.go @@ -9,8 +9,8 @@ package aes import ( "crypto/cipher" "crypto/subtle" - "encoding/binary" "errors" + "internal/byteorder" "runtime" ) @@ -66,14 +66,14 @@ func (c *aesCipherAsm) NewGCM(nonceSize, tagSize int) (cipher.AEAD, error) { // Reverse the bytes in each 8 byte chunk // Load little endian, store big endian if runtime.GOARCH == "ppc64le" { - h1 = binary.LittleEndian.Uint64(hle[:8]) - h2 = binary.LittleEndian.Uint64(hle[8:]) + h1 = byteorder.LeUint64(hle[:8]) + h2 = byteorder.LeUint64(hle[8:]) } else { - h1 = binary.BigEndian.Uint64(hle[:8]) - h2 = binary.BigEndian.Uint64(hle[8:]) + h1 = byteorder.BeUint64(hle[:8]) + h2 = byteorder.BeUint64(hle[8:]) } - binary.BigEndian.PutUint64(hle[:8], h1) - binary.BigEndian.PutUint64(hle[8:], h2) + byteorder.BePutUint64(hle[:8], h1) + byteorder.BePutUint64(hle[8:], h2) gcmInit(&g.productTable, hle) return g, nil @@ -126,8 +126,8 @@ func (g *gcmAsm) counterCrypt(out, in []byte, counter *[gcmBlockSize]byte) { // increments the rightmost 32-bits of the count value by 1. func gcmInc32(counterBlock *[16]byte) { c := counterBlock[len(counterBlock)-4:] - x := binary.BigEndian.Uint32(c) + 1 - binary.BigEndian.PutUint32(c, x) + x := byteorder.BeUint32(c) + 1 + byteorder.BePutUint32(c, x) } // paddedGHASH pads data with zeroes until its length is a multiple of diff --git a/src/crypto/aes/gcm_s390x.go b/src/crypto/aes/gcm_s390x.go index 9da3e1a478..492ae5d83b 100644 --- a/src/crypto/aes/gcm_s390x.go +++ b/src/crypto/aes/gcm_s390x.go @@ -10,8 +10,8 @@ import ( "crypto/cipher" "crypto/internal/alias" "crypto/subtle" - "encoding/binary" "errors" + "internal/byteorder" "internal/cpu" ) @@ -25,14 +25,14 @@ type gcmCount [16]byte // inc increments the rightmost 32-bits of the count value by 1. func (x *gcmCount) inc() { - binary.BigEndian.PutUint32(x[len(x)-4:], binary.BigEndian.Uint32(x[len(x)-4:])+1) + byteorder.BePutUint32(x[len(x)-4:], byteorder.BeUint32(x[len(x)-4:])+1) } // gcmLengths writes len0 || len1 as big-endian values to a 16-byte array. func gcmLengths(len0, len1 uint64) [16]byte { v := [16]byte{} - binary.BigEndian.PutUint64(v[0:], len0) - binary.BigEndian.PutUint64(v[8:], len1) + byteorder.BePutUint64(v[0:], len0) + byteorder.BePutUint64(v[8:], len1) return v } diff --git a/src/crypto/cipher/gcm.go b/src/crypto/cipher/gcm.go index 5b28b61f70..505be50c6a 100644 --- a/src/crypto/cipher/gcm.go +++ b/src/crypto/cipher/gcm.go @@ -7,8 +7,8 @@ package cipher import ( "crypto/internal/alias" "crypto/subtle" - "encoding/binary" "errors" + "internal/byteorder" ) // AEAD is a cipher mode providing authenticated encryption with associated @@ -137,8 +137,8 @@ func newGCMWithNonceAndTagSize(cipher Block, nonceSize, tagSize int) (AEAD, erro // would expect, say, 4*key to be in index 4 of the table but due to // this bit ordering it will actually be in index 0010 (base 2) = 2. x := gcmFieldElement{ - binary.BigEndian.Uint64(key[:8]), - binary.BigEndian.Uint64(key[8:]), + byteorder.BeUint64(key[:8]), + byteorder.BeUint64(key[8:]), } g.productTable[reverseBits(1)] = x @@ -321,8 +321,8 @@ func (g *gcm) mul(y *gcmFieldElement) { // Horner's rule. There must be a multiple of gcmBlockSize bytes in blocks. func (g *gcm) updateBlocks(y *gcmFieldElement, blocks []byte) { for len(blocks) > 0 { - y.low ^= binary.BigEndian.Uint64(blocks) - y.high ^= binary.BigEndian.Uint64(blocks[8:]) + y.low ^= byteorder.BeUint64(blocks) + y.high ^= byteorder.BeUint64(blocks[8:]) g.mul(y) blocks = blocks[gcmBlockSize:] } @@ -345,7 +345,7 @@ func (g *gcm) update(y *gcmFieldElement, data []byte) { // and increments it. func gcmInc32(counterBlock *[16]byte) { ctr := counterBlock[len(counterBlock)-4:] - binary.BigEndian.PutUint32(ctr, binary.BigEndian.Uint32(ctr)+1) + byteorder.BePutUint32(ctr, byteorder.BeUint32(ctr)+1) } // sliceForAppend takes a slice and a requested number of bytes. It returns a @@ -401,8 +401,8 @@ func (g *gcm) deriveCounter(counter *[gcmBlockSize]byte, nonce []byte) { g.update(&y, nonce) y.high ^= uint64(len(nonce)) * 8 g.mul(&y) - binary.BigEndian.PutUint64(counter[:8], y.low) - binary.BigEndian.PutUint64(counter[8:], y.high) + byteorder.BePutUint64(counter[:8], y.low) + byteorder.BePutUint64(counter[8:], y.high) } } @@ -418,8 +418,8 @@ func (g *gcm) auth(out, ciphertext, additionalData []byte, tagMask *[gcmTagSize] g.mul(&y) - binary.BigEndian.PutUint64(out, y.low) - binary.BigEndian.PutUint64(out[8:], y.high) + byteorder.BePutUint64(out, y.low) + byteorder.BePutUint64(out[8:], y.high) subtle.XORBytes(out, out, tagMask[:]) } diff --git a/src/crypto/des/block.go b/src/crypto/des/block.go index c525ab0e5c..7a68a472b4 100644 --- a/src/crypto/des/block.go +++ b/src/crypto/des/block.go @@ -5,12 +5,12 @@ package des import ( - "encoding/binary" + "internal/byteorder" "sync" ) func cryptBlock(subkeys []uint64, dst, src []byte, decrypt bool) { - b := binary.BigEndian.Uint64(src) + b := byteorder.BeUint64(src) b = permuteInitialBlock(b) left, right := uint32(b>>32), uint32(b) @@ -32,7 +32,7 @@ func cryptBlock(subkeys []uint64, dst, src []byte, decrypt bool) { // switch left & right and perform final permutation preOutput := (uint64(right) << 32) | uint64(left) - binary.BigEndian.PutUint64(dst, permuteFinalBlock(preOutput)) + byteorder.BePutUint64(dst, permuteFinalBlock(preOutput)) } // DES Feistel function. feistelBox must be initialized via @@ -218,7 +218,7 @@ func (c *desCipher) generateSubkeys(keyBytes []byte) { feistelBoxOnce.Do(initFeistelBox) // apply PC1 permutation to key - key := binary.BigEndian.Uint64(keyBytes) + key := byteorder.BeUint64(keyBytes) permutedKey := permuteBlock(key, permutedChoice1[:]) // rotate halves of permuted key according to the rotation schedule diff --git a/src/crypto/des/cipher.go b/src/crypto/des/cipher.go index b0f456e692..04b73e7d3b 100644 --- a/src/crypto/des/cipher.go +++ b/src/crypto/des/cipher.go @@ -7,7 +7,7 @@ package des import ( "crypto/cipher" "crypto/internal/alias" - "encoding/binary" + "internal/byteorder" "strconv" ) @@ -95,7 +95,7 @@ func (c *tripleDESCipher) Encrypt(dst, src []byte) { panic("crypto/des: invalid buffer overlap") } - b := binary.BigEndian.Uint64(src) + b := byteorder.BeUint64(src) b = permuteInitialBlock(b) left, right := uint32(b>>32), uint32(b) @@ -116,7 +116,7 @@ func (c *tripleDESCipher) Encrypt(dst, src []byte) { right = (right << 31) | (right >> 1) preOutput := (uint64(right) << 32) | uint64(left) - binary.BigEndian.PutUint64(dst, permuteFinalBlock(preOutput)) + byteorder.BePutUint64(dst, permuteFinalBlock(preOutput)) } func (c *tripleDESCipher) Decrypt(dst, src []byte) { @@ -130,7 +130,7 @@ func (c *tripleDESCipher) Decrypt(dst, src []byte) { panic("crypto/des: invalid buffer overlap") } - b := binary.BigEndian.Uint64(src) + b := byteorder.BeUint64(src) b = permuteInitialBlock(b) left, right := uint32(b>>32), uint32(b) @@ -151,5 +151,5 @@ func (c *tripleDESCipher) Decrypt(dst, src []byte) { right = (right << 31) | (right >> 1) preOutput := (uint64(right) << 32) | uint64(left) - binary.BigEndian.PutUint64(dst, permuteFinalBlock(preOutput)) + byteorder.BePutUint64(dst, permuteFinalBlock(preOutput)) } diff --git a/src/crypto/ecdh/nist.go b/src/crypto/ecdh/nist.go index b366491544..b91e8f38a5 100644 --- a/src/crypto/ecdh/nist.go +++ b/src/crypto/ecdh/nist.go @@ -8,8 +8,8 @@ import ( "crypto/internal/boring" "crypto/internal/nistec" "crypto/internal/randutil" - "encoding/binary" "errors" + "internal/byteorder" "io" "math/bits" ) @@ -156,7 +156,7 @@ func isLess(a, b []byte) bool { // Perform a subtraction with borrow. var borrow uint64 for i := 0; i < len(bufA); i += 8 { - limbA, limbB := binary.LittleEndian.Uint64(bufA[i:]), binary.LittleEndian.Uint64(bufB[i:]) + limbA, limbB := byteorder.LeUint64(bufA[i:]), byteorder.LeUint64(bufB[i:]) _, borrow = bits.Sub64(limbA, limbB, borrow) } diff --git a/src/crypto/internal/bigmod/nat.go b/src/crypto/internal/bigmod/nat.go index a16a24305d..5cbae40efe 100644 --- a/src/crypto/internal/bigmod/nat.go +++ b/src/crypto/internal/bigmod/nat.go @@ -5,8 +5,8 @@ package bigmod import ( - "encoding/binary" "errors" + "internal/byteorder" "math/big" "math/bits" ) @@ -170,9 +170,9 @@ func (x *Nat) SetOverflowingBytes(b []byte, m *Modulus) (*Nat, error) { // big-endian encoded uint value. func bigEndianUint(buf []byte) uint { if _W == 64 { - return uint(binary.BigEndian.Uint64(buf)) + return uint(byteorder.BeUint64(buf)) } - return uint(binary.BigEndian.Uint32(buf)) + return uint(byteorder.BeUint32(buf)) } func (x *Nat) setBytes(b []byte, m *Modulus) error { diff --git a/src/crypto/internal/edwards25519/field/fe.go b/src/crypto/internal/edwards25519/field/fe.go index 5518ef2b90..8a531f078e 100644 --- a/src/crypto/internal/edwards25519/field/fe.go +++ b/src/crypto/internal/edwards25519/field/fe.go @@ -7,8 +7,8 @@ package field import ( "crypto/subtle" - "encoding/binary" "errors" + "internal/byteorder" "math/bits" ) @@ -201,20 +201,20 @@ func (v *Element) SetBytes(x []byte) (*Element, error) { } // Bits 0:51 (bytes 0:8, bits 0:64, shift 0, mask 51). - v.l0 = binary.LittleEndian.Uint64(x[0:8]) + v.l0 = byteorder.LeUint64(x[0:8]) v.l0 &= maskLow51Bits // Bits 51:102 (bytes 6:14, bits 48:112, shift 3, mask 51). - v.l1 = binary.LittleEndian.Uint64(x[6:14]) >> 3 + v.l1 = byteorder.LeUint64(x[6:14]) >> 3 v.l1 &= maskLow51Bits // Bits 102:153 (bytes 12:20, bits 96:160, shift 6, mask 51). - v.l2 = binary.LittleEndian.Uint64(x[12:20]) >> 6 + v.l2 = byteorder.LeUint64(x[12:20]) >> 6 v.l2 &= maskLow51Bits // Bits 153:204 (bytes 19:27, bits 152:216, shift 1, mask 51). - v.l3 = binary.LittleEndian.Uint64(x[19:27]) >> 1 + v.l3 = byteorder.LeUint64(x[19:27]) >> 1 v.l3 &= maskLow51Bits // Bits 204:255 (bytes 24:32, bits 192:256, shift 12, mask 51). // Note: not bytes 25:33, shift 4, to avoid overread. - v.l4 = binary.LittleEndian.Uint64(x[24:32]) >> 12 + v.l4 = byteorder.LeUint64(x[24:32]) >> 12 v.l4 &= maskLow51Bits return v, nil @@ -235,7 +235,7 @@ func (v *Element) bytes(out *[32]byte) []byte { var buf [8]byte for i, l := range [5]uint64{t.l0, t.l1, t.l2, t.l3, t.l4} { bitsOffset := i * 51 - binary.LittleEndian.PutUint64(buf[:], l<= len(out) { diff --git a/src/crypto/internal/edwards25519/scalar.go b/src/crypto/internal/edwards25519/scalar.go index 3fd1653877..9f652faca1 100644 --- a/src/crypto/internal/edwards25519/scalar.go +++ b/src/crypto/internal/edwards25519/scalar.go @@ -5,8 +5,8 @@ package edwards25519 import ( - "encoding/binary" "errors" + "internal/byteorder" ) // A Scalar is an integer modulo @@ -271,7 +271,7 @@ func (s *Scalar) nonAdjacentForm(w uint) [256]int8 { var digits [5]uint64 for i := 0; i < 4; i++ { - digits[i] = binary.LittleEndian.Uint64(b[i*8:]) + digits[i] = byteorder.LeUint64(b[i*8:]) } width := uint64(1 << w) diff --git a/src/crypto/internal/mlkem768/mlkem768.go b/src/crypto/internal/mlkem768/mlkem768.go index 24bedea84f..76c6e80b4e 100644 --- a/src/crypto/internal/mlkem768/mlkem768.go +++ b/src/crypto/internal/mlkem768/mlkem768.go @@ -30,8 +30,8 @@ package mlkem768 import ( "crypto/rand" "crypto/subtle" - "encoding/binary" "errors" + "internal/byteorder" "golang.org/x/crypto/sha3" ) @@ -864,8 +864,8 @@ func sampleNTT(rho []byte, ii, jj byte) nttElement { B.Read(buf[:]) off = 0 } - d1 := binary.LittleEndian.Uint16(buf[off:]) & 0b1111_1111_1111 - d2 := binary.LittleEndian.Uint16(buf[off+1:]) >> 4 + d1 := byteorder.LeUint16(buf[off:]) & 0b1111_1111_1111 + d2 := byteorder.LeUint16(buf[off+1:]) >> 4 off += 3 if d1 < q { a[j] = fieldElement(d1) diff --git a/src/crypto/internal/nistec/p256_asm.go b/src/crypto/internal/nistec/p256_asm.go index 1a523cc13c..5dbd7efbd5 100644 --- a/src/crypto/internal/nistec/p256_asm.go +++ b/src/crypto/internal/nistec/p256_asm.go @@ -16,8 +16,8 @@ package nistec import ( _ "embed" - "encoding/binary" "errors" + "internal/byteorder" "math/bits" "runtime" "unsafe" @@ -327,7 +327,7 @@ func init() { if runtime.GOARCH == "s390x" { var newTable [43 * 32 * 2 * 4]uint64 for i, x := range (*[43 * 32 * 2 * 4][8]byte)(*p256PrecomputedPtr) { - newTable[i] = binary.LittleEndian.Uint64(x[:]) + newTable[i] = byteorder.LeUint64(x[:]) } newTablePtr := unsafe.Pointer(&newTable) p256PrecomputedPtr = &newTablePtr diff --git a/src/crypto/md5/gen.go b/src/crypto/md5/gen.go index cd2700a5cf..5290c3627c 100644 --- a/src/crypto/md5/gen.go +++ b/src/crypto/md5/gen.go @@ -201,7 +201,7 @@ var program = `// Copyright 2013 The Go Authors. All rights reserved. package md5 import ( - "encoding/binary" + "internal/byteorder" "math/bits" ) @@ -219,7 +219,7 @@ func blockGeneric(dig *digest, p []byte) { // load input block {{range $i := seq 16 -}} - {{printf "x%x := binary.LittleEndian.Uint32(q[4*%#x:])" $i $i}} + {{printf "x%x := byteorder.LeUint32(q[4*%#x:])" $i $i}} {{end}} // round 1 @@ -227,19 +227,19 @@ func blockGeneric(dig *digest, p []byte) { {{printf "arg0 = arg1 + bits.RotateLeft32((((arg2^arg3)&arg1)^arg3)+arg0+x%x+%#08x, %d)" (idx 1 $i) (index $.Table1 $i) $s | relabel}} {{rotate -}} {{end}} - + // round 2 {{range $i, $s := dup 4 .Shift2 -}} {{printf "arg0 = arg1 + bits.RotateLeft32((((arg1^arg2)&arg3)^arg2)+arg0+x%x+%#08x, %d)" (idx 2 $i) (index $.Table2 $i) $s | relabel}} {{rotate -}} {{end}} - + // round 3 {{range $i, $s := dup 4 .Shift3 -}} {{printf "arg0 = arg1 + bits.RotateLeft32((arg1^arg2^arg3)+arg0+x%x+%#08x, %d)" (idx 3 $i) (index $.Table3 $i) $s | relabel}} {{rotate -}} {{end}} - + // round 4 {{range $i, $s := dup 4 .Shift4 -}} {{printf "arg0 = arg1 + bits.RotateLeft32((arg2^(arg1|^arg3))+arg0+x%x+%#08x, %d)" (idx 4 $i) (index $.Table4 $i) $s | relabel}} diff --git a/src/crypto/md5/md5.go b/src/crypto/md5/md5.go index 83e9e4c07a..843678702b 100644 --- a/src/crypto/md5/md5.go +++ b/src/crypto/md5/md5.go @@ -12,9 +12,9 @@ package md5 import ( "crypto" - "encoding/binary" "errors" "hash" + "internal/byteorder" ) func init() { @@ -59,13 +59,13 @@ const ( func (d *digest) MarshalBinary() ([]byte, error) { b := make([]byte, 0, marshaledSize) b = append(b, magic...) - b = binary.BigEndian.AppendUint32(b, d.s[0]) - b = binary.BigEndian.AppendUint32(b, d.s[1]) - b = binary.BigEndian.AppendUint32(b, d.s[2]) - b = binary.BigEndian.AppendUint32(b, d.s[3]) + b = byteorder.BeAppendUint32(b, d.s[0]) + b = byteorder.BeAppendUint32(b, d.s[1]) + b = byteorder.BeAppendUint32(b, d.s[2]) + b = byteorder.BeAppendUint32(b, d.s[3]) b = append(b, d.x[:d.nx]...) b = b[:len(b)+len(d.x)-d.nx] // already zero - b = binary.BigEndian.AppendUint64(b, d.len) + b = byteorder.BeAppendUint64(b, d.len) return b, nil } @@ -88,11 +88,11 @@ func (d *digest) UnmarshalBinary(b []byte) error { } func consumeUint64(b []byte) ([]byte, uint64) { - return b[8:], binary.BigEndian.Uint64(b[0:8]) + return b[8:], byteorder.BeUint64(b[0:8]) } func consumeUint32(b []byte) ([]byte, uint32) { - return b[4:], binary.BigEndian.Uint32(b[0:4]) + return b[4:], byteorder.BeUint32(b[0:4]) } // New returns a new hash.Hash computing the MD5 checksum. The Hash also @@ -156,8 +156,8 @@ func (d *digest) checkSum() [Size]byte { // // 1 byte end marker :: 0-63 padding bytes :: 8 byte length tmp := [1 + 63 + 8]byte{0x80} - pad := (55 - d.len) % 64 // calculate number of padding bytes - binary.LittleEndian.PutUint64(tmp[1+pad:], d.len<<3) // append length in bits + pad := (55 - d.len) % 64 // calculate number of padding bytes + byteorder.LePutUint64(tmp[1+pad:], d.len<<3) // append length in bits d.Write(tmp[:1+pad+8]) // The previous write ensures that a whole number of @@ -167,10 +167,10 @@ func (d *digest) checkSum() [Size]byte { } var digest [Size]byte - binary.LittleEndian.PutUint32(digest[0:], d.s[0]) - binary.LittleEndian.PutUint32(digest[4:], d.s[1]) - binary.LittleEndian.PutUint32(digest[8:], d.s[2]) - binary.LittleEndian.PutUint32(digest[12:], d.s[3]) + byteorder.LePutUint32(digest[0:], d.s[0]) + byteorder.LePutUint32(digest[4:], d.s[1]) + byteorder.LePutUint32(digest[8:], d.s[2]) + byteorder.LePutUint32(digest[12:], d.s[3]) return digest } diff --git a/src/crypto/md5/md5block.go b/src/crypto/md5/md5block.go index 4ff289e860..473496b8d0 100644 --- a/src/crypto/md5/md5block.go +++ b/src/crypto/md5/md5block.go @@ -7,7 +7,7 @@ package md5 import ( - "encoding/binary" + "internal/byteorder" "math/bits" ) @@ -24,22 +24,22 @@ func blockGeneric(dig *digest, p []byte) { aa, bb, cc, dd := a, b, c, d // load input block - x0 := binary.LittleEndian.Uint32(q[4*0x0:]) - x1 := binary.LittleEndian.Uint32(q[4*0x1:]) - x2 := binary.LittleEndian.Uint32(q[4*0x2:]) - x3 := binary.LittleEndian.Uint32(q[4*0x3:]) - x4 := binary.LittleEndian.Uint32(q[4*0x4:]) - x5 := binary.LittleEndian.Uint32(q[4*0x5:]) - x6 := binary.LittleEndian.Uint32(q[4*0x6:]) - x7 := binary.LittleEndian.Uint32(q[4*0x7:]) - x8 := binary.LittleEndian.Uint32(q[4*0x8:]) - x9 := binary.LittleEndian.Uint32(q[4*0x9:]) - xa := binary.LittleEndian.Uint32(q[4*0xa:]) - xb := binary.LittleEndian.Uint32(q[4*0xb:]) - xc := binary.LittleEndian.Uint32(q[4*0xc:]) - xd := binary.LittleEndian.Uint32(q[4*0xd:]) - xe := binary.LittleEndian.Uint32(q[4*0xe:]) - xf := binary.LittleEndian.Uint32(q[4*0xf:]) + x0 := byteorder.LeUint32(q[4*0x0:]) + x1 := byteorder.LeUint32(q[4*0x1:]) + x2 := byteorder.LeUint32(q[4*0x2:]) + x3 := byteorder.LeUint32(q[4*0x3:]) + x4 := byteorder.LeUint32(q[4*0x4:]) + x5 := byteorder.LeUint32(q[4*0x5:]) + x6 := byteorder.LeUint32(q[4*0x6:]) + x7 := byteorder.LeUint32(q[4*0x7:]) + x8 := byteorder.LeUint32(q[4*0x8:]) + x9 := byteorder.LeUint32(q[4*0x9:]) + xa := byteorder.LeUint32(q[4*0xa:]) + xb := byteorder.LeUint32(q[4*0xb:]) + xc := byteorder.LeUint32(q[4*0xc:]) + xd := byteorder.LeUint32(q[4*0xd:]) + xe := byteorder.LeUint32(q[4*0xe:]) + xf := byteorder.LeUint32(q[4*0xf:]) // round 1 a = b + bits.RotateLeft32((((c^d)&b)^d)+a+x0+0xd76aa478, 7) diff --git a/src/crypto/rand/rand_plan9.go b/src/crypto/rand/rand_plan9.go index 8db19157a7..d5320210fd 100644 --- a/src/crypto/rand/rand_plan9.go +++ b/src/crypto/rand/rand_plan9.go @@ -9,7 +9,7 @@ package rand import ( "crypto/aes" - "encoding/binary" + "internal/byteorder" "io" "os" "sync" @@ -66,7 +66,7 @@ func (r *reader) Read(b []byte) (n int, err error) { if counter == 0 { panic("crypto/rand counter wrapped") } - binary.LittleEndian.PutUint64(block[:], counter) + byteorder.LePutUint64(block[:], counter) } blockCipher.Encrypt(r.key[:aes.BlockSize], block[:]) inc() diff --git a/src/crypto/sha1/sha1.go b/src/crypto/sha1/sha1.go index ac10fa1557..c0742b9d83 100644 --- a/src/crypto/sha1/sha1.go +++ b/src/crypto/sha1/sha1.go @@ -11,9 +11,9 @@ package sha1 import ( "crypto" "crypto/internal/boring" - "encoding/binary" "errors" "hash" + "internal/byteorder" ) func init() { @@ -51,14 +51,14 @@ const ( func (d *digest) MarshalBinary() ([]byte, error) { b := make([]byte, 0, marshaledSize) b = append(b, magic...) - b = binary.BigEndian.AppendUint32(b, d.h[0]) - b = binary.BigEndian.AppendUint32(b, d.h[1]) - b = binary.BigEndian.AppendUint32(b, d.h[2]) - b = binary.BigEndian.AppendUint32(b, d.h[3]) - b = binary.BigEndian.AppendUint32(b, d.h[4]) + b = byteorder.BeAppendUint32(b, d.h[0]) + b = byteorder.BeAppendUint32(b, d.h[1]) + b = byteorder.BeAppendUint32(b, d.h[2]) + b = byteorder.BeAppendUint32(b, d.h[3]) + b = byteorder.BeAppendUint32(b, d.h[4]) b = append(b, d.x[:d.nx]...) b = b[:len(b)+len(d.x)-d.nx] // already zero - b = binary.BigEndian.AppendUint64(b, d.len) + b = byteorder.BeAppendUint64(b, d.len) return b, nil } @@ -82,16 +82,11 @@ func (d *digest) UnmarshalBinary(b []byte) error { } func consumeUint64(b []byte) ([]byte, uint64) { - _ = b[7] - x := uint64(b[7]) | uint64(b[6])<<8 | uint64(b[5])<<16 | uint64(b[4])<<24 | - uint64(b[3])<<32 | uint64(b[2])<<40 | uint64(b[1])<<48 | uint64(b[0])<<56 - return b[8:], x + return b[8:], byteorder.BeUint64(b) } func consumeUint32(b []byte) ([]byte, uint32) { - _ = b[3] - x := uint32(b[3]) | uint32(b[2])<<8 | uint32(b[1])<<16 | uint32(b[0])<<24 - return b[4:], x + return b[4:], byteorder.BeUint32(b) } func (d *digest) Reset() { @@ -167,7 +162,7 @@ func (d *digest) checkSum() [Size]byte { // Length in bits. len <<= 3 padlen := tmp[:t+8] - binary.BigEndian.PutUint64(padlen[t:], len) + byteorder.BePutUint64(padlen[t:], len) d.Write(padlen) if d.nx != 0 { @@ -176,11 +171,11 @@ func (d *digest) checkSum() [Size]byte { var digest [Size]byte - binary.BigEndian.PutUint32(digest[0:], d.h[0]) - binary.BigEndian.PutUint32(digest[4:], d.h[1]) - binary.BigEndian.PutUint32(digest[8:], d.h[2]) - binary.BigEndian.PutUint32(digest[12:], d.h[3]) - binary.BigEndian.PutUint32(digest[16:], d.h[4]) + byteorder.BePutUint32(digest[0:], d.h[0]) + byteorder.BePutUint32(digest[4:], d.h[1]) + byteorder.BePutUint32(digest[8:], d.h[2]) + byteorder.BePutUint32(digest[12:], d.h[3]) + byteorder.BePutUint32(digest[16:], d.h[4]) return digest } diff --git a/src/crypto/sha256/sha256.go b/src/crypto/sha256/sha256.go index 0cc7fca0a6..68244fd63b 100644 --- a/src/crypto/sha256/sha256.go +++ b/src/crypto/sha256/sha256.go @@ -9,9 +9,9 @@ package sha256 import ( "crypto" "crypto/internal/boring" - "encoding/binary" "errors" "hash" + "internal/byteorder" ) func init() { @@ -70,17 +70,17 @@ func (d *digest) MarshalBinary() ([]byte, error) { } else { b = append(b, magic256...) } - b = binary.BigEndian.AppendUint32(b, d.h[0]) - b = binary.BigEndian.AppendUint32(b, d.h[1]) - b = binary.BigEndian.AppendUint32(b, d.h[2]) - b = binary.BigEndian.AppendUint32(b, d.h[3]) - b = binary.BigEndian.AppendUint32(b, d.h[4]) - b = binary.BigEndian.AppendUint32(b, d.h[5]) - b = binary.BigEndian.AppendUint32(b, d.h[6]) - b = binary.BigEndian.AppendUint32(b, d.h[7]) + b = byteorder.BeAppendUint32(b, d.h[0]) + b = byteorder.BeAppendUint32(b, d.h[1]) + b = byteorder.BeAppendUint32(b, d.h[2]) + b = byteorder.BeAppendUint32(b, d.h[3]) + b = byteorder.BeAppendUint32(b, d.h[4]) + b = byteorder.BeAppendUint32(b, d.h[5]) + b = byteorder.BeAppendUint32(b, d.h[6]) + b = byteorder.BeAppendUint32(b, d.h[7]) b = append(b, d.x[:d.nx]...) b = b[:len(b)+len(d.x)-d.nx] // already zero - b = binary.BigEndian.AppendUint64(b, d.len) + b = byteorder.BeAppendUint64(b, d.len) return b, nil } @@ -107,16 +107,11 @@ func (d *digest) UnmarshalBinary(b []byte) error { } func consumeUint64(b []byte) ([]byte, uint64) { - _ = b[7] - x := uint64(b[7]) | uint64(b[6])<<8 | uint64(b[5])<<16 | uint64(b[4])<<24 | - uint64(b[3])<<32 | uint64(b[2])<<40 | uint64(b[1])<<48 | uint64(b[0])<<56 - return b[8:], x + return b[8:], byteorder.BeUint64(b) } func consumeUint32(b []byte) ([]byte, uint32) { - _ = b[3] - x := uint32(b[3]) | uint32(b[2])<<8 | uint32(b[1])<<16 | uint32(b[0])<<24 - return b[4:], x + return b[4:], byteorder.BeUint32(b) } func (d *digest) Reset() { @@ -226,7 +221,7 @@ func (d *digest) checkSum() [Size]byte { // Length in bits. len <<= 3 padlen := tmp[:t+8] - binary.BigEndian.PutUint64(padlen[t+0:], len) + byteorder.BePutUint64(padlen[t+0:], len) d.Write(padlen) if d.nx != 0 { @@ -235,15 +230,15 @@ func (d *digest) checkSum() [Size]byte { var digest [Size]byte - binary.BigEndian.PutUint32(digest[0:], d.h[0]) - binary.BigEndian.PutUint32(digest[4:], d.h[1]) - binary.BigEndian.PutUint32(digest[8:], d.h[2]) - binary.BigEndian.PutUint32(digest[12:], d.h[3]) - binary.BigEndian.PutUint32(digest[16:], d.h[4]) - binary.BigEndian.PutUint32(digest[20:], d.h[5]) - binary.BigEndian.PutUint32(digest[24:], d.h[6]) + byteorder.BePutUint32(digest[0:], d.h[0]) + byteorder.BePutUint32(digest[4:], d.h[1]) + byteorder.BePutUint32(digest[8:], d.h[2]) + byteorder.BePutUint32(digest[12:], d.h[3]) + byteorder.BePutUint32(digest[16:], d.h[4]) + byteorder.BePutUint32(digest[20:], d.h[5]) + byteorder.BePutUint32(digest[24:], d.h[6]) if !d.is224 { - binary.BigEndian.PutUint32(digest[28:], d.h[7]) + byteorder.BePutUint32(digest[28:], d.h[7]) } return digest diff --git a/src/crypto/sha512/sha512.go b/src/crypto/sha512/sha512.go index 9ae1b3aae2..dde83625f7 100644 --- a/src/crypto/sha512/sha512.go +++ b/src/crypto/sha512/sha512.go @@ -13,9 +13,9 @@ package sha512 import ( "crypto" "crypto/internal/boring" - "encoding/binary" "errors" "hash" + "internal/byteorder" ) func init() { @@ -153,17 +153,17 @@ func (d *digest) MarshalBinary() ([]byte, error) { default: return nil, errors.New("crypto/sha512: invalid hash function") } - b = binary.BigEndian.AppendUint64(b, d.h[0]) - b = binary.BigEndian.AppendUint64(b, d.h[1]) - b = binary.BigEndian.AppendUint64(b, d.h[2]) - b = binary.BigEndian.AppendUint64(b, d.h[3]) - b = binary.BigEndian.AppendUint64(b, d.h[4]) - b = binary.BigEndian.AppendUint64(b, d.h[5]) - b = binary.BigEndian.AppendUint64(b, d.h[6]) - b = binary.BigEndian.AppendUint64(b, d.h[7]) + b = byteorder.BeAppendUint64(b, d.h[0]) + b = byteorder.BeAppendUint64(b, d.h[1]) + b = byteorder.BeAppendUint64(b, d.h[2]) + b = byteorder.BeAppendUint64(b, d.h[3]) + b = byteorder.BeAppendUint64(b, d.h[4]) + b = byteorder.BeAppendUint64(b, d.h[5]) + b = byteorder.BeAppendUint64(b, d.h[6]) + b = byteorder.BeAppendUint64(b, d.h[7]) b = append(b, d.x[:d.nx]...) b = b[:len(b)+len(d.x)-d.nx] // already zero - b = binary.BigEndian.AppendUint64(b, d.len) + b = byteorder.BeAppendUint64(b, d.len) return b, nil } @@ -198,10 +198,7 @@ func (d *digest) UnmarshalBinary(b []byte) error { } func consumeUint64(b []byte) ([]byte, uint64) { - _ = b[7] - x := uint64(b[7]) | uint64(b[6])<<8 | uint64(b[5])<<16 | uint64(b[4])<<24 | - uint64(b[3])<<32 | uint64(b[2])<<40 | uint64(b[1])<<48 | uint64(b[0])<<56 - return b[8:], x + return b[8:], byteorder.BeUint64(b) } // New returns a new hash.Hash computing the SHA-512 checksum. @@ -316,8 +313,8 @@ func (d *digest) checkSum() [Size]byte { padlen := tmp[:t+16] // Upper 64 bits are always zero, because len variable has type uint64, // and tmp is already zeroed at that index, so we can skip updating it. - // binary.BigEndian.PutUint64(padlen[t+0:], 0) - binary.BigEndian.PutUint64(padlen[t+8:], len) + // byteorder.BePutUint64(padlen[t+0:], 0) + byteorder.BePutUint64(padlen[t+8:], len) d.Write(padlen) if d.nx != 0 { @@ -325,15 +322,15 @@ func (d *digest) checkSum() [Size]byte { } var digest [Size]byte - binary.BigEndian.PutUint64(digest[0:], d.h[0]) - binary.BigEndian.PutUint64(digest[8:], d.h[1]) - binary.BigEndian.PutUint64(digest[16:], d.h[2]) - binary.BigEndian.PutUint64(digest[24:], d.h[3]) - binary.BigEndian.PutUint64(digest[32:], d.h[4]) - binary.BigEndian.PutUint64(digest[40:], d.h[5]) + byteorder.BePutUint64(digest[0:], d.h[0]) + byteorder.BePutUint64(digest[8:], d.h[1]) + byteorder.BePutUint64(digest[16:], d.h[2]) + byteorder.BePutUint64(digest[24:], d.h[3]) + byteorder.BePutUint64(digest[32:], d.h[4]) + byteorder.BePutUint64(digest[40:], d.h[5]) if d.function != crypto.SHA384 { - binary.BigEndian.PutUint64(digest[48:], d.h[6]) - binary.BigEndian.PutUint64(digest[56:], d.h[7]) + byteorder.BePutUint64(digest[48:], d.h[6]) + byteorder.BePutUint64(digest[56:], d.h[7]) } return digest diff --git a/src/crypto/tls/bogo_shim_test.go b/src/crypto/tls/bogo_shim_test.go index 731fcd6d95..e1a393c8bf 100644 --- a/src/crypto/tls/bogo_shim_test.go +++ b/src/crypto/tls/bogo_shim_test.go @@ -2,11 +2,11 @@ package tls import ( "crypto/x509" - "encoding/binary" "encoding/json" "encoding/pem" "flag" "fmt" + "internal/byteorder" "internal/testenv" "io" "log" @@ -186,7 +186,7 @@ func bogoShim() { // Write the shim ID we were passed as a little endian uint64 shimIDBytes := make([]byte, 8) - binary.LittleEndian.PutUint64(shimIDBytes, *shimID) + byteorder.LePutUint64(shimIDBytes, *shimID) if _, err := conn.Write(shimIDBytes); err != nil { log.Fatalf("failed to write shim id: %s", err) } diff --git a/src/crypto/tls/handshake_client.go b/src/crypto/tls/handshake_client.go index 0b35deefa1..d046c86679 100644 --- a/src/crypto/tls/handshake_client.go +++ b/src/crypto/tls/handshake_client.go @@ -526,7 +526,7 @@ func (hs *clientHandshakeState) pickCipherSuite() error { return errors.New("tls: server chose an unconfigured cipher suite") } - if hs.c.config.CipherSuites == nil && rsaKexCiphers[hs.suite.id] { + if hs.c.config.CipherSuites == nil && !needFIPS() && rsaKexCiphers[hs.suite.id] { tlsrsakex.IncNonDefault() } diff --git a/src/crypto/tls/handshake_client_test.go b/src/crypto/tls/handshake_client_test.go index ee9e79afab..157c67ff86 100644 --- a/src/crypto/tls/handshake_client_test.go +++ b/src/crypto/tls/handshake_client_test.go @@ -10,10 +10,10 @@ import ( "crypto/rsa" "crypto/x509" "encoding/base64" - "encoding/binary" "encoding/pem" "errors" "fmt" + "internal/byteorder" "io" "math/big" "net" @@ -202,7 +202,7 @@ func (test *clientTest) connFromCommand() (conn *recordingConn, child *exec.Cmd, var serverInfo bytes.Buffer for _, ext := range test.extensions { pem.Encode(&serverInfo, &pem.Block{ - Type: fmt.Sprintf("SERVERINFO FOR EXTENSION %d", binary.BigEndian.Uint16(ext)), + Type: fmt.Sprintf("SERVERINFO FOR EXTENSION %d", byteorder.BeUint16(ext)), Bytes: ext, }) } diff --git a/src/crypto/tls/handshake_server.go b/src/crypto/tls/handshake_server.go index eb87ee038c..d5f8cc843e 100644 --- a/src/crypto/tls/handshake_server.go +++ b/src/crypto/tls/handshake_server.go @@ -370,7 +370,7 @@ func (hs *serverHandshakeState) pickCipherSuite() error { } c.cipherSuite = hs.suite.id - if c.config.CipherSuites == nil && rsaKexCiphers[hs.suite.id] { + if c.config.CipherSuites == nil && !needFIPS() && rsaKexCiphers[hs.suite.id] { tlsrsakex.IncNonDefault() } diff --git a/src/crypto/tls/handshake_server_tls13.go b/src/crypto/tls/handshake_server_tls13.go index 60a3883023..7f15d05b28 100644 --- a/src/crypto/tls/handshake_server_tls13.go +++ b/src/crypto/tls/handshake_server_tls13.go @@ -10,9 +10,9 @@ import ( "crypto" "crypto/hmac" "crypto/rsa" - "encoding/binary" "errors" "hash" + "internal/byteorder" "io" "time" ) @@ -866,7 +866,7 @@ func (c *Conn) sendSessionTicket(earlyData bool) error { if _, err := c.config.rand().Read(ageAdd); err != nil { return err } - m.ageAdd = binary.LittleEndian.Uint32(ageAdd) + m.ageAdd = byteorder.LeUint32(ageAdd) if earlyData { // RFC 9001, Section 4.6.1 diff --git a/src/crypto/x509/oid.go b/src/crypto/x509/oid.go index 5359af624b..b00c35e696 100644 --- a/src/crypto/x509/oid.go +++ b/src/crypto/x509/oid.go @@ -24,6 +24,12 @@ type OID struct { der []byte } +// ParseOID parses a Object Identifier string, represented by ASCII numbers separated by dots. +func ParseOID(oid string) (OID, error) { + var o OID + return o, o.unmarshalOIDText(oid) +} + func newOIDFromDER(der []byte) (OID, bool) { if len(der) == 0 || der[len(der)-1]&0x80 != 0 { return OID{}, false @@ -83,6 +89,112 @@ func appendBase128Int(dst []byte, n uint64) []byte { return dst } +func base128BigIntLength(n *big.Int) int { + if n.Cmp(big.NewInt(0)) == 0 { + return 1 + } + return (n.BitLen() + 6) / 7 +} + +func appendBase128BigInt(dst []byte, n *big.Int) []byte { + if n.Cmp(big.NewInt(0)) == 0 { + return append(dst, 0) + } + + for i := base128BigIntLength(n) - 1; i >= 0; i-- { + o := byte(big.NewInt(0).Rsh(n, uint(i)*7).Bits()[0]) + o &= 0x7f + if i != 0 { + o |= 0x80 + } + dst = append(dst, o) + } + return dst +} + +// MarshalText implements [encoding.TextMarshaler] +func (o OID) MarshalText() ([]byte, error) { + return []byte(o.String()), nil +} + +// UnmarshalText implements [encoding.TextUnmarshaler] +func (o *OID) UnmarshalText(text []byte) error { + return o.unmarshalOIDText(string(text)) +} + +func (o *OID) unmarshalOIDText(oid string) error { + // (*big.Int).SetString allows +/- signs, but we don't want + // to allow them in the string representation of Object Identifier, so + // reject such encodings. + for _, c := range oid { + isDigit := c >= '0' && c <= '9' + if !isDigit && c != '.' { + return errInvalidOID + } + } + + var ( + firstNum string + secondNum string + ) + + var nextComponentExists bool + firstNum, oid, nextComponentExists = strings.Cut(oid, ".") + if !nextComponentExists { + return errInvalidOID + } + secondNum, oid, nextComponentExists = strings.Cut(oid, ".") + + var ( + first = big.NewInt(0) + second = big.NewInt(0) + ) + + if _, ok := first.SetString(firstNum, 10); !ok { + return errInvalidOID + } + if _, ok := second.SetString(secondNum, 10); !ok { + return errInvalidOID + } + + if first.Cmp(big.NewInt(2)) > 0 || (first.Cmp(big.NewInt(2)) < 0 && second.Cmp(big.NewInt(40)) >= 0) { + return errInvalidOID + } + + firstComponent := first.Mul(first, big.NewInt(40)) + firstComponent.Add(firstComponent, second) + + der := appendBase128BigInt(make([]byte, 0, 32), firstComponent) + + for nextComponentExists { + var strNum string + strNum, oid, nextComponentExists = strings.Cut(oid, ".") + b, ok := big.NewInt(0).SetString(strNum, 10) + if !ok { + return errInvalidOID + } + der = appendBase128BigInt(der, b) + } + + o.der = der + return nil +} + +// MarshalBinary implements [encoding.BinaryMarshaler] +func (o OID) MarshalBinary() ([]byte, error) { + return bytes.Clone(o.der), nil +} + +// UnmarshalBinary implements [encoding.BinaryUnmarshaler] +func (o *OID) UnmarshalBinary(b []byte) error { + oid, ok := newOIDFromDER(bytes.Clone(b)) + if !ok { + return errInvalidOID + } + *o = oid + return nil +} + // Equal returns true when oid and other represents the same Object Identifier. func (oid OID) Equal(other OID) bool { // There is only one possible DER encoding of diff --git a/src/crypto/x509/oid_test.go b/src/crypto/x509/oid_test.go index eb47244a73..cbb3406424 100644 --- a/src/crypto/x509/oid_test.go +++ b/src/crypto/x509/oid_test.go @@ -5,54 +5,54 @@ package x509 import ( + "encoding" "encoding/asn1" "math" "testing" ) +var oidTests = []struct { + raw []byte + valid bool + str string + ints []uint64 +}{ + {[]byte{}, false, "", nil}, + {[]byte{0x80, 0x01}, false, "", nil}, + {[]byte{0x01, 0x80, 0x01}, false, "", nil}, + + {[]byte{1, 2, 3}, true, "0.1.2.3", []uint64{0, 1, 2, 3}}, + {[]byte{41, 2, 3}, true, "1.1.2.3", []uint64{1, 1, 2, 3}}, + {[]byte{86, 2, 3}, true, "2.6.2.3", []uint64{2, 6, 2, 3}}, + + {[]byte{41, 255, 255, 255, 127}, true, "1.1.268435455", []uint64{1, 1, 268435455}}, + {[]byte{41, 0x87, 255, 255, 255, 127}, true, "1.1.2147483647", []uint64{1, 1, 2147483647}}, + {[]byte{41, 255, 255, 255, 255, 127}, true, "1.1.34359738367", []uint64{1, 1, 34359738367}}, + {[]byte{42, 255, 255, 255, 255, 255, 255, 255, 255, 127}, true, "1.2.9223372036854775807", []uint64{1, 2, 9223372036854775807}}, + {[]byte{43, 0x81, 255, 255, 255, 255, 255, 255, 255, 255, 127}, true, "1.3.18446744073709551615", []uint64{1, 3, 18446744073709551615}}, + {[]byte{44, 0x83, 255, 255, 255, 255, 255, 255, 255, 255, 127}, true, "1.4.36893488147419103231", nil}, + {[]byte{85, 255, 255, 255, 255, 255, 255, 255, 255, 255, 127}, true, "2.5.1180591620717411303423", nil}, + {[]byte{85, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 127}, true, "2.5.19342813113834066795298815", nil}, + + {[]byte{255, 255, 255, 127}, true, "2.268435375", []uint64{2, 268435375}}, + {[]byte{0x87, 255, 255, 255, 127}, true, "2.2147483567", []uint64{2, 2147483567}}, + {[]byte{255, 127}, true, "2.16303", []uint64{2, 16303}}, + {[]byte{255, 255, 255, 255, 127}, true, "2.34359738287", []uint64{2, 34359738287}}, + {[]byte{255, 255, 255, 255, 255, 255, 255, 255, 127}, true, "2.9223372036854775727", []uint64{2, 9223372036854775727}}, + {[]byte{0x81, 255, 255, 255, 255, 255, 255, 255, 255, 127}, true, "2.18446744073709551535", []uint64{2, 18446744073709551535}}, + {[]byte{0x83, 255, 255, 255, 255, 255, 255, 255, 255, 127}, true, "2.36893488147419103151", nil}, + {[]byte{255, 255, 255, 255, 255, 255, 255, 255, 255, 127}, true, "2.1180591620717411303343", nil}, + {[]byte{255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 127}, true, "2.19342813113834066795298735", nil}, + + {[]byte{41, 0x80 | 66, 0x80 | 44, 0x80 | 11, 33}, true, "1.1.139134369", []uint64{1, 1, 139134369}}, + {[]byte{0x80 | 66, 0x80 | 44, 0x80 | 11, 33}, true, "2.139134289", []uint64{2, 139134289}}, +} + func TestOID(t *testing.T) { - var tests = []struct { - raw []byte - valid bool - str string - ints []uint64 - }{ - {[]byte{}, false, "", nil}, - {[]byte{0x80, 0x01}, false, "", nil}, - {[]byte{0x01, 0x80, 0x01}, false, "", nil}, - - {[]byte{1, 2, 3}, true, "0.1.2.3", []uint64{0, 1, 2, 3}}, - {[]byte{41, 2, 3}, true, "1.1.2.3", []uint64{1, 1, 2, 3}}, - {[]byte{86, 2, 3}, true, "2.6.2.3", []uint64{2, 6, 2, 3}}, - - {[]byte{41, 255, 255, 255, 127}, true, "1.1.268435455", []uint64{1, 1, 268435455}}, - {[]byte{41, 0x87, 255, 255, 255, 127}, true, "1.1.2147483647", []uint64{1, 1, 2147483647}}, - {[]byte{41, 255, 255, 255, 255, 127}, true, "1.1.34359738367", []uint64{1, 1, 34359738367}}, - {[]byte{42, 255, 255, 255, 255, 255, 255, 255, 255, 127}, true, "1.2.9223372036854775807", []uint64{1, 2, 9223372036854775807}}, - {[]byte{43, 0x81, 255, 255, 255, 255, 255, 255, 255, 255, 127}, true, "1.3.18446744073709551615", []uint64{1, 3, 18446744073709551615}}, - {[]byte{44, 0x83, 255, 255, 255, 255, 255, 255, 255, 255, 127}, true, "1.4.36893488147419103231", nil}, - {[]byte{85, 255, 255, 255, 255, 255, 255, 255, 255, 255, 127}, true, "2.5.1180591620717411303423", nil}, - {[]byte{85, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 127}, true, "2.5.19342813113834066795298815", nil}, - - {[]byte{255, 255, 255, 127}, true, "2.268435375", []uint64{2, 268435375}}, - {[]byte{0x87, 255, 255, 255, 127}, true, "2.2147483567", []uint64{2, 2147483567}}, - {[]byte{255, 127}, true, "2.16303", []uint64{2, 16303}}, - {[]byte{255, 255, 255, 255, 127}, true, "2.34359738287", []uint64{2, 34359738287}}, - {[]byte{255, 255, 255, 255, 255, 255, 255, 255, 127}, true, "2.9223372036854775727", []uint64{2, 9223372036854775727}}, - {[]byte{0x81, 255, 255, 255, 255, 255, 255, 255, 255, 127}, true, "2.18446744073709551535", []uint64{2, 18446744073709551535}}, - {[]byte{0x83, 255, 255, 255, 255, 255, 255, 255, 255, 127}, true, "2.36893488147419103151", nil}, - {[]byte{255, 255, 255, 255, 255, 255, 255, 255, 255, 127}, true, "2.1180591620717411303343", nil}, - {[]byte{255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 127}, true, "2.19342813113834066795298735", nil}, - } - - for _, v := range tests { + for _, v := range oidTests { oid, ok := newOIDFromDER(v.raw) if ok != v.valid { - if ok { - t.Errorf("%v: unexpected success while parsing: %v", v.raw, oid) - } else { - t.Errorf("%v: unexpected failure while parsing", v.raw) - } + t.Errorf("newOIDFromDER(%v) = (%v, %v); want = (OID, %v)", v.raw, oid, ok, v.valid) continue } @@ -61,7 +61,7 @@ func TestOID(t *testing.T) { } if str := oid.String(); str != v.str { - t.Errorf("%v: oid.String() = %v, want; %v", v.raw, str, v.str) + t.Errorf("(%#v).String() = %v, want; %v", oid, str, v.str) } var asn1OID asn1.ObjectIdentifier @@ -75,33 +75,186 @@ func TestOID(t *testing.T) { o, ok := oid.toASN1OID() if shouldOk := asn1OID != nil; shouldOk != ok { - if ok { - t.Errorf("%v: oid.toASN1OID() unexpected success", v.raw) - } else { - t.Errorf("%v: oid.toASN1OID() unexpected failure", v.raw) - } + t.Errorf("(%#v).toASN1OID() = (%v, %v); want = (%v, %v)", oid, o, ok, asn1OID, shouldOk) continue } - if asn1OID != nil { - if !o.Equal(asn1OID) { - t.Errorf("%v: oid.toASN1OID(asn1OID).Equal(oid) = false, want: true", v.raw) - } + if asn1OID != nil && !o.Equal(asn1OID) { + t.Errorf("(%#v).toASN1OID() = (%v, true); want = (%v, true)", oid, o, asn1OID) } if v.ints != nil { oid2, err := OIDFromInts(v.ints) if err != nil { - t.Errorf("%v: OIDFromInts() unexpected error: %v", v.raw, err) + t.Errorf("OIDFromInts(%v) = (%v, %v); want = (%v, nil)", v.ints, oid2, err, oid) } if !oid2.Equal(oid) { - t.Errorf("%v: %#v.Equal(%#v) = false, want: true", v.raw, oid2, oid) + t.Errorf("OIDFromInts(%v) = (%v, nil); want = (%v, nil)", v.ints, oid2, oid) } } } } -func mustNewOIDFromInts(t *testing.T, ints []uint64) OID { +func TestInvalidOID(t *testing.T) { + cases := []struct { + str string + ints []uint64 + }{ + {str: "", ints: []uint64{}}, + {str: "1", ints: []uint64{1}}, + {str: "3", ints: []uint64{3}}, + {str: "3.100.200", ints: []uint64{3, 100, 200}}, + {str: "1.81", ints: []uint64{1, 81}}, + {str: "1.81.200", ints: []uint64{1, 81, 200}}, + } + + for _, tt := range cases { + oid, err := OIDFromInts(tt.ints) + if err == nil { + t.Errorf("OIDFromInts(%v) = (%v, %v); want = (OID{}, %v)", tt.ints, oid, err, errInvalidOID) + } + + oid2, err := ParseOID(tt.str) + if err == nil { + t.Errorf("ParseOID(%v) = (%v, %v); want = (OID{}, %v)", tt.str, oid2, err, errInvalidOID) + } + + var oid3 OID + err = oid3.UnmarshalText([]byte(tt.str)) + if err == nil { + t.Errorf("(*OID).UnmarshalText(%v) = (%v, %v); want = (OID{}, %v)", tt.str, oid3, err, errInvalidOID) + } + } +} + +var ( + _ encoding.BinaryMarshaler = OID{} + _ encoding.BinaryUnmarshaler = new(OID) + _ encoding.TextMarshaler = OID{} + _ encoding.TextUnmarshaler = new(OID) +) + +func TestOIDMarshal(t *testing.T) { + cases := []struct { + in string + out OID + err error + }{ + {in: "", err: errInvalidOID}, + {in: "0", err: errInvalidOID}, + {in: "1", err: errInvalidOID}, + {in: ".1", err: errInvalidOID}, + {in: ".1.", err: errInvalidOID}, + {in: "1.", err: errInvalidOID}, + {in: "1..", err: errInvalidOID}, + {in: "1.2.", err: errInvalidOID}, + {in: "1.2.333.", err: errInvalidOID}, + {in: "1.2.333..", err: errInvalidOID}, + {in: "1.2..", err: errInvalidOID}, + {in: "+1.2", err: errInvalidOID}, + {in: "-1.2", err: errInvalidOID}, + {in: "1.-2", err: errInvalidOID}, + {in: "1.2.+333", err: errInvalidOID}, + } + + for _, v := range oidTests { + oid, ok := newOIDFromDER(v.raw) + if !ok { + continue + } + cases = append(cases, struct { + in string + out OID + err error + }{ + in: v.str, + out: oid, + err: nil, + }) + } + + for _, tt := range cases { + o, err := ParseOID(tt.in) + if err != tt.err { + t.Errorf("ParseOID(%q) = %v; want = %v", tt.in, err, tt.err) + continue + } + + var o2 OID + err = o2.UnmarshalText([]byte(tt.in)) + if err != tt.err { + t.Errorf("(*OID).UnmarshalText(%q) = %v; want = %v", tt.in, err, tt.err) + continue + } + + if err != nil { + continue + } + + if !o.Equal(tt.out) { + t.Errorf("(*OID).UnmarshalText(%q) = %v; want = %v", tt.in, o, tt.out) + continue + } + + if !o2.Equal(tt.out) { + t.Errorf("ParseOID(%q) = %v; want = %v", tt.in, o2, tt.out) + continue + } + + marshalled, err := o.MarshalText() + if string(marshalled) != tt.in || err != nil { + t.Errorf("(%#v).MarshalText() = (%v, %v); want = (%v, nil)", o, string(marshalled), err, tt.in) + continue + } + + binary, err := o.MarshalBinary() + if err != nil { + t.Errorf("(%#v).MarshalBinary() = %v; want = nil", o, err) + } + + var o3 OID + if err := o3.UnmarshalBinary(binary); err != nil { + t.Errorf("(*OID).UnmarshalBinary(%v) = %v; want = nil", binary, err) + } + + if !o3.Equal(tt.out) { + t.Errorf("(*OID).UnmarshalBinary(%v) = %v; want = %v", binary, o3, tt.out) + continue + } + } +} + +func TestOIDUnmarshalBinary(t *testing.T) { + for _, tt := range oidTests { + var o OID + err := o.UnmarshalBinary(tt.raw) + + expectErr := errInvalidOID + if tt.valid { + expectErr = nil + } + + if err != expectErr { + t.Errorf("(o *OID).UnmarshalBinary(%v) = %v; want = %v; (o = %v)", tt.raw, err, expectErr, o) + } + } +} + +func BenchmarkOIDMarshalUnmarshalText(b *testing.B) { + oid := mustNewOIDFromInts(b, []uint64{1, 2, 3, 9999, 1024}) + for range b.N { + text, err := oid.MarshalText() + if err != nil { + b.Fatal(err) + } + var o OID + if err := o.UnmarshalText(text); err != nil { + b.Fatal(err) + } + } +} + +func mustNewOIDFromInts(t testing.TB, ints []uint64) OID { oid, err := OIDFromInts(ints) if err != nil { t.Fatalf("OIDFromInts(%v) unexpected error: %v", ints, err) diff --git a/src/crypto/x509/x509.go b/src/crypto/x509/x509.go index 3e26941573..47bb428110 100644 --- a/src/crypto/x509/x509.go +++ b/src/crypto/x509/x509.go @@ -2111,8 +2111,16 @@ func CreateCertificateRequest(rand io.Reader, template *CertificateRequest, priv signed = h.Sum(nil) } + var signerOpts crypto.SignerOpts = hashFunc + if template.SignatureAlgorithm != 0 && template.SignatureAlgorithm.isRSAPSS() { + signerOpts = &rsa.PSSOptions{ + SaltLength: rsa.PSSSaltLengthEqualsHash, + Hash: hashFunc, + } + } + var signature []byte - signature, err = key.Sign(rand, signed, hashFunc) + signature, err = key.Sign(rand, signed, signerOpts) if err != nil { return } diff --git a/src/crypto/x509/x509_test.go b/src/crypto/x509/x509_test.go index a9dc145265..026367b167 100644 --- a/src/crypto/x509/x509_test.go +++ b/src/crypto/x509/x509_test.go @@ -1418,6 +1418,7 @@ func TestCreateCertificateRequest(t *testing.T) { sigAlgo SignatureAlgorithm }{ {"RSA", testPrivateKey, SHA256WithRSA}, + {"RSA-PSS-SHA256", testPrivateKey, SHA256WithRSAPSS}, {"ECDSA-256", ecdsa256Priv, ECDSAWithSHA256}, {"ECDSA-384", ecdsa384Priv, ECDSAWithSHA256}, {"ECDSA-521", ecdsa521Priv, ECDSAWithSHA256}, diff --git a/src/database/sql/sql.go b/src/database/sql/sql.go index fdbe4b2172..9373aa1c58 100644 --- a/src/database/sql/sql.go +++ b/src/database/sql/sql.go @@ -551,9 +551,9 @@ type driverConn struct { // guarded by db.mu inUse bool + dbmuClosed bool // same as closed, but guarded by db.mu, for removeClosedStmtLocked returnedAt time.Time // Time the connection was created or returned. onPut []func() // code (with db.mu held) run when conn is next returned - dbmuClosed bool // same as closed, but guarded by db.mu, for removeClosedStmtLocked } func (dc *driverConn) releaseConn(err error) { @@ -2923,19 +2923,8 @@ type Rows struct { // // closemu guards lasterr and closed. closemu sync.RWMutex - closed bool lasterr error // non-nil only if closed is true - - // lastcols is only used in Scan, Next, and NextResultSet which are expected - // not to be called concurrently. - lastcols []driver.Value - - // raw is a buffer for RawBytes that persists between Scan calls. - // This is used when the driver returns a mismatched type that requires - // a cloning allocation. For example, if the driver returns a *string and - // the user is scanning into a *RawBytes, we need to copy the string. - // The raw buffer here lets us reuse the memory for that copy across Scan calls. - raw []byte + closed bool // closemuScanHold is whether the previous call to Scan kept closemu RLock'ed // without unlocking it. It does that when the user passes a *RawBytes scan @@ -2951,6 +2940,17 @@ type Rows struct { // returning. It's only used by Next and Err which are // expected not to be called concurrently. hitEOF bool + + // lastcols is only used in Scan, Next, and NextResultSet which are expected + // not to be called concurrently. + lastcols []driver.Value + + // raw is a buffer for RawBytes that persists between Scan calls. + // This is used when the driver returns a mismatched type that requires + // a cloning allocation. For example, if the driver returns a *string and + // the user is scanning into a *RawBytes, we need to copy the string. + // The raw buffer here lets us reuse the memory for that copy across Scan calls. + raw []byte } // lasterrOrErrLocked returns either lasterr or the provided err. diff --git a/src/encoding/base32/base32.go b/src/encoding/base32/base32.go index 4a61199a59..9e988ef39b 100644 --- a/src/encoding/base32/base32.go +++ b/src/encoding/base32/base32.go @@ -467,7 +467,7 @@ func (d *decoder) Read(p []byte) (n int, err error) { } // Read a chunk. - nn := len(p) / 5 * 8 + nn := (len(p) + 4) / 5 * 8 if nn < 8 { nn = 8 } diff --git a/src/encoding/base32/base32_test.go b/src/encoding/base32/base32_test.go index 33638adeac..f5d3c49e38 100644 --- a/src/encoding/base32/base32_test.go +++ b/src/encoding/base32/base32_test.go @@ -92,6 +92,43 @@ func TestEncoderBuffering(t *testing.T) { } } +func TestDecoderBufferingWithPadding(t *testing.T) { + for bs := 0; bs <= 12; bs++ { + for _, s := range pairs { + decoder := NewDecoder(StdEncoding, strings.NewReader(s.encoded)) + buf := make([]byte, len(s.decoded)+bs) + + var n int + var err error + n, err = decoder.Read(buf) + + if err != nil && err != io.EOF { + t.Errorf("Read from %q at pos %d = %d, unexpected error %v", s.encoded, len(s.decoded), n, err) + } + testEqual(t, "Decoding/%d of %q = %q, want %q\n", bs, s.encoded, string(buf[:n]), s.decoded) + } + } +} + +func TestDecoderBufferingWithoutPadding(t *testing.T) { + for bs := 0; bs <= 12; bs++ { + for _, s := range pairs { + encoded := strings.TrimRight(s.encoded, "=") + decoder := NewDecoder(StdEncoding.WithPadding(NoPadding), strings.NewReader(encoded)) + buf := make([]byte, len(s.decoded)+bs) + + var n int + var err error + n, err = decoder.Read(buf) + + if err != nil && err != io.EOF { + t.Errorf("Read from %q at pos %d = %d, unexpected error %v", encoded, len(s.decoded), n, err) + } + testEqual(t, "Decoding/%d of %q = %q, want %q\n", bs, encoded, string(buf[:n]), s.decoded) + } + } +} + func TestDecode(t *testing.T) { for _, p := range pairs { dbuf := make([]byte, StdEncoding.DecodedLen(len(p.encoded))) diff --git a/src/encoding/binary/binary.go b/src/encoding/binary/binary.go index 634995a5bd..291e494dd4 100644 --- a/src/encoding/binary/binary.go +++ b/src/encoding/binary/binary.go @@ -722,49 +722,37 @@ func (e *encoder) value(v reflect.Value) { case reflect.Bool: e.bool(v.Bool()) - case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64: - switch v.Type().Kind() { - case reflect.Int8: - e.int8(int8(v.Int())) - case reflect.Int16: - e.int16(int16(v.Int())) - case reflect.Int32: - e.int32(int32(v.Int())) - case reflect.Int64: - e.int64(v.Int()) - } + case reflect.Int8: + e.int8(int8(v.Int())) + case reflect.Int16: + e.int16(int16(v.Int())) + case reflect.Int32: + e.int32(int32(v.Int())) + case reflect.Int64: + e.int64(v.Int()) - case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64, reflect.Uintptr: - switch v.Type().Kind() { - case reflect.Uint8: - e.uint8(uint8(v.Uint())) - case reflect.Uint16: - e.uint16(uint16(v.Uint())) - case reflect.Uint32: - e.uint32(uint32(v.Uint())) - case reflect.Uint64: - e.uint64(v.Uint()) - } + case reflect.Uint8: + e.uint8(uint8(v.Uint())) + case reflect.Uint16: + e.uint16(uint16(v.Uint())) + case reflect.Uint32: + e.uint32(uint32(v.Uint())) + case reflect.Uint64: + e.uint64(v.Uint()) - case reflect.Float32, reflect.Float64: - switch v.Type().Kind() { - case reflect.Float32: - e.uint32(math.Float32bits(float32(v.Float()))) - case reflect.Float64: - e.uint64(math.Float64bits(v.Float())) - } + case reflect.Float32: + e.uint32(math.Float32bits(float32(v.Float()))) + case reflect.Float64: + e.uint64(math.Float64bits(v.Float())) - case reflect.Complex64, reflect.Complex128: - switch v.Type().Kind() { - case reflect.Complex64: - x := v.Complex() - e.uint32(math.Float32bits(float32(real(x)))) - e.uint32(math.Float32bits(float32(imag(x)))) - case reflect.Complex128: - x := v.Complex() - e.uint64(math.Float64bits(real(x))) - e.uint64(math.Float64bits(imag(x))) - } + case reflect.Complex64: + x := v.Complex() + e.uint32(math.Float32bits(float32(real(x)))) + e.uint32(math.Float32bits(float32(imag(x)))) + case reflect.Complex128: + x := v.Complex() + e.uint64(math.Float64bits(real(x))) + e.uint64(math.Float64bits(imag(x))) } } diff --git a/src/encoding/csv/reader.go b/src/encoding/csv/reader.go index d9cab86572..df4702fede 100644 --- a/src/encoding/csv/reader.go +++ b/src/encoding/csv/reader.go @@ -4,7 +4,8 @@ // Package csv reads and writes comma-separated values (CSV) files. // There are many kinds of CSV files; this package supports the format -// described in RFC 4180. +// described in RFC 4180, except that [Writer] uses LF +// instead of CRLF as newline character by default. // // A csv file contains zero or more records of one or more fields per record. // Each record is separated by the newline character. The final record may diff --git a/src/encoding/gob/encode.go b/src/encoding/gob/encode.go index c83071c717..5f4d2539fa 100644 --- a/src/encoding/gob/encode.go +++ b/src/encoding/gob/encode.go @@ -601,7 +601,7 @@ func compileEnc(ut *userTypeInfo, building map[*typeInfo]bool) *encEngine { if ut.externalEnc == 0 && srt.Kind() == reflect.Struct { for fieldNum, wireFieldNum := 0, 0; fieldNum < srt.NumField(); fieldNum++ { f := srt.Field(fieldNum) - if !isSent(srt, &f) { + if !isSent(&f) { continue } op, indir := encOpFor(f.Type, seen, building) diff --git a/src/encoding/gob/type.go b/src/encoding/gob/type.go index 3b1dde492c..c3ac1dbd61 100644 --- a/src/encoding/gob/type.go +++ b/src/encoding/gob/type.go @@ -538,7 +538,7 @@ func newTypeObject(name string, ut *userTypeInfo, rt reflect.Type) (gobType, err idToTypeSlice[st.id()] = st for i := 0; i < t.NumField(); i++ { f := t.Field(i) - if !isSent(t, &f) { + if !isSent(&f) { continue } typ := userType(f.Type).base @@ -576,7 +576,7 @@ func isExported(name string) bool { // isSent reports whether this struct field is to be transmitted. // It will be transmitted only if it is exported and not a chan or func field // or pointer to chan or func. -func isSent(struct_ reflect.Type, field *reflect.StructField) bool { +func isSent(field *reflect.StructField) bool { if !isExported(field.Name) { return false } @@ -590,16 +590,6 @@ func isSent(struct_ reflect.Type, field *reflect.StructField) bool { return false } - // Special case for Go 1.22: the x509.Certificate.Policies - // field is unencodable but also unused by default. - // Ignore it, so that x509.Certificate continues to be encodeable. - // Go 1.23 will add the right methods so that gob can - // handle the Policies field, and then we can remove this check. - // See go.dev/issue/65633. - if field.Name == "Policies" && struct_.PkgPath() == "crypto/x509" && struct_.Name() == "Certificate" { - return false - } - return true } diff --git a/src/fmt/print.go b/src/fmt/print.go index 8d6d961228..f9f200499d 100644 --- a/src/fmt/print.go +++ b/src/fmt/print.go @@ -814,7 +814,7 @@ func (p *pp) printValue(value reflect.Value, verb rune, depth int) { p.buf.writeString(mapString) } sorted := fmtsort.Sort(f) - for i, key := range sorted.Key { + for i, m := range sorted { if i > 0 { if p.fmt.sharpV { p.buf.writeString(commaSpaceString) @@ -822,9 +822,9 @@ func (p *pp) printValue(value reflect.Value, verb rune, depth int) { p.buf.writeByte(' ') } } - p.printValue(key, verb, depth+1) + p.printValue(m.Key, verb, depth+1) p.buf.writeByte(':') - p.printValue(sorted.Value[i], verb, depth+1) + p.printValue(m.Value, verb, depth+1) } if p.fmt.sharpV { p.buf.writeByte('}') diff --git a/src/go/ast/example_test.go b/src/go/ast/example_test.go index 4ce42fb153..31b32efece 100644 --- a/src/go/ast/example_test.go +++ b/src/go/ast/example_test.go @@ -140,6 +140,41 @@ func main() { // 61 } } +func ExamplePreorder() { + src := ` +package p + +func f(x, y int) { + print(x + y) +} +` + + fset := token.NewFileSet() + f, err := parser.ParseFile(fset, "", src, 0) + if err != nil { + panic(err) + } + + // Print identifiers in order + for n := range ast.Preorder(f) { + id, ok := n.(*ast.Ident) + if !ok { + continue + } + fmt.Println(id.Name) + } + + // Output: + // p + // f + // x + // y + // int + // print + // x + // y +} + // This example illustrates how to remove a variable declaration // in a Go program while maintaining correct comment association // using an ast.CommentMap. diff --git a/src/go/ast/walk.go b/src/go/ast/walk.go index 59e6fc174d..ec9a8901c4 100644 --- a/src/go/ast/walk.go +++ b/src/go/ast/walk.go @@ -4,7 +4,10 @@ package ast -import "fmt" +import ( + "fmt" + "iter" +) // A Visitor's Visit method is invoked for each node encountered by [Walk]. // If the result visitor w is not nil, [Walk] visits each of the children @@ -368,3 +371,21 @@ func (f inspector) Visit(node Node) Visitor { func Inspect(node Node, f func(Node) bool) { Walk(inspector(f), node) } + +// Preorder returns an iterator over all the nodes of the syntax tree +// beneath (and including) the specified root, in depth-first +// preorder. +// +// For greater control over the traversal of each subtree, use [Inspect]. +func Preorder(root Node) iter.Seq[Node] { + return func(yield func(Node) bool) { + ok := true + Inspect(root, func(n Node) bool { + if n != nil { + // yield must not be called once ok is false. + ok = ok && yield(n) + } + return ok + }) + } +} diff --git a/src/go/ast/walk_test.go b/src/go/ast/walk_test.go new file mode 100644 index 0000000000..b8b4f958ec --- /dev/null +++ b/src/go/ast/walk_test.go @@ -0,0 +1,33 @@ +// Copyright 2024 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package ast_test + +import ( + "go/ast" + "go/parser" + "go/token" + "testing" +) + +func TestPreorderBreak(t *testing.T) { + // This test checks that Preorder correctly handles a break statement while + // in the middle of walking a node. Previously, incorrect handling of the + // boolean returned by the yield function resulted in the iterator calling + // yield for sibling nodes even after yield had returned false. With that + // bug, this test failed with a runtime panic. + src := "package p\ntype T struct {\n\tF int `json:\"f\"` // a field\n}\n" + + fset := token.NewFileSet() + f, err := parser.ParseFile(fset, "", src, 0) + if err != nil { + panic(err) + } + + for n := range ast.Preorder(f) { + if id, ok := n.(*ast.Ident); ok && id.Name == "F" { + break + } + } +} diff --git a/src/go/build/build.go b/src/go/build/build.go index 9ce3700dc4..43c74cb99a 100644 --- a/src/go/build/build.go +++ b/src/go/build/build.go @@ -286,6 +286,7 @@ func (ctxt *Context) SrcDirs() []string { // if set, or else the compiled code's GOARCH, GOOS, and GOROOT. var Default Context = defaultContext() +// Keep consistent with cmd/go/internal/cfg.defaultGOPATH. func defaultGOPATH() string { env := "HOME" if runtime.GOOS == "windows" { diff --git a/src/go/build/deps_test.go b/src/go/build/deps_test.go index f4973e92b1..ee53b31140 100644 --- a/src/go/build/deps_test.go +++ b/src/go/build/deps_test.go @@ -42,19 +42,20 @@ var depsRules = ` < cmp, container/list, container/ring, internal/cfg, internal/coverage, internal/coverage/rtcov, internal/coverage/uleb128, internal/coverage/calloc, - internal/cpu, internal/goarch, internal/godebugs, - internal/goexperiment, internal/goos, + internal/goarch, internal/godebugs, + internal/goexperiment, internal/goos, internal/byteorder, internal/goversion, internal/nettrace, internal/platform, internal/trace/traceviewer/format, log/internal, unicode/utf8, unicode/utf16, unicode, unsafe; - # These packages depend only on internal/goarch and unsafe. - internal/goarch, unsafe - < internal/abi, internal/chacha8rand; + # internal/abi depends only on internal/goarch and unsafe. + internal/goarch, unsafe < internal/abi; - unsafe < maps; + internal/byteorder, internal/goarch, unsafe < internal/chacha8rand; + + unsafe < internal/cpu, maps; # RUNTIME is the core runtime group of packages, all of them very light-weight. internal/abi, @@ -67,7 +68,6 @@ var depsRules = ` internal/goos < internal/bytealg < internal/stringslite - < internal/byteorder < internal/itoa < internal/unsafeheader < runtime/internal/sys @@ -249,7 +249,7 @@ var depsRules = ` < hash/adler32, hash/crc32, hash/crc64, hash/fnv; # math/big - FMT, encoding/binary, math/rand + FMT, math/rand < math/big; # compression @@ -430,10 +430,8 @@ var depsRules = ` crypto/internal/boring/sig, crypto/internal/boring/fipstls < crypto/tls/fipsonly; # CRYPTO is core crypto algorithms - no cgo, fmt, net. - # Unfortunately, stuck with reflect via encoding/binary. crypto/internal/boring/sig, crypto/internal/boring/syso, - encoding/binary, golang.org/x/sys/cpu, hash, embed < crypto @@ -455,12 +453,14 @@ var depsRules = ` crypto/boring < crypto/aes, crypto/des, crypto/hmac, crypto/md5, crypto/rc4, - crypto/sha1, crypto/sha256, crypto/sha512, - golang.org/x/crypto/sha3; + crypto/sha1, crypto/sha256, crypto/sha512; crypto/boring, crypto/internal/edwards25519/field < crypto/ecdh; + # Unfortunately, stuck with reflect via encoding/binary. + encoding/binary, crypto/boring < golang.org/x/crypto/sha3; + crypto/aes, crypto/des, crypto/ecdh, diff --git a/src/go/types/alias.go b/src/go/types/alias.go index 56d2ad0c97..3fdd12ea02 100644 --- a/src/go/types/alias.go +++ b/src/go/types/alias.go @@ -17,7 +17,9 @@ import "fmt" // which points directly to the actual (aliased) type. type Alias struct { obj *TypeName // corresponding declared alias object + orig *Alias // original, uninstantiated alias tparams *TypeParamList // type parameters, or nil + targs *TypeList // type arguments, or nil fromRHS Type // RHS of type alias declaration; may be an alias actual Type // actual (aliased) type; never an alias } @@ -31,9 +33,34 @@ func NewAlias(obj *TypeName, rhs Type) *Alias { return alias } -func (a *Alias) Obj() *TypeName { return a.obj } +func (a *Alias) Obj() *TypeName { return a.obj } +func (a *Alias) String() string { return TypeString(a, nil) } + +// Underlying returns the [underlying type] of the alias type a, which is the +// underlying type of the aliased type. Underlying types are never Named, +// TypeParam, or Alias types. +// +// [underlying type]: https://go.dev/ref/spec#Underlying_types. func (a *Alias) Underlying() Type { return unalias(a).Underlying() } -func (a *Alias) String() string { return TypeString(a, nil) } + +// Origin returns the generic Alias type of which a is an instance. +// If a is not an instance of a generic alias, Origin returns a. +func (a *Alias) Origin() *Alias { return a.orig } + +// TypeParams returns the type parameters of the alias type a, or nil. +// A generic Alias and its instances have the same type parameters. +func (a *Alias) TypeParams() *TypeParamList { return a.tparams } + +// SetTypeParams sets the type parameters of the alias type a. +// The alias a must not have type arguments. +func (a *Alias) SetTypeParams(tparams []*TypeParam) { + assert(a.targs == nil) + a.tparams = bindTParams(tparams) +} + +// TypeArgs returns the type arguments used to instantiate the Alias type. +// If a is not an instance of a generic alias, the result is nil. +func (a *Alias) TypeArgs() *TypeList { return a.targs } // Rhs returns the type R on the right-hand side of an alias // declaration "type A = R", which may be another alias. @@ -85,7 +112,10 @@ func asNamed(t Type) *Named { // rhs must not be nil. func (check *Checker) newAlias(obj *TypeName, rhs Type) *Alias { assert(rhs != nil) - a := &Alias{obj, nil, rhs, nil} + a := new(Alias) + a.obj = obj + a.orig = a + a.fromRHS = rhs if obj.typ == nil { obj.typ = a } diff --git a/src/go/types/api.go b/src/go/types/api.go index 2db67e5329..dea974bec8 100644 --- a/src/go/types/api.go +++ b/src/go/types/api.go @@ -36,6 +36,7 @@ import ( "go/constant" "go/token" . "internal/types/errors" + _ "unsafe" // for linkname ) // An Error describes a type-checking error; it implements the error interface. @@ -192,6 +193,9 @@ type Config struct { _EnableAlias bool } +// Linkname for use from srcimporter. +//go:linkname srcimporter_setUsesCgo + func srcimporter_setUsesCgo(conf *Config) { conf.go115UsesCgo = true } diff --git a/src/go/types/instantiate.go b/src/go/types/instantiate.go index d53f5d3fba..38a7e3ffe9 100644 --- a/src/go/types/instantiate.go +++ b/src/go/types/instantiate.go @@ -17,6 +17,12 @@ import ( . "internal/types/errors" ) +// A genericType implements access to its type parameters. +type genericType interface { + Type + TypeParams() *TypeParamList +} + // Instantiate instantiates the type orig with the given type arguments targs. // orig must be a *Named or a *Signature type. If there is no error, the // resulting Type is an instantiated type of the same kind (either a *Named or @@ -44,17 +50,15 @@ import ( // count is incorrect; for *Named types, a panic may occur later inside the // *Named API. func Instantiate(ctxt *Context, orig Type, targs []Type, validate bool) (Type, error) { + assert(len(targs) > 0) if ctxt == nil { ctxt = NewContext() } + orig_ := orig.(genericType) // signature of Instantiate must not change for backward-compatibility + if validate { - var tparams []*TypeParam - switch t := orig.(type) { - case *Named: - tparams = t.TypeParams().list() - case *Signature: - tparams = t.TypeParams().list() - } + tparams := orig_.TypeParams().list() + assert(len(tparams) > 0) if len(targs) != len(tparams) { return nil, fmt.Errorf("got %d type arguments but %s has %d type parameters", len(targs), orig, len(tparams)) } @@ -63,7 +67,7 @@ func Instantiate(ctxt *Context, orig Type, targs []Type, validate bool) (Type, e } } - inst := (*Checker)(nil).instance(nopos, orig, targs, nil, ctxt) + inst := (*Checker)(nil).instance(nopos, orig_, targs, nil, ctxt) return inst, nil } @@ -78,7 +82,7 @@ func Instantiate(ctxt *Context, orig Type, targs []Type, validate bool) (Type, e // must be non-nil. // // For Named types the resulting instance may be unexpanded. -func (check *Checker) instance(pos token.Pos, orig Type, targs []Type, expanding *Named, ctxt *Context) (res Type) { +func (check *Checker) instance(pos token.Pos, orig genericType, targs []Type, expanding *Named, ctxt *Context) (res Type) { // The order of the contexts below matters: we always prefer instances in the // expanding instance context in order to preserve reference cycles. // diff --git a/src/go/types/named.go b/src/go/types/named.go index b204b787db..b44fa9d788 100644 --- a/src/go/types/named.go +++ b/src/go/types/named.go @@ -488,9 +488,17 @@ func (t *Named) methodIndex(name string, foldCase bool) int { return -1 } -// TODO(gri) Investigate if Unalias can be moved to where underlying is set. -func (t *Named) Underlying() Type { return Unalias(t.resolve().underlying) } -func (t *Named) String() string { return TypeString(t, nil) } +// Underlying returns the [underlying type] of the named type t, resolving all +// forwarding declarations. Underlying types are never Named, TypeParam, or +// Alias types. +// +// [underlying type]: https://go.dev/ref/spec#Underlying_types. +func (t *Named) Underlying() Type { + // TODO(gri) Investigate if Unalias can be moved to where underlying is set. + return Unalias(t.resolve().underlying) +} + +func (t *Named) String() string { return TypeString(t, nil) } // ---------------------------------------------------------------------------- // Implementation diff --git a/src/go/types/stmt.go b/src/go/types/stmt.go index bfb51fd2e5..258ad1d327 100644 --- a/src/go/types/stmt.go +++ b/src/go/types/stmt.go @@ -898,7 +898,7 @@ func (check *Checker) rangeStmt(inner stmtContext, s *ast.RangeStmt) { lhs := [2]Expr{sKey, sValue} // sKey, sValue may be nil rhs := [2]Type{key, val} // key, val may be nil - constIntRange := x.mode == constant_ && isInteger(x.typ) + rangeOverInt := isInteger(x.typ) if isDef { // short variable declaration @@ -933,14 +933,15 @@ func (check *Checker) rangeStmt(inner stmtContext, s *ast.RangeStmt) { continue } - // initialize lhs variable - if constIntRange { + if rangeOverInt { + assert(i == 0) // at most one iteration variable (rhs[1] == nil for rangeOverInt) check.initVar(obj, &x, "range clause") } else { - x.mode = value - x.expr = lhs // we don't have a better rhs expression to use here - x.typ = typ - check.initVar(obj, &x, "assignment") // error is on variable, use "assignment" not "range clause" + var y operand + y.mode = value + y.expr = lhs // we don't have a better rhs expression to use here + y.typ = typ + check.initVar(obj, &y, "assignment") // error is on variable, use "assignment" not "range clause" } assert(obj.typ != nil) } @@ -967,21 +968,30 @@ func (check *Checker) rangeStmt(inner stmtContext, s *ast.RangeStmt) { continue } - if constIntRange { + if rangeOverInt { + assert(i == 0) // at most one iteration variable (rhs[1] == nil for rangeOverInt) check.assignVar(lhs, nil, &x, "range clause") + // If the assignment succeeded, if x was untyped before, it now + // has a type inferred via the assignment. It must be an integer. + // (go.dev/issues/67027) + if x.mode != invalid && !isInteger(x.typ) { + check.softErrorf(lhs, InvalidRangeExpr, "cannot use iteration variable of type %s", x.typ) + } } else { - x.mode = value - x.expr = lhs // we don't have a better rhs expression to use here - x.typ = typ - check.assignVar(lhs, nil, &x, "assignment") // error is on variable, use "assignment" not "range clause" + var y operand + y.mode = value + y.expr = lhs // we don't have a better rhs expression to use here + y.typ = typ + check.assignVar(lhs, nil, &y, "assignment") // error is on variable, use "assignment" not "range clause" } } - } else if constIntRange { + } else if rangeOverInt { // If we don't have any iteration variables, we still need to // check that a (possibly untyped) integer range expression x // is valid. // We do this by checking the assignment _ = x. This ensures - // that an untyped x can be converted to a value of type int. + // that an untyped x can be converted to a value of its default + // type (rune or int). check.assignment(&x, nil, "range clause") } @@ -1011,6 +1021,7 @@ func rangeKeyVal(typ Type, allowVersion func(goVersion) bool) (key, val Type, ca return Typ[Int], universeRune, "", false, true // use 'rune' name } if isInteger(typ) { + // untyped numeric constants may be representable as integer values if allowVersion != nil && !allowVersion(go1_22) { return bad("requires go1.22 or later") } diff --git a/src/go/types/type.go b/src/go/types/type.go index f6bd75908f..8fae93fb58 100644 --- a/src/go/types/type.go +++ b/src/go/types/type.go @@ -8,6 +8,9 @@ package types // All types implement the Type interface. type Type interface { // Underlying returns the underlying type of a type. + // Underlying types are never Named, TypeParam, or Alias types. + // + // See https://go.dev/ref/spec#Underlying_types. Underlying() Type // String returns a string representation of a type. diff --git a/src/go/types/typeparam.go b/src/go/types/typeparam.go index 8c960311cd..58a02de860 100644 --- a/src/go/types/typeparam.go +++ b/src/go/types/typeparam.go @@ -89,6 +89,10 @@ func (t *TypeParam) SetConstraint(bound Type) { t.iface() } +// Underlying returns the [underlying type] of the type parameter t, which is +// the underlying type of its constraint. This type is always an interface. +// +// [underlying type]: https://go.dev/ref/spec#Underlying_types. func (t *TypeParam) Underlying() Type { return t.iface() } diff --git a/src/image/gif/writer.go b/src/image/gif/writer.go index 0da47f3e35..0d2a1321c0 100644 --- a/src/image/gif/writer.go +++ b/src/image/gif/writer.go @@ -13,6 +13,7 @@ import ( "image/color" "image/color/palette" "image/draw" + "internal/byteorder" "io" ) @@ -33,12 +34,6 @@ func log2(x int) int { return -1 } -// Little-endian. -func writeUint16(b []uint8, u uint16) { - b[0] = uint8(u) - b[1] = uint8(u >> 8) -} - // writer is a buffered writer. type writer interface { Flush() error @@ -151,8 +146,8 @@ func (e *encoder) writeHeader() { } // Logical screen width and height. - writeUint16(e.buf[0:2], uint16(e.g.Config.Width)) - writeUint16(e.buf[2:4], uint16(e.g.Config.Height)) + byteorder.LePutUint16(e.buf[0:2], uint16(e.g.Config.Width)) + byteorder.LePutUint16(e.buf[2:4], uint16(e.g.Config.Height)) e.write(e.buf[:4]) if p, ok := e.g.Config.ColorModel.(color.Palette); ok && len(p) > 0 { @@ -190,7 +185,7 @@ func (e *encoder) writeHeader() { } e.buf[0] = 0x03 // Block Size. e.buf[1] = 0x01 // Sub-block Index. - writeUint16(e.buf[2:4], uint16(e.g.LoopCount)) + byteorder.LePutUint16(e.buf[2:4], uint16(e.g.LoopCount)) e.buf[4] = 0x00 // Block Terminator. e.write(e.buf[:5]) } @@ -276,7 +271,7 @@ func (e *encoder) writeImageBlock(pm *image.Paletted, delay int, disposal byte) } else { e.buf[3] = 0x00 | disposal<<2 } - writeUint16(e.buf[4:6], uint16(delay)) // Delay Time (1/100ths of a second) + byteorder.LePutUint16(e.buf[4:6], uint16(delay)) // Delay Time (1/100ths of a second) // Transparent color index. if transparentIndex != -1 { @@ -288,10 +283,10 @@ func (e *encoder) writeImageBlock(pm *image.Paletted, delay int, disposal byte) e.write(e.buf[:8]) } e.buf[0] = sImageDescriptor - writeUint16(e.buf[1:3], uint16(b.Min.X)) - writeUint16(e.buf[3:5], uint16(b.Min.Y)) - writeUint16(e.buf[5:7], uint16(b.Dx())) - writeUint16(e.buf[7:9], uint16(b.Dy())) + byteorder.LePutUint16(e.buf[1:3], uint16(b.Min.X)) + byteorder.LePutUint16(e.buf[3:5], uint16(b.Min.Y)) + byteorder.LePutUint16(e.buf[5:7], uint16(b.Dx())) + byteorder.LePutUint16(e.buf[7:9], uint16(b.Dy())) e.write(e.buf[:9]) // To determine whether or not this frame's palette is the same as the diff --git a/src/internal/chacha8rand/chacha8.go b/src/internal/chacha8rand/chacha8.go index 0e601c23ac..8f1b4e5315 100644 --- a/src/internal/chacha8rand/chacha8.go +++ b/src/internal/chacha8rand/chacha8.go @@ -4,9 +4,11 @@ // Package chacha8rand implements a pseudorandom generator // based on ChaCha8. It is used by both runtime and math/rand/v2 -// and must have no dependencies. +// and must have minimal dependencies. package chacha8rand +import "internal/byteorder" + const ( ctrInc = 4 // increment counter by 4 between block calls ctrMax = 16 // reseed when counter reaches 16 @@ -51,10 +53,10 @@ func (s *State) Next() (uint64, bool) { // Init seeds the State with the given seed value. func (s *State) Init(seed [32]byte) { s.Init64([4]uint64{ - leUint64(seed[0*8:]), - leUint64(seed[1*8:]), - leUint64(seed[2*8:]), - leUint64(seed[3*8:]), + byteorder.LeUint64(seed[0*8:]), + byteorder.LeUint64(seed[1*8:]), + byteorder.LeUint64(seed[2*8:]), + byteorder.LeUint64(seed[3*8:]), }) } @@ -122,9 +124,9 @@ func Marshal(s *State) []byte { data := make([]byte, 6*8) copy(data, "chacha8:") used := (s.c/ctrInc)*chunk + s.i - bePutUint64(data[1*8:], uint64(used)) + byteorder.BePutUint64(data[1*8:], uint64(used)) for i, seed := range s.seed { - lePutUint64(data[(2+i)*8:], seed) + byteorder.LePutUint64(data[(2+i)*8:], seed) } return data } @@ -140,12 +142,12 @@ func Unmarshal(s *State, data []byte) error { if len(data) != 6*8 || string(data[:8]) != "chacha8:" { return new(errUnmarshalChaCha8) } - used := beUint64(data[1*8:]) + used := byteorder.BeUint64(data[1*8:]) if used > (ctrMax/ctrInc)*chunk-reseed { return new(errUnmarshalChaCha8) } for i := range s.seed { - s.seed[i] = leUint64(data[(2+i)*8:]) + s.seed[i] = byteorder.LeUint64(data[(2+i)*8:]) } s.c = ctrInc * (uint32(used) / chunk) block(&s.seed, &s.buf, s.c) @@ -156,43 +158,3 @@ func Unmarshal(s *State, data []byte) error { } return nil } - -// binary.bigEndian.Uint64, copied to avoid dependency -func beUint64(b []byte) uint64 { - _ = b[7] // bounds check hint to compiler; see golang.org/issue/14808 - return uint64(b[7]) | uint64(b[6])<<8 | uint64(b[5])<<16 | uint64(b[4])<<24 | - uint64(b[3])<<32 | uint64(b[2])<<40 | uint64(b[1])<<48 | uint64(b[0])<<56 -} - -// binary.bigEndian.PutUint64, copied to avoid dependency -func bePutUint64(b []byte, v uint64) { - _ = b[7] // early bounds check to guarantee safety of writes below - b[0] = byte(v >> 56) - b[1] = byte(v >> 48) - b[2] = byte(v >> 40) - b[3] = byte(v >> 32) - b[4] = byte(v >> 24) - b[5] = byte(v >> 16) - b[6] = byte(v >> 8) - b[7] = byte(v) -} - -// binary.littleEndian.Uint64, copied to avoid dependency -func leUint64(b []byte) uint64 { - _ = b[7] // bounds check hint to compiler; see golang.org/issue/14808 - return uint64(b[0]) | uint64(b[1])<<8 | uint64(b[2])<<16 | uint64(b[3])<<24 | - uint64(b[4])<<32 | uint64(b[5])<<40 | uint64(b[6])<<48 | uint64(b[7])<<56 -} - -// binary.littleEndian.PutUint64, copied to avoid dependency -func lePutUint64(b []byte, v uint64) { - _ = b[7] // early bounds check to guarantee safety of writes below - b[0] = byte(v) - b[1] = byte(v >> 8) - b[2] = byte(v >> 16) - b[3] = byte(v >> 24) - b[4] = byte(v >> 32) - b[5] = byte(v >> 40) - b[6] = byte(v >> 48) - b[7] = byte(v >> 56) -} diff --git a/src/internal/cpu/cpu.go b/src/internal/cpu/cpu.go index d794e53cee..9be280c6ba 100644 --- a/src/internal/cpu/cpu.go +++ b/src/internal/cpu/cpu.go @@ -6,6 +6,8 @@ // used by the Go standard library. package cpu +import _ "unsafe" // for linkname + // DebugOptions is set to true by the runtime if the OS supports reading // GODEBUG early in runtime startup. // This should not be changed after it is initialized. @@ -121,6 +123,14 @@ var S390X struct { _ CacheLinePad } +// CPU feature variables are accessed by assembly code in various packages. +//go:linkname X86 +//go:linkname ARM +//go:linkname ARM64 +//go:linkname MIPS64X +//go:linkname PPC64 +//go:linkname S390X + // Initialize examines the processor and sets the relevant variables above. // This is called by the runtime package early in program initialization, // before normal init functions are run. env is set by runtime if the OS supports diff --git a/src/internal/fmtsort/sort.go b/src/internal/fmtsort/sort.go index 278a89bd75..ea042e1811 100644 --- a/src/internal/fmtsort/sort.go +++ b/src/internal/fmtsort/sort.go @@ -10,24 +10,21 @@ package fmtsort import ( "reflect" - "sort" + "slices" ) // Note: Throughout this package we avoid calling reflect.Value.Interface as // it is not always legal to do so and it's easier to avoid the issue than to face it. -// SortedMap represents a map's keys and values. The keys and values are -// aligned in index order: Value[i] is the value in the map corresponding to Key[i]. -type SortedMap struct { - Key []reflect.Value - Value []reflect.Value -} +// SortedMap is a slice of KeyValue pairs that simplifies sorting +// and iterating over map entries. +// +// Each KeyValue pair contains a map key and its corresponding value. +type SortedMap []KeyValue -func (o *SortedMap) Len() int { return len(o.Key) } -func (o *SortedMap) Less(i, j int) bool { return compare(o.Key[i], o.Key[j]) < 0 } -func (o *SortedMap) Swap(i, j int) { - o.Key[i], o.Key[j] = o.Key[j], o.Key[i] - o.Value[i], o.Value[j] = o.Value[j], o.Value[i] +// KeyValue holds a single key and value pair found in a map. +type KeyValue struct { + Key, Value reflect.Value } // Sort accepts a map and returns a SortedMap that has the same keys and @@ -48,7 +45,7 @@ func (o *SortedMap) Swap(i, j int) { // Otherwise identical arrays compare by length. // - interface values compare first by reflect.Type describing the concrete type // and then by concrete value as described in the previous rules. -func Sort(mapValue reflect.Value) *SortedMap { +func Sort(mapValue reflect.Value) SortedMap { if mapValue.Type().Kind() != reflect.Map { return nil } @@ -56,18 +53,14 @@ func Sort(mapValue reflect.Value) *SortedMap { // of a concurrent map update. The runtime is responsible for // yelling loudly if that happens. See issue 33275. n := mapValue.Len() - key := make([]reflect.Value, 0, n) - value := make([]reflect.Value, 0, n) + sorted := make(SortedMap, 0, n) iter := mapValue.MapRange() for iter.Next() { - key = append(key, iter.Key()) - value = append(value, iter.Value()) + sorted = append(sorted, KeyValue{iter.Key(), iter.Value()}) } - sorted := &SortedMap{ - Key: key, - Value: value, - } - sort.Stable(sorted) + slices.SortStableFunc(sorted, func(a, b KeyValue) int { + return compare(a.Key, b.Key) + }) return sorted } diff --git a/src/internal/fmtsort/sort_test.go b/src/internal/fmtsort/sort_test.go index 7d5de9f56b..29a9c2c43f 100644 --- a/src/internal/fmtsort/sort_test.go +++ b/src/internal/fmtsort/sort_test.go @@ -142,13 +142,13 @@ func sprint(data any) string { return "nil" } b := new(strings.Builder) - for i, key := range om.Key { + for i, m := range om { if i > 0 { b.WriteRune(' ') } - b.WriteString(sprintKey(key)) + b.WriteString(sprintKey(m.Key)) b.WriteRune(':') - fmt.Fprint(b, om.Value[i]) + fmt.Fprint(b, m.Value) } return b.String() } diff --git a/src/internal/poll/fd_wasip1.go b/src/internal/poll/fd_wasip1.go index aecd89669b..195aaa9517 100644 --- a/src/internal/poll/fd_wasip1.go +++ b/src/internal/poll/fd_wasip1.go @@ -5,6 +5,7 @@ package poll import ( + "internal/byteorder" "sync/atomic" "syscall" "unsafe" @@ -224,15 +225,11 @@ func readIntLE(b []byte, size uintptr) uint64 { case 1: return uint64(b[0]) case 2: - _ = b[1] // bounds check hint to compiler; see golang.org/issue/14808 - return uint64(b[0]) | uint64(b[1])<<8 + return uint64(byteorder.LeUint16(b)) case 4: - _ = b[3] // bounds check hint to compiler; see golang.org/issue/14808 - return uint64(b[0]) | uint64(b[1])<<8 | uint64(b[2])<<16 | uint64(b[3])<<24 + return uint64(byteorder.LeUint32(b)) case 8: - _ = b[7] // bounds check hint to compiler; see golang.org/issue/14808 - return uint64(b[0]) | uint64(b[1])<<8 | uint64(b[2])<<16 | uint64(b[3])<<24 | - uint64(b[4])<<32 | uint64(b[5])<<40 | uint64(b[6])<<48 | uint64(b[7])<<56 + return uint64(byteorder.LeUint64(b)) default: panic("internal/poll: readInt with unsupported size") } diff --git a/src/internal/runtime/atomic/atomic_386.go b/src/internal/runtime/atomic/atomic_386.go index e74dcaa92d..a023baddb7 100644 --- a/src/internal/runtime/atomic/atomic_386.go +++ b/src/internal/runtime/atomic/atomic_386.go @@ -12,6 +12,7 @@ import "unsafe" // //go:linkname Load //go:linkname Loadp +//go:linkname LoadAcquintptr //go:nosplit //go:noinline diff --git a/src/internal/runtime/atomic/atomic_arm.go b/src/internal/runtime/atomic/atomic_arm.go index 567e951244..b58f643ca3 100644 --- a/src/internal/runtime/atomic/atomic_arm.go +++ b/src/internal/runtime/atomic/atomic_arm.go @@ -19,6 +19,7 @@ const ( // //go:linkname Xchg //go:linkname Xchguintptr +//go:linkname Xadd type spinlock struct { v uint32 diff --git a/src/internal/runtime/atomic/atomic_wasm.go b/src/internal/runtime/atomic/atomic_wasm.go index 835fc43ccf..d1dcfec7ad 100644 --- a/src/internal/runtime/atomic/atomic_wasm.go +++ b/src/internal/runtime/atomic/atomic_wasm.go @@ -13,6 +13,7 @@ //go:linkname Loadint32 //go:linkname Loadint64 //go:linkname Loaduintptr +//go:linkname LoadAcquintptr //go:linkname Xadd //go:linkname Xaddint32 //go:linkname Xaddint64 @@ -33,6 +34,7 @@ //go:linkname Storeint32 //go:linkname Storeint64 //go:linkname Storeuintptr +//go:linkname StoreReluintptr package atomic diff --git a/src/internal/types/testdata/check/stmt0.go b/src/internal/types/testdata/check/stmt0.go index a6c47cb483..ea161279c6 100644 --- a/src/internal/types/testdata/check/stmt0.go +++ b/src/internal/types/testdata/check/stmt0.go @@ -953,10 +953,10 @@ func issue10148() { for y /* ERROR "declared and not used" */ := range "" { _ = "" /* ERROR "mismatched types untyped string and untyped int" */ + 1 } - for range 1.5 /* ERROR "cannot range over 1.5" */ { + for range 1.5 /* ERROR "cannot range over 1.5 (untyped float constant)" */ { _ = "" /* ERROR "mismatched types untyped string and untyped int" */ + 1 } - for y := range 1.5 /* ERROR "cannot range over 1.5" */ { + for y := range 1.5 /* ERROR "cannot range over 1.5 (untyped float constant)" */ { _ = "" /* ERROR "mismatched types untyped string and untyped int" */ + 1 } } diff --git a/src/internal/types/testdata/spec/range_int.go b/src/internal/types/testdata/spec/range_int.go index 7f722e2d99..766736cc15 100644 --- a/src/internal/types/testdata/spec/range_int.go +++ b/src/internal/types/testdata/spec/range_int.go @@ -129,3 +129,62 @@ func issue65133() { for u8 = range 256 /* ERROR "cannot use 256 (untyped int constant) as uint8 value in range clause (overflows)" */ { } } + +func issue64471() { + for i := range 'a' { + var _ *rune = &i // ensure i has type rune + } +} + +func issue66561() { + for range 10.0 /* ERROR "cannot range over 10.0 (untyped float constant 10)" */ { + } + for range 1e3 /* ERROR "cannot range over 1e3 (untyped float constant 1000)" */ { + } + for range 1 /* ERROR "cannot range over 1 + 0i (untyped complex constant (1 + 0i))" */ + 0i { + } + + for range 1.1 /* ERROR "cannot range over 1.1 (untyped float constant)" */ { + } + for range 1i /* ERROR "cannot range over 1i (untyped complex constant (0 + 1i))" */ { + } + + for i := range 10.0 /* ERROR "cannot range over 10.0 (untyped float constant 10)" */ { + _ = i + } + for i := range 1e3 /* ERROR "cannot range over 1e3 (untyped float constant 1000)" */ { + _ = i + } + for i := range 1 /* ERROR "cannot range over 1 + 0i (untyped complex constant (1 + 0i))" */ + 0i { + _ = i + } + + for i := range 1.1 /* ERROR "cannot range over 1.1 (untyped float constant)" */ { + _ = i + } + for i := range 1i /* ERROR "cannot range over 1i (untyped complex constant (0 + 1i))" */ { + _ = i + } + + var j float64 + _ = j + for j /* ERROR "cannot use iteration variable of type float64" */ = range 1 { + } + for j = range 1.1 /* ERROR "cannot range over 1.1 (untyped float constant)" */ { + } + for j = range 10.0 /* ERROR "cannot range over 10.0 (untyped float constant 10)" */ { + } + + // There shouldn't be assignment errors if there are more iteration variables than permitted. + var i int + _ = i + for i, j /* ERROR "range over 10 (untyped int constant) permits only one iteration variable" */ = range 10 { + } +} + +func issue67027() { + var i float64 + _ = i + for i /* ERROR "cannot use iteration variable of type float64" */ = range 10 { + } +} diff --git a/src/log/slog/value.go b/src/log/slog/value.go index d278d9b923..6b0768eb1d 100644 --- a/src/log/slog/value.go +++ b/src/log/slog/value.go @@ -90,7 +90,7 @@ func (v Value) Kind() Kind { return x case stringptr: return KindString - case timeLocation: + case timeLocation, timeTime: return KindTime case groupptr: return KindGroup @@ -139,9 +139,14 @@ func BoolValue(v bool) Value { return Value{num: u, any: KindBool} } -// Unexported version of *time.Location, just so we can store *time.Locations in -// Values. (No user-provided value has this type.) -type timeLocation *time.Location +type ( + // Unexported version of *time.Location, just so we can store *time.Locations in + // Values. (No user-provided value has this type.) + timeLocation *time.Location + + // timeTime is for times where UnixNano is undefined. + timeTime time.Time +) // TimeValue returns a [Value] for a [time.Time]. // It discards the monotonic portion. @@ -153,7 +158,15 @@ func TimeValue(v time.Time) Value { // mistaken for any other Value, time.Time or otherwise. return Value{any: timeLocation(nil)} } - return Value{num: uint64(v.UnixNano()), any: timeLocation(v.Location())} + nsec := v.UnixNano() + t := time.Unix(0, nsec) + if v.Equal(t) { + // UnixNano correctly represents the time, so use a zero-alloc representation. + return Value{num: uint64(nsec), any: timeLocation(v.Location())} + } + // Fall back to the general form. + // Strip the monotonic portion to match the other representation. + return Value{any: timeTime(v.Round(0))} } // DurationValue returns a [Value] for a [time.Duration]. @@ -368,12 +381,19 @@ func (v Value) Time() time.Time { return v.time() } +// See TimeValue to understand how times are represented. func (v Value) time() time.Time { - loc := v.any.(timeLocation) - if loc == nil { - return time.Time{} + switch a := v.any.(type) { + case timeLocation: + if a == nil { + return time.Time{} + } + return time.Unix(0, int64(v.num)).In(a) + case timeTime: + return time.Time(a) + default: + panic(fmt.Sprintf("bad time type %T", v.any)) } - return time.Unix(0, int64(v.num)).In(loc) } // LogValuer returns v's value as a LogValuer. It panics diff --git a/src/log/slog/value_test.go b/src/log/slog/value_test.go index 033f945407..3e191589c5 100644 --- a/src/log/slog/value_test.go +++ b/src/log/slog/value_test.go @@ -30,7 +30,10 @@ func TestValueEqual(t *testing.T) { BoolValue(true), BoolValue(false), TimeValue(testTime), + TimeValue(time.Time{}), TimeValue(time.Date(2001, 1, 2, 3, 4, 5, 0, time.UTC)), + TimeValue(time.Date(2300, 1, 1, 0, 0, 0, 0, time.UTC)), // overflows nanoseconds + TimeValue(time.Date(1715, 6, 13, 0, 25, 26, 290448384, time.UTC)), // overflowed value AnyValue(&x), AnyValue(&y), GroupValue(Bool("b", true), Int("i", 3)), @@ -229,11 +232,20 @@ func TestLogValue(t *testing.T) { } } -func TestZeroTime(t *testing.T) { - z := time.Time{} - got := TimeValue(z).Time() - if !got.IsZero() { - t.Errorf("got %s (%#[1]v), not zero time (%#v)", got, z) +func TestValueTime(t *testing.T) { + // Validate that all representations of times work correctly. + for _, tm := range []time.Time{ + time.Time{}, + time.Unix(0, 1e15), // UnixNanos is defined + time.Date(2300, 1, 1, 0, 0, 0, 0, time.UTC), // overflows UnixNanos + } { + got := TimeValue(tm).Time() + if !got.Equal(tm) { + t.Errorf("got %s (%#[1]v), want %s (%#[2]v)", got, tm) + } + if g, w := got.Location(), tm.Location(); g != w { + t.Errorf("%s: location: got %v, want %v", tm, g, w) + } } } diff --git a/src/math/big/floatmarsh.go b/src/math/big/floatmarsh.go index 8a908cef28..16be946971 100644 --- a/src/math/big/floatmarsh.go +++ b/src/math/big/floatmarsh.go @@ -7,9 +7,9 @@ package big import ( - "encoding/binary" "errors" "fmt" + "internal/byteorder" ) // Gob codec version. Permits backward-compatible changes to the encoding. @@ -48,10 +48,10 @@ func (x *Float) GobEncode() ([]byte, error) { b |= 1 } buf[1] = b - binary.BigEndian.PutUint32(buf[2:], x.prec) + byteorder.BePutUint32(buf[2:], x.prec) if x.form == finite { - binary.BigEndian.PutUint32(buf[6:], uint32(x.exp)) + byteorder.BePutUint32(buf[6:], uint32(x.exp)) x.mant[len(x.mant)-n:].bytes(buf[10:]) // cut off unused trailing words } @@ -84,13 +84,13 @@ func (z *Float) GobDecode(buf []byte) error { z.acc = Accuracy((b>>3)&3) - 1 z.form = form((b >> 1) & 3) z.neg = b&1 != 0 - z.prec = binary.BigEndian.Uint32(buf[2:]) + z.prec = byteorder.BeUint32(buf[2:]) if z.form == finite { if len(buf) < 10 { return errors.New("Float.GobDecode: buffer too small for finite form float") } - z.exp = int32(binary.BigEndian.Uint32(buf[6:])) + z.exp = int32(byteorder.BeUint32(buf[6:])) z.mant = z.mant.setBytes(buf[10:]) } diff --git a/src/math/big/nat.go b/src/math/big/nat.go index 1d702c7726..23b2a0b8dd 100644 --- a/src/math/big/nat.go +++ b/src/math/big/nat.go @@ -14,7 +14,7 @@ package big import ( - "encoding/binary" + "internal/byteorder" "math/bits" "math/rand" "sync" @@ -1321,9 +1321,9 @@ func (z nat) bytes(buf []byte) (i int) { // bigEndianWord returns the contents of buf interpreted as a big-endian encoded Word value. func bigEndianWord(buf []byte) Word { if _W == 64 { - return Word(binary.BigEndian.Uint64(buf)) + return Word(byteorder.BeUint64(buf)) } - return Word(binary.BigEndian.Uint32(buf)) + return Word(byteorder.BeUint32(buf)) } // setBytes interprets buf as the bytes of a big-endian unsigned diff --git a/src/math/big/ratmarsh.go b/src/math/big/ratmarsh.go index 033fb4459d..6962829453 100644 --- a/src/math/big/ratmarsh.go +++ b/src/math/big/ratmarsh.go @@ -7,9 +7,9 @@ package big import ( - "encoding/binary" "errors" "fmt" + "internal/byteorder" "math" ) @@ -29,7 +29,7 @@ func (x *Rat) GobEncode() ([]byte, error) { // this should never happen return nil, errors.New("Rat.GobEncode: numerator too large") } - binary.BigEndian.PutUint32(buf[j-4:j], uint32(n)) + byteorder.BePutUint32(buf[j-4:j], uint32(n)) j -= 1 + 4 b := ratGobVersion << 1 // make space for sign bit if x.a.neg { @@ -54,7 +54,7 @@ func (z *Rat) GobDecode(buf []byte) error { return fmt.Errorf("Rat.GobDecode: encoding version %d not supported", b>>1) } const j = 1 + 4 - ln := binary.BigEndian.Uint32(buf[j-4 : j]) + ln := byteorder.BeUint32(buf[j-4 : j]) if uint64(ln) > math.MaxInt-j { return errors.New("Rat.GobDecode: invalid length") } diff --git a/src/math/rand/v2/pcg.go b/src/math/rand/v2/pcg.go index 77708d799e..4ccd5e320b 100644 --- a/src/math/rand/v2/pcg.go +++ b/src/math/rand/v2/pcg.go @@ -6,6 +6,7 @@ package rand import ( "errors" + "internal/byteorder" "math/bits" ) @@ -30,32 +31,12 @@ func (p *PCG) Seed(seed1, seed2 uint64) { p.lo = seed2 } -// binary.bigEndian.Uint64, copied to avoid dependency -func beUint64(b []byte) uint64 { - _ = b[7] // bounds check hint to compiler; see golang.org/issue/14808 - return uint64(b[7]) | uint64(b[6])<<8 | uint64(b[5])<<16 | uint64(b[4])<<24 | - uint64(b[3])<<32 | uint64(b[2])<<40 | uint64(b[1])<<48 | uint64(b[0])<<56 -} - -// binary.bigEndian.PutUint64, copied to avoid dependency -func bePutUint64(b []byte, v uint64) { - _ = b[7] // early bounds check to guarantee safety of writes below - b[0] = byte(v >> 56) - b[1] = byte(v >> 48) - b[2] = byte(v >> 40) - b[3] = byte(v >> 32) - b[4] = byte(v >> 24) - b[5] = byte(v >> 16) - b[6] = byte(v >> 8) - b[7] = byte(v) -} - // MarshalBinary implements the encoding.BinaryMarshaler interface. func (p *PCG) MarshalBinary() ([]byte, error) { b := make([]byte, 20) copy(b, "pcg:") - bePutUint64(b[4:], p.hi) - bePutUint64(b[4+8:], p.lo) + byteorder.BePutUint64(b[4:], p.hi) + byteorder.BePutUint64(b[4+8:], p.lo) return b, nil } @@ -66,8 +47,8 @@ func (p *PCG) UnmarshalBinary(data []byte) error { if len(data) != 20 || string(data[:4]) != "pcg:" { return errUnmarshalPCG } - p.hi = beUint64(data[4:]) - p.lo = beUint64(data[4+8:]) + p.hi = byteorder.BeUint64(data[4:]) + p.lo = byteorder.BeUint64(data[4+8:]) return nil } diff --git a/src/math/rand/v2/rand_test.go b/src/math/rand/v2/rand_test.go index d223180fb6..e89ee29f60 100644 --- a/src/math/rand/v2/rand_test.go +++ b/src/math/rand/v2/rand_test.go @@ -31,13 +31,6 @@ type statsResults struct { maxError float64 } -func max(a, b float64) float64 { - if a > b { - return a - } - return b -} - func nearEqual(a, b, closeEnough, maxError float64) bool { absDiff := math.Abs(a - b) if absDiff < closeEnough { // Necessary when one value is zero and one value is close to zero. diff --git a/src/net/http/fs_test.go b/src/net/http/fs_test.go index 63278d890f..2c3426f735 100644 --- a/src/net/http/fs_test.go +++ b/src/net/http/fs_test.go @@ -1456,7 +1456,7 @@ func (d fileServerCleanPathDir) Open(path string) (File, error) { type panicOnSeek struct{ io.ReadSeeker } -func Test_scanETag(t *testing.T) { +func TestScanETag(t *testing.T) { tests := []struct { in string wantETag string diff --git a/src/net/http/request.go b/src/net/http/request.go index bdd18adf3f..f208b95c46 100644 --- a/src/net/http/request.go +++ b/src/net/http/request.go @@ -320,6 +320,10 @@ type Request struct { // redirects. Response *Response + // Pattern is the [ServeMux] pattern that matched the request. + // It is empty if the request was not matched against a pattern. + Pattern string + // ctx is either the client or server context. It should only // be modified via copying the whole Request using Clone or WithContext. // It is unexported to prevent people from using Context wrong diff --git a/src/net/http/request_test.go b/src/net/http/request_test.go index a7deba46e3..9b6eb6e1a8 100644 --- a/src/net/http/request_test.go +++ b/src/net/http/request_test.go @@ -1527,7 +1527,7 @@ func TestPathValueNoMatch(t *testing.T) { } } -func TestPathValue(t *testing.T) { +func TestPathValueAndPattern(t *testing.T) { for _, test := range []struct { pattern string url string @@ -1576,6 +1576,9 @@ func TestPathValue(t *testing.T) { t.Errorf("%q, %q: got %q, want %q", test.pattern, name, got, want) } } + if r.Pattern != test.pattern { + t.Errorf("pattern: got %s, want %s", r.Pattern, test.pattern) + } }) server := httptest.NewServer(mux) defer server.Close() diff --git a/src/net/http/serve_test.go b/src/net/http/serve_test.go index e21af8b159..f454dcdbed 100644 --- a/src/net/http/serve_test.go +++ b/src/net/http/serve_test.go @@ -7166,3 +7166,79 @@ func TestError(t *testing.T) { t.Errorf("X-Content-Type-Options: %q, want %q", v, "nosniff") } } + +func TestServerReadAfterWriteHeader100Continue(t *testing.T) { + run(t, testServerReadAfterWriteHeader100Continue) +} +func testServerReadAfterWriteHeader100Continue(t *testing.T, mode testMode) { + body := []byte("body") + cst := newClientServerTest(t, mode, HandlerFunc(func(w ResponseWriter, r *Request) { + w.WriteHeader(200) + NewResponseController(w).Flush() + io.ReadAll(r.Body) + w.Write(body) + })) + + req, _ := NewRequest("GET", cst.ts.URL, strings.NewReader("body")) + req.Header.Set("Expect", "100-continue") + res, err := cst.c.Do(req) + if err != nil { + t.Fatalf("Get(%q) = %v", cst.ts.URL, err) + } + defer res.Body.Close() + got, err := io.ReadAll(res.Body) + if err != nil { + t.Fatalf("io.ReadAll(res.Body) = %v", err) + } + if !bytes.Equal(got, body) { + t.Fatalf("response body = %q, want %q", got, body) + } +} + +func TestServerReadAfterHandlerDone100Continue(t *testing.T) { + run(t, testServerReadAfterHandlerDone100Continue) +} +func testServerReadAfterHandlerDone100Continue(t *testing.T, mode testMode) { + readyc := make(chan struct{}) + cst := newClientServerTest(t, mode, HandlerFunc(func(w ResponseWriter, r *Request) { + go func() { + <-readyc + io.ReadAll(r.Body) + <-readyc + }() + })) + + req, _ := NewRequest("GET", cst.ts.URL, strings.NewReader("body")) + req.Header.Set("Expect", "100-continue") + res, err := cst.c.Do(req) + if err != nil { + t.Fatalf("Get(%q) = %v", cst.ts.URL, err) + } + res.Body.Close() + readyc <- struct{}{} // server starts reading from the request body + readyc <- struct{}{} // server finishes reading from the request body +} + +func TestServerReadAfterHandlerAbort100Continue(t *testing.T) { + run(t, testServerReadAfterHandlerAbort100Continue) +} +func testServerReadAfterHandlerAbort100Continue(t *testing.T, mode testMode) { + readyc := make(chan struct{}) + cst := newClientServerTest(t, mode, HandlerFunc(func(w ResponseWriter, r *Request) { + go func() { + <-readyc + io.ReadAll(r.Body) + <-readyc + }() + panic(ErrAbortHandler) + })) + + req, _ := NewRequest("GET", cst.ts.URL, strings.NewReader("body")) + req.Header.Set("Expect", "100-continue") + res, err := cst.c.Do(req) + if err == nil { + res.Body.Close() + } + readyc <- struct{}{} // server starts reading from the request body + readyc <- struct{}{} // server finishes reading from the request body +} diff --git a/src/net/http/server.go b/src/net/http/server.go index a50b20b7da..9786a68129 100644 --- a/src/net/http/server.go +++ b/src/net/http/server.go @@ -425,7 +425,6 @@ type response struct { reqBody io.ReadCloser cancelCtx context.CancelFunc // when ServeHTTP exits wroteHeader bool // a non-1xx header has been (logically) written - wroteContinue bool // 100 Continue response was written wants10KeepAlive bool // HTTP/1.0 w/ Connection "keep-alive" wantsClose bool // HTTP request has Connection "close" @@ -436,8 +435,8 @@ type response struct { // These two fields together synchronize the body reader (the // expectContinueReader, which wants to write 100 Continue) // against the main writer. - canWriteContinue atomic.Bool writeContinueMu sync.Mutex + canWriteContinue atomic.Bool w *bufio.Writer // buffers output in chunks to chunkWriter cw chunkWriter @@ -565,6 +564,14 @@ func (w *response) requestTooLarge() { } } +// disableWriteContinue stops Request.Body.Read from sending an automatic 100-Continue. +// If a 100-Continue is being written, it waits for it to complete before continuing. +func (w *response) disableWriteContinue() { + w.writeContinueMu.Lock() + w.canWriteContinue.Store(false) + w.writeContinueMu.Unlock() +} + // writerOnly hides an io.Writer value's optional ReadFrom method // from io.Copy. type writerOnly struct { @@ -917,8 +924,7 @@ func (ecr *expectContinueReader) Read(p []byte) (n int, err error) { return 0, ErrBodyReadAfterClose } w := ecr.resp - if !w.wroteContinue && w.canWriteContinue.Load() && !w.conn.hijacked() { - w.wroteContinue = true + if w.canWriteContinue.Load() { w.writeContinueMu.Lock() if w.canWriteContinue.Load() { w.conn.bufw.WriteString("HTTP/1.1 100 Continue\r\n\r\n") @@ -1159,18 +1165,17 @@ func (w *response) WriteHeader(code int) { } checkWriteHeaderCode(code) + if code < 101 || code > 199 { + // Sending a 100 Continue or any non-1xx header disables the + // automatically-sent 100 Continue from Request.Body.Read. + w.disableWriteContinue() + } + // Handle informational headers. // // We shouldn't send any further headers after 101 Switching Protocols, // so it takes the non-informational path. if code >= 100 && code <= 199 && code != StatusSwitchingProtocols { - // Prevent a potential race with an automatically-sent 100 Continue triggered by Request.Body.Read() - if code == 100 && w.canWriteContinue.Load() { - w.writeContinueMu.Lock() - w.canWriteContinue.Store(false) - w.writeContinueMu.Unlock() - } - writeStatusLine(w.conn.bufw, w.req.ProtoAtLeast(1, 1), code, w.statusBuf[:]) // Per RFC 8297 we must not clear the current header map @@ -1378,14 +1383,20 @@ func (cw *chunkWriter) writeHeader(p []byte) { // // If full duplex mode has been enabled with ResponseController.EnableFullDuplex, // then leave the request body alone. + // + // We don't take this path when w.closeAfterReply is set. + // We may not need to consume the request to get ready for the next one + // (since we're closing the conn), but a client which sends a full request + // before reading a response may deadlock in this case. + // This behavior has been present since CL 5268043 (2011), however, + // so it doesn't seem to be causing problems. if w.req.ContentLength != 0 && !w.closeAfterReply && !w.fullDuplex { var discard, tooBig bool switch bdy := w.req.Body.(type) { case *expectContinueReader: - if bdy.resp.wroteContinue { - discard = true - } + // We only get here if we have already fully consumed the request body + // (see above). case *body: bdy.mu.Lock() switch { @@ -1626,13 +1637,8 @@ func (w *response) write(lenData int, dataB []byte, dataS string) (n int, err er } if w.canWriteContinue.Load() { - // Body reader wants to write 100 Continue but hasn't yet. - // Tell it not to. The store must be done while holding the lock - // because the lock makes sure that there is not an active write - // this very moment. - w.writeContinueMu.Lock() - w.canWriteContinue.Store(false) - w.writeContinueMu.Unlock() + // Body reader wants to write 100 Continue but hasn't yet. Tell it not to. + w.disableWriteContinue() } if !w.wroteHeader { @@ -1900,6 +1906,7 @@ func (c *conn) serve(ctx context.Context) { } if inFlightResponse != nil { inFlightResponse.cancelCtx() + inFlightResponse.disableWriteContinue() } if !c.hijacked() { if inFlightResponse != nil { @@ -2106,6 +2113,7 @@ func (w *response) Hijack() (rwc net.Conn, buf *bufio.ReadWriter, err error) { if w.handlerDone.Load() { panic("net/http: Hijack called after ServeHTTP finished") } + w.disableWriteContinue() if w.wroteHeader { w.cw.flush() } @@ -2695,7 +2703,7 @@ func (mux *ServeMux) ServeHTTP(w ResponseWriter, r *Request) { if use121 { h, _ = mux.mux121.findHandler(r) } else { - h, _, r.pat, r.matches = mux.findHandler(r) + h, r.Pattern, r.pat, r.matches = mux.findHandler(r) } h.ServeHTTP(w, r) } diff --git a/src/net/http/transport.go b/src/net/http/transport.go index e6a97a00c6..f7a7092ef7 100644 --- a/src/net/http/transport.go +++ b/src/net/http/transport.go @@ -100,7 +100,7 @@ type Transport struct { idleLRU connLRU reqMu sync.Mutex - reqCanceler map[cancelKey]func(error) + reqCanceler map[*Request]context.CancelCauseFunc altMu sync.Mutex // guards changing altProto only altProto atomic.Value // of nil or map[string]RoundTripper, key is URI scheme @@ -294,13 +294,6 @@ type Transport struct { ForceAttemptHTTP2 bool } -// A cancelKey is the key of the reqCanceler map. -// We wrap the *Request in this type since we want to use the original request, -// not any transient one created by roundTrip. -type cancelKey struct { - req *Request -} - func (t *Transport) writeBufferSize() int { if t.WriteBufferSize > 0 { return t.WriteBufferSize @@ -466,10 +459,12 @@ func ProxyURL(fixedURL *url.URL) func(*Request) (*url.URL, error) { // optional extra headers to write and stores any error to return // from roundTrip. type transportRequest struct { - *Request // original request, not to be mutated - extra Header // extra headers to write, or nil - trace *httptrace.ClientTrace // optional - cancelKey cancelKey + *Request // original request, not to be mutated + extra Header // extra headers to write, or nil + trace *httptrace.ClientTrace // optional + + ctx context.Context // canceled when we are done with the request + cancel context.CancelCauseFunc mu sync.Mutex // guards err err error // first setError value for mapRoundTripError to consider @@ -531,7 +526,7 @@ func validateHeaders(hdrs Header) string { } // roundTrip implements a RoundTripper over HTTP. -func (t *Transport) roundTrip(req *Request) (*Response, error) { +func (t *Transport) roundTrip(req *Request) (_ *Response, err error) { t.nextProtoOnce.Do(t.onceSetNextProtoDefaults) ctx := req.Context() trace := httptrace.ContextClientTrace(ctx) @@ -561,7 +556,6 @@ func (t *Transport) roundTrip(req *Request) (*Response, error) { } origReq := req - cancelKey := cancelKey{origReq} req = setupRewindBody(req) if altRT := t.alternateRoundTripper(req); altRT != nil { @@ -587,16 +581,44 @@ func (t *Transport) roundTrip(req *Request) (*Response, error) { return nil, errors.New("http: no Host in request URL") } + // Transport request context. + // + // If RoundTrip returns an error, it cancels this context before returning. + // + // If RoundTrip returns no error: + // - For an HTTP/1 request, persistConn.readLoop cancels this context + // after reading the request body. + // - For an HTTP/2 request, RoundTrip cancels this context after the HTTP/2 + // RoundTripper returns. + ctx, cancel := context.WithCancelCause(req.Context()) + + // Convert Request.Cancel into context cancelation. + if origReq.Cancel != nil { + go awaitLegacyCancel(ctx, cancel, origReq) + } + + // Convert Transport.CancelRequest into context cancelation. + // + // This is lamentably expensive. CancelRequest has been deprecated for a long time + // and doesn't work on HTTP/2 requests. Perhaps we should drop support for it entirely. + cancel = t.prepareTransportCancel(origReq, cancel) + + defer func() { + if err != nil { + cancel(err) + } + }() + for { select { case <-ctx.Done(): req.closeBody() - return nil, ctx.Err() + return nil, context.Cause(ctx) default: } // treq gets modified by roundTrip, so we need to recreate for each retry. - treq := &transportRequest{Request: req, trace: trace, cancelKey: cancelKey} + treq := &transportRequest{Request: req, trace: trace, ctx: ctx, cancel: cancel} cm, err := t.connectMethodForRequest(treq) if err != nil { req.closeBody() @@ -609,7 +631,6 @@ func (t *Transport) roundTrip(req *Request) (*Response, error) { // to send it requests. pconn, err := t.getConn(treq, cm) if err != nil { - t.setReqCanceler(cancelKey, nil) req.closeBody() return nil, err } @@ -617,12 +638,19 @@ func (t *Transport) roundTrip(req *Request) (*Response, error) { var resp *Response if pconn.alt != nil { // HTTP/2 path. - t.setReqCanceler(cancelKey, nil) // not cancelable with CancelRequest resp, err = pconn.alt.RoundTrip(req) } else { resp, err = pconn.roundTrip(treq) } if err == nil { + if pconn.alt != nil { + // HTTP/2 requests are not cancelable with CancelRequest, + // so we have no further need for the request context. + // + // On the HTTP/1 path, roundTrip takes responsibility for + // canceling the context after the response body is read. + cancel(errRequestDone) + } resp.Request = origReq return resp, nil } @@ -659,6 +687,14 @@ func (t *Transport) roundTrip(req *Request) (*Response, error) { } } +func awaitLegacyCancel(ctx context.Context, cancel context.CancelCauseFunc, req *Request) { + select { + case <-req.Cancel: + cancel(errRequestCanceled) + case <-ctx.Done(): + } +} + var errCannotRewind = errors.New("net/http: cannot rewind body after connection loss") type readTrackingBody struct { @@ -820,30 +856,42 @@ func (t *Transport) CloseIdleConnections() { } } +// prepareTransportCancel sets up state to convert Transport.CancelRequest into context cancelation. +func (t *Transport) prepareTransportCancel(req *Request, origCancel context.CancelCauseFunc) context.CancelCauseFunc { + // Historically, RoundTrip has not modified the Request in any way. + // We could avoid the need to keep a map of all in-flight requests by adding + // a field to the Request containing its cancel func, and setting that field + // while the request is in-flight. Callers aren't supposed to reuse a Request + // until after the response body is closed, so this wouldn't violate any + // concurrency guarantees. + cancel := func(err error) { + origCancel(err) + t.reqMu.Lock() + delete(t.reqCanceler, req) + t.reqMu.Unlock() + } + t.reqMu.Lock() + if t.reqCanceler == nil { + t.reqCanceler = make(map[*Request]context.CancelCauseFunc) + } + t.reqCanceler[req] = cancel + t.reqMu.Unlock() + return cancel +} + // CancelRequest cancels an in-flight request by closing its connection. // CancelRequest should only be called after [Transport.RoundTrip] has returned. // // Deprecated: Use [Request.WithContext] to create a request with a // cancelable context instead. CancelRequest cannot cancel HTTP/2 -// requests. +// requests. This may become a no-op in a future release of Go. func (t *Transport) CancelRequest(req *Request) { - t.cancelRequest(cancelKey{req}, errRequestCanceled) -} - -// Cancel an in-flight request, recording the error value. -// Returns whether the request was canceled. -func (t *Transport) cancelRequest(key cancelKey, err error) bool { - // This function must not return until the cancel func has completed. - // See: https://golang.org/issue/34658 t.reqMu.Lock() - defer t.reqMu.Unlock() - cancel := t.reqCanceler[key] - delete(t.reqCanceler, key) + cancel := t.reqCanceler[req] + t.reqMu.Unlock() if cancel != nil { - cancel(err) + cancel(errRequestCanceled) } - - return cancel != nil } // @@ -1170,38 +1218,6 @@ func (t *Transport) removeIdleConnLocked(pconn *persistConn) bool { return removed } -func (t *Transport) setReqCanceler(key cancelKey, fn func(error)) { - t.reqMu.Lock() - defer t.reqMu.Unlock() - if t.reqCanceler == nil { - t.reqCanceler = make(map[cancelKey]func(error)) - } - if fn != nil { - t.reqCanceler[key] = fn - } else { - delete(t.reqCanceler, key) - } -} - -// replaceReqCanceler replaces an existing cancel function. If there is no cancel function -// for the request, we don't set the function and return false. -// Since CancelRequest will clear the canceler, we can use the return value to detect if -// the request was canceled since the last setReqCancel call. -func (t *Transport) replaceReqCanceler(key cancelKey, fn func(error)) bool { - t.reqMu.Lock() - defer t.reqMu.Unlock() - _, ok := t.reqCanceler[key] - if !ok { - return false - } - if fn != nil { - t.reqCanceler[key] = fn - } else { - delete(t.reqCanceler, key) - } - return true -} - var zeroDialer net.Dialer func (t *Transport) dial(ctx context.Context, network, addr string) (net.Conn, error) { @@ -1442,19 +1458,8 @@ func (t *Transport) getConn(treq *transportRequest, cm connectMethod) (_ *persis } }() - var cancelc chan error - // Queue for idle connection. - if delivered := t.queueForIdleConn(w); delivered { - // set request canceler to some non-nil function so we - // can detect whether it was cleared between now and when - // we enter roundTrip - t.setReqCanceler(treq.cancelKey, func(error) {}) - } else { - cancelc = make(chan error, 1) - t.setReqCanceler(treq.cancelKey, func(err error) { cancelc <- err }) - - // Queue for permission to dial. + if delivered := t.queueForIdleConn(w); !delivered { t.queueForDial(w) } @@ -1479,11 +1484,8 @@ func (t *Transport) getConn(treq *transportRequest, cm connectMethod) (_ *persis // what caused r.err; if so, prefer to return the // cancellation error (see golang.org/issue/16049). select { - case <-req.Cancel: - return nil, errRequestCanceledConn - case <-req.Context().Done(): - return nil, req.Context().Err() - case err := <-cancelc: + case <-treq.ctx.Done(): + err := context.Cause(treq.ctx) if err == errRequestCanceled { err = errRequestCanceledConn } @@ -1493,11 +1495,8 @@ func (t *Transport) getConn(treq *transportRequest, cm connectMethod) (_ *persis } } return r.pc, r.err - case <-req.Cancel: - return nil, errRequestCanceledConn - case <-req.Context().Done(): - return nil, req.Context().Err() - case err := <-cancelc: + case <-treq.ctx.Done(): + err := context.Cause(treq.ctx) if err == errRequestCanceled { err = errRequestCanceledConn } @@ -2173,7 +2172,8 @@ func (pc *persistConn) readLoop() { pc.t.removeIdleConn(pc) }() - tryPutIdleConn := func(trace *httptrace.ClientTrace) bool { + tryPutIdleConn := func(treq *transportRequest) bool { + trace := treq.trace if err := pc.t.tryPutIdleConn(pc); err != nil { closeErr = err if trace != nil && trace.PutIdleConn != nil && err != errKeepAlivesDisabled { @@ -2212,7 +2212,7 @@ func (pc *persistConn) readLoop() { pc.mu.Unlock() rc := <-pc.reqch - trace := httptrace.ContextClientTrace(rc.req.Context()) + trace := rc.treq.trace var resp *Response if err == nil { @@ -2241,9 +2241,9 @@ func (pc *persistConn) readLoop() { pc.mu.Unlock() bodyWritable := resp.bodyIsWritable() - hasBody := rc.req.Method != "HEAD" && resp.ContentLength != 0 + hasBody := rc.treq.Request.Method != "HEAD" && resp.ContentLength != 0 - if resp.Close || rc.req.Close || resp.StatusCode <= 199 || bodyWritable { + if resp.Close || rc.treq.Request.Close || resp.StatusCode <= 199 || bodyWritable { // Don't do keep-alive on error if either party requested a close // or we get an unexpected informational (1xx) response. // StatusCode 100 is already handled above. @@ -2251,8 +2251,6 @@ func (pc *persistConn) readLoop() { } if !hasBody || bodyWritable { - replaced := pc.t.replaceReqCanceler(rc.cancelKey, nil) - // Put the idle conn back into the pool before we send the response // so if they process it quickly and make another request, they'll // get this same conn. But we use the unbuffered channel 'rc' @@ -2261,7 +2259,7 @@ func (pc *persistConn) readLoop() { alive = alive && !pc.sawEOF && pc.wroteRequest() && - replaced && tryPutIdleConn(trace) + tryPutIdleConn(rc.treq) if bodyWritable { closeErr = errCallerOwnsConn @@ -2273,6 +2271,8 @@ func (pc *persistConn) readLoop() { return } + rc.treq.cancel(errRequestDone) + // Now that they've read from the unbuffered channel, they're safely // out of the select that also waits on this goroutine to die, so // we're allowed to exit now if needed (if alive is false) @@ -2323,26 +2323,22 @@ func (pc *persistConn) readLoop() { // reading the response body. (or for cancellation or death) select { case bodyEOF := <-waitForBodyRead: - replaced := pc.t.replaceReqCanceler(rc.cancelKey, nil) // before pc might return to idle pool alive = alive && bodyEOF && !pc.sawEOF && pc.wroteRequest() && - replaced && tryPutIdleConn(trace) + tryPutIdleConn(rc.treq) if bodyEOF { eofc <- struct{}{} } - case <-rc.req.Cancel: + case <-rc.treq.ctx.Done(): alive = false - pc.t.cancelRequest(rc.cancelKey, errRequestCanceled) - case <-rc.req.Context().Done(): - alive = false - pc.t.cancelRequest(rc.cancelKey, rc.req.Context().Err()) + pc.cancelRequest(errRequestCanceled) case <-pc.closech: alive = false - pc.t.setReqCanceler(rc.cancelKey, nil) } + rc.treq.cancel(errRequestDone) testHookReadLoopBeforeNextRead() } } @@ -2395,7 +2391,7 @@ func (pc *persistConn) readResponse(rc requestAndChan, trace *httptrace.ClientTr continueCh := rc.continueCh for { - resp, err = ReadResponse(pc.br, rc.req) + resp, err = ReadResponse(pc.br, rc.treq.Request) if err != nil { return } @@ -2587,10 +2583,9 @@ type responseAndError struct { } type requestAndChan struct { - _ incomparable - req *Request - cancelKey cancelKey - ch chan responseAndError // unbuffered; always send in select on callerGone + _ incomparable + treq *transportRequest + ch chan responseAndError // unbuffered; always send in select on callerGone // whether the Transport (as opposed to the user client code) // added the Accept-Encoding gzip header. If the Transport @@ -2638,6 +2633,10 @@ var errTimeout error = &timeoutError{"net/http: timeout awaiting response header var errRequestCanceled = http2errRequestCanceled var errRequestCanceledConn = errors.New("net/http: request canceled while waiting for connection") // TODO: unify? +// errRequestDone is used to cancel the round trip Context after a request is successfully done. +// It should not be seen by the user. +var errRequestDone = errors.New("net/http: request completed") + func nop() {} // testHooks. Always non-nil. @@ -2654,10 +2653,6 @@ var ( func (pc *persistConn) roundTrip(req *transportRequest) (resp *Response, err error) { testHookEnterRoundTrip() - if !pc.t.replaceReqCanceler(req.cancelKey, pc.cancelRequest) { - pc.t.putOrCloseIdleConn(pc) - return nil, errRequestCanceled - } pc.mu.Lock() pc.numExpectedResponses++ headerFn := pc.mutateHeaderFunc @@ -2706,12 +2701,6 @@ func (pc *persistConn) roundTrip(req *transportRequest) (resp *Response, err err gone := make(chan struct{}) defer close(gone) - defer func() { - if err != nil { - pc.t.setReqCanceler(req.cancelKey, nil) - } - }() - const debugRoundTrip = false // Write the request concurrently with waiting for a response, @@ -2723,19 +2712,29 @@ func (pc *persistConn) roundTrip(req *transportRequest) (resp *Response, err err resc := make(chan responseAndError) pc.reqch <- requestAndChan{ - req: req.Request, - cancelKey: req.cancelKey, + treq: req, ch: resc, addedGzip: requestedGzip, continueCh: continueCh, callerGone: gone, } + handleResponse := func(re responseAndError) (*Response, error) { + if (re.res == nil) == (re.err == nil) { + panic(fmt.Sprintf("internal error: exactly one of res or err should be set; nil=%v", re.res == nil)) + } + if debugRoundTrip { + req.logf("resc recv: %p, %T/%#v", re.res, re.err, re.err) + } + if re.err != nil { + return nil, pc.mapRoundTripError(req, startBytesWritten, re.err) + } + return re.res, nil + } + var respHeaderTimer <-chan time.Time - cancelChan := req.Request.Cancel - ctxDoneChan := req.Context().Done() + ctxDoneChan := req.ctx.Done() pcClosed := pc.closech - canceled := false for { testHookWaitResLoop() select { @@ -2756,13 +2755,18 @@ func (pc *persistConn) roundTrip(req *transportRequest) (resp *Response, err err respHeaderTimer = timer.C } case <-pcClosed: - pcClosed = nil - if canceled || pc.t.replaceReqCanceler(req.cancelKey, nil) { - if debugRoundTrip { - req.logf("closech recv: %T %#v", pc.closed, pc.closed) - } - return nil, pc.mapRoundTripError(req, startBytesWritten, pc.closed) + select { + case re := <-resc: + // The pconn closing raced with the response to the request, + // probably after the server wrote a response and immediately + // closed the connection. Use the response. + return handleResponse(re) + default: } + if debugRoundTrip { + req.logf("closech recv: %T %#v", pc.closed, pc.closed) + } + return nil, pc.mapRoundTripError(req, startBytesWritten, pc.closed) case <-respHeaderTimer: if debugRoundTrip { req.logf("timeout waiting for response headers.") @@ -2770,23 +2774,17 @@ func (pc *persistConn) roundTrip(req *transportRequest) (resp *Response, err err pc.close(errTimeout) return nil, errTimeout case re := <-resc: - if (re.res == nil) == (re.err == nil) { - panic(fmt.Sprintf("internal error: exactly one of res or err should be set; nil=%v", re.res == nil)) - } - if debugRoundTrip { - req.logf("resc recv: %p, %T/%#v", re.res, re.err, re.err) - } - if re.err != nil { - return nil, pc.mapRoundTripError(req, startBytesWritten, re.err) - } - return re.res, nil - case <-cancelChan: - canceled = pc.t.cancelRequest(req.cancelKey, errRequestCanceled) - cancelChan = nil + return handleResponse(re) case <-ctxDoneChan: - canceled = pc.t.cancelRequest(req.cancelKey, req.Context().Err()) - cancelChan = nil - ctxDoneChan = nil + select { + case re := <-resc: + // readLoop is responsible for canceling req.ctx after + // it reads the response body. Check for a response racing + // the context close, and use the response if available. + return handleResponse(re) + default: + } + pc.cancelRequest(context.Cause(req.ctx)) } } } diff --git a/src/net/http/transport_internal_test.go b/src/net/http/transport_internal_test.go index dc3259fadf..f86970b248 100644 --- a/src/net/http/transport_internal_test.go +++ b/src/net/http/transport_internal_test.go @@ -8,6 +8,7 @@ package http import ( "bytes" + "context" "crypto/tls" "errors" "io" @@ -36,7 +37,8 @@ func TestTransportPersistConnReadLoopEOF(t *testing.T) { tr := new(Transport) req, _ := NewRequest("GET", "http://"+ln.Addr().String(), nil) req = req.WithT(t) - treq := &transportRequest{Request: req} + ctx, cancel := context.WithCancelCause(context.Background()) + treq := &transportRequest{Request: req, ctx: ctx, cancel: cancel} cm := connectMethod{targetScheme: "http", targetAddr: ln.Addr().String()} pc, err := tr.getConn(treq, cm) if err != nil { diff --git a/src/net/iprawsock_posix.go b/src/net/iprawsock_posix.go index 73b41ab522..b25cb648c3 100644 --- a/src/net/iprawsock_posix.go +++ b/src/net/iprawsock_posix.go @@ -124,7 +124,7 @@ func (sd *sysDialer) dialIP(ctx context.Context, laddr, raddr *IPAddr) (*IPConn, } ctrlCtxFn := sd.Dialer.ControlContext if ctrlCtxFn == nil && sd.Dialer.Control != nil { - ctrlCtxFn = func(cxt context.Context, network, address string, c syscall.RawConn) error { + ctrlCtxFn = func(ctx context.Context, network, address string, c syscall.RawConn) error { return sd.Dialer.Control(network, address, c) } } @@ -145,9 +145,9 @@ func (sl *sysListener) listenIP(ctx context.Context, laddr *IPAddr) (*IPConn, er default: return nil, UnknownNetworkError(sl.network) } - var ctrlCtxFn func(cxt context.Context, network, address string, c syscall.RawConn) error + var ctrlCtxFn func(ctx context.Context, network, address string, c syscall.RawConn) error if sl.ListenConfig.Control != nil { - ctrlCtxFn = func(cxt context.Context, network, address string, c syscall.RawConn) error { + ctrlCtxFn = func(ctx context.Context, network, address string, c syscall.RawConn) error { return sl.ListenConfig.Control(network, address, c) } } diff --git a/src/net/netip/inlining_test.go b/src/net/netip/inlining_test.go index b521eeebfd..f5eb30df90 100644 --- a/src/net/netip/inlining_test.go +++ b/src/net/netip/inlining_test.go @@ -80,6 +80,7 @@ func TestInlining(t *testing.T) { case "amd64", "arm64": // These don't inline on 32-bit. wantInlinable = append(wantInlinable, + "Addr.AsSlice", "Addr.Next", "Addr.Prev", ) diff --git a/src/net/netip/leaf_alts.go b/src/net/netip/leaf_alts.go deleted file mode 100644 index d887bed627..0000000000 --- a/src/net/netip/leaf_alts.go +++ /dev/null @@ -1,45 +0,0 @@ -// Copyright 2021 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -// Stuff that exists in std, but we can't use due to being a dependency -// of net, for go/build deps_test policy reasons. - -package netip - -func beUint64(b []byte) uint64 { - _ = b[7] // bounds check hint to compiler; see golang.org/issue/14808 - return uint64(b[7]) | uint64(b[6])<<8 | uint64(b[5])<<16 | uint64(b[4])<<24 | - uint64(b[3])<<32 | uint64(b[2])<<40 | uint64(b[1])<<48 | uint64(b[0])<<56 -} - -func bePutUint64(b []byte, v uint64) { - _ = b[7] // early bounds check to guarantee safety of writes below - b[0] = byte(v >> 56) - b[1] = byte(v >> 48) - b[2] = byte(v >> 40) - b[3] = byte(v >> 32) - b[4] = byte(v >> 24) - b[5] = byte(v >> 16) - b[6] = byte(v >> 8) - b[7] = byte(v) -} - -func bePutUint32(b []byte, v uint32) { - _ = b[3] // early bounds check to guarantee safety of writes below - b[0] = byte(v >> 24) - b[1] = byte(v >> 16) - b[2] = byte(v >> 8) - b[3] = byte(v) -} - -func leUint16(b []byte) uint16 { - _ = b[1] // bounds check hint to compiler; see golang.org/issue/14808 - return uint16(b[0]) | uint16(b[1])<<8 -} - -func lePutUint16(b []byte, v uint16) { - _ = b[1] // early bounds check to guarantee safety of writes below - b[0] = byte(v) - b[1] = byte(v >> 8) -} diff --git a/src/net/netip/netip.go b/src/net/netip/netip.go index 1912561c74..4b0a61dd98 100644 --- a/src/net/netip/netip.go +++ b/src/net/netip/netip.go @@ -14,12 +14,12 @@ package netip import ( "cmp" "errors" + "internal/bytealg" + "internal/byteorder" + "internal/itoa" "math" "strconv" "unique" - - "internal/bytealg" - "internal/itoa" ) // Sizes: (64-bit) @@ -60,7 +60,7 @@ type Addr struct { // addrDetail represents the details of an Addr, like address family and IPv6 zone. type addrDetail struct { IsV6 bool // IPv4 is false, IPv6 is true. - ZoneV6 string // != nil only if IsV6 is true. + ZoneV6 string // != "" only if IsV6 is true. } // z0, z4, and z6noz are sentinel Addr.z values. @@ -102,8 +102,8 @@ func AddrFrom4(addr [4]byte) Addr { func AddrFrom16(addr [16]byte) Addr { return Addr{ addr: uint128{ - beUint64(addr[:8]), - beUint64(addr[8:]), + byteorder.BeUint64(addr[:8]), + byteorder.BeUint64(addr[8:]), }, z: z6noz, } @@ -676,8 +676,8 @@ func (ip Addr) Prefix(b int) (Prefix, error) { // [Addr.Zone] method to get it). // The ip zero value returns all zeroes. func (ip Addr) As16() (a16 [16]byte) { - bePutUint64(a16[:8], ip.addr.hi) - bePutUint64(a16[8:], ip.addr.lo) + byteorder.BePutUint64(a16[:8], ip.addr.hi) + byteorder.BePutUint64(a16[8:], ip.addr.lo) return a16 } @@ -686,7 +686,7 @@ func (ip Addr) As16() (a16 [16]byte) { // Note that 0.0.0.0 is not the zero Addr. func (ip Addr) As4() (a4 [4]byte) { if ip.z == z4 || ip.Is4In6() { - bePutUint32(a4[:], uint32(ip.addr.lo)) + byteorder.BePutUint32(a4[:], uint32(ip.addr.lo)) return a4 } if ip.z == z0 { @@ -702,12 +702,12 @@ func (ip Addr) AsSlice() []byte { return nil case z4: var ret [4]byte - bePutUint32(ret[:], uint32(ip.addr.lo)) + byteorder.BePutUint32(ret[:], uint32(ip.addr.lo)) return ret[:] default: var ret [16]byte - bePutUint64(ret[:8], ip.addr.hi) - bePutUint64(ret[8:], ip.addr.lo) + byteorder.BePutUint64(ret[:8], ip.addr.hi) + byteorder.BePutUint64(ret[8:], ip.addr.lo) return ret[:] } } @@ -987,12 +987,12 @@ func (ip Addr) marshalBinaryWithTrailingBytes(trailingBytes int) []byte { b = make([]byte, trailingBytes) case z4: b = make([]byte, 4+trailingBytes) - bePutUint32(b, uint32(ip.addr.lo)) + byteorder.BePutUint32(b, uint32(ip.addr.lo)) default: z := ip.Zone() b = make([]byte, 16+len(z)+trailingBytes) - bePutUint64(b[:8], ip.addr.hi) - bePutUint64(b[8:], ip.addr.lo) + byteorder.BePutUint64(b[:8], ip.addr.hi) + byteorder.BePutUint64(b[8:], ip.addr.lo) copy(b[16:], z) } return b @@ -1209,7 +1209,7 @@ func (p *AddrPort) UnmarshalText(text []byte) error { // containing the port in little-endian. func (p AddrPort) MarshalBinary() ([]byte, error) { b := p.Addr().marshalBinaryWithTrailingBytes(2) - lePutUint16(b[len(b)-2:], p.Port()) + byteorder.LePutUint16(b[len(b)-2:], p.Port()) return b, nil } @@ -1224,7 +1224,7 @@ func (p *AddrPort) UnmarshalBinary(b []byte) error { if err != nil { return err } - *p = AddrPortFrom(addr, leUint16(b[len(b)-2:])) + *p = AddrPortFrom(addr, byteorder.LeUint16(b[len(b)-2:])) return nil } diff --git a/src/net/tcpsock_posix.go b/src/net/tcpsock_posix.go index a25494d9c0..7bca8dca55 100644 --- a/src/net/tcpsock_posix.go +++ b/src/net/tcpsock_posix.go @@ -78,7 +78,7 @@ func (sd *sysDialer) doDialTCP(ctx context.Context, laddr, raddr *TCPAddr) (*TCP func (sd *sysDialer) doDialTCPProto(ctx context.Context, laddr, raddr *TCPAddr, proto int) (*TCPConn, error) { ctrlCtxFn := sd.Dialer.ControlContext if ctrlCtxFn == nil && sd.Dialer.Control != nil { - ctrlCtxFn = func(cxt context.Context, network, address string, c syscall.RawConn) error { + ctrlCtxFn = func(ctx context.Context, network, address string, c syscall.RawConn) error { return sd.Dialer.Control(network, address, c) } } @@ -180,9 +180,9 @@ func (sl *sysListener) listenTCP(ctx context.Context, laddr *TCPAddr) (*TCPListe } func (sl *sysListener) listenTCPProto(ctx context.Context, laddr *TCPAddr, proto int) (*TCPListener, error) { - var ctrlCtxFn func(cxt context.Context, network, address string, c syscall.RawConn) error + var ctrlCtxFn func(ctx context.Context, network, address string, c syscall.RawConn) error if sl.ListenConfig.Control != nil { - ctrlCtxFn = func(cxt context.Context, network, address string, c syscall.RawConn) error { + ctrlCtxFn = func(ctx context.Context, network, address string, c syscall.RawConn) error { return sl.ListenConfig.Control(network, address, c) } } diff --git a/src/net/textproto/reader.go b/src/net/textproto/reader.go index 1a81453559..f98e05bd1d 100644 --- a/src/net/textproto/reader.go +++ b/src/net/textproto/reader.go @@ -14,6 +14,7 @@ import ( "strconv" "strings" "sync" + _ "unsafe" // for linkname ) // TODO: This should be a distinguishable error (ErrMessageTooLarge) @@ -501,6 +502,9 @@ func (r *Reader) ReadMIMEHeader() (MIMEHeader, error) { return readMIMEHeader(r, math.MaxInt64, math.MaxInt64) } +// readMIMEHeader is accessed from mime/multipart. +//go:linkname readMIMEHeader + // readMIMEHeader is a version of ReadMIMEHeader which takes a limit on the header size. // It is called by the mime/multipart package. func readMIMEHeader(r *Reader, maxMemory, maxHeaders int64) (MIMEHeader, error) { diff --git a/src/net/udpsock_posix.go b/src/net/udpsock_posix.go index 5035059831..3cd1d0a762 100644 --- a/src/net/udpsock_posix.go +++ b/src/net/udpsock_posix.go @@ -205,7 +205,7 @@ func (c *UDPConn) writeMsgAddrPort(b, oob []byte, addr netip.AddrPort) (n, oobn func (sd *sysDialer) dialUDP(ctx context.Context, laddr, raddr *UDPAddr) (*UDPConn, error) { ctrlCtxFn := sd.Dialer.ControlContext if ctrlCtxFn == nil && sd.Dialer.Control != nil { - ctrlCtxFn = func(cxt context.Context, network, address string, c syscall.RawConn) error { + ctrlCtxFn = func(ctx context.Context, network, address string, c syscall.RawConn) error { return sd.Dialer.Control(network, address, c) } } @@ -217,9 +217,9 @@ func (sd *sysDialer) dialUDP(ctx context.Context, laddr, raddr *UDPAddr) (*UDPCo } func (sl *sysListener) listenUDP(ctx context.Context, laddr *UDPAddr) (*UDPConn, error) { - var ctrlCtxFn func(cxt context.Context, network, address string, c syscall.RawConn) error + var ctrlCtxFn func(ctx context.Context, network, address string, c syscall.RawConn) error if sl.ListenConfig.Control != nil { - ctrlCtxFn = func(cxt context.Context, network, address string, c syscall.RawConn) error { + ctrlCtxFn = func(ctx context.Context, network, address string, c syscall.RawConn) error { return sl.ListenConfig.Control(network, address, c) } } @@ -231,9 +231,9 @@ func (sl *sysListener) listenUDP(ctx context.Context, laddr *UDPAddr) (*UDPConn, } func (sl *sysListener) listenMulticastUDP(ctx context.Context, ifi *Interface, gaddr *UDPAddr) (*UDPConn, error) { - var ctrlCtxFn func(cxt context.Context, network, address string, c syscall.RawConn) error + var ctrlCtxFn func(ctx context.Context, network, address string, c syscall.RawConn) error if sl.ListenConfig.Control != nil { - ctrlCtxFn = func(cxt context.Context, network, address string, c syscall.RawConn) error { + ctrlCtxFn = func(ctx context.Context, network, address string, c syscall.RawConn) error { return sl.ListenConfig.Control(network, address, c) } } diff --git a/src/net/unixsock_posix.go b/src/net/unixsock_posix.go index f6c8e8f0b0..dc01b3874a 100644 --- a/src/net/unixsock_posix.go +++ b/src/net/unixsock_posix.go @@ -157,7 +157,7 @@ func (c *UnixConn) writeMsg(b, oob []byte, addr *UnixAddr) (n, oobn int, err err func (sd *sysDialer) dialUnix(ctx context.Context, laddr, raddr *UnixAddr) (*UnixConn, error) { ctrlCtxFn := sd.Dialer.ControlContext if ctrlCtxFn == nil && sd.Dialer.Control != nil { - ctrlCtxFn = func(cxt context.Context, network, address string, c syscall.RawConn) error { + ctrlCtxFn = func(ctx context.Context, network, address string, c syscall.RawConn) error { return sd.Dialer.Control(network, address, c) } } @@ -217,9 +217,9 @@ func (l *UnixListener) SetUnlinkOnClose(unlink bool) { } func (sl *sysListener) listenUnix(ctx context.Context, laddr *UnixAddr) (*UnixListener, error) { - var ctrlCtxFn func(cxt context.Context, network, address string, c syscall.RawConn) error + var ctrlCtxFn func(ctx context.Context, network, address string, c syscall.RawConn) error if sl.ListenConfig.Control != nil { - ctrlCtxFn = func(cxt context.Context, network, address string, c syscall.RawConn) error { + ctrlCtxFn = func(ctx context.Context, network, address string, c syscall.RawConn) error { return sl.ListenConfig.Control(network, address, c) } } @@ -231,9 +231,9 @@ func (sl *sysListener) listenUnix(ctx context.Context, laddr *UnixAddr) (*UnixLi } func (sl *sysListener) listenUnixgram(ctx context.Context, laddr *UnixAddr) (*UnixConn, error) { - var ctrlCtxFn func(cxt context.Context, network, address string, c syscall.RawConn) error + var ctrlCtxFn func(ctx context.Context, network, address string, c syscall.RawConn) error if sl.ListenConfig.Control != nil { - ctrlCtxFn = func(cxt context.Context, network, address string, c syscall.RawConn) error { + ctrlCtxFn = func(ctx context.Context, network, address string, c syscall.RawConn) error { return sl.ListenConfig.Control(network, address, c) } } diff --git a/src/os/dir_unix.go b/src/os/dir_unix.go index 7680be7799..d8b4faa057 100644 --- a/src/os/dir_unix.go +++ b/src/os/dir_unix.go @@ -7,6 +7,7 @@ package os import ( + "internal/byteorder" "internal/goarch" "io" "runtime" @@ -174,15 +175,11 @@ func readIntBE(b []byte, size uintptr) uint64 { case 1: return uint64(b[0]) case 2: - _ = b[1] // bounds check hint to compiler; see golang.org/issue/14808 - return uint64(b[1]) | uint64(b[0])<<8 + return uint64(byteorder.BeUint16(b)) case 4: - _ = b[3] // bounds check hint to compiler; see golang.org/issue/14808 - return uint64(b[3]) | uint64(b[2])<<8 | uint64(b[1])<<16 | uint64(b[0])<<24 + return uint64(byteorder.BeUint32(b)) case 8: - _ = b[7] // bounds check hint to compiler; see golang.org/issue/14808 - return uint64(b[7]) | uint64(b[6])<<8 | uint64(b[5])<<16 | uint64(b[4])<<24 | - uint64(b[3])<<32 | uint64(b[2])<<40 | uint64(b[1])<<48 | uint64(b[0])<<56 + return uint64(byteorder.BeUint64(b)) default: panic("syscall: readInt with unsupported size") } @@ -193,15 +190,11 @@ func readIntLE(b []byte, size uintptr) uint64 { case 1: return uint64(b[0]) case 2: - _ = b[1] // bounds check hint to compiler; see golang.org/issue/14808 - return uint64(b[0]) | uint64(b[1])<<8 + return uint64(byteorder.LeUint16(b)) case 4: - _ = b[3] // bounds check hint to compiler; see golang.org/issue/14808 - return uint64(b[0]) | uint64(b[1])<<8 | uint64(b[2])<<16 | uint64(b[3])<<24 + return uint64(byteorder.LeUint32(b)) case 8: - _ = b[7] // bounds check hint to compiler; see golang.org/issue/14808 - return uint64(b[0]) | uint64(b[1])<<8 | uint64(b[2])<<16 | uint64(b[3])<<24 | - uint64(b[4])<<32 | uint64(b[5])<<40 | uint64(b[6])<<48 | uint64(b[7])<<56 + return uint64(byteorder.LeUint64(b)) default: panic("syscall: readInt with unsupported size") } diff --git a/src/os/executable_darwin.go b/src/os/executable_darwin.go index dae9f4ee18..2bb50ab3fe 100644 --- a/src/os/executable_darwin.go +++ b/src/os/executable_darwin.go @@ -4,8 +4,12 @@ package os -import "errors" +import ( + "errors" + _ "unsafe" // for linkname +) +//go:linkname executablePath var executablePath string // set by ../runtime/os_darwin.go var initCwd, initCwdErr = Getwd() diff --git a/src/os/executable_solaris.go b/src/os/executable_solaris.go index b145980c56..8ee897f4b0 100644 --- a/src/os/executable_solaris.go +++ b/src/os/executable_solaris.go @@ -4,8 +4,12 @@ package os -import "syscall" +import ( + "syscall" + _ "unsafe" // for linkname +) +//go:linkname executablePath var executablePath string // set by sysauxv in ../runtime/os3_solaris.go var initCwd, initCwdErr = Getwd() diff --git a/src/reflect/type_test.go b/src/reflect/type_test.go index 40ae7131c3..51abc0776c 100644 --- a/src/reflect/type_test.go +++ b/src/reflect/type_test.go @@ -118,7 +118,7 @@ func BenchmarkTypeForError(b *testing.B) { } } -func Test_Type_CanSeq(t *testing.T) { +func TestType_CanSeq(t *testing.T) { tests := []struct { name string tr reflect.Type @@ -143,7 +143,7 @@ func Test_Type_CanSeq(t *testing.T) { } } -func Test_Type_CanSeq2(t *testing.T) { +func TestType_CanSeq2(t *testing.T) { tests := []struct { name string tr reflect.Type diff --git a/src/run.bash b/src/run.bash index badb8c60e2..cb474325c4 100755 --- a/src/run.bash +++ b/src/run.bash @@ -41,15 +41,7 @@ export CC ulimit -c 0 # Raise soft limits to hard limits for NetBSD/OpenBSD. -# We need at least 256 files and ~300 MB of bss. -# On OS X ulimit -S -n rejects 'unlimited'. -# -# Note that ulimit -S -n may fail if ulimit -H -n is set higher than a -# non-root process is allowed to set the high limit. -# This is a system misconfiguration and should be fixed on the -# broken system, not "fixed" by ignoring the failure here. -# See longer discussion on golang.org/issue/7381. -[ "$(ulimit -H -n)" = "unlimited" ] || ulimit -S -n $(ulimit -H -n) +# We need at least ~300 MB of bss. [ "$(ulimit -H -d)" = "unlimited" ] || ulimit -S -d $(ulimit -H -d) # Thread count limit on NetBSD 7. diff --git a/src/runtime/asm.s b/src/runtime/asm.s index b4bcb04cd1..f487e44100 100644 --- a/src/runtime/asm.s +++ b/src/runtime/asm.s @@ -13,34 +13,3 @@ TEXT ·sigpanic0(SB),NOSPLIT,$0-0 TEXT ·mapinitnoop(SB),NOSPLIT,$0-0 RET -#ifndef GOARCH_386 -#ifndef GOARCH_arm -#ifndef GOARCH_amd64 -#ifndef GOARCH_arm64 -#ifndef GOARCH_loong64 -#ifndef GOARCH_mips -#ifndef GOARCH_mipsle -#ifndef GOARCH_mips64 -#ifndef GOARCH_mips64le -#ifndef GOARCH_ppc64 -#ifndef GOARCH_ppc64le -#ifndef GOARCH_riscv64 -#ifndef GOARCH_s390x -#ifndef GOARCH_wasm -// stub to appease shared build mode. -TEXT ·switchToCrashStack0(SB),NOSPLIT,$0-0 - UNDEF -#endif -#endif -#endif -#endif -#endif -#endif -#endif -#endif -#endif -#endif -#endif -#endif -#endif -#endif diff --git a/src/runtime/cgo/gcc_stack_darwin.c b/src/runtime/cgo/gcc_stack_darwin.c index 0a9038eb3b..28364c7420 100644 --- a/src/runtime/cgo/gcc_stack_darwin.c +++ b/src/runtime/cgo/gcc_stack_darwin.c @@ -15,6 +15,11 @@ x_cgo_getstackbound(uintptr bounds[2]) p = pthread_self(); addr = pthread_get_stackaddr_np(p); // high address (!) size = pthread_get_stacksize_np(p); + + // bounds points into the Go stack. TSAN can't see the synchronization + // in Go around stack reuse. + _cgo_tsan_acquire(); bounds[0] = (uintptr)addr - size; bounds[1] = (uintptr)addr; + _cgo_tsan_release(); } diff --git a/src/runtime/cgo/gcc_stack_unix.c b/src/runtime/cgo/gcc_stack_unix.c index 67efd9bc63..eb1d7f9ec5 100644 --- a/src/runtime/cgo/gcc_stack_unix.c +++ b/src/runtime/cgo/gcc_stack_unix.c @@ -37,6 +37,10 @@ x_cgo_getstackbound(uintptr bounds[2]) #endif pthread_attr_destroy(&attr); + // bounds points into the Go stack. TSAN can't see the synchronization + // in Go around stack reuse. + _cgo_tsan_acquire(); bounds[0] = (uintptr)addr; bounds[1] = (uintptr)addr + size; + _cgo_tsan_release(); } diff --git a/src/runtime/cgocall.go b/src/runtime/cgocall.go index 8f09b6831b..071643614b 100644 --- a/src/runtime/cgocall.go +++ b/src/runtime/cgocall.go @@ -221,15 +221,18 @@ func cgocall(fn, arg unsafe.Pointer) int32 { //go:nosplit func callbackUpdateSystemStack(mp *m, sp uintptr, signal bool) { g0 := mp.g0 - if sp > g0.stack.lo && sp <= g0.stack.hi { - // Stack already in bounds, nothing to do. - return - } - if mp.ncgo > 0 { + inBound := sp > g0.stack.lo && sp <= g0.stack.hi + if mp.ncgo > 0 && !inBound { // ncgo > 0 indicates that this M was in Go further up the stack - // (it called C and is now receiving a callback). It is not - // safe for the C call to change the stack out from under us. + // (it called C and is now receiving a callback). + // + // !inBound indicates that we were called with SP outside the + // expected system stack bounds (C changed the stack out from + // under us between the cgocall and cgocallback?). + // + // It is not safe for the C call to change the stack out from + // under us, so throw. // Note that this case isn't possible for signal == true, as // that is always passing a new M from needm. @@ -247,12 +250,26 @@ func callbackUpdateSystemStack(mp *m, sp uintptr, signal bool) { exit(2) } + if !mp.isextra { + // We allocated the stack for standard Ms. Don't replace the + // stack bounds with estimated ones when we already initialized + // with the exact ones. + return + } + // This M does not have Go further up the stack. However, it may have // previously called into Go, initializing the stack bounds. Between // that call returning and now the stack may have changed (perhaps the // C thread is running a coroutine library). We need to update the // stack bounds for this case. // + // N.B. we need to update the stack bounds even if SP appears to + // already be in bounds. Our "bounds" may actually be estimated dummy + // bounds (below). The actual stack bounds could have shifted but still + // have partial overlap with our dummy bounds. If we failed to update + // in that case, we could find ourselves seemingly called near the + // bottom of the stack bounds, where we quickly run out of space. + // Set the stack bounds to match the current stack. If we don't // actually know how big the stack is, like we don't know how big any // scheduling stack is, but we assume there's at least 32 kB. If we diff --git a/src/runtime/coro.go b/src/runtime/coro.go index 98e789f133..b2bc801940 100644 --- a/src/runtime/coro.go +++ b/src/runtime/coro.go @@ -46,8 +46,6 @@ func newcoro(f func(*coro)) *coro { return c } -//go:linkname corostart - // corostart is the entry func for a new coroutine. // It runs the coroutine user function f passed to corostart // and then calls coroexit to remove the extra concurrency. diff --git a/src/runtime/coverage/emit.go b/src/runtime/coverage/emit.go index 6fe04daea8..6510c889ea 100644 --- a/src/runtime/coverage/emit.go +++ b/src/runtime/coverage/emit.go @@ -574,6 +574,9 @@ func (s *emitState) emitCounterDataFile(finalHash [16]byte, w io.Writer) error { return nil } +// markProfileEmitted is injected to testmain via linkname. +//go:linkname markProfileEmitted + // markProfileEmitted signals the runtime/coverage machinery that // coverage data output files have already been written out, and there // is no need to take any additional action at exit time. This diff --git a/src/runtime/coverage/testsupport.go b/src/runtime/coverage/testsupport.go index 4b00f3a0f7..b673d3cd2c 100644 --- a/src/runtime/coverage/testsupport.go +++ b/src/runtime/coverage/testsupport.go @@ -22,6 +22,9 @@ import ( "unsafe" ) +// processCoverTestDir is injected in testmain. +//go:linkname processCoverTestDir + // processCoverTestDir is called (via a linknamed reference) from // testmain code when "go test -cover" is in effect. It is not // intended to be used other than internally by the Go command's @@ -277,6 +280,9 @@ func (ts *tstate) readAuxMetaFiles(metafiles string, importpaths map[string]stru return nil } +// snapshot is injected in testmain. +//go:linkname snapshot + // snapshot returns a snapshot of coverage percentage at a moment of // time within a running test, so as to support the testing.Coverage() // function. This version doesn't examine coverage meta-data, so the diff --git a/src/runtime/debug/example_monitor_test.go b/src/runtime/debug/example_monitor_test.go index 5a1f4e1417..b077e7adb3 100644 --- a/src/runtime/debug/example_monitor_test.go +++ b/src/runtime/debug/example_monitor_test.go @@ -91,7 +91,7 @@ func monitor() { if err != nil { log.Fatalf("StdinPipe: %v", err) } - debug.SetCrashOutput(pipe.(*os.File)) // (this conversion is safe) + debug.SetCrashOutput(pipe.(*os.File), debug.CrashOptions{}) // (this conversion is safe) if err := cmd.Start(); err != nil { log.Fatalf("can't start monitor: %v", err) } diff --git a/src/runtime/debug/stack.go b/src/runtime/debug/stack.go index 8dfea52d34..d7a860b7dc 100644 --- a/src/runtime/debug/stack.go +++ b/src/runtime/debug/stack.go @@ -31,6 +31,12 @@ func Stack() []byte { } } +// CrashOptions provides options that control the formatting of the +// fatal crash message. +type CrashOptions struct { + /* for future expansion */ +} + // SetCrashOutput configures a single additional file where unhandled // panics and other fatal errors are printed, in addition to standard error. // There is only one additional file: calling SetCrashOutput again overrides @@ -40,7 +46,7 @@ func Stack() []byte { // To disable this additional crash output, call SetCrashOutput(nil). // If called concurrently with a crash, some in-progress output may be written // to the old file even after an overriding SetCrashOutput returns. -func SetCrashOutput(f *os.File) error { +func SetCrashOutput(f *os.File, opts CrashOptions) error { fd := ^uintptr(0) if f != nil { // The runtime will write to this file descriptor from diff --git a/src/runtime/debug/stack_test.go b/src/runtime/debug/stack_test.go index 289749ccb4..e1559303f0 100644 --- a/src/runtime/debug/stack_test.go +++ b/src/runtime/debug/stack_test.go @@ -13,6 +13,7 @@ import ( "os/exec" "path/filepath" "runtime" + "runtime/debug" . "runtime/debug" "strings" "testing" @@ -29,7 +30,7 @@ func TestMain(m *testing.M) { if err != nil { log.Fatal(err) } - if err := SetCrashOutput(f); err != nil { + if err := SetCrashOutput(f, debug.CrashOptions{}); err != nil { log.Fatal(err) // e.g. EMFILE } println("hello") diff --git a/src/runtime/linkname.go b/src/runtime/linkname.go new file mode 100644 index 0000000000..0f02c6b4e3 --- /dev/null +++ b/src/runtime/linkname.go @@ -0,0 +1,49 @@ +// Copyright 2024 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package runtime + +import _ "unsafe" + +// used in time and internal/poll +//go:linkname nanotime + +// used in internal/godebug and syscall +//go:linkname write + +// used in internal/runtime/atomic +//go:linkname goarm + +// used by cgo +//go:linkname cgocall +//go:linkname _cgo_panic_internal +//go:linkname cgoAlwaysFalse +//go:linkname cgoUse +//go:linkname cgoCheckPointer +//go:linkname cgoCheckResult +//go:linkname cgoNoCallback +//go:linkname gobytes +//go:linkname gostringn +//go:linkname throw + +// used in plugin +//go:linkname doInit + +// used in math/bits +//go:linkname overflowError +//go:linkname divideError + +// used in runtime/coverage and in tests +//go:linkname addExitHook + +// used in x/sys/cpu +//go:linkname getAuxv + +// used in tests +//go:linkname extraMInUse +//go:linkname getm +//go:linkname blockevent +//go:linkname haveHighResSleep +//go:linkname blockUntilEmptyFinalizerQueue +//go:linkname lockedOSThread diff --git a/src/runtime/linkname_unix.go b/src/runtime/linkname_unix.go new file mode 100644 index 0000000000..65f876fa4b --- /dev/null +++ b/src/runtime/linkname_unix.go @@ -0,0 +1,12 @@ +// Copyright 2024 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +//go:build unix + +package runtime + +import _ "unsafe" + +// used in internal/syscall/unix +//go:linkname fcntl diff --git a/src/runtime/mprof.go b/src/runtime/mprof.go index d4d5e285fd..b51edcbcab 100644 --- a/src/runtime/mprof.go +++ b/src/runtime/mprof.go @@ -43,7 +43,10 @@ const ( // Note that it's only used internally as a guard against // wildly out-of-bounds slicing of the PCs that come after // a bucket struct, and it could increase in the future. - maxStack = 32 + // The "+ 1" is to account for the first stack entry being + // taken up by a "skip" sentinel value for profilers which + // defer inline frame expansion until the profile is reported. + maxStack = 32 + 1 ) type bucketType int @@ -502,14 +505,40 @@ func blocksampled(cycles, rate int64) bool { return true } +// saveblockevent records a profile event of the type specified by which. +// cycles is the quantity associated with this event and rate is the sampling rate, +// used to adjust the cycles value in the manner determined by the profile type. +// skip is the number of frames to omit from the traceback associated with the event. +// The traceback will be recorded from the stack of the goroutine associated with the current m. +// skip should be positive if this event is recorded from the current stack +// (e.g. when this is not called from a system stack) func saveblockevent(cycles, rate int64, skip int, which bucketType) { - var nstk int gp := getg() mp := acquirem() // we must not be preempted while accessing profstack - if gp.m.curg == nil || gp.m.curg == gp { - nstk = callers(skip, mp.profStack) + nstk := 1 + if tracefpunwindoff() || gp.m.hasCgoOnStack() { + mp.profStack[0] = logicalStackSentinel + if gp.m.curg == nil || gp.m.curg == gp { + nstk = callers(skip, mp.profStack[1:]) + } else { + nstk = gcallers(gp.m.curg, skip, mp.profStack[1:]) + } } else { - nstk = gcallers(gp.m.curg, skip, mp.profStack) + mp.profStack[0] = uintptr(skip) + if gp.m.curg == nil || gp.m.curg == gp { + if skip > 0 { + // We skip one fewer frame than the provided value for frame + // pointer unwinding because the skip value includes the current + // frame, whereas the saved frame pointer will give us the + // caller's return address first (so, not including + // saveblockevent) + mp.profStack[0] -= 1 + } + nstk += fpTracebackPCs(unsafe.Pointer(getfp()), mp.profStack[1:]) + } else { + mp.profStack[1] = gp.m.curg.sched.pc + nstk += 1 + fpTracebackPCs(unsafe.Pointer(gp.m.curg.sched.bp), mp.profStack[2:]) + } } saveBlockEventStack(cycles, rate, mp.profStack[:nstk], which) @@ -689,9 +718,10 @@ func (prof *mLockProfile) captureStack() { } prof.pending = 0 + prof.stack[0] = logicalStackSentinel if debug.runtimeContentionStacks.Load() == 0 { - prof.stack[0] = abi.FuncPCABIInternal(_LostContendedRuntimeLock) + sys.PCQuantum - prof.stack[1] = 0 + prof.stack[1] = abi.FuncPCABIInternal(_LostContendedRuntimeLock) + sys.PCQuantum + prof.stack[2] = 0 return } @@ -702,7 +732,7 @@ func (prof *mLockProfile) captureStack() { systemstack(func() { var u unwinder u.initAt(pc, sp, 0, gp, unwindSilentErrors|unwindJumpStack) - nstk = tracebackPCs(&u, skip, prof.stack) + nstk = 1 + tracebackPCs(&u, skip, prof.stack[1:]) }) if nstk < len(prof.stack) { prof.stack[nstk] = 0 @@ -732,6 +762,7 @@ func (prof *mLockProfile) store() { saveBlockEventStack(cycles, rate, prof.stack[:nstk], mutexProfile) if lost > 0 { lostStk := [...]uintptr{ + logicalStackSentinel, abi.FuncPCABIInternal(_LostContendedRuntimeLock) + sys.PCQuantum, } saveBlockEventStack(lost, rate, lostStk[:], mutexProfile) @@ -952,8 +983,8 @@ func record(r *MemProfileRecord, b *bucket) { if asanenabled { asanwrite(unsafe.Pointer(&r.Stack0[0]), unsafe.Sizeof(r.Stack0)) } - copy(r.Stack0[:], b.stk()) - clear(r.Stack0[b.nstk:]) + i := copy(r.Stack0[:], b.stk()) + clear(r.Stack0[i:]) } func iterate_memprof(fn func(*bucket, uintptr, *uintptr, uintptr, uintptr, uintptr)) { @@ -1008,7 +1039,7 @@ func BlockProfile(p []BlockProfileRecord) (n int, ok bool) { if asanenabled { asanwrite(unsafe.Pointer(&r.Stack0[0]), unsafe.Sizeof(r.Stack0)) } - i := copy(r.Stack0[:], b.stk()) + i := fpunwindExpand(r.Stack0[:], b.stk()) clear(r.Stack0[i:]) p = p[1:] } @@ -1036,7 +1067,7 @@ func MutexProfile(p []BlockProfileRecord) (n int, ok bool) { r := &p[0] r.Count = int64(bp.count) r.Cycles = bp.cycles - i := copy(r.Stack0[:], b.stk()) + i := fpunwindExpand(r.Stack0[:], b.stk()) clear(r.Stack0[i:]) p = p[1:] } diff --git a/src/runtime/netpoll.go b/src/runtime/netpoll.go index bbfef80aec..7b37d91b24 100644 --- a/src/runtime/netpoll.go +++ b/src/runtime/netpoll.go @@ -207,6 +207,9 @@ var ( netpollWaiters atomic.Uint32 ) +// netpollWaiters is accessed in tests +//go:linkname netpollWaiters + //go:linkname poll_runtime_pollServerInit internal/poll.runtime_pollServerInit func poll_runtime_pollServerInit() { netpollGenericInit() diff --git a/src/runtime/panic.go b/src/runtime/panic.go index 122fc30df2..ff9c64113f 100644 --- a/src/runtime/panic.go +++ b/src/runtime/panic.go @@ -1014,13 +1014,12 @@ func sync_fatal(s string) { // issue #67274, so as to fix longtest builders. // //go:nosplit -//go:noinline func throw(s string) { // Everything throw does should be recursively nosplit so it // can be called even when it's unsafe to grow the stack. systemstack(func() { print("fatal error: ") - printpanicval(s) + printindented(s) // logically printpanicval(s), but avoids convTstring write barrier print("\n") }) @@ -1041,7 +1040,7 @@ func fatal(s string) { // can be called even when it's unsafe to grow the stack. systemstack(func() { print("fatal error: ") - printpanicval(s) + printindented(s) // logically printpanicval(s), but avoids convTstring write barrier print("\n") }) diff --git a/src/runtime/proc.go b/src/runtime/proc.go index 13a8d70186..140f06d03d 100644 --- a/src/runtime/proc.go +++ b/src/runtime/proc.go @@ -578,7 +578,7 @@ func switchToCrashStack(fn func()) { // Disable crash stack on Windows for now. Apparently, throwing an exception // on a non-system-allocated crash stack causes EXCEPTION_STACK_OVERFLOW and // hangs the process (see issue 63938). -const crashStackImplemented = (GOARCH == "386" || GOARCH == "amd64" || GOARCH == "arm" || GOARCH == "arm64" || GOARCH == "loong64" || GOARCH == "mips" || GOARCH == "mipsle" || GOARCH == "mips64" || GOARCH == "mips64le" || GOARCH == "ppc64" || GOARCH == "ppc64le" || GOARCH == "riscv64" || GOARCH == "s390x" || GOARCH == "wasm") && GOOS != "windows" +const crashStackImplemented = GOOS != "windows" //go:noescape func switchToCrashStack0(fn func()) // in assembly diff --git a/src/runtime/string.go b/src/runtime/string.go index e01b7fc744..81d1b80e56 100644 --- a/src/runtime/string.go +++ b/src/runtime/string.go @@ -312,7 +312,7 @@ func gobytes(p *byte, n int) (b []byte) { return } -// This is exported via linkname to assembly in syscall (for Plan9). +// This is exported via linkname to assembly in syscall (for Plan9) and cgo. // //go:linkname gostring func gostring(p *byte) string { diff --git a/src/runtime/traceback_system_test.go b/src/runtime/traceback_system_test.go index 5131e44e64..ece58e806d 100644 --- a/src/runtime/traceback_system_test.go +++ b/src/runtime/traceback_system_test.go @@ -28,7 +28,7 @@ func crash() { // Ensure that we get pc=0x%x values in the traceback. debug.SetTraceback("system") writeSentinel(os.Stdout) - debug.SetCrashOutput(os.Stdout) + debug.SetCrashOutput(os.Stdout, debug.CrashOptions{}) go func() { // This call is typically inlined. diff --git a/src/runtime/tracestack.go b/src/runtime/tracestack.go index 04b935a2c9..477526d7cb 100644 --- a/src/runtime/tracestack.go +++ b/src/runtime/tracestack.go @@ -147,20 +147,22 @@ func (t *traceStackTable) put(pcs []uintptr) uint64 { // releases all memory and resets state. It must only be called once the caller // can guarantee that there are no more writers to the table. func (t *traceStackTable) dump(gen uintptr) { + stackBuf := make([]uintptr, traceStackSize) w := unsafeTraceWriter(gen, nil) if root := (*traceMapNode)(t.tab.root.Load()); root != nil { - w = dumpStacksRec(root, w) + w = dumpStacksRec(root, w, stackBuf) } w.flush().end() t.tab.reset() } -func dumpStacksRec(node *traceMapNode, w traceWriter) traceWriter { +func dumpStacksRec(node *traceMapNode, w traceWriter, stackBuf []uintptr) traceWriter { stack := unsafe.Slice((*uintptr)(unsafe.Pointer(&node.data[0])), uintptr(len(node.data))/unsafe.Sizeof(uintptr(0))) // N.B. This might allocate, but that's OK because we're not writing to the M's buffer, // but one we're about to create (with ensure). - frames := makeTraceFrames(w.gen, fpunwindExpand(stack)) + n := fpunwindExpand(stackBuf, stack) + frames := makeTraceFrames(w.gen, stackBuf[:n]) // The maximum number of bytes required to hold the encoded stack, given that // it contains N frames. @@ -194,7 +196,7 @@ func dumpStacksRec(node *traceMapNode, w traceWriter) traceWriter { if child == nil { continue } - w = dumpStacksRec((*traceMapNode)(child), w) + w = dumpStacksRec((*traceMapNode)(child), w, stackBuf) } return w } @@ -260,31 +262,36 @@ func fpTracebackPCs(fp unsafe.Pointer, pcBuf []uintptr) (i int) { return i } +// fpunwindExpand expands a call stack from pcBuf into dst, +// returning the number of PCs written to dst. +// pcBuf and dst should not overlap. +// // fpunwindExpand checks if pcBuf contains logical frames (which include inlined // frames) or physical frames (produced by frame pointer unwinding) using a // sentinel value in pcBuf[0]. Logical frames are simply returned without the // sentinel. Physical frames are turned into logical frames via inline unwinding // and by applying the skip value that's stored in pcBuf[0]. -func fpunwindExpand(pcBuf []uintptr) []uintptr { +func fpunwindExpand(dst, pcBuf []uintptr) int { if len(pcBuf) > 0 && pcBuf[0] == logicalStackSentinel { // pcBuf contains logical rather than inlined frames, skip has already been // applied, just return it without the sentinel value in pcBuf[0]. - return pcBuf[1:] + return copy(dst, pcBuf[1:]) } var ( + n int lastFuncID = abi.FuncIDNormal - newPCBuf = make([]uintptr, 0, traceStackSize) skip = pcBuf[0] // skipOrAdd skips or appends retPC to newPCBuf and returns true if more // pcs can be added. skipOrAdd = func(retPC uintptr) bool { if skip > 0 { skip-- - } else { - newPCBuf = append(newPCBuf, retPC) + } else if n < len(dst) { + dst[n] = retPC + n++ } - return len(newPCBuf) < cap(newPCBuf) + return n < len(dst) } ) @@ -312,7 +319,7 @@ outer: lastFuncID = sf.funcID } } - return newPCBuf + return n } // startPCForTrace returns the start PC of a goroutine for tracing purposes. diff --git a/src/runtime/vdso_linux_amd64.go b/src/runtime/vdso_linux_amd64.go index 4e9f748f4a..9c56409137 100644 --- a/src/runtime/vdso_linux_amd64.go +++ b/src/runtime/vdso_linux_amd64.go @@ -4,6 +4,8 @@ package runtime +import _ "unsafe" // for linkname + const ( // vdsoArrayMax is the byte-size of a maximally sized array on this architecture. // See cmd/compile/internal/amd64/galign.go arch.MAXWIDTH initialization. @@ -21,3 +23,6 @@ var ( vdsoGettimeofdaySym uintptr vdsoClockgettimeSym uintptr ) + +// vdsoGettimeofdaySym is accessed from the syscall package. +//go:linkname vdsoGettimeofdaySym diff --git a/src/slices/iter.go b/src/slices/iter.go index a0f642e423..131cece3a0 100644 --- a/src/slices/iter.go +++ b/src/slices/iter.go @@ -33,7 +33,7 @@ func Backward[Slice ~[]E, E any](s Slice) iter.Seq2[int, E] { } } -// Values returns an iterator over the slice elements. +// Values returns an iterator over the slice elements, // starting with s[0]. func Values[Slice ~[]E, E any](s Slice) iter.Seq[E] { return func(yield func(E) bool) { diff --git a/src/syscall/dirent.go b/src/syscall/dirent.go index a9eab15736..e4f36a191b 100644 --- a/src/syscall/dirent.go +++ b/src/syscall/dirent.go @@ -7,6 +7,7 @@ package syscall import ( + "internal/byteorder" "internal/goarch" "runtime" "unsafe" @@ -28,15 +29,11 @@ func readIntBE(b []byte, size uintptr) uint64 { case 1: return uint64(b[0]) case 2: - _ = b[1] // bounds check hint to compiler; see golang.org/issue/14808 - return uint64(b[1]) | uint64(b[0])<<8 + return uint64(byteorder.BeUint16(b)) case 4: - _ = b[3] // bounds check hint to compiler; see golang.org/issue/14808 - return uint64(b[3]) | uint64(b[2])<<8 | uint64(b[1])<<16 | uint64(b[0])<<24 + return uint64(byteorder.BeUint32(b)) case 8: - _ = b[7] // bounds check hint to compiler; see golang.org/issue/14808 - return uint64(b[7]) | uint64(b[6])<<8 | uint64(b[5])<<16 | uint64(b[4])<<24 | - uint64(b[3])<<32 | uint64(b[2])<<40 | uint64(b[1])<<48 | uint64(b[0])<<56 + return uint64(byteorder.BeUint64(b)) default: panic("syscall: readInt with unsupported size") } @@ -47,15 +44,11 @@ func readIntLE(b []byte, size uintptr) uint64 { case 1: return uint64(b[0]) case 2: - _ = b[1] // bounds check hint to compiler; see golang.org/issue/14808 - return uint64(b[0]) | uint64(b[1])<<8 + return uint64(byteorder.LeUint16(b)) case 4: - _ = b[3] // bounds check hint to compiler; see golang.org/issue/14808 - return uint64(b[0]) | uint64(b[1])<<8 | uint64(b[2])<<16 | uint64(b[3])<<24 + return uint64(byteorder.LeUint32(b)) case 8: - _ = b[7] // bounds check hint to compiler; see golang.org/issue/14808 - return uint64(b[0]) | uint64(b[1])<<8 | uint64(b[2])<<16 | uint64(b[3])<<24 | - uint64(b[4])<<32 | uint64(b[5])<<40 | uint64(b[6])<<48 | uint64(b[7])<<56 + return uint64(byteorder.LeUint64(b)) default: panic("syscall: readInt with unsupported size") } diff --git a/src/syscall/exec_linux.go b/src/syscall/exec_linux.go index e6d6343ed8..e4b9ce1bf4 100644 --- a/src/syscall/exec_linux.go +++ b/src/syscall/exec_linux.go @@ -53,6 +53,10 @@ const ( // SysProcIDMap holds Container ID to Host ID mappings used for User Namespaces in Linux. // See user_namespaces(7). +// +// Note that User Namespaces are not available on a number of popular Linux +// versions (due to security issues), or are available but subject to AppArmor +// restrictions like in Ubuntu 24.04. type SysProcIDMap struct { ContainerID int // Container ID. HostID int // Host ID. diff --git a/src/syscall/exec_linux_test.go b/src/syscall/exec_linux_test.go index 5ec1a24ba4..079220eab1 100644 --- a/src/syscall/exec_linux_test.go +++ b/src/syscall/exec_linux_test.go @@ -642,6 +642,10 @@ func TestAmbientCaps(t *testing.T) { } func TestAmbientCapsUserns(t *testing.T) { + b, err := os.ReadFile("/proc/sys/kernel/apparmor_restrict_unprivileged_userns") + if err == nil && strings.TrimSpace(string(b)) == "1" { + t.Skip("AppArmor restriction for unprivileged user namespaces is enabled") + } testAmbientCaps(t, true) } diff --git a/src/syscall/export_linux_test.go b/src/syscall/export_linux_test.go index 9bcf73e771..d9309bf234 100644 --- a/src/syscall/export_linux_test.go +++ b/src/syscall/export_linux_test.go @@ -11,6 +11,7 @@ import ( var ( RawSyscallNoError = rawSyscallNoError ForceClone3 = &forceClone3 + Prlimit = prlimit ) const ( diff --git a/src/syscall/export_rlimit_test.go b/src/syscall/export_rlimit_test.go index 8b1545cb03..f584ac410d 100644 --- a/src/syscall/export_rlimit_test.go +++ b/src/syscall/export_rlimit_test.go @@ -6,6 +6,12 @@ package syscall +import "sync/atomic" + func OrigRlimitNofile() *Rlimit { return origRlimitNofile.Load() } + +func GetInternalOrigRlimitNofile() *atomic.Pointer[Rlimit] { + return &origRlimitNofile +} diff --git a/src/syscall/fs_wasip1.go b/src/syscall/fs_wasip1.go index f19e8f3b3c..fc361ee898 100644 --- a/src/syscall/fs_wasip1.go +++ b/src/syscall/fs_wasip1.go @@ -288,12 +288,18 @@ func fd_fdstat_get(fd int32, buf unsafe.Pointer) Errno //go:noescape func fd_fdstat_set_flags(fd int32, flags fdflags) Errno +// fd_fdstat_get_flags is accessed from internal/syscall/unix +//go:linkname fd_fdstat_get_flags + func fd_fdstat_get_flags(fd int) (uint32, error) { var stat fdstat errno := fd_fdstat_get(int32(fd), unsafe.Pointer(&stat)) return uint32(stat.fdflags), errnoErr(errno) } +// fd_fdstat_get_type is accessed from net +//go:linkname fd_fdstat_get_type + func fd_fdstat_get_type(fd int) (uint8, error) { var stat fdstat errno := fd_fdstat_get(int32(fd), unsafe.Pointer(&stat)) diff --git a/src/syscall/linkname_bsd.go b/src/syscall/linkname_bsd.go new file mode 100644 index 0000000000..65ef900241 --- /dev/null +++ b/src/syscall/linkname_bsd.go @@ -0,0 +1,15 @@ +// Copyright 2024 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +//go:build darwin || dragonfly || freebsd || netbsd || openbsd + +package syscall + +import _ "unsafe" + +// used by internal/syscall/unix +//go:linkname ioctlPtr + +// used by x/net/route +//go:linkname sysctl diff --git a/src/syscall/linkname_darwin.go b/src/syscall/linkname_darwin.go new file mode 100644 index 0000000000..2ed83a4fad --- /dev/null +++ b/src/syscall/linkname_darwin.go @@ -0,0 +1,23 @@ +// Copyright 2024 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package syscall + +import _ "unsafe" + +// used by os +//go:linkname closedir +//go:linkname readdir_r + +// used by internal/poll +//go:linkname fdopendir + +// used by internal/syscall/unix +//go:linkname unlinkat +//go:linkname openat +//go:linkname fstatat + +// used by cmd/link +//go:linkname msync +//go:linkname fcntl diff --git a/src/syscall/linkname_libc.go b/src/syscall/linkname_libc.go new file mode 100644 index 0000000000..1e7b4880d6 --- /dev/null +++ b/src/syscall/linkname_libc.go @@ -0,0 +1,12 @@ +// Copyright 2024 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +//go:build aix || darwin || (openbsd && !mips64) || solaris + +package syscall + +import _ "unsafe" + +// used by internal/poll +//go:linkname writev diff --git a/src/syscall/linkname_openbsd.go b/src/syscall/linkname_openbsd.go new file mode 100644 index 0000000000..5f5c517ab5 --- /dev/null +++ b/src/syscall/linkname_openbsd.go @@ -0,0 +1,15 @@ +// Copyright 2024 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +//go:build openbsd && !mips64 + +package syscall + +import _ "unsafe" + +// used by internal/syscall/unix +//go:linkname unlinkat +//go:linkname openat +//go:linkname fstatat +//go:linkname getentropy diff --git a/src/syscall/syscall_linux.go b/src/syscall/syscall_linux.go index 6547c517a7..28727dc98a 100644 --- a/src/syscall/syscall_linux.go +++ b/src/syscall/syscall_linux.go @@ -1284,12 +1284,15 @@ func Munmap(b []byte) (err error) { //sys Mlockall(flags int) (err error) //sys Munlockall() (err error) +// prlimit is accessed from x/sys/unix. +//go:linkname prlimit + // prlimit changes a resource limit. We use a single definition so that // we can tell StartProcess to not restore the original NOFILE limit. // This is unexported but can be called from x/sys/unix. func prlimit(pid int, resource int, newlimit *Rlimit, old *Rlimit) (err error) { err = prlimit1(pid, resource, newlimit, old) - if err == nil && newlimit != nil && resource == RLIMIT_NOFILE { + if err == nil && newlimit != nil && resource == RLIMIT_NOFILE && (pid == 0 || pid == Getpid()) { origRlimitNofile.Store(nil) } return err diff --git a/src/syscall/syscall_linux_test.go b/src/syscall/syscall_linux_test.go index 1300fc046e..675406fba0 100644 --- a/src/syscall/syscall_linux_test.go +++ b/src/syscall/syscall_linux_test.go @@ -654,3 +654,69 @@ func TestAllThreadsSyscallBlockedSyscall(t *testing.T) { wr.Close() wg.Wait() } + +func TestPrlimitSelf(t *testing.T) { + origLimit := syscall.OrigRlimitNofile() + origRlimitNofile := syscall.GetInternalOrigRlimitNofile() + + if origLimit == nil { + defer origRlimitNofile.Store(origLimit) + origRlimitNofile.Store(&syscall.Rlimit{ + Cur: 1024, + Max: 65536, + }) + } + + // Get current process's nofile limit + var lim syscall.Rlimit + if err := syscall.Prlimit(0, syscall.RLIMIT_NOFILE, nil, &lim); err != nil { + t.Fatalf("Failed to get the current nofile limit: %v", err) + } + // Set current process's nofile limit through prlimit + if err := syscall.Prlimit(0, syscall.RLIMIT_NOFILE, &lim, nil); err != nil { + t.Fatalf("Prlimit self failed: %v", err) + } + + rlimLater := origRlimitNofile.Load() + if rlimLater != nil { + t.Fatalf("origRlimitNofile got=%v, want=nil", rlimLater) + } +} + +func TestPrlimitOtherProcess(t *testing.T) { + origLimit := syscall.OrigRlimitNofile() + origRlimitNofile := syscall.GetInternalOrigRlimitNofile() + + if origLimit == nil { + defer origRlimitNofile.Store(origLimit) + origRlimitNofile.Store(&syscall.Rlimit{ + Cur: 1024, + Max: 65536, + }) + } + rlimOrig := origRlimitNofile.Load() + + // Start a child process firstly, + // so we can use Prlimit to set it's nofile limit. + cmd := exec.Command("sleep", "infinity") + cmd.Start() + defer func() { + cmd.Process.Kill() + cmd.Process.Wait() + }() + + // Get child process's current nofile limit + var lim syscall.Rlimit + if err := syscall.Prlimit(cmd.Process.Pid, syscall.RLIMIT_NOFILE, nil, &lim); err != nil { + t.Fatalf("Failed to get the current nofile limit: %v", err) + } + // Set child process's nofile rlimit through prlimit + if err := syscall.Prlimit(cmd.Process.Pid, syscall.RLIMIT_NOFILE, &lim, nil); err != nil { + t.Fatalf("Prlimit(%d) failed: %v", cmd.Process.Pid, err) + } + + rlimLater := origRlimitNofile.Load() + if rlimLater != rlimOrig { + t.Fatalf("origRlimitNofile got=%v, want=%v", rlimLater, rlimOrig) + } +} diff --git a/src/testing/newcover.go b/src/testing/newcover.go index 6199f3bd7b..7a70dcfffa 100644 --- a/src/testing/newcover.go +++ b/src/testing/newcover.go @@ -10,6 +10,7 @@ import ( "fmt" "internal/goexperiment" "os" + _ "unsafe" // for linkname ) // cover2 variable stores the current coverage mode and a @@ -20,6 +21,9 @@ var cover2 struct { snapshotcov func() float64 } +// registerCover2 is injected in testmain. +//go:linkname registerCover2 + // registerCover2 is invoked during "go test -cover" runs by the test harness // code in _testmain.go; it is used to record a 'tear down' function // (to be called when the test is complete) and the coverage mode. @@ -42,6 +46,9 @@ func coverReport2() { } } +// testGoCoverDir is used in runtime/coverage tests. +//go:linkname testGoCoverDir + // testGoCoverDir returns the value passed to the -test.gocoverdir // flag by the Go command, if goexperiment.CoverageRedesign is // in effect. diff --git a/src/text/template/exec.go b/src/text/template/exec.go index 4c899b1c79..1a8f2fa0df 100644 --- a/src/text/template/exec.go +++ b/src/text/template/exec.go @@ -408,8 +408,8 @@ func (s *state) walkRange(dot reflect.Value, r *parse.RangeNode) { break } om := fmtsort.Sort(val) - for i, key := range om.Key { - oneIteration(key, om.Value[i]) + for _, m := range om { + oneIteration(m.Key, m.Value) } return case reflect.Chan: diff --git a/src/time/tick_test.go b/src/time/tick_test.go index 4aaf6a2b80..750aa90f4d 100644 --- a/src/time/tick_test.go +++ b/src/time/tick_test.go @@ -361,7 +361,11 @@ func testTimerChan(t *testing.T, tim timer, C <-chan Time, synctimerchan bool) { drainTries = 5 ) - drain := func() { + // drain1 removes one potential stale time value + // from the timer/ticker channel after Reset. + // When using Go 1.23 sync timers/tickers, draining is never needed + // (that's the whole point of the sync timer/ticker change). + drain1 := func() { for range drainTries { select { case <-C: @@ -371,6 +375,34 @@ func testTimerChan(t *testing.T, tim timer, C <-chan Time, synctimerchan bool) { Sleep(sched) } } + + // drainAsync removes potential stale time values after Stop/Reset. + // When using Go 1 async timers, draining one or two values + // may be needed after Reset or Stop (see comments in body for details). + drainAsync := func() { + if synctimerchan { + // sync timers must have the right semantics without draining: + // there are no stale values. + return + } + + // async timers can send one stale value (then the timer is disabled). + drain1() + if isTicker { + // async tickers can send two stale values: there may be one + // sitting in the channel buffer, and there may also be one + // send racing with the Reset/Stop+drain that arrives after + // the first drain1 has pulled the value out. + // This is rare, but it does happen on overloaded builder machines. + // It can also be reproduced on an M3 MacBook Pro using: + // + // go test -c strings + // stress ./strings.test & # chew up CPU + // go test -c -race time + // stress -p 48 ./time.test -test.count=10 -test.run=TestChan/asynctimerchan=1/Ticker + drain1() + } + } noTick := func() { t.Helper() select { @@ -439,9 +471,7 @@ func testTimerChan(t *testing.T, tim timer, C <-chan Time, synctimerchan bool) { assertTick() Sleep(sched) tim.Reset(10000 * Second) - if isTicker { - drain() - } + drainAsync() noTick() // Test that len sees an immediate tick arrive @@ -453,9 +483,7 @@ func testTimerChan(t *testing.T, tim timer, C <-chan Time, synctimerchan bool) { // Test that len sees an immediate tick arrive // for Reset of timer NOT in heap. tim.Stop() - if !synctimerchan { - drain() - } + drainAsync() tim.Reset(1) assertLen() assertTick() @@ -465,9 +493,7 @@ func testTimerChan(t *testing.T, tim timer, C <-chan Time, synctimerchan bool) { // Test that Reset does not lose the tick that should have happened. Sleep(sched) tim.Reset(10000 * Second) - if !synctimerchan && isTicker { - drain() - } + drainAsync() noTick() notDone := func(done chan bool) { @@ -494,9 +520,7 @@ func testTimerChan(t *testing.T, tim timer, C <-chan Time, synctimerchan bool) { // Reset timer in heap (already reset above, but just in case). tim.Reset(10000 * Second) - if !synctimerchan { - drain() - } + drainAsync() // Test stop while timer in heap (because goroutine is blocked on <-C). done := make(chan bool) @@ -526,17 +550,12 @@ func testTimerChan(t *testing.T, tim timer, C <-chan Time, synctimerchan bool) { } tim.Stop() - if isTicker || !synctimerchan { - t.Logf("drain") - drain() - } + drainAsync() noTick() // Again using select and with two goroutines waiting. tim.Reset(10000 * Second) - if !synctimerchan { - drain() - } + drainAsync() done = make(chan bool, 2) done1 := make(chan bool) done2 := make(chan bool) diff --git a/src/time/zoneinfo_read.go b/src/time/zoneinfo_read.go index 9ce735d279..5314b6ff9a 100644 --- a/src/time/zoneinfo_read.go +++ b/src/time/zoneinfo_read.go @@ -14,10 +14,13 @@ import ( "internal/bytealg" "runtime" "syscall" + _ "unsafe" // for linkname ) // registerLoadFromEmbeddedTZData is called by the time/tzdata package, // if it is imported. +// +//go:linkname registerLoadFromEmbeddedTZData func registerLoadFromEmbeddedTZData(f func(string) (string, error)) { loadFromEmbeddedTZData = f } diff --git a/test/alias2.go b/test/alias2.go index 2846e5dc31..95eb25a94b 100644 --- a/test/alias2.go +++ b/test/alias2.go @@ -47,7 +47,7 @@ var _ T0 = A0{} // But aliases and original types cannot be used with new types based on them. var _ N0 = T0{} // ERROR "cannot use T0{} \(value of type T0\) as N0 value in variable declaration" -var _ N0 = A0{} // ERROR "cannot use A0{} \(value of type T0\) as N0 value in variable declaration" +var _ N0 = A0{} // ERROR "cannot use A0{} \(value of type A0\) as N0 value in variable declaration" var _ A5 = Value{} @@ -83,7 +83,7 @@ func _() { var _ T0 = A0{} var _ N0 = T0{} // ERROR "cannot use T0{} \(value of type T0\) as N0 value in variable declaration" - var _ N0 = A0{} // ERROR "cannot use A0{} \(value of type T0\) as N0 value in variable declaration" + var _ N0 = A0{} // ERROR "cannot use A0{} \(value of type A0\) as N0 value in variable declaration" var _ A5 = Value{} // ERROR "cannot use Value{} \(value of type reflect\.Value\) as A5 value in variable declaration" } @@ -92,10 +92,10 @@ func _() { type _ = reflect.ValueOf // ERROR "reflect.ValueOf .*is not a type|expected type" -func (A1) m() {} // ERROR "cannot define new methods on non-local type int|may not define methods on non-local type" +func (A1) m() {} // ERROR "cannot define new methods on non-local type|may not define methods on non-local type" func (A2) m() {} // ERROR "invalid receiver type" -func (A3) m() {} // ERROR "cannot define new methods on non-local type reflect.Value|may not define methods on non-local type" -func (A4) m() {} // ERROR "cannot define new methods on non-local type reflect.Value|may not define methods on non-local type" +func (A3) m() {} // ERROR "cannot define new methods on non-local type|may not define methods on non-local type" +func (A4) m() {} // ERROR "cannot define new methods on non-local type|may not define methods on non-local type" type B1 = struct{} diff --git a/test/codegen/shift.go b/test/codegen/shift.go index 5bd7acc063..dd91a1db98 100644 --- a/test/codegen/shift.go +++ b/test/codegen/shift.go @@ -467,11 +467,32 @@ func checkMergedShifts64(a [256]uint32, b [256]uint64, v uint64) { // ppc64x: "SRD", "CLRLSLDI", -"RLWNM" a[5] = a[(v>>32)&0x01] // ppc64x: "SRD", "CLRLSLDI", -"RLWNM" - a[5] = a[(v>>34)&0x03] + a[6] = a[(v>>34)&0x03] // ppc64x: -"CLRLSLDI", "RLWNM\t[$]12, R[0-9]+, [$]21, [$]28, R[0-9]+" b[0] = b[uint8(v>>23)] // ppc64x: -"CLRLSLDI", "RLWNM\t[$]15, R[0-9]+, [$]21, [$]28, R[0-9]+" b[1] = b[(v>>20)&0xFF] + // ppc64x: "RLWNM", -"SLD" + b[2] = b[((uint64((uint32(v) >> 21)) & 0x3f) << 4)] +} + +func checkShiftMask(a uint32, b uint64, z []uint32, y []uint64) { + _ = y[128] + _ = z[128] + // ppc64x: -"MOVBZ", -"SRW", "RLWNM" + z[0] = uint32(uint8(a >> 5)) + // ppc64x: -"MOVBZ", -"SRW", "RLWNM" + z[1] = uint32(uint8((a >> 4) & 0x7e)) + // ppc64x: "RLWNM\t[$]25, R[0-9]+, [$]27, [$]29, R[0-9]+" + z[2] = uint32(uint8(a>>7)) & 0x1c + // ppc64x: -"MOVWZ" + y[0] = uint64((a >> 6) & 0x1c) + // ppc64x: -"MOVWZ" + y[1] = uint64(uint32(b)<<6) + 1 + // ppc64x: -"MOVHZ", -"MOVWZ" + y[2] = uint64((uint16(a) >> 9) & 0x1F) + // ppc64x: -"MOVHZ", -"MOVWZ", -"ANDCC" + y[3] = uint64(((uint16(a) & 0xFF0) >> 9) & 0x1F) } // 128 bit shifts diff --git a/test/fixedbugs/issue65893.go b/test/fixedbugs/issue65893.go new file mode 100644 index 0000000000..6f015feaed --- /dev/null +++ b/test/fixedbugs/issue65893.go @@ -0,0 +1,16 @@ +// compile + +// Copyright 2024 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package p + +type ( + s = struct{ f func(s1) } + s1 = struct{ i I } +) + +type I interface { + S() *s +} diff --git a/test/fixedbugs/issue66873.go b/test/fixedbugs/issue66873.go new file mode 100644 index 0000000000..2f49cc933d --- /dev/null +++ b/test/fixedbugs/issue66873.go @@ -0,0 +1,15 @@ +// compile + +// Copyright 2024 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package p + +func f(A) {} + +type T int + +type A = T + +func (A) m() {} diff --git a/test/linkname3.go b/test/linkname3.go index df110cd064..0d5df0b86e 100644 --- a/test/linkname3.go +++ b/test/linkname3.go @@ -13,13 +13,17 @@ type t int var x, y int +func F[T any](T) {} + //go:linkname x ok // ERROR "//go:linkname must refer to declared function or variable" // ERROR "//go:linkname must refer to declared function or variable" // ERROR "duplicate //go:linkname for x" +// ERROR "//go:linkname reference of an instantiation is not allowed" -//line linkname3.go:18 +//line linkname3.go:20 //go:linkname nonexist nonexist //go:linkname t notvarfunc //go:linkname x duplicate +//go:linkname i F[go.shape.int]