From 7b895318605c17dd93af65eeb388f34009406f7c Mon Sep 17 00:00:00 2001 From: Than McIntosh Date: Mon, 24 Apr 2023 10:33:01 -0400 Subject: [PATCH 001/299] internal/coverage/slicewriter: fix off-by-1 error in seek utilities The slicewriter Seek method was being too restrictive on offsets accepted, due to an off-by-one problem in the error checking code. This fixes the problem and touches up the unit tests. Change-Id: I75d6121551de19ec9275f0e331810db231db6ea9 Reviewed-on: https://go-review.googlesource.com/c/go/+/488116 Run-TryBot: Than McIntosh Reviewed-by: Cherry Mui TryBot-Result: Gopher Robot --- src/internal/coverage/slicewriter/slicewriter.go | 6 +++--- src/internal/coverage/slicewriter/slw_test.go | 11 +++++++---- 2 files changed, 10 insertions(+), 7 deletions(-) diff --git a/src/internal/coverage/slicewriter/slicewriter.go b/src/internal/coverage/slicewriter/slicewriter.go index 3522bf5770..460e9dc98c 100644 --- a/src/internal/coverage/slicewriter/slicewriter.go +++ b/src/internal/coverage/slicewriter/slicewriter.go @@ -38,21 +38,21 @@ func (sws *WriteSeeker) Write(p []byte) (n int, err error) { func (sws *WriteSeeker) Seek(offset int64, whence int) (int64, error) { switch whence { case io.SeekStart: - if sws.off != offset && (offset < 0 || offset >= int64(len(sws.payload))) { + if sws.off != offset && (offset < 0 || offset > int64(len(sws.payload))) { return 0, fmt.Errorf("invalid seek: new offset %d (out of range [0 %d]", offset, len(sws.payload)) } sws.off = offset return offset, nil case io.SeekCurrent: newoff := sws.off + offset - if newoff != sws.off && (newoff < 0 || newoff >= int64(len(sws.payload))) { + if newoff != sws.off && (newoff < 0 || newoff > int64(len(sws.payload))) { return 0, fmt.Errorf("invalid seek: new offset %d (out of range [0 %d]", newoff, len(sws.payload)) } sws.off += offset return sws.off, nil case io.SeekEnd: newoff := int64(len(sws.payload)) + offset - if newoff != sws.off && (newoff < 0 || newoff >= int64(len(sws.payload))) { + if newoff != sws.off && (newoff < 0 || newoff > int64(len(sws.payload))) { return 0, fmt.Errorf("invalid seek: new offset %d (out of range [0 %d]", newoff, len(sws.payload)) } sws.off = newoff diff --git a/src/internal/coverage/slicewriter/slw_test.go b/src/internal/coverage/slicewriter/slw_test.go index f4e75f40d9..9e267670e0 100644 --- a/src/internal/coverage/slicewriter/slw_test.go +++ b/src/internal/coverage/slicewriter/slw_test.go @@ -47,12 +47,13 @@ func TestSliceWriter(t *testing.T) { sleq(t, b, p) } - sk := func(t *testing.T, ws *WriteSeeker, offset int64, whence int) { + sk := func(t *testing.T, ws *WriteSeeker, offset int64, whence int) int64 { t.Helper() - _, err := ws.Seek(offset, whence) + off, err := ws.Seek(offset, whence) if err != nil { t.Fatalf("unexpected seek error: %v", err) } + return off } wp1 := []byte{1, 2} @@ -80,6 +81,8 @@ func TestSliceWriter(t *testing.T) { rf(t, ws, []byte{2, 7}) sk(t, ws, -4, io.SeekEnd) rf(t, ws, []byte{2, 7}) + off := sk(t, ws, 0, io.SeekEnd) + sk(t, ws, off, io.SeekStart) // seek back and overwrite sk(t, ws, 1, io.SeekStart) @@ -98,7 +101,7 @@ func TestSliceWriter(t *testing.T) { if err == nil { t.Fatalf("expected error on invalid -1 seek") } - _, err = ws.Seek(int64(len(ws.BytesWritten())), io.SeekStart) + _, err = ws.Seek(int64(len(ws.BytesWritten())+1), io.SeekStart) if err == nil { t.Fatalf("expected error on invalid %d seek", len(ws.BytesWritten())) } @@ -108,7 +111,7 @@ func TestSliceWriter(t *testing.T) { if err == nil { t.Fatalf("expected error on invalid -1 seek") } - _, err = ws.Seek(int64(len(ws.BytesWritten())), io.SeekCurrent) + _, err = ws.Seek(int64(len(ws.BytesWritten())+1), io.SeekCurrent) if err == nil { t.Fatalf("expected error on invalid %d seek", len(ws.BytesWritten())) } From 39957b5d89fec8bc3a79f4a982452c6e3d5b3ad3 Mon Sep 17 00:00:00 2001 From: Than McIntosh Date: Thu, 13 Apr 2023 14:27:33 -0400 Subject: [PATCH 002/299] coverage: fix count vs emit discrepancy in coverage counter data writing This patch revises the way coverage counter data writing takes place to avoid problems where useful counter data (for user-written functions) is skipped in favor of counter data from stdlib functions that are executed "late in the game", during the counter writing process itself. Reading counter values from a running "--coverpkg=all" program is an inherently racy operation; while the the code that scans the coverage counter segment is reading values, the program is still executing, potentially updating those values, and updates can include execution of previously un-executed functions. The existing counter data writing code was using a two-pass model (initial sweep over the counter segment to count live functions, second sweep to actually write data), and wasn't properly accounting for the fact that the second pass could see more functions than the first. In the bug in question, the first pass discovered an initial set of 1240 functions, but by the time the second pass kicked in, several additional new functions were also live. The second pass scanned the counter segment again to write out exactly 1240 functions, but since some of the counters for the newly executed functions were earlier in the segment (due to linker layout quirks) than the user's selected function, the sweep terminated before writing out counters for the function of interest. The fix rewrites the counter data file encoder to make a single sweep over the counter segment instead of using a two-pass scheme. Fixes #59563. Change-Id: I5e908e226bb224adb90a2fb783013e52deb341da Reviewed-on: https://go-review.googlesource.com/c/go/+/484535 Reviewed-by: Cherry Mui TryBot-Result: Gopher Robot Run-TryBot: Than McIntosh --- src/cmd/covdata/metamerge.go | 11 - src/internal/coverage/encodecounter/encode.go | 71 +- src/internal/coverage/test/counter_test.go | 4 - src/runtime/coverage/emit.go | 46 - src/runtime/coverage/emitdata_test.go | 49 ++ .../coverage/testdata/issue59563/repro.go | 823 ++++++++++++++++++ .../testdata/issue59563/repro_test.go | 14 + 7 files changed, 925 insertions(+), 93 deletions(-) create mode 100644 src/runtime/coverage/testdata/issue59563/repro.go create mode 100644 src/runtime/coverage/testdata/issue59563/repro_test.go diff --git a/src/cmd/covdata/metamerge.go b/src/cmd/covdata/metamerge.go index b224984f68..c5a1a73a48 100644 --- a/src/cmd/covdata/metamerge.go +++ b/src/cmd/covdata/metamerge.go @@ -273,17 +273,6 @@ func (mm *metaMerge) emitCounters(outdir string, metaHash [16]byte) { mm.astate = &argstate{} } -// NumFuncs is used while writing the counter data files; it -// implements the 'NumFuncs' method required by the interface -// internal/coverage/encodecounter/CounterVisitor. -func (mm *metaMerge) NumFuncs() (int, error) { - rval := 0 - for _, p := range mm.pkgs { - rval += len(p.ctab) - } - return rval, nil -} - // VisitFuncs is used while writing the counter data files; it // implements the 'VisitFuncs' method required by the interface // internal/coverage/encodecounter/CounterVisitor. diff --git a/src/internal/coverage/encodecounter/encode.go b/src/internal/coverage/encodecounter/encode.go index 8db4f514e8..1ff6cb1f9a 100644 --- a/src/internal/coverage/encodecounter/encode.go +++ b/src/internal/coverage/encodecounter/encode.go @@ -26,7 +26,9 @@ import ( type CoverageDataWriter struct { stab *stringtab.Writer w *bufio.Writer + csh coverage.CounterSegmentHeader tmp []byte + nfuncs uint64 cflavor coverage.CounterFlavor segs uint32 debug bool @@ -47,13 +49,10 @@ func NewCoverageDataWriter(w io.Writer, flav coverage.CounterFlavor) *CoverageDa // CounterVisitor describes a helper object used during counter file // writing; when writing counter data files, clients pass a -// CounterVisitor to the write/emit routines. The writers will then -// first invoke the visitor's NumFuncs() method to find out how many -// function's worth of data to write, then it will invoke VisitFuncs. -// The expectation is that the VisitFuncs method will then invoke the -// callback "f" with data for each function to emit to the file. +// CounterVisitor to the write/emit routines, then the expectation is +// that the VisitFuncs method will then invoke the callback "f" with +// data for each function to emit to the file. type CounterVisitor interface { - NumFuncs() (int, error) VisitFuncs(f CounterVisitorFn) error } @@ -86,23 +85,35 @@ func padToFourByteBoundary(ws *slicewriter.WriteSeeker) error { return nil } -func (cfw *CoverageDataWriter) writeSegmentPreamble(args map[string]string, visitor CounterVisitor) error { - var csh coverage.CounterSegmentHeader - if nf, err := visitor.NumFuncs(); err != nil { - return err - } else { - csh.FcnEntries = uint64(nf) +func (cfw *CoverageDataWriter) patchSegmentHeader(ws *slicewriter.WriteSeeker) error { + if _, err := ws.Seek(0, io.SeekStart); err != nil { + return fmt.Errorf("error seeking in patchSegmentHeader: %v", err) } + cfw.csh.FcnEntries = cfw.nfuncs + cfw.nfuncs = 0 + if cfw.debug { + fmt.Fprintf(os.Stderr, "=-= writing counter segment header: %+v", cfw.csh) + } + if err := binary.Write(ws, binary.LittleEndian, cfw.csh); err != nil { + return err + } + return nil +} + +func (cfw *CoverageDataWriter) writeSegmentPreamble(args map[string]string, ws *slicewriter.WriteSeeker) error { + if err := binary.Write(ws, binary.LittleEndian, cfw.csh); err != nil { + return err + } + hdrsz := uint32(len(ws.BytesWritten())) // Write string table and args to a byte slice (since we need // to capture offsets at various points), then emit the slice // once we are done. cfw.stab.Freeze() - ws := &slicewriter.WriteSeeker{} if err := cfw.stab.Write(ws); err != nil { return err } - csh.StrTabLen = uint32(len(ws.BytesWritten())) + cfw.csh.StrTabLen = uint32(len(ws.BytesWritten())) - hdrsz akeys := make([]string, 0, len(args)) for k := range args { @@ -138,21 +149,8 @@ func (cfw *CoverageDataWriter) writeSegmentPreamble(args map[string]string, visi if err := padToFourByteBoundary(ws); err != nil { return err } - csh.ArgsLen = uint32(len(ws.BytesWritten())) - csh.StrTabLen + cfw.csh.ArgsLen = uint32(len(ws.BytesWritten())) - (cfw.csh.StrTabLen + hdrsz) - if cfw.debug { - fmt.Fprintf(os.Stderr, "=-= counter segment header: %+v", csh) - fmt.Fprintf(os.Stderr, " FcnEntries=0x%x StrTabLen=0x%x ArgsLen=0x%x\n", - csh.FcnEntries, csh.StrTabLen, csh.ArgsLen) - } - - // At this point we can now do the actual write. - if err := binary.Write(cfw.w, binary.LittleEndian, csh); err != nil { - return err - } - if err := cfw.writeBytes(ws.BytesWritten()); err != nil { - return err - } return nil } @@ -169,10 +167,18 @@ func (cfw *CoverageDataWriter) AppendSegment(args map[string]string, visitor Cou cfw.stab.Lookup(v) } - if err = cfw.writeSegmentPreamble(args, visitor); err != nil { + var swws slicewriter.WriteSeeker + ws := &swws + if err = cfw.writeSegmentPreamble(args, ws); err != nil { return err } - if err = cfw.writeCounters(visitor); err != nil { + if err = cfw.writeCounters(visitor, ws); err != nil { + return err + } + if err = cfw.patchSegmentHeader(ws); err != nil { + return err + } + if err := cfw.writeBytes(ws.BytesWritten()); err != nil { return err } if err = cfw.writeFooter(); err != nil { @@ -214,7 +220,7 @@ func (cfw *CoverageDataWriter) writeBytes(b []byte) error { return nil } -func (cfw *CoverageDataWriter) writeCounters(visitor CounterVisitor) error { +func (cfw *CoverageDataWriter) writeCounters(visitor CounterVisitor, ws *slicewriter.WriteSeeker) error { // Notes: // - this version writes everything little-endian, which means // a call is needed to encode every value (expensive) @@ -237,7 +243,7 @@ func (cfw *CoverageDataWriter) writeCounters(visitor CounterVisitor) error { } else { panic("internal error: bad counter flavor") } - if sz, err := cfw.w.Write(buf); err != nil { + if sz, err := ws.Write(buf); err != nil { return err } else if sz != towr { return fmt.Errorf("writing counters: short write") @@ -247,6 +253,7 @@ func (cfw *CoverageDataWriter) writeCounters(visitor CounterVisitor) error { // Write out entries for each live function. emitter := func(pkid uint32, funcid uint32, counters []uint32) error { + cfw.nfuncs++ if err := wrval(uint32(len(counters))); err != nil { return err } diff --git a/src/internal/coverage/test/counter_test.go b/src/internal/coverage/test/counter_test.go index 3fc111ea12..e29baeddc0 100644 --- a/src/internal/coverage/test/counter_test.go +++ b/src/internal/coverage/test/counter_test.go @@ -19,10 +19,6 @@ type ctrVis struct { funcs []decodecounter.FuncPayload } -func (v *ctrVis) NumFuncs() (int, error) { - return len(v.funcs), nil -} - func (v *ctrVis) VisitFuncs(f encodecounter.CounterVisitorFn) error { for _, fn := range v.funcs { if err := f(fn.PkgIdx, fn.FuncIdx, fn.Counters); err != nil { diff --git a/src/runtime/coverage/emit.go b/src/runtime/coverage/emit.go index 0f77ce287b..bb0c6fb6a2 100644 --- a/src/runtime/coverage/emit.go +++ b/src/runtime/coverage/emit.go @@ -463,52 +463,6 @@ func writeMetaData(w io.Writer, metalist []rtcov.CovMetaBlob, cmode coverage.Cou return mfw.Write(finalHash, blobs, cmode, gran) } -func (s *emitState) NumFuncs() (int, error) { - var sd []atomic.Uint32 - bufHdr := (*reflect.SliceHeader)(unsafe.Pointer(&sd)) - - totalFuncs := 0 - for _, c := range s.counterlist { - bufHdr.Data = uintptr(unsafe.Pointer(c.Counters)) - bufHdr.Len = int(c.Len) - bufHdr.Cap = int(c.Len) - for i := 0; i < len(sd); i++ { - // Skip ahead until the next non-zero value. - sdi := sd[i].Load() - if sdi == 0 { - continue - } - - // We found a function that was executed. - nCtrs := sdi - - // Check to make sure that we have at least one live - // counter. See the implementation note in ClearCoverageCounters - // for a description of why this is needed. - isLive := false - st := i + coverage.FirstCtrOffset - counters := sd[st : st+int(nCtrs)] - for i := 0; i < len(counters); i++ { - if counters[i].Load() != 0 { - isLive = true - break - } - } - if !isLive { - // Skip this function. - i += coverage.FirstCtrOffset + int(nCtrs) - 1 - continue - } - - totalFuncs++ - - // Move to the next function. - i += coverage.FirstCtrOffset + int(nCtrs) - 1 - } - } - return totalFuncs, nil -} - func (s *emitState) VisitFuncs(f encodecounter.CounterVisitorFn) error { var sd []atomic.Uint32 var tcounters []uint32 diff --git a/src/runtime/coverage/emitdata_test.go b/src/runtime/coverage/emitdata_test.go index f6c47e998d..b0bd0e5c21 100644 --- a/src/runtime/coverage/emitdata_test.go +++ b/src/runtime/coverage/emitdata_test.go @@ -496,3 +496,52 @@ func TestIssue56006EmitDataRaceCoverRunningGoroutine(t *testing.T) { } } } + +func TestIssue59563TruncatedCoverPkgAll(t *testing.T) { + if testing.Short() { + t.Skipf("skipping test: too long for short mode") + } + testenv.MustHaveGoRun(t) + + tmpdir := t.TempDir() + ppath := filepath.Join(tmpdir, "foo.cov") + + cmd := exec.Command(testenv.GoToolPath(t), "test", "-coverpkg=all", "-coverprofile="+ppath) + cmd.Dir = filepath.Join("testdata", "issue59563") + b, err := cmd.CombinedOutput() + if err != nil { + t.Fatalf("go test -cover failed: %v", err) + } + + cmd = exec.Command(testenv.GoToolPath(t), "tool", "cover", "-func="+ppath) + b, err = cmd.CombinedOutput() + if err != nil { + t.Fatalf("go tool cover -func failed: %v", err) + } + + lines := strings.Split(string(b), "\n") + nfound := 0 + bad := false + for _, line := range lines { + f := strings.Fields(line) + if len(f) == 0 { + continue + } + if !strings.HasPrefix(f[0], "runtime/coverage/testdata/issue59563/repro.go") { + continue + } + nfound++ + want := "100.0%" + if f[len(f)-1] != want { + t.Errorf("wanted %s got: %q\n", want, line) + bad = true + } + } + if nfound != 2 { + t.Errorf("wanted 2 found, got %d\n", nfound) + bad = true + } + if bad { + t.Logf("func output:\n%s\n", string(b)) + } +} diff --git a/src/runtime/coverage/testdata/issue59563/repro.go b/src/runtime/coverage/testdata/issue59563/repro.go new file mode 100644 index 0000000000..d054567dc5 --- /dev/null +++ b/src/runtime/coverage/testdata/issue59563/repro.go @@ -0,0 +1,823 @@ +// Copyright 2023 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package repro + +import ( + "fmt" + "net/http" +) + +func small() { + go func() { + fmt.Println(http.ListenAndServe("localhost:7070", nil)) + }() +} + +func large(x int) int { + if x == 0 { + x += 0 + } else if x == 1 { + x += 1 + } else if x == 2 { + x += 2 + } else if x == 3 { + x += 3 + } else if x == 4 { + x += 4 + } else if x == 5 { + x += 5 + } else if x == 6 { + x += 6 + } else if x == 7 { + x += 7 + } else if x == 8 { + x += 8 + } else if x == 9 { + x += 9 + } else if x == 10 { + x += 10 + } else if x == 11 { + x += 11 + } else if x == 12 { + x += 12 + } else if x == 13 { + x += 13 + } else if x == 14 { + x += 14 + } else if x == 15 { + x += 15 + } else if x == 16 { + x += 16 + } else if x == 17 { + x += 17 + } else if x == 18 { + x += 18 + } else if x == 19 { + x += 19 + } else if x == 20 { + x += 20 + } else if x == 21 { + x += 21 + } else if x == 22 { + x += 22 + } else if x == 23 { + x += 23 + } else if x == 24 { + x += 24 + } else if x == 25 { + x += 25 + } else if x == 26 { + x += 26 + } else if x == 27 { + x += 27 + } else if x == 28 { + x += 28 + } else if x == 29 { + x += 29 + } else if x == 30 { + x += 30 + } else if x == 31 { + x += 31 + } else if x == 32 { + x += 32 + } else if x == 33 { + x += 33 + } else if x == 34 { + x += 34 + } else if x == 35 { + x += 35 + } else if x == 36 { + x += 36 + } else if x == 37 { + x += 37 + } else if x == 38 { + x += 38 + } else if x == 39 { + x += 39 + } else if x == 40 { + x += 40 + } else if x == 41 { + x += 41 + } else if x == 42 { + x += 42 + } else if x == 43 { + x += 43 + } else if x == 44 { + x += 44 + } else if x == 45 { + x += 45 + } else if x == 46 { + x += 46 + } else if x == 47 { + x += 47 + } else if x == 48 { + x += 48 + } else if x == 49 { + x += 49 + } else if x == 50 { + x += 50 + } else if x == 51 { + x += 51 + } else if x == 52 { + x += 52 + } else if x == 53 { + x += 53 + } else if x == 54 { + x += 54 + } else if x == 55 { + x += 55 + } else if x == 56 { + x += 56 + } else if x == 57 { + x += 57 + } else if x == 58 { + x += 58 + } else if x == 59 { + x += 59 + } else if x == 60 { + x += 60 + } else if x == 61 { + x += 61 + } else if x == 62 { + x += 62 + } else if x == 63 { + x += 63 + } else if x == 64 { + x += 64 + } else if x == 65 { + x += 65 + } else if x == 66 { + x += 66 + } else if x == 67 { + x += 67 + } else if x == 68 { + x += 68 + } else if x == 69 { + x += 69 + } else if x == 70 { + x += 70 + } else if x == 71 { + x += 71 + } else if x == 72 { + x += 72 + } else if x == 73 { + x += 73 + } else if x == 74 { + x += 74 + } else if x == 75 { + x += 75 + } else if x == 76 { + x += 76 + } else if x == 77 { + x += 77 + } else if x == 78 { + x += 78 + } else if x == 79 { + x += 79 + } else if x == 80 { + x += 80 + } else if x == 81 { + x += 81 + } else if x == 82 { + x += 82 + } else if x == 83 { + x += 83 + } else if x == 84 { + x += 84 + } else if x == 85 { + x += 85 + } else if x == 86 { + x += 86 + } else if x == 87 { + x += 87 + } else if x == 88 { + x += 88 + } else if x == 89 { + x += 89 + } else if x == 90 { + x += 90 + } else if x == 91 { + x += 91 + } else if x == 92 { + x += 92 + } else if x == 93 { + x += 93 + } else if x == 94 { + x += 94 + } else if x == 95 { + x += 95 + } else if x == 96 { + x += 96 + } else if x == 97 { + x += 97 + } else if x == 98 { + x += 98 + } else if x == 99 { + x += 99 + } else if x == 100 { + x += 100 + } else if x == 101 { + x += 101 + } else if x == 102 { + x += 102 + } else if x == 103 { + x += 103 + } else if x == 104 { + x += 104 + } else if x == 105 { + x += 105 + } else if x == 106 { + x += 106 + } else if x == 107 { + x += 107 + } else if x == 108 { + x += 108 + } else if x == 109 { + x += 109 + } else if x == 110 { + x += 110 + } else if x == 111 { + x += 111 + } else if x == 112 { + x += 112 + } else if x == 113 { + x += 113 + } else if x == 114 { + x += 114 + } else if x == 115 { + x += 115 + } else if x == 116 { + x += 116 + } else if x == 117 { + x += 117 + } else if x == 118 { + x += 118 + } else if x == 119 { + x += 119 + } else if x == 120 { + x += 120 + } else if x == 121 { + x += 121 + } else if x == 122 { + x += 122 + } else if x == 123 { + x += 123 + } else if x == 124 { + x += 124 + } else if x == 125 { + x += 125 + } else if x == 126 { + x += 126 + } else if x == 127 { + x += 127 + } else if x == 128 { + x += 128 + } else if x == 129 { + x += 129 + } else if x == 130 { + x += 130 + } else if x == 131 { + x += 131 + } else if x == 132 { + x += 132 + } else if x == 133 { + x += 133 + } else if x == 134 { + x += 134 + } else if x == 135 { + x += 135 + } else if x == 136 { + x += 136 + } else if x == 137 { + x += 137 + } else if x == 138 { + x += 138 + } else if x == 139 { + x += 139 + } else if x == 140 { + x += 140 + } else if x == 141 { + x += 141 + } else if x == 142 { + x += 142 + } else if x == 143 { + x += 143 + } else if x == 144 { + x += 144 + } else if x == 145 { + x += 145 + } else if x == 146 { + x += 146 + } else if x == 147 { + x += 147 + } else if x == 148 { + x += 148 + } else if x == 149 { + x += 149 + } else if x == 150 { + x += 150 + } else if x == 151 { + x += 151 + } else if x == 152 { + x += 152 + } else if x == 153 { + x += 153 + } else if x == 154 { + x += 154 + } else if x == 155 { + x += 155 + } else if x == 156 { + x += 156 + } else if x == 157 { + x += 157 + } else if x == 158 { + x += 158 + } else if x == 159 { + x += 159 + } else if x == 160 { + x += 160 + } else if x == 161 { + x += 161 + } else if x == 162 { + x += 162 + } else if x == 163 { + x += 163 + } else if x == 164 { + x += 164 + } else if x == 165 { + x += 165 + } else if x == 166 { + x += 166 + } else if x == 167 { + x += 167 + } else if x == 168 { + x += 168 + } else if x == 169 { + x += 169 + } else if x == 170 { + x += 170 + } else if x == 171 { + x += 171 + } else if x == 172 { + x += 172 + } else if x == 173 { + x += 173 + } else if x == 174 { + x += 174 + } else if x == 175 { + x += 175 + } else if x == 176 { + x += 176 + } else if x == 177 { + x += 177 + } else if x == 178 { + x += 178 + } else if x == 179 { + x += 179 + } else if x == 180 { + x += 180 + } else if x == 181 { + x += 181 + } else if x == 182 { + x += 182 + } else if x == 183 { + x += 183 + } else if x == 184 { + x += 184 + } else if x == 185 { + x += 185 + } else if x == 186 { + x += 186 + } else if x == 187 { + x += 187 + } else if x == 188 { + x += 188 + } else if x == 189 { + x += 189 + } else if x == 190 { + x += 190 + } else if x == 191 { + x += 191 + } else if x == 192 { + x += 192 + } else if x == 193 { + x += 193 + } else if x == 194 { + x += 194 + } else if x == 195 { + x += 195 + } else if x == 196 { + x += 196 + } else if x == 197 { + x += 197 + } else if x == 198 { + x += 198 + } else if x == 199 { + x += 199 + } else if x == 200 { + x += 200 + } else if x == 201 { + x += 201 + } else if x == 202 { + x += 202 + } else if x == 203 { + x += 203 + } else if x == 204 { + x += 204 + } else if x == 205 { + x += 205 + } else if x == 206 { + x += 206 + } else if x == 207 { + x += 207 + } else if x == 208 { + x += 208 + } else if x == 209 { + x += 209 + } else if x == 210 { + x += 210 + } else if x == 211 { + x += 211 + } else if x == 212 { + x += 212 + } else if x == 213 { + x += 213 + } else if x == 214 { + x += 214 + } else if x == 215 { + x += 215 + } else if x == 216 { + x += 216 + } else if x == 217 { + x += 217 + } else if x == 218 { + x += 218 + } else if x == 219 { + x += 219 + } else if x == 220 { + x += 220 + } else if x == 221 { + x += 221 + } else if x == 222 { + x += 222 + } else if x == 223 { + x += 223 + } else if x == 224 { + x += 224 + } else if x == 225 { + x += 225 + } else if x == 226 { + x += 226 + } else if x == 227 { + x += 227 + } else if x == 228 { + x += 228 + } else if x == 229 { + x += 229 + } else if x == 230 { + x += 230 + } else if x == 231 { + x += 231 + } else if x == 232 { + x += 232 + } else if x == 233 { + x += 233 + } else if x == 234 { + x += 234 + } else if x == 235 { + x += 235 + } else if x == 236 { + x += 236 + } else if x == 237 { + x += 237 + } else if x == 238 { + x += 238 + } else if x == 239 { + x += 239 + } else if x == 240 { + x += 240 + } else if x == 241 { + x += 241 + } else if x == 242 { + x += 242 + } else if x == 243 { + x += 243 + } else if x == 244 { + x += 244 + } else if x == 245 { + x += 245 + } else if x == 246 { + x += 246 + } else if x == 247 { + x += 247 + } else if x == 248 { + x += 248 + } else if x == 249 { + x += 249 + } else if x == 250 { + x += 250 + } else if x == 251 { + x += 251 + } else if x == 252 { + x += 252 + } else if x == 253 { + x += 253 + } else if x == 254 { + x += 254 + } else if x == 255 { + x += 255 + } else if x == 256 { + x += 256 + } else if x == 257 { + x += 257 + } else if x == 258 { + x += 258 + } else if x == 259 { + x += 259 + } else if x == 260 { + x += 260 + } else if x == 261 { + x += 261 + } else if x == 262 { + x += 262 + } else if x == 263 { + x += 263 + } else if x == 264 { + x += 264 + } else if x == 265 { + x += 265 + } else if x == 266 { + x += 266 + } else if x == 267 { + x += 267 + } else if x == 268 { + x += 268 + } else if x == 269 { + x += 269 + } else if x == 270 { + x += 270 + } else if x == 271 { + x += 271 + } else if x == 272 { + x += 272 + } else if x == 273 { + x += 273 + } else if x == 274 { + x += 274 + } else if x == 275 { + x += 275 + } else if x == 276 { + x += 276 + } else if x == 277 { + x += 277 + } else if x == 278 { + x += 278 + } else if x == 279 { + x += 279 + } else if x == 280 { + x += 280 + } else if x == 281 { + x += 281 + } else if x == 282 { + x += 282 + } else if x == 283 { + x += 283 + } else if x == 284 { + x += 284 + } else if x == 285 { + x += 285 + } else if x == 286 { + x += 286 + } else if x == 287 { + x += 287 + } else if x == 288 { + x += 288 + } else if x == 289 { + x += 289 + } else if x == 290 { + x += 290 + } else if x == 291 { + x += 291 + } else if x == 292 { + x += 292 + } else if x == 293 { + x += 293 + } else if x == 294 { + x += 294 + } else if x == 295 { + x += 295 + } else if x == 296 { + x += 296 + } else if x == 297 { + x += 297 + } else if x == 298 { + x += 298 + } else if x == 299 { + x += 299 + } else if x == 300 { + x += 300 + } else if x == 301 { + x += 301 + } else if x == 302 { + x += 302 + } else if x == 303 { + x += 303 + } else if x == 304 { + x += 304 + } else if x == 305 { + x += 305 + } else if x == 306 { + x += 306 + } else if x == 307 { + x += 307 + } else if x == 308 { + x += 308 + } else if x == 309 { + x += 309 + } else if x == 310 { + x += 310 + } else if x == 311 { + x += 311 + } else if x == 312 { + x += 312 + } else if x == 313 { + x += 313 + } else if x == 314 { + x += 314 + } else if x == 315 { + x += 315 + } else if x == 316 { + x += 316 + } else if x == 317 { + x += 317 + } else if x == 318 { + x += 318 + } else if x == 319 { + x += 319 + } else if x == 320 { + x += 320 + } else if x == 321 { + x += 321 + } else if x == 322 { + x += 322 + } else if x == 323 { + x += 323 + } else if x == 324 { + x += 324 + } else if x == 325 { + x += 325 + } else if x == 326 { + x += 326 + } else if x == 327 { + x += 327 + } else if x == 328 { + x += 328 + } else if x == 329 { + x += 329 + } else if x == 330 { + x += 330 + } else if x == 331 { + x += 331 + } else if x == 332 { + x += 332 + } else if x == 333 { + x += 333 + } else if x == 334 { + x += 334 + } else if x == 335 { + x += 335 + } else if x == 336 { + x += 336 + } else if x == 337 { + x += 337 + } else if x == 338 { + x += 338 + } else if x == 339 { + x += 339 + } else if x == 340 { + x += 340 + } else if x == 341 { + x += 341 + } else if x == 342 { + x += 342 + } else if x == 343 { + x += 343 + } else if x == 344 { + x += 344 + } else if x == 345 { + x += 345 + } else if x == 346 { + x += 346 + } else if x == 347 { + x += 347 + } else if x == 348 { + x += 348 + } else if x == 349 { + x += 349 + } else if x == 350 { + x += 350 + } else if x == 351 { + x += 351 + } else if x == 352 { + x += 352 + } else if x == 353 { + x += 353 + } else if x == 354 { + x += 354 + } else if x == 355 { + x += 355 + } else if x == 356 { + x += 356 + } else if x == 357 { + x += 357 + } else if x == 358 { + x += 358 + } else if x == 359 { + x += 359 + } else if x == 360 { + x += 360 + } else if x == 361 { + x += 361 + } else if x == 362 { + x += 362 + } else if x == 363 { + x += 363 + } else if x == 364 { + x += 364 + } else if x == 365 { + x += 365 + } else if x == 366 { + x += 366 + } else if x == 367 { + x += 367 + } else if x == 368 { + x += 368 + } else if x == 369 { + x += 369 + } else if x == 370 { + x += 370 + } else if x == 371 { + x += 371 + } else if x == 372 { + x += 372 + } else if x == 373 { + x += 373 + } else if x == 374 { + x += 374 + } else if x == 375 { + x += 375 + } else if x == 376 { + x += 376 + } else if x == 377 { + x += 377 + } else if x == 378 { + x += 378 + } else if x == 379 { + x += 379 + } else if x == 380 { + x += 380 + } else if x == 381 { + x += 381 + } else if x == 382 { + x += 382 + } else if x == 383 { + x += 383 + } else if x == 384 { + x += 384 + } else if x == 385 { + x += 385 + } else if x == 386 { + x += 386 + } else if x == 387 { + x += 387 + } else if x == 388 { + x += 388 + } else if x == 389 { + x += 389 + } else if x == 390 { + x += 390 + } else if x == 391 { + x += 391 + } else if x == 392 { + x += 392 + } else if x == 393 { + x += 393 + } else if x == 394 { + x += 394 + } else if x == 395 { + x += 395 + } else if x == 396 { + x += 396 + } else if x == 397 { + x += 397 + } else if x == 398 { + x += 398 + } else if x == 399 { + x += 399 + } else if x == 400 { + x += 400 + } + return x * x +} diff --git a/src/runtime/coverage/testdata/issue59563/repro_test.go b/src/runtime/coverage/testdata/issue59563/repro_test.go new file mode 100644 index 0000000000..15c8e01a28 --- /dev/null +++ b/src/runtime/coverage/testdata/issue59563/repro_test.go @@ -0,0 +1,14 @@ +// Copyright 2023 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package repro + +import "testing" + +func TestSomething(t *testing.T) { + small() + for i := 0; i < 1001; i++ { + large(i) + } +} From d816f85f787bfa5114787687b085194d1cd3b468 Mon Sep 17 00:00:00 2001 From: "Paul E. Murphy" Date: Tue, 25 Apr 2023 16:41:51 -0500 Subject: [PATCH 003/299] cmd/link/internal/loadelf: set AttrExternal on text section symbols PPC64 processes external object relocations against the section symbols. This needs to be set correctly to determine the type of PLT stub to generate when both Go and External code make PLT calls. Change-Id: I5abdd5a0473866164083c33e80324dffcc1707f0 Reviewed-on: https://go-review.googlesource.com/c/go/+/488895 Reviewed-by: Than McIntosh Reviewed-by: Dmitri Shuralyov Run-TryBot: Paul Murphy Reviewed-by: Cherry Mui TryBot-Result: Gopher Robot --- src/cmd/link/internal/loadelf/ldelf.go | 1 + 1 file changed, 1 insertion(+) diff --git a/src/cmd/link/internal/loadelf/ldelf.go b/src/cmd/link/internal/loadelf/ldelf.go index 7ac7699996..c1bfec059d 100644 --- a/src/cmd/link/internal/loadelf/ldelf.go +++ b/src/cmd/link/internal/loadelf/ldelf.go @@ -540,6 +540,7 @@ func Load(l *loader.Loader, arch *sys.Arch, localSymVersion int, f *bio.Reader, } if sect.type_ == elf.SHT_PROGBITS { sb.SetData(sect.base[:sect.size]) + sb.SetExternal(true) } sb.SetSize(int64(sect.size)) From 7b874619beec9ec88928f72efa8dc5bc44fec2d7 Mon Sep 17 00:00:00 2001 From: Michael Pratt Date: Mon, 17 Apr 2023 14:51:28 -0400 Subject: [PATCH 004/299] runtime/cgo: store M for C-created thread in pthread key This reapplies CL 481061, with the followup fixes in CL 482975, CL 485315, and CL 485316 incorporated. CL 481061, by doujiang24 , speed up C to Go calls by binding the M to the C thread. See below for its description. CL 482975 is a followup fix to a C declaration in testprogcgo. CL 485315 is a followup fix for x_cgo_getstackbound on Illumos. CL 485316 is a followup cleanup for ppc64 assembly. [Original CL 481061 description] This reapplies CL 392854, with the followup fixes in CL 479255, CL 479915, and CL 481057 incorporated. CL 392854, by doujiang24 , speed up C to Go calls by binding the M to the C thread. See below for its description. CL 479255 is a followup fix for a small bug in ARM assembly code. CL 479915 is another followup fix to address C to Go calls after the C code uses some stack, but that CL is also buggy. CL 481057, by Michael Knyszek, is a followup fix for a memory leak bug of CL 479915. [Original CL 392854 description] In a C thread, it's necessary to acquire an extra M by using needm while invoking a Go function from C. But, needm and dropm are heavy costs due to the signal-related syscalls. So, we change to not dropm while returning back to C, which means binding the extra M to the C thread until it exits, to avoid needm and dropm on each C to Go call. Instead, we only dropm while the C thread exits, so the extra M won't leak. When invoking a Go function from C: Allocate a pthread variable using pthread_key_create, only once per shared object, and register a thread-exit-time destructor. And store the g0 of the current m into the thread-specified value of the pthread key, only once per C thread, so that the destructor will put the extra M back onto the extra M list while the C thread exits. When returning back to C: Skip dropm in cgocallback, when the pthread variable has been created, so that the extra M will be reused the next time invoke a Go function from C. This is purely a performance optimization. The old version, in which needm & dropm happen on each cgo call, is still correct too, and we have to keep the old version on systems with cgo but without pthreads, like Windows. This optimization is significant, and the specific value depends on the OS system and CPU, but in general, it can be considered as 10x faster, for a simple Go function call from a C thread. For the newly added BenchmarkCGoInCThread, some benchmark results: 1. it's 28x faster, from 3395 ns/op to 121 ns/op, in darwin OS & Intel(R) Core(TM) i7-9750H CPU @ 2.60GHz 2. it's 6.5x faster, from 1495 ns/op to 230 ns/op, in Linux OS & Intel(R) Xeon(R) CPU E5-2630 0 @ 2.30GHz [CL 479915 description] Currently, when C calls into Go the first time, we grab an M using needm, which sets m.g0's stack bounds using the SP. We don't know how big the stack is, so we simply assume 32K. Previously, when the Go function returns to C, we drop the M, and the next time C calls into Go, we put a new stack bound on the g0 based on the current SP. After CL 392854, we don't drop the M, and the next time C calls into Go, we reuse the same g0, without recomputing the stack bounds. If the C code uses quite a bit of stack space before calling into Go, the SP may be well below the 32K stack bound we assumed, so the runtime thinks the g0 stack overflows. This CL makes needm get a more accurate stack bound from pthread. (In some platforms this may still be a guess as we don't know exactly where we are in the C stack), but it is probably better than simply assuming 32K. [CL 485500 description] CL 479915 passed the G to _cgo_getstackbound for direct updates to gp.stack.lo. A G can be reused on a new thread after the previous thread exited. This could trigger the C TSAN race detector because it couldn't see the synchronization in Go (lockextra) preventing the same G from being used on multiple threads at the same time. We work around this by passing the address of a stack variable to _cgo_getstackbound rather than the G. The stack is generally unique per thread, so TSAN won't see the same address from multiple threads. Even if stacks are reused across threads by pthread, C TSAN should see the synchonization in the stack allocator. A regression test is added to misc/cgo/testsanitizer. Fixes #51676. Fixes #59294. Fixes #59678. Change-Id: Ic62be31a06ee83568215e875a891df37084e08ca Reviewed-on: https://go-review.googlesource.com/c/go/+/485500 TryBot-Result: Gopher Robot Reviewed-by: Cherry Mui Run-TryBot: Michael Pratt --- misc/cgo/test/cgo_test.go | 7 +- misc/cgo/test/cthread_unix.c | 24 +++++ misc/cgo/test/cthread_windows.c | 22 ++++ misc/cgo/test/testx.go | 14 +++ misc/cgo/testcarchive/carchive_test.go | 54 ++++++++++ misc/cgo/testcarchive/testdata/libgo9/a.go | 14 +++ misc/cgo/testcarchive/testdata/main9.c | 24 +++++ misc/cgo/testsanitizers/testdata/tsan14.go | 53 +++++++++ misc/cgo/testsanitizers/tsan_test.go | 1 + src/runtime/asm_386.s | 41 +++++-- src/runtime/asm_amd64.s | 38 ++++++- src/runtime/asm_arm.s | 38 ++++++- src/runtime/asm_arm64.s | 32 +++++- src/runtime/asm_loong64.s | 32 +++++- src/runtime/asm_mips64x.s | 32 +++++- src/runtime/asm_mipsx.s | 32 +++++- src/runtime/asm_ppc64x.s | 35 +++++- src/runtime/asm_riscv64.s | 32 +++++- src/runtime/asm_s390x.s | 32 +++++- src/runtime/cgo.go | 9 ++ src/runtime/cgo/asm_386.s | 8 ++ src/runtime/cgo/asm_amd64.s | 8 ++ src/runtime/cgo/asm_arm.s | 8 ++ src/runtime/cgo/asm_arm64.s | 8 ++ src/runtime/cgo/asm_loong64.s | 8 ++ src/runtime/cgo/asm_mips64x.s | 8 ++ src/runtime/cgo/asm_mipsx.s | 8 ++ src/runtime/cgo/asm_ppc64x.s | 23 ++++ src/runtime/cgo/asm_riscv64.s | 8 ++ src/runtime/cgo/asm_s390x.s | 8 ++ src/runtime/cgo/asm_wasm.s | 3 + src/runtime/cgo/callbacks.go | 45 ++++++++ src/runtime/cgo/gcc_libinit.c | 35 ++++++ src/runtime/cgo/gcc_libinit_windows.c | 9 ++ src/runtime/cgo/gcc_stack_darwin.c | 19 ++++ src/runtime/cgo/gcc_stack_unix.c | 39 +++++++ src/runtime/cgo/gcc_stack_windows.c | 7 ++ src/runtime/cgo/libcgo.h | 5 + src/runtime/cgocall.go | 6 ++ src/runtime/crash_cgo_test.go | 13 +++ src/runtime/proc.go | 118 ++++++++++++++++++--- src/runtime/runtime2.go | 1 + src/runtime/signal_unix.go | 20 +++- src/runtime/stubs.go | 3 + src/runtime/testdata/testprogcgo/bindm.c | 34 ++++++ src/runtime/testdata/testprogcgo/bindm.go | 61 +++++++++++ 46 files changed, 1012 insertions(+), 67 deletions(-) create mode 100644 misc/cgo/testcarchive/testdata/libgo9/a.go create mode 100644 misc/cgo/testcarchive/testdata/main9.c create mode 100644 misc/cgo/testsanitizers/testdata/tsan14.go create mode 100644 src/runtime/cgo/gcc_stack_darwin.c create mode 100644 src/runtime/cgo/gcc_stack_unix.c create mode 100644 src/runtime/cgo/gcc_stack_windows.c create mode 100644 src/runtime/testdata/testprogcgo/bindm.c create mode 100644 src/runtime/testdata/testprogcgo/bindm.go diff --git a/misc/cgo/test/cgo_test.go b/misc/cgo/test/cgo_test.go index 5b298954f5..0c3980c12d 100644 --- a/misc/cgo/test/cgo_test.go +++ b/misc/cgo/test/cgo_test.go @@ -104,6 +104,7 @@ func TestThreadLock(t *testing.T) { testThreadLockFunc(t) } func TestUnsignedInt(t *testing.T) { testUnsignedInt(t) } func TestZeroArgCallback(t *testing.T) { testZeroArgCallback(t) } -func BenchmarkCgoCall(b *testing.B) { benchCgoCall(b) } -func BenchmarkGoString(b *testing.B) { benchGoString(b) } -func BenchmarkCGoCallback(b *testing.B) { benchCallback(b) } +func BenchmarkCgoCall(b *testing.B) { benchCgoCall(b) } +func BenchmarkGoString(b *testing.B) { benchGoString(b) } +func BenchmarkCGoCallback(b *testing.B) { benchCallback(b) } +func BenchmarkCGoInCThread(b *testing.B) { benchCGoInCthread(b) } diff --git a/misc/cgo/test/cthread_unix.c b/misc/cgo/test/cthread_unix.c index b6ec39816b..d0da643158 100644 --- a/misc/cgo/test/cthread_unix.c +++ b/misc/cgo/test/cthread_unix.c @@ -32,3 +32,27 @@ doAdd(int max, int nthread) for(i=0; i _cgo_getstackbound +// to update gp.stack.lo with the stack bounds. If the G itself is passed to +// _cgo_getstackbound, then writes to the same G can be seen on multiple +// threads (when the G is reused after thread exit). This would trigger TSAN. + +/* +#include + +void go_callback(); + +static void *thr(void *arg) { + go_callback(); + return 0; +} + +static void foo() { + pthread_t th; + pthread_attr_t attr; + pthread_attr_init(&attr); + pthread_attr_setstacksize(&attr, 256 << 10); + pthread_create(&th, &attr, thr, 0); + pthread_join(th, 0); +} +*/ +import "C" + +import ( + "time" +) + +//export go_callback +func go_callback() { +} + +func main() { + for i := 0; i < 2; i++ { + go func() { + for { + C.foo() + } + }() + } + + time.Sleep(1000*time.Millisecond) +} diff --git a/misc/cgo/testsanitizers/tsan_test.go b/misc/cgo/testsanitizers/tsan_test.go index f65d842363..dffed96373 100644 --- a/misc/cgo/testsanitizers/tsan_test.go +++ b/misc/cgo/testsanitizers/tsan_test.go @@ -47,6 +47,7 @@ func TestTSAN(t *testing.T) { {src: "tsan11.go", needsRuntime: true}, {src: "tsan12.go", needsRuntime: true}, {src: "tsan13.go", needsRuntime: true}, + {src: "tsan14.go", needsRuntime: true}, } for _, tc := range cases { tc := tc diff --git a/src/runtime/asm_386.s b/src/runtime/asm_386.s index febe27089f..5fd0ab9817 100644 --- a/src/runtime/asm_386.s +++ b/src/runtime/asm_386.s @@ -689,7 +689,20 @@ nosave: TEXT ·cgocallback(SB),NOSPLIT,$12-12 // Frame size must match commented places below NO_LOCAL_POINTERS - // If g is nil, Go did not create the current thread. + // Skip cgocallbackg, just dropm when fn is nil, and frame is the saved g. + // It is used to dropm while thread is exiting. + MOVL fn+0(FP), AX + CMPL AX, $0 + JNE loadg + // Restore the g from frame. + get_tls(CX) + MOVL frame+4(FP), BX + MOVL BX, g(CX) + JMP dropm + +loadg: + // If g is nil, Go did not create the current thread, + // or if this thread never called into Go on pthread platforms. // Call needm to obtain one for temporary use. // In this case, we're running on the thread stack, so there's // lots of space, but the linker doesn't know. Hide the call from @@ -707,9 +720,9 @@ TEXT ·cgocallback(SB),NOSPLIT,$12-12 // Frame size must match commented places MOVL BP, savedm-4(SP) // saved copy of oldm JMP havem needm: - MOVL $runtime·needm(SB), AX + MOVL $runtime·needAndBindM(SB), AX CALL AX - MOVL $0, savedm-4(SP) // dropm on return + MOVL $0, savedm-4(SP) get_tls(CX) MOVL g(CX), BP MOVL g_m(BP), BP @@ -784,13 +797,29 @@ havem: MOVL 0(SP), AX MOVL AX, (g_sched+gobuf_sp)(SI) - // If the m on entry was nil, we called needm above to borrow an m - // for the duration of the call. Since the call is over, return it with dropm. + // If the m on entry was nil, we called needm above to borrow an m, + // 1. for the duration of the call on non-pthread platforms, + // 2. or the duration of the C thread alive on pthread platforms. + // If the m on entry wasn't nil, + // 1. the thread might be a Go thread, + // 2. or it's wasn't the first call from a C thread on pthread platforms, + // since the we skip dropm to resue the m in the first call. MOVL savedm-4(SP), DX CMPL DX, $0 - JNE 3(PC) + JNE droppedm + + // Skip dropm to reuse it in the next call, when a pthread key has been created. + MOVL _cgo_pthread_key_created(SB), DX + // It means cgo is disabled when _cgo_pthread_key_created is a nil pointer, need dropm. + CMPL DX, $0 + JEQ dropm + CMPL (DX), $0 + JNE droppedm + +dropm: MOVL $runtime·dropm(SB), AX CALL AX +droppedm: // Done! RET diff --git a/src/runtime/asm_amd64.s b/src/runtime/asm_amd64.s index 7fb1ae2cff..7fe8528d19 100644 --- a/src/runtime/asm_amd64.s +++ b/src/runtime/asm_amd64.s @@ -918,7 +918,20 @@ GLOBL zeroTLS<>(SB),RODATA,$const_tlsSize TEXT ·cgocallback(SB),NOSPLIT,$24-24 NO_LOCAL_POINTERS - // If g is nil, Go did not create the current thread. + // Skip cgocallbackg, just dropm when fn is nil, and frame is the saved g. + // It is used to dropm while thread is exiting. + MOVQ fn+0(FP), AX + CMPQ AX, $0 + JNE loadg + // Restore the g from frame. + get_tls(CX) + MOVQ frame+8(FP), BX + MOVQ BX, g(CX) + JMP dropm + +loadg: + // If g is nil, Go did not create the current thread, + // or if this thread never called into Go on pthread platforms. // Call needm to obtain one m for temporary use. // In this case, we're running on the thread stack, so there's // lots of space, but the linker doesn't know. Hide the call from @@ -956,9 +969,9 @@ needm: // a bad value in there, in case needm tries to use it. XORPS X15, X15 XORQ R14, R14 - MOVQ $runtime·needm(SB), AX + MOVQ $runtime·needAndBindM(SB), AX CALL AX - MOVQ $0, savedm-8(SP) // dropm on return + MOVQ $0, savedm-8(SP) get_tls(CX) MOVQ g(CX), BX MOVQ g_m(BX), BX @@ -1047,11 +1060,26 @@ havem: MOVQ 0(SP), AX MOVQ AX, (g_sched+gobuf_sp)(SI) - // If the m on entry was nil, we called needm above to borrow an m - // for the duration of the call. Since the call is over, return it with dropm. + // If the m on entry was nil, we called needm above to borrow an m, + // 1. for the duration of the call on non-pthread platforms, + // 2. or the duration of the C thread alive on pthread platforms. + // If the m on entry wasn't nil, + // 1. the thread might be a Go thread, + // 2. or it's wasn't the first call from a C thread on pthread platforms, + // since the we skip dropm to resue the m in the first call. MOVQ savedm-8(SP), BX CMPQ BX, $0 JNE done + + // Skip dropm to reuse it in the next call, when a pthread key has been created. + MOVQ _cgo_pthread_key_created(SB), AX + // It means cgo is disabled when _cgo_pthread_key_created is a nil pointer, need dropm. + CMPQ AX, $0 + JEQ dropm + CMPQ (AX), $0 + JNE done + +dropm: MOVQ $runtime·dropm(SB), AX CALL AX #ifdef GOOS_windows diff --git a/src/runtime/asm_arm.s b/src/runtime/asm_arm.s index 01621245dc..cd692e51a3 100644 --- a/src/runtime/asm_arm.s +++ b/src/runtime/asm_arm.s @@ -630,6 +630,16 @@ nosave: TEXT ·cgocallback(SB),NOSPLIT,$12-12 NO_LOCAL_POINTERS + // Skip cgocallbackg, just dropm when fn is nil, and frame is the saved g. + // It is used to dropm while thread is exiting. + MOVW fn+0(FP), R1 + CMP $0, R1 + B.NE loadg + // Restore the g from frame. + MOVW frame+4(FP), g + B dropm + +loadg: // Load m and g from thread-local storage. #ifdef GOOS_openbsd BL runtime·load_g(SB) @@ -639,7 +649,8 @@ TEXT ·cgocallback(SB),NOSPLIT,$12-12 BL.NE runtime·load_g(SB) #endif - // If g is nil, Go did not create the current thread. + // If g is nil, Go did not create the current thread, + // or if this thread never called into Go on pthread platforms. // Call needm to obtain one for temporary use. // In this case, we're running on the thread stack, so there's // lots of space, but the linker doesn't know. Hide the call from @@ -653,7 +664,7 @@ TEXT ·cgocallback(SB),NOSPLIT,$12-12 needm: MOVW g, savedm-4(SP) // g is zero, so is m. - MOVW $runtime·needm(SB), R0 + MOVW $runtime·needAndBindM(SB), R0 BL (R0) // Set m->g0->sched.sp = SP, so that if a panic happens @@ -724,14 +735,31 @@ havem: MOVW savedsp-12(SP), R4 // must match frame size MOVW R4, (g_sched+gobuf_sp)(g) - // If the m on entry was nil, we called needm above to borrow an m - // for the duration of the call. Since the call is over, return it with dropm. + // If the m on entry was nil, we called needm above to borrow an m, + // 1. for the duration of the call on non-pthread platforms, + // 2. or the duration of the C thread alive on pthread platforms. + // If the m on entry wasn't nil, + // 1. the thread might be a Go thread, + // 2. or it's wasn't the first call from a C thread on pthread platforms, + // since the we skip dropm to resue the m in the first call. MOVW savedm-4(SP), R6 CMP $0, R6 - B.NE 3(PC) + B.NE done + + // Skip dropm to reuse it in the next call, when a pthread key has been created. + MOVW _cgo_pthread_key_created(SB), R6 + // It means cgo is disabled when _cgo_pthread_key_created is a nil pointer, need dropm. + CMP $0, R6 + B.EQ dropm + MOVW (R6), R6 + CMP $0, R6 + B.NE done + +dropm: MOVW $runtime·dropm(SB), R0 BL (R0) +done: // Done! RET diff --git a/src/runtime/asm_arm64.s b/src/runtime/asm_arm64.s index 6fe04a6445..5cce33d7fe 100644 --- a/src/runtime/asm_arm64.s +++ b/src/runtime/asm_arm64.s @@ -1015,10 +1015,20 @@ nosave: TEXT ·cgocallback(SB),NOSPLIT,$24-24 NO_LOCAL_POINTERS + // Skip cgocallbackg, just dropm when fn is nil, and frame is the saved g. + // It is used to dropm while thread is exiting. + MOVD fn+0(FP), R1 + CBNZ R1, loadg + // Restore the g from frame. + MOVD frame+8(FP), g + B dropm + +loadg: // Load g from thread-local storage. BL runtime·load_g(SB) - // If g is nil, Go did not create the current thread. + // If g is nil, Go did not create the current thread, + // or if this thread never called into Go on pthread platforms. // Call needm to obtain one for temporary use. // In this case, we're running on the thread stack, so there's // lots of space, but the linker doesn't know. Hide the call from @@ -1031,7 +1041,7 @@ TEXT ·cgocallback(SB),NOSPLIT,$24-24 needm: MOVD g, savedm-8(SP) // g is zero, so is m. - MOVD $runtime·needm(SB), R0 + MOVD $runtime·needAndBindM(SB), R0 BL (R0) // Set m->g0->sched.sp = SP, so that if a panic happens @@ -1112,10 +1122,24 @@ havem: MOVD savedsp-16(SP), R4 MOVD R4, (g_sched+gobuf_sp)(g) - // If the m on entry was nil, we called needm above to borrow an m - // for the duration of the call. Since the call is over, return it with dropm. + // If the m on entry was nil, we called needm above to borrow an m, + // 1. for the duration of the call on non-pthread platforms, + // 2. or the duration of the C thread alive on pthread platforms. + // If the m on entry wasn't nil, + // 1. the thread might be a Go thread, + // 2. or it's wasn't the first call from a C thread on pthread platforms, + // since the we skip dropm to resue the m in the first call. MOVD savedm-8(SP), R6 CBNZ R6, droppedm + + // Skip dropm to reuse it in the next call, when a pthread key has been created. + MOVD _cgo_pthread_key_created(SB), R6 + // It means cgo is disabled when _cgo_pthread_key_created is a nil pointer, need dropm. + CBZ R6, dropm + MOVD (R6), R6 + CBNZ R6, droppedm + +dropm: MOVD $runtime·dropm(SB), R0 BL (R0) droppedm: diff --git a/src/runtime/asm_loong64.s b/src/runtime/asm_loong64.s index 6029dbc8c3..b93ad3316d 100644 --- a/src/runtime/asm_loong64.s +++ b/src/runtime/asm_loong64.s @@ -460,13 +460,23 @@ g0: TEXT ·cgocallback(SB),NOSPLIT,$24-24 NO_LOCAL_POINTERS + // Skip cgocallbackg, just dropm when fn is nil, and frame is the saved g. + // It is used to dropm while thread is exiting. + MOVV fn+0(FP), R5 + BNE R5, loadg + // Restore the g from frame. + MOVV frame+8(FP), g + JMP dropm + +loadg: // Load m and g from thread-local storage. MOVB runtime·iscgo(SB), R19 BEQ R19, nocgo JAL runtime·load_g(SB) nocgo: - // If g is nil, Go did not create the current thread. + // If g is nil, Go did not create the current thread, + // or if this thread never called into Go on pthread platforms. // Call needm to obtain one for temporary use. // In this case, we're running on the thread stack, so there's // lots of space, but the linker doesn't know. Hide the call from @@ -479,7 +489,7 @@ nocgo: needm: MOVV g, savedm-8(SP) // g is zero, so is m. - MOVV $runtime·needm(SB), R4 + MOVV $runtime·needAndBindM(SB), R4 JAL (R4) // Set m->sched.sp = SP, so that if a panic happens @@ -551,10 +561,24 @@ havem: MOVV savedsp-24(SP), R13 // must match frame size MOVV R13, (g_sched+gobuf_sp)(g) - // If the m on entry was nil, we called needm above to borrow an m - // for the duration of the call. Since the call is over, return it with dropm. + // If the m on entry was nil, we called needm above to borrow an m, + // 1. for the duration of the call on non-pthread platforms, + // 2. or the duration of the C thread alive on pthread platforms. + // If the m on entry wasn't nil, + // 1. the thread might be a Go thread, + // 2. or it's wasn't the first call from a C thread on pthread platforms, + // since the we skip dropm to resue the m in the first call. MOVV savedm-8(SP), R12 BNE R12, droppedm + + // Skip dropm to reuse it in the next call, when a pthread key has been created. + MOVV _cgo_pthread_key_created(SB), R12 + // It means cgo is disabled when _cgo_pthread_key_created is a nil pointer, need dropm. + BEQ R12, dropm + MOVV (R12), R12 + BNE R12, droppedm + +dropm: MOVV $runtime·dropm(SB), R4 JAL (R4) droppedm: diff --git a/src/runtime/asm_mips64x.s b/src/runtime/asm_mips64x.s index e6eb13f00a..1da90f7777 100644 --- a/src/runtime/asm_mips64x.s +++ b/src/runtime/asm_mips64x.s @@ -469,13 +469,23 @@ g0: TEXT ·cgocallback(SB),NOSPLIT,$24-24 NO_LOCAL_POINTERS + // Skip cgocallbackg, just dropm when fn is nil, and frame is the saved g. + // It is used to dropm while thread is exiting. + MOVV fn+0(FP), R5 + BNE R5, loadg + // Restore the g from frame. + MOVV frame+8(FP), g + JMP dropm + +loadg: // Load m and g from thread-local storage. MOVB runtime·iscgo(SB), R1 BEQ R1, nocgo JAL runtime·load_g(SB) nocgo: - // If g is nil, Go did not create the current thread. + // If g is nil, Go did not create the current thread, + // or if this thread never called into Go on pthread platforms. // Call needm to obtain one for temporary use. // In this case, we're running on the thread stack, so there's // lots of space, but the linker doesn't know. Hide the call from @@ -488,7 +498,7 @@ nocgo: needm: MOVV g, savedm-8(SP) // g is zero, so is m. - MOVV $runtime·needm(SB), R4 + MOVV $runtime·needAndBindM(SB), R4 JAL (R4) // Set m->sched.sp = SP, so that if a panic happens @@ -559,10 +569,24 @@ havem: MOVV savedsp-24(SP), R2 // must match frame size MOVV R2, (g_sched+gobuf_sp)(g) - // If the m on entry was nil, we called needm above to borrow an m - // for the duration of the call. Since the call is over, return it with dropm. + // If the m on entry was nil, we called needm above to borrow an m, + // 1. for the duration of the call on non-pthread platforms, + // 2. or the duration of the C thread alive on pthread platforms. + // If the m on entry wasn't nil, + // 1. the thread might be a Go thread, + // 2. or it's wasn't the first call from a C thread on pthread platforms, + // since the we skip dropm to resue the m in the first call. MOVV savedm-8(SP), R3 BNE R3, droppedm + + // Skip dropm to reuse it in the next call, when a pthread key has been created. + MOVV _cgo_pthread_key_created(SB), R3 + // It means cgo is disabled when _cgo_pthread_key_created is a nil pointer, need dropm. + BEQ R3, dropm + MOVV (R3), R3 + BNE R3, droppedm + +dropm: MOVV $runtime·dropm(SB), R4 JAL (R4) droppedm: diff --git a/src/runtime/asm_mipsx.s b/src/runtime/asm_mipsx.s index fc81e76354..49f96044c4 100644 --- a/src/runtime/asm_mipsx.s +++ b/src/runtime/asm_mipsx.s @@ -459,13 +459,23 @@ g0: TEXT ·cgocallback(SB),NOSPLIT,$12-12 NO_LOCAL_POINTERS + // Skip cgocallbackg, just dropm when fn is nil, and frame is the saved g. + // It is used to dropm while thread is exiting. + MOVW fn+0(FP), R5 + BNE R5, loadg + // Restore the g from frame. + MOVW frame+4(FP), g + JMP dropm + +loadg: // Load m and g from thread-local storage. MOVB runtime·iscgo(SB), R1 BEQ R1, nocgo JAL runtime·load_g(SB) nocgo: - // If g is nil, Go did not create the current thread. + // If g is nil, Go did not create the current thread, + // or if this thread never called into Go on pthread platforms. // Call needm to obtain one for temporary use. // In this case, we're running on the thread stack, so there's // lots of space, but the linker doesn't know. Hide the call from @@ -478,7 +488,7 @@ nocgo: needm: MOVW g, savedm-4(SP) // g is zero, so is m. - MOVW $runtime·needm(SB), R4 + MOVW $runtime·needAndBindM(SB), R4 JAL (R4) // Set m->sched.sp = SP, so that if a panic happens @@ -549,10 +559,24 @@ havem: MOVW savedsp-12(SP), R2 // must match frame size MOVW R2, (g_sched+gobuf_sp)(g) - // If the m on entry was nil, we called needm above to borrow an m - // for the duration of the call. Since the call is over, return it with dropm. + // If the m on entry was nil, we called needm above to borrow an m, + // 1. for the duration of the call on non-pthread platforms, + // 2. or the duration of the C thread alive on pthread platforms. + // If the m on entry wasn't nil, + // 1. the thread might be a Go thread, + // 2. or it's wasn't the first call from a C thread on pthread platforms, + // since the we skip dropm to resue the m in the first call. MOVW savedm-4(SP), R3 BNE R3, droppedm + + // Skip dropm to reuse it in the next call, when a pthread key has been created. + MOVW _cgo_pthread_key_created(SB), R3 + // It means cgo is disabled when _cgo_pthread_key_created is a nil pointer, need dropm. + BEQ R3, dropm + MOVW (R3), R3 + BNE R3, droppedm + +dropm: MOVW $runtime·dropm(SB), R4 JAL (R4) droppedm: diff --git a/src/runtime/asm_ppc64x.s b/src/runtime/asm_ppc64x.s index 1e17291d78..d5be18e853 100644 --- a/src/runtime/asm_ppc64x.s +++ b/src/runtime/asm_ppc64x.s @@ -628,6 +628,16 @@ g0: TEXT ·cgocallback(SB),NOSPLIT,$24-24 NO_LOCAL_POINTERS + // Skip cgocallbackg, just dropm when fn is nil, and frame is the saved g. + // It is used to dropm while thread is exiting. + MOVD fn+0(FP), R5 + CMP R5, $0 + BNE loadg + // Restore the g from frame. + MOVD frame+8(FP), g + BR dropm + +loadg: // Load m and g from thread-local storage. MOVBZ runtime·iscgo(SB), R3 CMP R3, $0 @@ -635,7 +645,8 @@ TEXT ·cgocallback(SB),NOSPLIT,$24-24 BL runtime·load_g(SB) nocgo: - // If g is nil, Go did not create the current thread. + // If g is nil, Go did not create the current thread, + // or if this thread never called into Go on pthread platforms. // Call needm to obtain one for temporary use. // In this case, we're running on the thread stack, so there's // lots of space, but the linker doesn't know. Hide the call from @@ -649,7 +660,7 @@ nocgo: needm: MOVD g, savedm-8(SP) // g is zero, so is m. - MOVD $runtime·needm(SB), R12 + MOVD $runtime·needAndBindM(SB), R12 MOVD R12, CTR BL (CTR) @@ -724,11 +735,27 @@ havem: MOVD savedsp-24(SP), R4 // must match frame size MOVD R4, (g_sched+gobuf_sp)(g) - // If the m on entry was nil, we called needm above to borrow an m - // for the duration of the call. Since the call is over, return it with dropm. + // If the m on entry was nil, we called needm above to borrow an m, + // 1. for the duration of the call on non-pthread platforms, + // 2. or the duration of the C thread alive on pthread platforms. + // If the m on entry wasn't nil, + // 1. the thread might be a Go thread, + // 2. or it's wasn't the first call from a C thread on pthread platforms, + // since the we skip dropm to resue the m in the first call. MOVD savedm-8(SP), R6 CMP R6, $0 BNE droppedm + + // Skip dropm to reuse it in the next call, when a pthread key has been created. + MOVD _cgo_pthread_key_created(SB), R6 + // It means cgo is disabled when _cgo_pthread_key_created is a nil pointer, need dropm. + CMP R6, $0 + BEQ dropm + MOVD (R6), R6 + CMP R6, $0 + BNE droppedm + +dropm: MOVD $runtime·dropm(SB), R12 MOVD R12, CTR BL (CTR) diff --git a/src/runtime/asm_riscv64.s b/src/runtime/asm_riscv64.s index 759bae24b5..0a34a591fd 100644 --- a/src/runtime/asm_riscv64.s +++ b/src/runtime/asm_riscv64.s @@ -519,13 +519,23 @@ TEXT runtime·goexit(SB),NOSPLIT|NOFRAME|TOPFRAME,$0-0 TEXT ·cgocallback(SB),NOSPLIT,$24-24 NO_LOCAL_POINTERS + // Skip cgocallbackg, just dropm when fn is nil, and frame is the saved g. + // It is used to dropm while thread is exiting. + MOV fn+0(FP), X7 + BNE ZERO, X7, loadg + // Restore the g from frame. + MOV frame+8(FP), g + JMP dropm + +loadg: // Load m and g from thread-local storage. MOVBU runtime·iscgo(SB), X5 BEQ ZERO, X5, nocgo CALL runtime·load_g(SB) nocgo: - // If g is nil, Go did not create the current thread. + // If g is nil, Go did not create the current thread, + // or if this thread never called into Go on pthread platforms. // Call needm to obtain one for temporary use. // In this case, we're running on the thread stack, so there's // lots of space, but the linker doesn't know. Hide the call from @@ -538,7 +548,7 @@ nocgo: needm: MOV g, savedm-8(SP) // g is zero, so is m. - MOV $runtime·needm(SB), X6 + MOV $runtime·needAndBindM(SB), X6 JALR RA, X6 // Set m->sched.sp = SP, so that if a panic happens @@ -609,10 +619,24 @@ havem: MOV savedsp-24(SP), X6 // must match frame size MOV X6, (g_sched+gobuf_sp)(g) - // If the m on entry was nil, we called needm above to borrow an m - // for the duration of the call. Since the call is over, return it with dropm. + // If the m on entry was nil, we called needm above to borrow an m, + // 1. for the duration of the call on non-pthread platforms, + // 2. or the duration of the C thread alive on pthread platforms. + // If the m on entry wasn't nil, + // 1. the thread might be a Go thread, + // 2. or it's wasn't the first call from a C thread on pthread platforms, + // since the we skip dropm to resue the m in the first call. MOV savedm-8(SP), X5 BNE ZERO, X5, droppedm + + // Skip dropm to reuse it in the next call, when a pthread key has been created. + MOV _cgo_pthread_key_created(SB), X5 + // It means cgo is disabled when _cgo_pthread_key_created is a nil pointer, need dropm. + BEQ ZERO, X5, dropm + MOV (X5), X5 + BNE ZERO, X5, droppedm + +dropm: MOV $runtime·dropm(SB), X6 JALR RA, X6 droppedm: diff --git a/src/runtime/asm_s390x.s b/src/runtime/asm_s390x.s index d427c07de4..4c4a42e00a 100644 --- a/src/runtime/asm_s390x.s +++ b/src/runtime/asm_s390x.s @@ -564,13 +564,23 @@ g0: TEXT ·cgocallback(SB),NOSPLIT,$24-24 NO_LOCAL_POINTERS + // Skip cgocallbackg, just dropm when fn is nil, and frame is the saved g. + // It is used to dropm while thread is exiting. + MOVD fn+0(FP), R1 + CMPBNE R1, $0, loadg + // Restore the g from frame. + MOVD frame+8(FP), g + BR dropm + +loadg: // Load m and g from thread-local storage. MOVB runtime·iscgo(SB), R3 CMPBEQ R3, $0, nocgo BL runtime·load_g(SB) nocgo: - // If g is nil, Go did not create the current thread. + // If g is nil, Go did not create the current thread, + // or if this thread never called into Go on pthread platforms. // Call needm to obtain one for temporary use. // In this case, we're running on the thread stack, so there's // lots of space, but the linker doesn't know. Hide the call from @@ -583,7 +593,7 @@ nocgo: needm: MOVD g, savedm-8(SP) // g is zero, so is m. - MOVD $runtime·needm(SB), R3 + MOVD $runtime·needAndBindM(SB), R3 BL (R3) // Set m->sched.sp = SP, so that if a panic happens @@ -654,10 +664,24 @@ havem: MOVD savedsp-24(SP), R4 // must match frame size MOVD R4, (g_sched+gobuf_sp)(g) - // If the m on entry was nil, we called needm above to borrow an m - // for the duration of the call. Since the call is over, return it with dropm. + // If the m on entry was nil, we called needm above to borrow an m, + // 1. for the duration of the call on non-pthread platforms, + // 2. or the duration of the C thread alive on pthread platforms. + // If the m on entry wasn't nil, + // 1. the thread might be a Go thread, + // 2. or it's wasn't the first call from a C thread on pthread platforms, + // since the we skip dropm to resue the m in the first call. MOVD savedm-8(SP), R6 CMPBNE R6, $0, droppedm + + // Skip dropm to reuse it in the next call, when a pthread key has been created. + MOVD _cgo_pthread_key_created(SB), R6 + // It means cgo is disabled when _cgo_pthread_key_created is a nil pointer, need dropm. + CMPBEQ R6, $0, dropm + MOVD (R6), R6 + CMPBNE R6, $0, droppedm + +dropm: MOVD $runtime·dropm(SB), R3 BL (R3) droppedm: diff --git a/src/runtime/cgo.go b/src/runtime/cgo.go index d90468240d..395303552c 100644 --- a/src/runtime/cgo.go +++ b/src/runtime/cgo.go @@ -17,6 +17,9 @@ import "unsafe" //go:linkname _cgo_callers _cgo_callers //go:linkname _cgo_set_context_function _cgo_set_context_function //go:linkname _cgo_yield _cgo_yield +//go:linkname _cgo_pthread_key_created _cgo_pthread_key_created +//go:linkname _cgo_bindm _cgo_bindm +//go:linkname _cgo_getstackbound _cgo_getstackbound var ( _cgo_init unsafe.Pointer @@ -26,11 +29,17 @@ var ( _cgo_callers unsafe.Pointer _cgo_set_context_function unsafe.Pointer _cgo_yield unsafe.Pointer + _cgo_pthread_key_created unsafe.Pointer + _cgo_bindm unsafe.Pointer + _cgo_getstackbound unsafe.Pointer ) // iscgo is set to true by the runtime/cgo package var iscgo bool +// set_crosscall2 is set by the runtime/cgo package +var set_crosscall2 func() + // cgoHasExtraM is set on startup when an extra M is created for cgo. // The extra M must be created before any C/C++ code calls cgocallback. var cgoHasExtraM bool diff --git a/src/runtime/cgo/asm_386.s b/src/runtime/cgo/asm_386.s index 2e7e9512e2..086e20b02f 100644 --- a/src/runtime/cgo/asm_386.s +++ b/src/runtime/cgo/asm_386.s @@ -4,6 +4,14 @@ #include "textflag.h" +// Set the x_crosscall2_ptr C function pointer variable point to crosscall2. +// It's such a pointer chain: _crosscall2_ptr -> x_crosscall2_ptr -> crosscall2 +TEXT ·set_crosscall2(SB),NOSPLIT,$0-0 + MOVL _crosscall2_ptr(SB), AX + MOVL $crosscall2(SB), BX + MOVL BX, (AX) + RET + // Called by C code generated by cmd/cgo. // func crosscall2(fn, a unsafe.Pointer, n int32, ctxt uintptr) // Saves C callee-saved registers and calls cgocallback with three arguments. diff --git a/src/runtime/cgo/asm_amd64.s b/src/runtime/cgo/asm_amd64.s index e223a6c870..f254622f23 100644 --- a/src/runtime/cgo/asm_amd64.s +++ b/src/runtime/cgo/asm_amd64.s @@ -5,6 +5,14 @@ #include "textflag.h" #include "abi_amd64.h" +// Set the x_crosscall2_ptr C function pointer variable point to crosscall2. +// It's such a pointer chain: _crosscall2_ptr -> x_crosscall2_ptr -> crosscall2 +TEXT ·set_crosscall2(SB),NOSPLIT,$0-0 + MOVQ _crosscall2_ptr(SB), AX + MOVQ $crosscall2(SB), BX + MOVQ BX, (AX) + RET + // Called by C code generated by cmd/cgo. // func crosscall2(fn, a unsafe.Pointer, n int32, ctxt uintptr) // Saves C callee-saved registers and calls cgocallback with three arguments. diff --git a/src/runtime/cgo/asm_arm.s b/src/runtime/cgo/asm_arm.s index ea55e173c1..f7f99772a6 100644 --- a/src/runtime/cgo/asm_arm.s +++ b/src/runtime/cgo/asm_arm.s @@ -4,6 +4,14 @@ #include "textflag.h" +// Set the x_crosscall2_ptr C function pointer variable point to crosscall2. +// It's such a pointer chain: _crosscall2_ptr -> x_crosscall2_ptr -> crosscall2 +TEXT ·set_crosscall2(SB),NOSPLIT,$0-0 + MOVW _crosscall2_ptr(SB), R1 + MOVW $crosscall2(SB), R2 + MOVW R2, (R1) + RET + // Called by C code generated by cmd/cgo. // func crosscall2(fn, a unsafe.Pointer, n int32, ctxt uintptr) // Saves C callee-saved registers and calls cgocallback with three arguments. diff --git a/src/runtime/cgo/asm_arm64.s b/src/runtime/cgo/asm_arm64.s index e808dedcfc..ce8909b492 100644 --- a/src/runtime/cgo/asm_arm64.s +++ b/src/runtime/cgo/asm_arm64.s @@ -5,6 +5,14 @@ #include "textflag.h" #include "abi_arm64.h" +// Set the x_crosscall2_ptr C function pointer variable point to crosscall2. +// It's such a pointer chain: _crosscall2_ptr -> x_crosscall2_ptr -> crosscall2 +TEXT ·set_crosscall2(SB),NOSPLIT,$0-0 + MOVD _crosscall2_ptr(SB), R1 + MOVD $crosscall2(SB), R2 + MOVD R2, (R1) + RET + // Called by C code generated by cmd/cgo. // func crosscall2(fn, a unsafe.Pointer, n int32, ctxt uintptr) // Saves C callee-saved registers and calls cgocallback with three arguments. diff --git a/src/runtime/cgo/asm_loong64.s b/src/runtime/cgo/asm_loong64.s index aea4f8e6b9..3b514ffc4a 100644 --- a/src/runtime/cgo/asm_loong64.s +++ b/src/runtime/cgo/asm_loong64.s @@ -5,6 +5,14 @@ #include "textflag.h" #include "abi_loong64.h" +// Set the x_crosscall2_ptr C function pointer variable point to crosscall2. +// It's such a pointer chain: _crosscall2_ptr -> x_crosscall2_ptr -> crosscall2 +TEXT ·set_crosscall2(SB),NOSPLIT,$0-0 + MOVV _crosscall2_ptr(SB), R5 + MOVV $crosscall2(SB), R6 + MOVV R6, (R5) + RET + // Called by C code generated by cmd/cgo. // func crosscall2(fn, a unsafe.Pointer, n int32, ctxt uintptr) // Saves C callee-saved registers and calls cgocallback with three arguments. diff --git a/src/runtime/cgo/asm_mips64x.s b/src/runtime/cgo/asm_mips64x.s index 904f781d87..0a8fbbbef0 100644 --- a/src/runtime/cgo/asm_mips64x.s +++ b/src/runtime/cgo/asm_mips64x.s @@ -6,6 +6,14 @@ #include "textflag.h" +// Set the x_crosscall2_ptr C function pointer variable point to crosscall2. +// It's such a pointer chain: _crosscall2_ptr -> x_crosscall2_ptr -> crosscall2 +TEXT ·set_crosscall2(SB),NOSPLIT,$0-0 + MOVV _crosscall2_ptr(SB), R5 + MOVV $crosscall2(SB), R6 + MOVV R6, (R5) + RET + // Called by C code generated by cmd/cgo. // func crosscall2(fn, a unsafe.Pointer, n int32, ctxt uintptr) // Saves C callee-saved registers and calls cgocallback with three arguments. diff --git a/src/runtime/cgo/asm_mipsx.s b/src/runtime/cgo/asm_mipsx.s index 5e2db0b56e..a57ae97d7e 100644 --- a/src/runtime/cgo/asm_mipsx.s +++ b/src/runtime/cgo/asm_mipsx.s @@ -6,6 +6,14 @@ #include "textflag.h" +// Set the x_crosscall2_ptr C function pointer variable point to crosscall2. +// It's such a pointer chain: _crosscall2_ptr -> x_crosscall2_ptr -> crosscall2 +TEXT ·set_crosscall2(SB),NOSPLIT,$0-0 + MOVW _crosscall2_ptr(SB), R5 + MOVW $crosscall2(SB), R6 + MOVW R6, (R5) + RET + // Called by C code generated by cmd/cgo. // func crosscall2(fn, a unsafe.Pointer, n int32, ctxt uintptr) // Saves C callee-saved registers and calls cgocallback with three arguments. diff --git a/src/runtime/cgo/asm_ppc64x.s b/src/runtime/cgo/asm_ppc64x.s index fea749670b..7752feb650 100644 --- a/src/runtime/cgo/asm_ppc64x.s +++ b/src/runtime/cgo/asm_ppc64x.s @@ -7,6 +7,25 @@ #include "textflag.h" #include "asm_ppc64x.h" +#ifdef GO_PPC64X_HAS_FUNCDESC +// crosscall2 is marked with go:cgo_export_static. On AIX, this creates and exports +// the symbol name and descriptor as the AIX linker expects, but does not work if +// referenced from within Go. Create and use an aliased descriptor of crosscall2 +// to workaround this. +DEFINE_PPC64X_FUNCDESC(_crosscall2<>, crosscall2) +#define CROSSCALL2_FPTR $_crosscall2<>(SB) +#else +#define CROSSCALL2_FPTR $crosscall2(SB) +#endif + +// Set the x_crosscall2_ptr C function pointer variable point to crosscall2. +// It's such a pointer chain: _crosscall2_ptr -> x_crosscall2_ptr -> crosscall2 +TEXT ·set_crosscall2(SB),NOSPLIT,$0-0 + MOVD _crosscall2_ptr(SB), R5 + MOVD CROSSCALL2_FPTR, R6 + MOVD R6, (R5) + RET + // Called by C code generated by cmd/cgo. // func crosscall2(fn, a unsafe.Pointer, n int32, ctxt uintptr) // Saves C callee-saved registers and calls cgocallback with three arguments. @@ -32,8 +51,12 @@ TEXT crosscall2(SB),NOSPLIT|NOFRAME,$0 #ifdef GO_PPC64X_HAS_FUNCDESC // Load the real entry address from the first slot of the function descriptor. + // The first argument fn might be null, that means dropm in pthread key destructor. + CMP R3, $0 + BEQ nil_fn MOVD 8(R3), R2 MOVD (R3), R3 +nil_fn: #endif MOVD R3, FIXED_FRAME+0(R1) // fn unsafe.Pointer MOVD R4, FIXED_FRAME+8(R1) // a unsafe.Pointer diff --git a/src/runtime/cgo/asm_riscv64.s b/src/runtime/cgo/asm_riscv64.s index 45151bf02b..08c4ed8466 100644 --- a/src/runtime/cgo/asm_riscv64.s +++ b/src/runtime/cgo/asm_riscv64.s @@ -4,6 +4,14 @@ #include "textflag.h" +// Set the x_crosscall2_ptr C function pointer variable point to crosscall2. +// It's such a pointer chain: _crosscall2_ptr -> x_crosscall2_ptr -> crosscall2 +TEXT ·set_crosscall2(SB),NOSPLIT,$0-0 + MOV _crosscall2_ptr(SB), X7 + MOV $crosscall2(SB), X8 + MOV X8, (X7) + RET + // Called by C code generated by cmd/cgo. // func crosscall2(fn, a unsafe.Pointer, n int32, ctxt uintptr) // Saves C callee-saved registers and calls cgocallback with three arguments. diff --git a/src/runtime/cgo/asm_s390x.s b/src/runtime/cgo/asm_s390x.s index 8bf16e75e2..bb0dfc1e31 100644 --- a/src/runtime/cgo/asm_s390x.s +++ b/src/runtime/cgo/asm_s390x.s @@ -4,6 +4,14 @@ #include "textflag.h" +// Set the x_crosscall2_ptr C function pointer variable point to crosscall2. +// It's such a pointer chain: _crosscall2_ptr -> x_crosscall2_ptr -> crosscall2 +TEXT ·set_crosscall2(SB),NOSPLIT,$0-0 + MOVD _crosscall2_ptr(SB), R1 + MOVD $crosscall2(SB), R2 + MOVD R2, (R1) + RET + // Called by C code generated by cmd/cgo. // func crosscall2(fn, a unsafe.Pointer, n int32, ctxt uintptr) // Saves C callee-saved registers and calls cgocallback with three arguments. diff --git a/src/runtime/cgo/asm_wasm.s b/src/runtime/cgo/asm_wasm.s index cb140eb7b8..e7f01bdc56 100644 --- a/src/runtime/cgo/asm_wasm.s +++ b/src/runtime/cgo/asm_wasm.s @@ -4,5 +4,8 @@ #include "textflag.h" +TEXT ·set_crosscall2(SB),NOSPLIT,$0-0 + UNDEF + TEXT crosscall2(SB), NOSPLIT, $0 UNDEF diff --git a/src/runtime/cgo/callbacks.go b/src/runtime/cgo/callbacks.go index e7c8ef3e07..3c246a88b6 100644 --- a/src/runtime/cgo/callbacks.go +++ b/src/runtime/cgo/callbacks.go @@ -71,6 +71,42 @@ var _cgo_thread_start = &x_cgo_thread_start var x_cgo_sys_thread_create byte var _cgo_sys_thread_create = &x_cgo_sys_thread_create +// Indicates whether a dummy thread key has been created or not. +// +// When calling go exported function from C, we register a destructor +// callback, for a dummy thread key, by using pthread_key_create. + +//go:cgo_import_static x_cgo_pthread_key_created +//go:linkname x_cgo_pthread_key_created x_cgo_pthread_key_created +//go:linkname _cgo_pthread_key_created _cgo_pthread_key_created +var x_cgo_pthread_key_created byte +var _cgo_pthread_key_created = &x_cgo_pthread_key_created + +// Export crosscall2 to a c function pointer variable. +// Used to dropm in pthread key destructor, while C thread is exiting. + +//go:cgo_import_static x_crosscall2_ptr +//go:linkname x_crosscall2_ptr x_crosscall2_ptr +//go:linkname _crosscall2_ptr _crosscall2_ptr +var x_crosscall2_ptr byte +var _crosscall2_ptr = &x_crosscall2_ptr + +// Set the x_crosscall2_ptr C function pointer variable point to crosscall2. +// It's for the runtime package to call at init time. +func set_crosscall2() + +//go:linkname _set_crosscall2 runtime.set_crosscall2 +var _set_crosscall2 = set_crosscall2 + +// Store the g into the thread-specific value. +// So that pthread_key_destructor will dropm when the thread is exiting. + +//go:cgo_import_static x_cgo_bindm +//go:linkname x_cgo_bindm x_cgo_bindm +//go:linkname _cgo_bindm _cgo_bindm +var x_cgo_bindm byte +var _cgo_bindm = &x_cgo_bindm + // Notifies that the runtime has been initialized. // // We currently block at every CGO entry point (via _cgo_wait_runtime_init_done) @@ -105,3 +141,12 @@ var _cgo_yield unsafe.Pointer //go:cgo_export_static _cgo_topofstack //go:cgo_export_dynamic _cgo_topofstack + +// x_cgo_getstackbound gets the thread's C stack size and +// set the G's stack bound based on the stack size. + +//go:cgo_import_static x_cgo_getstackbound +//go:linkname x_cgo_getstackbound x_cgo_getstackbound +//go:linkname _cgo_getstackbound _cgo_getstackbound +var x_cgo_getstackbound byte +var _cgo_getstackbound = &x_cgo_getstackbound diff --git a/src/runtime/cgo/gcc_libinit.c b/src/runtime/cgo/gcc_libinit.c index 57620fe4de..9676593211 100644 --- a/src/runtime/cgo/gcc_libinit.c +++ b/src/runtime/cgo/gcc_libinit.c @@ -17,6 +17,14 @@ static pthread_cond_t runtime_init_cond = PTHREAD_COND_INITIALIZER; static pthread_mutex_t runtime_init_mu = PTHREAD_MUTEX_INITIALIZER; static int runtime_init_done; +// pthread_g is a pthread specific key, for storing the g that binded to the C thread. +// The registered pthread_key_destructor will dropm, when the pthread-specified value g is not NULL, +// while a C thread is exiting. +static pthread_key_t pthread_g; +static void pthread_key_destructor(void* g); +uintptr_t x_cgo_pthread_key_created; +void (*x_crosscall2_ptr)(void (*fn)(void *), void *, int, size_t); + // The context function, used when tracing back C calls into Go. static void (*cgo_context_function)(struct context_arg*); @@ -39,6 +47,12 @@ _cgo_wait_runtime_init_done(void) { pthread_cond_wait(&runtime_init_cond, &runtime_init_mu); } + // The key and x_cgo_pthread_key_created are for the whole program, + // whereas the specific and destructor is per thread. + if (x_cgo_pthread_key_created == 0 && pthread_key_create(&pthread_g, pthread_key_destructor) == 0) { + x_cgo_pthread_key_created = 1; + } + // TODO(iant): For the case of a new C thread calling into Go, such // as when using -buildmode=c-archive, we know that Go runtime // initialization is complete but we do not know that all Go init @@ -61,6 +75,16 @@ _cgo_wait_runtime_init_done(void) { return 0; } +// Store the g into a thread-specific value associated with the pthread key pthread_g. +// And pthread_key_destructor will dropm when the thread is exiting. +void x_cgo_bindm(void* g) { + // We assume this will always succeed, otherwise, there might be extra M leaking, + // when a C thread exits after a cgo call. + // We only invoke this function once per thread in runtime.needAndBindM, + // and the next calls just reuse the bound m. + pthread_setspecific(pthread_g, g); +} + void x_cgo_notify_runtime_init_done(void* dummy __attribute__ ((unused))) { pthread_mutex_lock(&runtime_init_mu); @@ -110,3 +134,14 @@ _cgo_try_pthread_create(pthread_t* thread, const pthread_attr_t* attr, void* (*p } return EAGAIN; } + +static void +pthread_key_destructor(void* g) { + if (x_crosscall2_ptr != NULL) { + // fn == NULL means dropm. + // We restore g by using the stored g, before dropm in runtime.cgocallback, + // since the g stored in the TLS by Go might be cleared in some platforms, + // before this destructor invoked. + x_crosscall2_ptr(NULL, g, 0, 0); + } +} diff --git a/src/runtime/cgo/gcc_libinit_windows.c b/src/runtime/cgo/gcc_libinit_windows.c index fdcf027424..9a8c65ea29 100644 --- a/src/runtime/cgo/gcc_libinit_windows.c +++ b/src/runtime/cgo/gcc_libinit_windows.c @@ -30,6 +30,9 @@ static CRITICAL_SECTION runtime_init_cs; static HANDLE runtime_init_wait; static int runtime_init_done; +uintptr_t x_cgo_pthread_key_created; +void (*x_crosscall2_ptr)(void (*fn)(void *), void *, int, size_t); + // Pre-initialize the runtime synchronization objects void _cgo_preinit_init() { @@ -91,6 +94,12 @@ _cgo_wait_runtime_init_done(void) { return 0; } +// Should not be used since x_cgo_pthread_key_created will always be zero. +void x_cgo_bindm(void* dummy) { + fprintf(stderr, "unexpected cgo_bindm on Windows\n"); + abort(); +} + void x_cgo_notify_runtime_init_done(void* dummy) { _cgo_maybe_run_preinit(); diff --git a/src/runtime/cgo/gcc_stack_darwin.c b/src/runtime/cgo/gcc_stack_darwin.c new file mode 100644 index 0000000000..700d2eb5a5 --- /dev/null +++ b/src/runtime/cgo/gcc_stack_darwin.c @@ -0,0 +1,19 @@ +// Copyright 2023 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +#include +#include "libcgo.h" + +void +x_cgo_getstackbound(uintptr *low) +{ + void* addr; + size_t size; + pthread_t p; + + p = pthread_self(); + addr = pthread_get_stackaddr_np(p); // high address (!) + size = pthread_get_stacksize_np(p); + *low = (uintptr)addr - size; +} diff --git a/src/runtime/cgo/gcc_stack_unix.c b/src/runtime/cgo/gcc_stack_unix.c new file mode 100644 index 0000000000..6fe46c56d7 --- /dev/null +++ b/src/runtime/cgo/gcc_stack_unix.c @@ -0,0 +1,39 @@ +// Copyright 2023 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +//go:build unix && !darwin + +#ifndef _GNU_SOURCE // pthread_getattr_np +#define _GNU_SOURCE +#endif + +#include +#include "libcgo.h" + +void +x_cgo_getstackbound(uintptr *low) +{ + pthread_attr_t attr; + void *addr; + size_t size; + +#if defined(__GLIBC__) || (defined(__sun) && !defined(__illumos__)) + // pthread_getattr_np is a GNU extension supported in glibc. + // Solaris is not glibc but does support pthread_getattr_np + // (and the fallback doesn't work...). Illumos does not. + pthread_getattr_np(pthread_self(), &attr); // GNU extension + pthread_attr_getstack(&attr, &addr, &size); // low address +#elif defined(__illumos__) + pthread_attr_init(&attr); + pthread_attr_get_np(pthread_self(), &attr); + pthread_attr_getstack(&attr, &addr, &size); // low address +#else + pthread_attr_init(&attr); + pthread_attr_getstacksize(&attr, &size); + addr = __builtin_frame_address(0) + 4096 - size; +#endif + pthread_attr_destroy(&attr); + + *low = (uintptr)addr; +} diff --git a/src/runtime/cgo/gcc_stack_windows.c b/src/runtime/cgo/gcc_stack_windows.c new file mode 100644 index 0000000000..14604e1e4f --- /dev/null +++ b/src/runtime/cgo/gcc_stack_windows.c @@ -0,0 +1,7 @@ +// Copyright 2023 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +#include "libcgo.h" + +void x_cgo_getstackbound(uintptr *low) {} // no-op for now diff --git a/src/runtime/cgo/libcgo.h b/src/runtime/cgo/libcgo.h index af4960e7e9..04755f0f20 100644 --- a/src/runtime/cgo/libcgo.h +++ b/src/runtime/cgo/libcgo.h @@ -51,6 +51,11 @@ extern void (*_cgo_thread_start)(ThreadStart *ts); */ extern void (*_cgo_sys_thread_create)(void* (*func)(void*), void* arg); +/* + * Indicates whether a dummy pthread per-thread variable is allocated. + */ +extern uintptr_t *_cgo_pthread_key_created; + /* * Creates the new operating system thread (OS, arch dependent). */ diff --git a/src/runtime/cgocall.go b/src/runtime/cgocall.go index f9d79eca4b..7f1a02fb4b 100644 --- a/src/runtime/cgocall.go +++ b/src/runtime/cgocall.go @@ -229,6 +229,9 @@ func cgocallbackg(fn, frame unsafe.Pointer, ctxt uintptr) { savedpc := gp.syscallpc exitsyscall() // coming out of cgo call gp.m.incgo = false + if gp.m.isextra { + gp.m.isExtraInC = false + } osPreemptExtExit(gp.m) @@ -239,6 +242,9 @@ func cgocallbackg(fn, frame unsafe.Pointer, ctxt uintptr) { // This is enforced by checking incgo in the schedule function. gp.m.incgo = true + if gp.m.isextra { + gp.m.isExtraInC = true + } if gp.m != checkm { throw("m changed unexpectedly in cgocallbackg") diff --git a/src/runtime/crash_cgo_test.go b/src/runtime/crash_cgo_test.go index 6fe9364122..258c0ffc17 100644 --- a/src/runtime/crash_cgo_test.go +++ b/src/runtime/crash_cgo_test.go @@ -826,3 +826,16 @@ func TestDestructorCallbackRace(t *testing.T) { t.Errorf("expected %q, but got:\n%s", want, got) } } + +func TestEnsureBindM(t *testing.T) { + t.Parallel() + switch runtime.GOOS { + case "windows", "plan9": + t.Skipf("skipping bindm test on %s", runtime.GOOS) + } + got := runTestProg(t, "testprogcgo", "EnsureBindM") + want := "OK\n" + if got != want { + t.Errorf("expected %q, got %v", want, got) + } +} diff --git a/src/runtime/proc.go b/src/runtime/proc.go index 26bab27cb1..cef9680db2 100644 --- a/src/runtime/proc.go +++ b/src/runtime/proc.go @@ -209,6 +209,10 @@ func main() { main_init_done = make(chan bool) if iscgo { + if _cgo_pthread_key_created == nil { + throw("_cgo_pthread_key_created missing") + } + if _cgo_thread_start == nil { throw("_cgo_thread_start missing") } @@ -223,6 +227,13 @@ func main() { if _cgo_notify_runtime_init_done == nil { throw("_cgo_notify_runtime_init_done missing") } + + // Set the x_crosscall2_ptr C function pointer variable point to crosscall2. + if set_crosscall2 == nil { + throw("set_crosscall2 missing") + } + set_crosscall2() + // Start the template thread in case we enter Go from // a C-created thread and need to create a new thread. startTemplateThread() @@ -1880,11 +1891,15 @@ func allocm(pp *p, fn func(), id int64) *m { // pressed into service as the scheduling stack and current // goroutine for the duration of the cgo callback. // -// When the callback is done with the m, it calls dropm to -// put the m back on the list. +// It calls dropm to put the m back on the list, +// 1. when the callback is done with the m in non-pthread platforms, +// 2. or when the C thread exiting on pthread platforms. +// +// The signal argument indicates whether we're called from a signal +// handler. // //go:nosplit -func needm() { +func needm(signal bool) { if (iscgo || GOOS == "windows") && !cgoHasExtraM { // Can happen if C/C++ code calls Go from a global ctor. // Can also happen on Windows if a global ctor uses a @@ -1933,16 +1948,36 @@ func needm() { osSetupTLS(mp) // Install g (= m->g0) and set the stack bounds - // to match the current stack. We don't actually know + // to match the current stack. If we don't actually know // how big the stack is, like we don't know how big any - // scheduling stack is, but we assume there's at least 32 kB, - // which is more than enough for us. + // scheduling stack is, but we assume there's at least 32 kB. + // If we can get a more accurate stack bound from pthread, + // use that. setg(mp.g0) gp := getg() gp.stack.hi = getcallersp() + 1024 gp.stack.lo = getcallersp() - 32*1024 + if !signal && _cgo_getstackbound != nil { + // Don't adjust if called from the signal handler. + // We are on the signal stack, not the pthread stack. + // (We could get the stack bounds from sigaltstack, but + // we're getting out of the signal handler very soon + // anyway. Not worth it.) + var low uintptr + asmcgocall(_cgo_getstackbound, unsafe.Pointer(&low)) + // getstackbound is an unsupported no-op on Windows. + if low != 0 { + gp.stack.lo = low + // TODO: Also get gp.stack.hi from getstackbound. + } + } gp.stackguard0 = gp.stack.lo + stackGuard + // Should mark we are already in Go now. + // Otherwise, we may call needm again when we get a signal, before cgocallbackg1, + // which means the extram list may be empty, that will cause a deadlock. + mp.isExtraInC = false + // Initialize this thread to use the m. asminit() minit() @@ -1952,6 +1987,17 @@ func needm() { sched.ngsys.Add(-1) } +// Acquire an extra m and bind it to the C thread when a pthread key has been created. +// +//go:nosplit +func needAndBindM() { + needm(false) + + if _cgo_pthread_key_created != nil && *(*uintptr)(_cgo_pthread_key_created) != 0 { + cgoBindM() + } +} + // newextram allocates m's and puts them on the extra list. // It is called with a working local m, so that it can do things // like call schedlock and allocate. @@ -1996,6 +2042,8 @@ func oneNewExtraM() { gp.m = mp mp.curg = gp mp.isextra = true + // mark we are in C by default. + mp.isExtraInC = true mp.lockedInt++ mp.lockedg.set(gp) gp.lockedm.set(mp) @@ -2028,9 +2076,11 @@ func oneNewExtraM() { unlockextra(mp) } +// dropm puts the current m back onto the extra list. +// +// 1. On systems without pthreads, like Windows // dropm is called when a cgo callback has called needm but is now // done with the callback and returning back into the non-Go thread. -// It puts the current m back onto the extra list. // // The main expense here is the call to signalstack to release the // m's signal stack, and then the call to needm on the next callback @@ -2042,15 +2092,18 @@ func oneNewExtraM() { // call. These should typically not be scheduling operations, just a few // atomics, so the cost should be small. // -// TODO(rsc): An alternative would be to allocate a dummy pthread per-thread -// variable using pthread_key_create. Unlike the pthread keys we already use -// on OS X, this dummy key would never be read by Go code. It would exist -// only so that we could register at thread-exit-time destructor. -// That destructor would put the m back onto the extra list. -// This is purely a performance optimization. The current version, -// in which dropm happens on each cgo call, is still correct too. -// We may have to keep the current version on systems with cgo -// but without pthreads, like Windows. +// 2. On systems with pthreads +// dropm is called while a non-Go thread is exiting. +// We allocate a pthread per-thread variable using pthread_key_create, +// to register a thread-exit-time destructor. +// And store the g into a thread-specific value associated with the pthread key, +// when first return back to C. +// So that the destructor would invoke dropm while the non-Go thread is exiting. +// This is much faster since it avoids expensive signal-related syscalls. +// +// NOTE: this always runs without a P, so, nowritebarrierrec required. +// +//go:nowritebarrierrec func dropm() { // Clear m and g, and return m to the extra list. // After the call to setg we can only call nosplit functions @@ -2082,6 +2135,39 @@ func dropm() { msigrestore(sigmask) } +// bindm store the g0 of the current m into a thread-specific value. +// +// We allocate a pthread per-thread variable using pthread_key_create, +// to register a thread-exit-time destructor. +// We are here setting the thread-specific value of the pthread key, to enable the destructor. +// So that the pthread_key_destructor would dropm while the C thread is exiting. +// +// And the saved g will be used in pthread_key_destructor, +// since the g stored in the TLS by Go might be cleared in some platforms, +// before the destructor invoked, so, we restore g by the stored g, before dropm. +// +// We store g0 instead of m, to make the assembly code simpler, +// since we need to restore g0 in runtime.cgocallback. +// +// On systems without pthreads, like Windows, bindm shouldn't be used. +// +// NOTE: this always runs without a P, so, nowritebarrierrec required. +// +//go:nosplit +//go:nowritebarrierrec +func cgoBindM() { + if GOOS == "windows" || GOOS == "plan9" { + fatal("bindm in unexpected GOOS") + } + g := getg() + if g.m.g0 != g { + fatal("the current g is not g0") + } + if _cgo_bindm != nil { + asmcgocall(_cgo_bindm, unsafe.Pointer(g)) + } +} + // A helper function for EnsureDropM. func getm() uintptr { return uintptr(unsafe.Pointer(getg().m)) diff --git a/src/runtime/runtime2.go b/src/runtime/runtime2.go index eb9a1693ba..4fd5bba74d 100644 --- a/src/runtime/runtime2.go +++ b/src/runtime/runtime2.go @@ -562,6 +562,7 @@ type m struct { printlock int8 incgo bool // m is executing a cgo call isextra bool // m is an extra m + isExtraInC bool // m is an extra m that is not executing Go code freeWait atomic.Uint32 // Whether it is safe to free g0 and delete m (one of freeMRef, freeMStack, freeMWait) fastrand uint64 needextram bool diff --git a/src/runtime/signal_unix.go b/src/runtime/signal_unix.go index 8b0d281ac9..7fb629f4d5 100644 --- a/src/runtime/signal_unix.go +++ b/src/runtime/signal_unix.go @@ -435,7 +435,7 @@ func sigtrampgo(sig uint32, info *siginfo, ctx unsafe.Pointer) { c := &sigctxt{info, ctx} gp := sigFetchG(c) setg(gp) - if gp == nil { + if gp == nil || (gp.m != nil && gp.m.isExtraInC) { if sig == _SIGPROF { // Some platforms (Linux) have per-thread timers, which we use in // combination with the process-wide timer. Avoid double-counting. @@ -458,7 +458,18 @@ func sigtrampgo(sig uint32, info *siginfo, ctx unsafe.Pointer) { return } c.fixsigcode(sig) + // Set g to nil here and badsignal will use g0 by needm. + // TODO: reuse the current m here by using the gsignal and adjustSignalStack, + // since the current g maybe a normal goroutine and actually running on the signal stack, + // it may hit stack split that is not expected here. + if gp != nil { + setg(nil) + } badsignal(uintptr(sig), c) + // Restore g + if gp != nil { + setg(gp) + } return } @@ -574,7 +585,7 @@ func adjustSignalStack(sig uint32, mp *m, gsigStack *gsignalStack) bool { // sp is not within gsignal stack, g0 stack, or sigaltstack. Bad. setg(nil) - needm() + needm(true) if st.ss_flags&_SS_DISABLE != 0 { noSignalStack(sig) } else { @@ -1047,7 +1058,7 @@ func badsignal(sig uintptr, c *sigctxt) { exit(2) *(*uintptr)(unsafe.Pointer(uintptr(123))) = 2 } - needm() + needm(true) if !sigsend(uint32(sig)) { // A foreign thread received the signal sig, and the // Go code does not want to handle it. @@ -1115,8 +1126,9 @@ func sigfwdgo(sig uint32, info *siginfo, ctx unsafe.Pointer) bool { // (1) we weren't in VDSO page, // (2) we were in a goroutine (i.e., m.curg != nil), and // (3) we weren't in CGO. + // (4) we weren't in dropped extra m. gp := sigFetchG(c) - if gp != nil && gp.m != nil && gp.m.curg != nil && !gp.m.incgo { + if gp != nil && gp.m != nil && gp.m.curg != nil && !gp.m.isExtraInC && !gp.m.incgo { return false } diff --git a/src/runtime/stubs.go b/src/runtime/stubs.go index 373445d613..65b7299f74 100644 --- a/src/runtime/stubs.go +++ b/src/runtime/stubs.go @@ -237,6 +237,9 @@ func noEscapePtr[T any](p *T) *T { // cgocallback is not called from Go, only from crosscall2. // This in turn calls cgocallbackg, which is where we'll find // pointer-declared arguments. +// +// When fn is nil (frame is saved g), call dropm instead, +// this is used when the C thread is exiting. func cgocallback(fn, frame, ctxt uintptr) func gogo(buf *gobuf) diff --git a/src/runtime/testdata/testprogcgo/bindm.c b/src/runtime/testdata/testprogcgo/bindm.c new file mode 100644 index 0000000000..815d8a75f2 --- /dev/null +++ b/src/runtime/testdata/testprogcgo/bindm.c @@ -0,0 +1,34 @@ +// Copyright 2023 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +//go:build !plan9 && !windows + +#include +#include +#include +#include "_cgo_export.h" + +#define CTHREADS 2 +#define CHECKCALLS 100 + +static void* checkBindMThread(void* thread) { + int i; + for (i = 0; i < CHECKCALLS; i++) { + GoCheckBindM((uintptr_t)thread); + usleep(1); + } + return NULL; +} + +void CheckBindM() { + int i; + pthread_t s[CTHREADS]; + + for (i = 0; i < CTHREADS; i++) { + pthread_create(&s[i], NULL, checkBindMThread, &s[i]); + } + for (i = 0; i < CTHREADS; i++) { + pthread_join(s[i], NULL); + } +} diff --git a/src/runtime/testdata/testprogcgo/bindm.go b/src/runtime/testdata/testprogcgo/bindm.go new file mode 100644 index 0000000000..c2003c2093 --- /dev/null +++ b/src/runtime/testdata/testprogcgo/bindm.go @@ -0,0 +1,61 @@ +// Copyright 2023 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +//go:build !plan9 && !windows + +// Test that callbacks from C to Go in the same C-thread always get the same m. +// Make sure the extra M bind to the C-thread. + +package main + +/* +extern void CheckBindM(); +*/ +import "C" + +import ( + "fmt" + "os" + "runtime" + "sync" + "sync/atomic" +) + +var ( + mutex = sync.Mutex{} + cThreadToM = map[uintptr]uintptr{} + started = atomic.Uint32{} +) + +// same as CTHREADS in C, make sure all the C threads are actually started. +const cThreadNum = 2 + +func init() { + register("EnsureBindM", EnsureBindM) +} + +//export GoCheckBindM +func GoCheckBindM(thread uintptr) { + // Wait all threads start + if started.Load() != cThreadNum { + // Only once for each thread, since it will wait all threads start. + started.Add(1) + for started.Load() < cThreadNum { + runtime.Gosched() + } + } + m := runtime_getm_for_test() + mutex.Lock() + defer mutex.Unlock() + if savedM, ok := cThreadToM[thread]; ok && savedM != m { + fmt.Printf("m == %x want %x\n", m, savedM) + os.Exit(1) + } + cThreadToM[thread] = m +} + +func EnsureBindM() { + C.CheckBindM() + fmt.Println("OK") +} From fa80fe7b1cc770d37b2263eb5d81c87db75ceb80 Mon Sep 17 00:00:00 2001 From: Brad Fitzpatrick Date: Wed, 26 Apr 2023 10:45:01 -0700 Subject: [PATCH 005/299] cmd/go: simplify code that still assumed the build cache could be nil cache.Default always returns a non-nil value since Go 1.12; the docs were updated in https://go.dev/cl/465555. This updates all the callers of cache.Default that were checking whether the result was nil so the code isn't misleading/confusing to readers. Change-Id: Ia63567dd70affef6041c744259f65cea79a2752e Reviewed-on: https://go-review.googlesource.com/c/go/+/489355 Auto-Submit: Brad Fitzpatrick Reviewed-by: Bryan Mills Run-TryBot: Brad Fitzpatrick TryBot-Result: Gopher Robot Reviewed-by: Dmitri Shuralyov --- src/cmd/go/internal/list/list.go | 14 ----- src/cmd/go/internal/test/test.go | 8 --- src/cmd/go/internal/work/buildid.go | 80 ++++++++++++++--------------- 3 files changed, 38 insertions(+), 64 deletions(-) diff --git a/src/cmd/go/internal/list/list.go b/src/cmd/go/internal/list/list.go index dd3e5cd06f..be2dd60dff 100644 --- a/src/cmd/go/internal/list/list.go +++ b/src/cmd/go/internal/list/list.go @@ -624,20 +624,6 @@ func runList(ctx context.Context, cmd *base.Command, args []string) { base.ExitIfErrors() } - if cache.Default() == nil { - // These flags return file names pointing into the build cache, - // so the build cache must exist. - if *listCompiled { - base.Fatalf("go list -compiled requires build cache") - } - if *listExport { - base.Fatalf("go list -export requires build cache") - } - if *listTest { - base.Fatalf("go list -test requires build cache") - } - } - if *listTest { c := cache.Default() // Add test binaries to packages to be listed. diff --git a/src/cmd/go/internal/test/test.go b/src/cmd/go/internal/test/test.go index a986718abf..e82ea72094 100644 --- a/src/cmd/go/internal/test/test.go +++ b/src/cmd/go/internal/test/test.go @@ -1537,14 +1537,6 @@ func (c *runCache) tryCacheWithID(b *work.Builder, a *work.Action, id string) bo } } - if cache.Default() == nil { - if cache.DebugTest { - fmt.Fprintf(os.Stderr, "testcache: GOCACHE=off\n") - } - c.disableCache = true - return false - } - // The test cache result fetch is a two-level lookup. // // First, we use the content hash of the test binary diff --git a/src/cmd/go/internal/work/buildid.go b/src/cmd/go/internal/work/buildid.go index db56714788..ea3240412c 100644 --- a/src/cmd/go/internal/work/buildid.go +++ b/src/cmd/go/internal/work/buildid.go @@ -437,6 +437,8 @@ func (b *Builder) useCache(a *Action, actionHash cache.ActionID, target string, return false } + c := cache.Default() + if target != "" { buildID, _ := buildid.ReadFile(target) if strings.HasPrefix(buildID, actionID+buildIDSeparator) { @@ -474,10 +476,8 @@ func (b *Builder) useCache(a *Action, actionHash cache.ActionID, target string, // If it doesn't work, it doesn't work: reusing the cached binary is more // important than reprinting diagnostic information. if printOutput { - if c := cache.Default(); c != nil { - showStdout(b, c, a.actionID, "stdout") // compile output - showStdout(b, c, a.actionID, "link-stdout") // link output - } + showStdout(b, c, a.actionID, "stdout") // compile output + showStdout(b, c, a.actionID, "link-stdout") // link output } // Poison a.Target to catch uses later in the build. @@ -504,10 +504,8 @@ func (b *Builder) useCache(a *Action, actionHash cache.ActionID, target string, // If it doesn't work, it doesn't work: reusing the test result is more // important than reprinting diagnostic information. if printOutput { - if c := cache.Default(); c != nil { - showStdout(b, c, a.Deps[0].actionID, "stdout") // compile output - showStdout(b, c, a.Deps[0].actionID, "link-stdout") // link output - } + showStdout(b, c, a.Deps[0].actionID, "stdout") // compile output + showStdout(b, c, a.Deps[0].actionID, "link-stdout") // link output } // Poison a.Target to catch uses later in the build. @@ -517,25 +515,23 @@ func (b *Builder) useCache(a *Action, actionHash cache.ActionID, target string, } // Check to see if the action output is cached. - if c := cache.Default(); c != nil { - if file, _, err := c.GetFile(actionHash); err == nil { - if buildID, err := buildid.ReadFile(file); err == nil { - if printOutput { - showStdout(b, c, a.actionID, "stdout") - } - a.built = file - a.Target = "DO NOT USE - using cache" - a.buildID = buildID - if a.json != nil { - a.json.BuildID = a.buildID - } - if p := a.Package; p != nil && target != "" { - p.Stale = true - // Clearer than explaining that something else is stale. - p.StaleReason = "not installed but available in build cache" - } - return true + if file, _, err := c.GetFile(actionHash); err == nil { + if buildID, err := buildid.ReadFile(file); err == nil { + if printOutput { + showStdout(b, c, a.actionID, "stdout") } + a.built = file + a.Target = "DO NOT USE - using cache" + a.buildID = buildID + if a.json != nil { + a.json.BuildID = a.buildID + } + if p := a.Package; p != nil && target != "" { + p.Stale = true + // Clearer than explaining that something else is stale. + p.StaleReason = "not installed but available in build cache" + } + return true } } @@ -609,22 +605,22 @@ func (b *Builder) updateBuildID(a *Action, target string, rewrite bool) error { } } + c := cache.Default() + // Cache output from compile/link, even if we don't do the rest. - if c := cache.Default(); c != nil { - switch a.Mode { - case "build": - c.PutBytes(cache.Subkey(a.actionID, "stdout"), a.output) - case "link": - // Even though we don't cache the binary, cache the linker text output. - // We might notice that an installed binary is up-to-date but still - // want to pretend to have run the linker. - // Store it under the main package's action ID - // to make it easier to find when that's all we have. - for _, a1 := range a.Deps { - if p1 := a1.Package; p1 != nil && p1.Name == "main" { - c.PutBytes(cache.Subkey(a1.actionID, "link-stdout"), a.output) - break - } + switch a.Mode { + case "build": + c.PutBytes(cache.Subkey(a.actionID, "stdout"), a.output) + case "link": + // Even though we don't cache the binary, cache the linker text output. + // We might notice that an installed binary is up-to-date but still + // want to pretend to have run the linker. + // Store it under the main package's action ID + // to make it easier to find when that's all we have. + for _, a1 := range a.Deps { + if p1 := a1.Package; p1 != nil && p1.Name == "main" { + c.PutBytes(cache.Subkey(a1.actionID, "link-stdout"), a.output) + break } } } @@ -682,7 +678,7 @@ func (b *Builder) updateBuildID(a *Action, target string, rewrite bool) error { // that will mean the go process is itself writing a binary // and then executing it, so we will need to defend against // ETXTBSY problems as discussed in exec.go and golang.org/issue/22220. - if c := cache.Default(); c != nil && a.Mode == "build" { + if a.Mode == "build" { r, err := os.Open(target) if err == nil { if a.output == nil { From ffa663c9b3c942cbde590bb792179dcca52224b7 Mon Sep 17 00:00:00 2001 From: "Bryan C. Mills" Date: Wed, 26 Apr 2023 15:38:08 -0400 Subject: [PATCH 006/299] net/http/cgi: propagate LD_LIBRARY_PATH on Android Android is functionally a variant on linux, and should be treated as such. Change-Id: I08056f00bf98c1935c8cc3c859a6c72fe1a48efa Reviewed-on: https://go-review.googlesource.com/c/go/+/489395 Run-TryBot: Bryan Mills TryBot-Result: Gopher Robot Reviewed-by: Ian Lance Taylor --- src/net/http/cgi/host.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/net/http/cgi/host.go b/src/net/http/cgi/host.go index 349dda15ac..073952a7bd 100644 --- a/src/net/http/cgi/host.go +++ b/src/net/http/cgi/host.go @@ -39,7 +39,7 @@ var osDefaultInheritEnv = func() []string { switch runtime.GOOS { case "darwin", "ios": return []string{"DYLD_LIBRARY_PATH"} - case "linux", "freebsd", "netbsd", "openbsd": + case "android", "linux", "freebsd", "netbsd", "openbsd": return []string{"LD_LIBRARY_PATH"} case "hpux": return []string{"LD_LIBRARY_PATH", "SHLIB_PATH"} From 527745c4d55d07279125e1ad5442a86d2a631250 Mon Sep 17 00:00:00 2001 From: "Bryan C. Mills" Date: Wed, 26 Apr 2023 15:54:28 -0400 Subject: [PATCH 007/299] misc/wasm: default to /tmp if TMPDIR is unset Change-Id: Ibf460d86ced08687099725bcd8ea8f38d7e8484c Reviewed-on: https://go-review.googlesource.com/c/go/+/489435 Reviewed-by: Dmitri Shuralyov Auto-Submit: Bryan Mills Reviewed-by: Johan Brandhorst-Satzkorn TryBot-Result: Gopher Robot Run-TryBot: Bryan Mills --- misc/wasm/go_wasip1_wasm_exec | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/misc/wasm/go_wasip1_wasm_exec b/misc/wasm/go_wasip1_wasm_exec index 9838212d98..3e1fc62156 100755 --- a/misc/wasm/go_wasip1_wasm_exec +++ b/misc/wasm/go_wasip1_wasm_exec @@ -8,7 +8,7 @@ case "$GOWASIRUNTIME" in exec wasmtime run --dir=/ --env PWD="$PWD" --max-wasm-stack 1048576 "$1" -- "${@:2}" ;; "wazero" | "") - exec wazero run -mount /:/ -env-inherit -cachedir "${TMPDIR}"/wazero "$1" "${@:2}" + exec wazero run -mount /:/ -env-inherit -cachedir "${TMPDIR:-/tmp}"/wazero "$1" "${@:2}" ;; *) echo "Unknown Go WASI runtime specified: $GOWASIRUNTIME" From b4eed1c3414b6def26820a9189cb65cb070540be Mon Sep 17 00:00:00 2001 From: cui fliter Date: Thu, 6 Apr 2023 17:08:49 +0800 Subject: [PATCH 008/299] crypto: use t.Parallel() to increase speed It is necessary to invoke the t.Parallel() method in both the top-level test function and its subtest function to maximize parallelism. In doing so, all subtest functions calling the t.Parallel() method in the package will work in parallel. On my machine, the execution time of this test file was cut in half. Change-Id: If09147a2a9969bb044932d71e6bfea29492866d6 Reviewed-on: https://go-review.googlesource.com/c/go/+/482755 Run-TryBot: shuang cui Reviewed-by: Ian Lance Taylor Auto-Submit: Roland Shoemaker TryBot-Result: Gopher Robot Reviewed-by: Roland Shoemaker --- src/crypto/elliptic/elliptic_test.go | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/src/crypto/elliptic/elliptic_test.go b/src/crypto/elliptic/elliptic_test.go index 34d70f6a47..aedbefc4ca 100644 --- a/src/crypto/elliptic/elliptic_test.go +++ b/src/crypto/elliptic/elliptic_test.go @@ -48,6 +48,7 @@ func testAllCurves(t *testing.T, f func(*testing.T, Curve)) { } func TestOnCurve(t *testing.T) { + t.Parallel() testAllCurves(t, func(t *testing.T, curve Curve) { if !curve.IsOnCurve(curve.Params().Gx, curve.Params().Gy) { t.Error("basepoint is not on the curve") @@ -56,6 +57,7 @@ func TestOnCurve(t *testing.T) { } func TestOffCurve(t *testing.T) { + t.Parallel() testAllCurves(t, func(t *testing.T, curve Curve) { x, y := new(big.Int).SetInt64(1), new(big.Int).SetInt64(1) if curve.IsOnCurve(x, y) { @@ -76,6 +78,7 @@ func TestOffCurve(t *testing.T) { } func TestInfinity(t *testing.T) { + t.Parallel() testAllCurves(t, testInfinity) } @@ -150,6 +153,7 @@ func testInfinity(t *testing.T, curve Curve) { } func TestMarshal(t *testing.T) { + t.Parallel() testAllCurves(t, func(t *testing.T, curve Curve) { _, x, y, err := GenerateKey(curve, rand.Reader) if err != nil { @@ -167,6 +171,7 @@ func TestMarshal(t *testing.T) { } func TestUnmarshalToLargeCoordinates(t *testing.T) { + t.Parallel() // See https://golang.org/issues/20482. testAllCurves(t, testUnmarshalToLargeCoordinates) } @@ -216,6 +221,7 @@ func testUnmarshalToLargeCoordinates(t *testing.T, curve Curve) { // (negative or bigger than P). They are expected to return false from // IsOnCurve, all other behavior is undefined. func TestInvalidCoordinates(t *testing.T) { + t.Parallel() testAllCurves(t, testInvalidCoordinates) } @@ -268,6 +274,7 @@ func testInvalidCoordinates(t *testing.T, curve Curve) { } func TestMarshalCompressed(t *testing.T) { + t.Parallel() t.Run("P-256/03", func(t *testing.T) { data, _ := hex.DecodeString("031e3987d9f9ea9d7dd7155a56a86b2009e1e0ab332f962d10d8beb6406ab1ad79") x, _ := new(big.Int).SetString("13671033352574878777044637384712060483119675368076128232297328793087057702265", 10) @@ -326,6 +333,7 @@ func testMarshalCompressed(t *testing.T, curve Curve, x, y *big.Int, want []byte } func TestLargeIsOnCurve(t *testing.T) { + t.Parallel() testAllCurves(t, func(t *testing.T, curve Curve) { large := big.NewInt(1) large.Lsh(large, 1000) From 0972096c5c2facec2d5c6db08a2df32684f41caa Mon Sep 17 00:00:00 2001 From: Dmitri Shuralyov Date: Mon, 24 Apr 2023 11:26:23 -0400 Subject: [PATCH 009/299] cmd/api: make api/next/* entries required, not optional We want the API check to catch if some API present in api/next/* files is no longer implemented in the tree, and report it in the same CL that is making the change (by failing loudly). Arguably this should've been the case since CL 315350, but I didn't notice it at the time. Do it now. For #43956. Change-Id: I73330dd5fd3f5706a1fdf13b2bf8e0f24c6b48e5 Reviewed-on: https://go-review.googlesource.com/c/go/+/488135 Reviewed-by: Dmitri Shuralyov Run-TryBot: Dmitri Shuralyov TryBot-Result: Gopher Robot Reviewed-by: Bryan Mills Auto-Submit: Dmitri Shuralyov --- src/cmd/api/api.go | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/src/cmd/api/api.go b/src/cmd/api/api.go index f0d48e3ccd..4dd33b13a5 100644 --- a/src/cmd/api/api.go +++ b/src/cmd/api/api.go @@ -193,13 +193,12 @@ func Check(t *testing.T) { bw := bufio.NewWriter(os.Stdout) defer bw.Flush() - var required []string + var required, optional []string for _, file := range checkFiles { required = append(required, fileFeatures(file, needApproval(file))...) } - var optional []string for _, file := range nextFiles { - optional = append(optional, fileFeatures(file, true)...) + required = append(required, fileFeatures(file, true)...) } exception := fileFeatures(filepath.Join(testenv.GOROOT(t), "api/except.txt"), false) From e8c8b79f000515e086012df632f01fc0ec21076b Mon Sep 17 00:00:00 2001 From: Dmitri Shuralyov Date: Wed, 26 Apr 2023 16:41:41 -0400 Subject: [PATCH 010/299] cmd/api: remove unused functionality We no longer use the optional parameter to compareAPI. We now always set allowAdd to false. (Except in tests, making them less useful than they could be.) Flags and parsing their value are no more. Remove all the unused functionality and update test cases so they're closer to what the API checker does when it runs for real. Order the features, required, exception variables and fields more consistently. For #43956. Change-Id: Iaa4656a89a3fca3129742165a448d385e55e4a98 Reviewed-on: https://go-review.googlesource.com/c/go/+/489436 Auto-Submit: Dmitri Shuralyov Reviewed-by: Bryan Mills Run-TryBot: Dmitri Shuralyov Reviewed-by: Dmitri Shuralyov TryBot-Result: Gopher Robot --- src/cmd/api/api.go | 65 +++++++---------------------------------- src/cmd/api/api_test.go | 63 ++++++++++++++++++++++++--------------- 2 files changed, 50 insertions(+), 78 deletions(-) diff --git a/src/cmd/api/api.go b/src/cmd/api/api.go index 4dd33b13a5..376dc53fdc 100644 --- a/src/cmd/api/api.go +++ b/src/cmd/api/api.go @@ -10,7 +10,6 @@ import ( "bufio" "bytes" "encoding/json" - "flag" "fmt" "go/ast" "go/build" @@ -46,8 +45,7 @@ func goCmd() string { return "go" } -// contexts are the default contexts which are scanned, unless -// overridden by the -contexts flag. +// contexts are the default contexts which are scanned. var contexts = []*build.Context{ {GOOS: "linux", GOARCH: "386", CgoEnabled: true}, {GOOS: "linux", GOARCH: "386"}, @@ -96,25 +94,6 @@ func contextName(c *build.Context) string { return s } -func parseContext(c string) *build.Context { - parts := strings.Split(c, "-") - if len(parts) < 2 { - log.Fatalf("bad context: %q", c) - } - bc := &build.Context{ - GOOS: parts[0], - GOARCH: parts[1], - } - if len(parts) == 3 { - if parts[2] == "cgo" { - bc.CgoEnabled = true - } else { - log.Fatalf("bad context: %q", c) - } - } - return bc -} - var internalPkg = regexp.MustCompile(`(^|/)internal($|/)`) var exitCode = 0 @@ -152,12 +131,7 @@ func Check(t *testing.T) { var featureCtx = make(map[string]map[string]bool) // feature -> context name -> true for _, w := range walkers { - pkgNames := w.stdPackages - if flag.NArg() > 0 { - pkgNames = flag.Args() - } - - for _, name := range pkgNames { + for _, name := range w.stdPackages { pkg, err := w.import_(name) if _, nogo := err.(*build.NoGoError); nogo { continue @@ -193,7 +167,7 @@ func Check(t *testing.T) { bw := bufio.NewWriter(os.Stdout) defer bw.Flush() - var required, optional []string + var required []string for _, file := range checkFiles { required = append(required, fileFeatures(file, needApproval(file))...) } @@ -205,7 +179,7 @@ func Check(t *testing.T) { if exitCode == 1 { t.Errorf("API database problems found") } - if !compareAPI(bw, features, required, optional, exception, false) { + if !compareAPI(bw, features, required, exception) { t.Errorf("API differences found") } } @@ -251,12 +225,11 @@ func portRemoved(feature string) bool { strings.Contains(feature, "(darwin-386-cgo)") } -func compareAPI(w io.Writer, features, required, optional, exception []string, allowAdd bool) (ok bool) { +func compareAPI(w io.Writer, features, required, exception []string) (ok bool) { ok = true - optionalSet := set(optional) - exceptionSet := set(exception) featureSet := set(features) + exceptionSet := set(exception) sort.Strings(features) sort.Strings(required) @@ -267,7 +240,7 @@ func compareAPI(w io.Writer, features, required, optional, exception []string, a return s } - for len(required) > 0 || len(features) > 0 { + for len(features) > 0 || len(required) > 0 { switch { case len(features) == 0 || (len(required) > 0 && required[0] < features[0]): feature := take(&required) @@ -288,33 +261,15 @@ func compareAPI(w io.Writer, features, required, optional, exception []string, a } case len(required) == 0 || (len(features) > 0 && required[0] > features[0]): newFeature := take(&features) - if optionalSet[newFeature] { - // Known added feature to the upcoming release. - // Delete it from the map so we can detect any upcoming features - // which were never seen. (so we can clean up the nextFile) - delete(optionalSet, newFeature) - } else { - fmt.Fprintf(w, "+%s\n", newFeature) - if !allowAdd { - ok = false // we're in lock-down mode for next release - } - } + fmt.Fprintf(w, "+%s\n", newFeature) + ok = false // feature not in api/next/* default: take(&required) take(&features) } } - // In next file, but not in API. - var missing []string - for feature := range optionalSet { - missing = append(missing, feature) - } - sort.Strings(missing) - for _, feature := range missing { - fmt.Fprintf(w, "±%s\n", feature) - } - return + return ok } // aliasReplacer applies type aliases to earlier API files, diff --git a/src/cmd/api/api_test.go b/src/cmd/api/api_test.go index 5f9aa6d297..142cbb4339 100644 --- a/src/cmd/api/api_test.go +++ b/src/cmd/api/api_test.go @@ -115,16 +115,23 @@ func TestGolden(t *testing.T) { func TestCompareAPI(t *testing.T) { tests := []struct { - name string - features, required, optional, exception []string - ok bool // want - out string // want + name string + features, required, exception []string + ok bool // want + out string // want }{ + { + name: "equal", + features: []string{"A", "B", "C"}, + required: []string{"A", "B", "C"}, + ok: true, + out: "", + }, { name: "feature added", features: []string{"A", "B", "C", "D", "E", "F"}, required: []string{"B", "D"}, - ok: true, + ok: false, out: "+A\n+C\n+E\n+F\n", }, { @@ -134,42 +141,52 @@ func TestCompareAPI(t *testing.T) { ok: false, out: "-B\n", }, - { - name: "feature added then removed", - features: []string{"A", "C"}, - optional: []string{"B"}, - required: []string{"A", "C"}, - ok: true, - out: "±B\n", - }, { name: "exception removal", - required: []string{"A", "B", "C"}, features: []string{"A", "C"}, + required: []string{"A", "B", "C"}, exception: []string{"B"}, ok: true, out: "", }, + + // Test that a feature required on a subset of ports is implicitly satisfied + // by the same feature being implemented on all ports. That is, it shouldn't + // say "pkg syscall (darwin-amd64), type RawSockaddrInet6 struct" is missing. + // See https://go.dev/issue/4303. { - // https://golang.org/issue/4303 - name: "contexts reconverging", - required: []string{ - "A", - "pkg syscall (darwin-amd64), type RawSockaddrInet6 struct", - }, + name: "contexts reconverging after api/next/* update", features: []string{ "A", "pkg syscall, type RawSockaddrInet6 struct", }, + required: []string{ + "A", + "pkg syscall (darwin-amd64), type RawSockaddrInet6 struct", // api/go1.n.txt + "pkg syscall, type RawSockaddrInet6 struct", // api/next/n.txt + }, ok: true, + out: "", + }, + { + name: "contexts reconverging before api/next/* update", + features: []string{ + "A", + "pkg syscall, type RawSockaddrInet6 struct", + }, + required: []string{ + "A", + "pkg syscall (darwin-amd64), type RawSockaddrInet6 struct", + }, + ok: false, out: "+pkg syscall, type RawSockaddrInet6 struct\n", }, } for _, tt := range tests { buf := new(strings.Builder) - gotok := compareAPI(buf, tt.features, tt.required, tt.optional, tt.exception, true) - if gotok != tt.ok { - t.Errorf("%s: ok = %v; want %v", tt.name, gotok, tt.ok) + gotOK := compareAPI(buf, tt.features, tt.required, tt.exception) + if gotOK != tt.ok { + t.Errorf("%s: ok = %v; want %v", tt.name, gotOK, tt.ok) } if got := buf.String(); got != tt.out { t.Errorf("%s: output differs\nGOT:\n%s\nWANT:\n%s", tt.name, got, tt.out) From 42f558bd56b043905e36702af649ae46b2aeec8a Mon Sep 17 00:00:00 2001 From: cui fliter Date: Thu, 27 Apr 2023 20:25:06 +0800 Subject: [PATCH 011/299] all: remove repeated definite articles Change-Id: Idea3e6ca6e62bd5a5ff6e6d5c3f39efb7628f0ec Reviewed-on: https://go-review.googlesource.com/c/go/+/489635 Run-TryBot: Michael Pratt Run-TryBot: shuang cui TryBot-Result: Gopher Robot Reviewed-by: Michael Pratt Reviewed-by: Dmitri Shuralyov Auto-Submit: Michael Pratt --- src/cmd/compile/internal/ssa/_gen/generic.rules | 2 +- src/internal/coverage/decodecounter/decodecounterfile.go | 2 +- src/internal/poll/fd_windows_test.go | 2 +- src/runtime/coverage/apis.go | 2 +- src/runtime/metrics/description.go | 2 +- 5 files changed, 5 insertions(+), 5 deletions(-) diff --git a/src/cmd/compile/internal/ssa/_gen/generic.rules b/src/cmd/compile/internal/ssa/_gen/generic.rules index 175a7456b1..c7a525abb7 100644 --- a/src/cmd/compile/internal/ssa/_gen/generic.rules +++ b/src/cmd/compile/internal/ssa/_gen/generic.rules @@ -2601,7 +2601,7 @@ (SelectN [0] call:(StaticLECall {sym} a x)) && needRaceCleanup(sym, call) && clobber(call) => x (SelectN [0] call:(StaticLECall {sym} x)) && needRaceCleanup(sym, call) && clobber(call) => x -// When rewriting append to growslice, we use as the the new length the result of +// When rewriting append to growslice, we use as the new length the result of // growslice so that we don't have to spill/restore the new length around the growslice call. // The exception here is that if the new length is a constant, avoiding spilling it // is pointless and its constantness is sometimes useful for subsequent optimizations. diff --git a/src/internal/coverage/decodecounter/decodecounterfile.go b/src/internal/coverage/decodecounter/decodecounterfile.go index fce060aaba..83934fe68b 100644 --- a/src/internal/coverage/decodecounter/decodecounterfile.go +++ b/src/internal/coverage/decodecounter/decodecounterfile.go @@ -236,7 +236,7 @@ func (cdr *CounterDataReader) NumSegments() uint32 { return cdr.ftr.NumSegments } -// BeginNextSegment sets up the the reader to read the next segment, +// BeginNextSegment sets up the reader to read the next segment, // returning TRUE if we do have another segment to read, or FALSE // if we're done with all the segments (also an error if // something went wrong). diff --git a/src/internal/poll/fd_windows_test.go b/src/internal/poll/fd_windows_test.go index 9f1db1000e..f0697a0d7b 100644 --- a/src/internal/poll/fd_windows_test.go +++ b/src/internal/poll/fd_windows_test.go @@ -144,7 +144,7 @@ func TestWSASocketConflict(t *testing.T) { t.Fatalf("could not create the event!") } - // Set the low bit of the Event Handle so that the the completion + // Set the low bit of the Event Handle so that the completion // of the overlapped I/O event will not trigger a completion event // on any I/O completion port associated with the handle. ovs[0].HEvent |= 0x1 diff --git a/src/runtime/coverage/apis.go b/src/runtime/coverage/apis.go index 4366ef47ab..05da345ede 100644 --- a/src/runtime/coverage/apis.go +++ b/src/runtime/coverage/apis.go @@ -27,7 +27,7 @@ func WriteMetaDir(dir string) error { // WriteMeta writes the meta-data content (the payload that would // normally be emitted to a meta-data file) for the currently running -// program to the the writer 'w'. An error will be returned if the +// program to the writer 'w'. An error will be returned if the // operation can't be completed successfully (for example, if the // currently running program was not built with "-cover", or if a // write fails). diff --git a/src/runtime/metrics/description.go b/src/runtime/metrics/description.go index ad69d424c2..2d5b0f2195 100644 --- a/src/runtime/metrics/description.go +++ b/src/runtime/metrics/description.go @@ -380,7 +380,7 @@ var allDesc = []Description{ } func init() { - // Insert all the the non-default-reporting GODEBUGs into the table, + // Insert all the non-default-reporting GODEBUGs into the table, // preserving the overall sort order. i := 0 for i < len(allDesc) && allDesc[i].Name < "/godebug/" { From 954ff15dbe5fc722f8965c80934774a3fa2d8c71 Mon Sep 17 00:00:00 2001 From: "Bryan C. Mills" Date: Mon, 24 Apr 2023 13:48:29 -0400 Subject: [PATCH 012/299] cmd/go: skip the 'git' part of get_insecure_redirect in short mode Invoking 'git' adds about 200ms to this test on a fast machine, probably more on a slow one. (As a general habit we skip the 'git' tests uniformly in short mode.) For #52545. Change-Id: Iea6d86a8c9c8b0f1fe51888faf7f5fe7dd8f1eb3 Reviewed-on: https://go-review.googlesource.com/c/go/+/488236 Auto-Submit: Bryan Mills Reviewed-by: Than McIntosh Run-TryBot: Bryan Mills TryBot-Result: Gopher Robot --- src/cmd/go/testdata/script/get_insecure_redirect.txt | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/cmd/go/testdata/script/get_insecure_redirect.txt b/src/cmd/go/testdata/script/get_insecure_redirect.txt index 2a37902215..2e53c5857d 100644 --- a/src/cmd/go/testdata/script/get_insecure_redirect.txt +++ b/src/cmd/go/testdata/script/get_insecure_redirect.txt @@ -8,5 +8,7 @@ env GO111MODULE=off ! go get -d vcs-test.golang.org/insecure/go/insecure stderr 'redirected .* to insecure URL' +[short] stop 'builds a git repo' + env GOINSECURE=vcs-test.golang.org/insecure/go/insecure go get -d vcs-test.golang.org/insecure/go/insecure From 3a7806d387e8dc62a327ce9d2c7a3ea913f1efde Mon Sep 17 00:00:00 2001 From: Guoqi Chen Date: Wed, 26 Apr 2023 11:47:16 +0800 Subject: [PATCH 013/299] cmd/dist,internal: enable buildmode=c-archive for linux/loong64 Now the shared flag is supported on the linux/loong64 platform and misc/cgo/testcarchive has been passed, buildmode=c-archive can be used. Change-Id: Ice450dc11fcb91942fdf2ddd34bb163853267e01 Reviewed-on: https://go-review.googlesource.com/c/go/+/489576 Reviewed-by: Dmitri Shuralyov Reviewed-by: Meidan Li Auto-Submit: Ian Lance Taylor Reviewed-by: WANG Xuerui Reviewed-by: Ian Lance Taylor TryBot-Result: Gopher Robot Run-TryBot: Ian Lance Taylor Run-TryBot: WANG Xuerui --- src/cmd/dist/test.go | 2 +- src/internal/platform/supported.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/cmd/dist/test.go b/src/cmd/dist/test.go index cc96223aa6..31eb69113a 100644 --- a/src/cmd/dist/test.go +++ b/src/cmd/dist/test.go @@ -1694,7 +1694,7 @@ func buildModeSupported(compiler, buildmode, goos, goarch string) bool { return true case "linux": switch goarch { - case "386", "amd64", "arm", "armbe", "arm64", "arm64be", "ppc64le", "riscv64", "s390x": + case "386", "amd64", "arm", "armbe", "arm64", "arm64be", "loong64", "ppc64le", "riscv64", "s390x": // linux/ppc64 not supported because it does // not support external linking mode yet. return true diff --git a/src/internal/platform/supported.go b/src/internal/platform/supported.go index 57a86b054d..8eb0657d4c 100644 --- a/src/internal/platform/supported.go +++ b/src/internal/platform/supported.go @@ -140,7 +140,7 @@ func BuildModeSupported(compiler, buildmode, goos, goarch string) bool { return true case "linux": switch goarch { - case "386", "amd64", "arm", "armbe", "arm64", "arm64be", "ppc64le", "riscv64", "s390x": + case "386", "amd64", "arm", "armbe", "arm64", "arm64be", "loong64", "ppc64le", "riscv64", "s390x": // linux/ppc64 not supported because it does // not support external linking mode yet. return true From 635839a17aeb086b3e1eeba73973aac6ccdb8df2 Mon Sep 17 00:00:00 2001 From: Keith Randall Date: Thu, 20 Apr 2023 08:55:12 -0700 Subject: [PATCH 014/299] cmd/compile: constant-fold loads from constant dictionaries and types Update #59591 Change-Id: Id250a7779c5b53776fff73f3e678fec54d92a8e3 Reviewed-on: https://go-review.googlesource.com/c/go/+/486895 Reviewed-by: Cuong Manh Le Reviewed-by: Dmitri Shuralyov Run-TryBot: Keith Randall Reviewed-by: Matthew Dempsky Auto-Submit: Keith Randall TryBot-Result: Gopher Robot --- src/cmd/compile/internal/noder/reader.go | 2 +- .../compile/internal/reflectdata/reflect.go | 13 +- .../compile/internal/ssa/_gen/generic.rules | 27 +- src/cmd/compile/internal/ssa/rewrite.go | 91 ++++- .../compile/internal/ssa/rewritegeneric.go | 357 ++++++++++++++++-- src/cmd/internal/obj/link.go | 18 +- 6 files changed, 453 insertions(+), 55 deletions(-) diff --git a/src/cmd/compile/internal/noder/reader.go b/src/cmd/compile/internal/noder/reader.go index 6098c92ac9..9a07ff7666 100644 --- a/src/cmd/compile/internal/noder/reader.go +++ b/src/cmd/compile/internal/noder/reader.go @@ -3988,7 +3988,7 @@ func setBasePos(pos src.XPos) { // // N.B., this variable name is known to Delve: // https://github.com/go-delve/delve/blob/cb91509630529e6055be845688fd21eb89ae8714/pkg/proc/eval.go#L28 -const dictParamName = ".dict" +const dictParamName = typecheck.LocalDictName // shapeSig returns a copy of fn's signature, except adding a // dictionary parameter and promoting the receiver parameter (if any) diff --git a/src/cmd/compile/internal/reflectdata/reflect.go b/src/cmd/compile/internal/reflectdata/reflect.go index cde8c68876..73bbff94d8 100644 --- a/src/cmd/compile/internal/reflectdata/reflect.go +++ b/src/cmd/compile/internal/reflectdata/reflect.go @@ -844,7 +844,14 @@ func TypeLinksymLookup(name string) *obj.LSym { } func TypeLinksym(t *types.Type) *obj.LSym { - return TypeSym(t).Linksym() + lsym := TypeSym(t).Linksym() + signatmu.Lock() + if lsym.Extra == nil { + ti := lsym.NewTypeInfo() + ti.Type = t + } + signatmu.Unlock() + return lsym } // Deprecated: Use TypePtrAt instead. @@ -1878,7 +1885,9 @@ func MarkTypeUsedInInterface(t *types.Type, from *obj.LSym) { // Shape types shouldn't be put in interfaces, so we shouldn't ever get here. base.Fatalf("shape types have no methods %+v", t) } - tsym := TypeLinksym(t) + MarkTypeSymUsedInInterface(TypeLinksym(t), from) +} +func MarkTypeSymUsedInInterface(tsym *obj.LSym, from *obj.LSym) { // Emit a marker relocation. The linker will know the type is converted // to an interface if "from" is reachable. r := obj.Addrel(from) diff --git a/src/cmd/compile/internal/ssa/_gen/generic.rules b/src/cmd/compile/internal/ssa/_gen/generic.rules index c7a525abb7..bea743e94a 100644 --- a/src/cmd/compile/internal/ssa/_gen/generic.rules +++ b/src/cmd/compile/internal/ssa/_gen/generic.rules @@ -2061,6 +2061,10 @@ && warnRule(fe.Debug_checknil(), v, "removed nil check") => (Invalid) +// Addresses of globals are always non-nil. +(NilCheck (Addr {_} (SB)) _) => (Invalid) +(NilCheck (Convert (Addr {_} (SB)) _) _) => (Invalid) + // for late-expanded calls, recognize memequal applied to a single constant byte // Support is limited by 1, 2, 4, 8 byte sizes (StaticLECall {callAux} sptr (Addr {scon} (SB)) (Const64 [1]) mem) @@ -2148,6 +2152,8 @@ (NeqPtr (OffPtr [o1] p1) (OffPtr [o2] p2)) && isSamePtr(p1, p2) => (ConstBool [o1 != o2]) (EqPtr (Const(32|64) [c]) (Const(32|64) [d])) => (ConstBool [c == d]) (NeqPtr (Const(32|64) [c]) (Const(32|64) [d])) => (ConstBool [c != d]) +(EqPtr (Convert (Addr {x} _) _) (Addr {y} _)) => (ConstBool [x==y]) +(NeqPtr (Convert (Addr {x} _) _) (Addr {y} _)) => (ConstBool [x!=y]) (EqPtr (LocalAddr _ _) (Addr _)) => (ConstBool [false]) (EqPtr (OffPtr (LocalAddr _ _)) (Addr _)) => (ConstBool [false]) @@ -2169,7 +2175,8 @@ // Evaluate constant user nil checks. (IsNonNil (ConstNil)) => (ConstBool [false]) (IsNonNil (Const(32|64) [c])) => (ConstBool [c != 0]) -(IsNonNil (Addr _)) => (ConstBool [true]) +(IsNonNil (Addr _) ) => (ConstBool [true]) +(IsNonNil (Convert (Addr _) _)) => (ConstBool [true]) (IsNonNil (LocalAddr _ _)) => (ConstBool [true]) // Inline small or disjoint runtime.memmove calls with constant length. @@ -2212,11 +2219,7 @@ => (Move {types.Types[types.TUINT8]} [int64(sz)] dst src mem) // De-virtualize late-expanded interface calls into late-expanded static calls. -// Note that (ITab (IMake)) doesn't get rewritten until after the first opt pass, -// so this rule should trigger reliably. -// devirtLECall removes the first argument, adds the devirtualized symbol to the AuxCall, and changes the opcode -(InterLECall [argsize] {auxCall} (Load (OffPtr [off] (ITab (IMake (Addr {itab} (SB)) _))) _) ___) && devirtLESym(v, auxCall, itab, off) != - nil => devirtLECall(v, devirtLESym(v, auxCall, itab, off)) +(InterLECall [argsize] {auxCall} (Addr {fn} (SB)) ___) => devirtLECall(v, fn.(*obj.LSym)) // Move and Zero optimizations. // Move source and destination may overlap. @@ -2730,3 +2733,15 @@ (RotateLeft(64|32|16|8) (RotateLeft(64|32|16|8) x c) d) && c.Type.Size() == 4 && d.Type.Size() == 4 => (RotateLeft(64|32|16|8) x (Add32 c d)) (RotateLeft(64|32|16|8) (RotateLeft(64|32|16|8) x c) d) && c.Type.Size() == 2 && d.Type.Size() == 2 => (RotateLeft(64|32|16|8) x (Add16 c d)) (RotateLeft(64|32|16|8) (RotateLeft(64|32|16|8) x c) d) && c.Type.Size() == 1 && d.Type.Size() == 1 => (RotateLeft(64|32|16|8) x (Add8 c d)) + +// Loading constant values from dictionaries and itabs. +(Load (OffPtr [off] (Addr {s} sb) ) _) && t.IsUintptr() && isFixedSym(s, off) => (Addr {fixedSym(b.Func, s, off)} sb) +(Load (OffPtr [off] (Convert (Addr {s} sb) _) ) _) && t.IsUintptr() && isFixedSym(s, off) => (Addr {fixedSym(b.Func, s, off)} sb) +(Load (OffPtr [off] (ITab (IMake (Addr {s} sb) _))) _) && t.IsUintptr() && isFixedSym(s, off) => (Addr {fixedSym(b.Func, s, off)} sb) +(Load (OffPtr [off] (ITab (IMake (Convert (Addr {s} sb) _) _))) _) && t.IsUintptr() && isFixedSym(s, off) => (Addr {fixedSym(b.Func, s, off)} sb) + +// Loading constant values from runtime._type.hash. +(Load (OffPtr [off] (Addr {sym} _) ) _) && t.IsInteger() && t.Size() == 4 && isFixed32(config, sym, off) => (Const32 [fixed32(config, sym, off)]) +(Load (OffPtr [off] (Convert (Addr {sym} _) _) ) _) && t.IsInteger() && t.Size() == 4 && isFixed32(config, sym, off) => (Const32 [fixed32(config, sym, off)]) +(Load (OffPtr [off] (ITab (IMake (Addr {sym} _) _))) _) && t.IsInteger() && t.Size() == 4 && isFixed32(config, sym, off) => (Const32 [fixed32(config, sym, off)]) +(Load (OffPtr [off] (ITab (IMake (Convert (Addr {sym} _) _) _))) _) && t.IsInteger() && t.Size() == 4 && isFixed32(config, sym, off) => (Const32 [fixed32(config, sym, off)]) diff --git a/src/cmd/compile/internal/ssa/rewrite.go b/src/cmd/compile/internal/ssa/rewrite.go index 58813d2fbe..d16fee0ba2 100644 --- a/src/cmd/compile/internal/ssa/rewrite.go +++ b/src/cmd/compile/internal/ssa/rewrite.go @@ -7,6 +7,7 @@ package ssa import ( "cmd/compile/internal/base" "cmd/compile/internal/logopt" + "cmd/compile/internal/reflectdata" "cmd/compile/internal/types" "cmd/internal/obj" "cmd/internal/obj/s390x" @@ -19,6 +20,7 @@ import ( "math/bits" "os" "path/filepath" + "strings" ) type deadValueChoice bool @@ -799,25 +801,6 @@ func loadLSymOffset(lsym *obj.LSym, offset int64) *obj.LSym { return nil } -// de-virtualize an InterLECall -// 'sym' is the symbol for the itab. -func devirtLESym(v *Value, aux Aux, sym Sym, offset int64) *obj.LSym { - n, ok := sym.(*obj.LSym) - if !ok { - return nil - } - - lsym := loadLSymOffset(n, offset) - if f := v.Block.Func; f.pass.debug > 0 { - if lsym != nil { - f.Warnl(v.Pos, "de-virtualizing call") - } else { - f.Warnl(v.Pos, "couldn't de-virtualize call") - } - } - return lsym -} - func devirtLECall(v *Value, sym *obj.LSym) *Value { v.Op = OpStaticLECall auxcall := v.Aux.(*AuxCall) @@ -827,6 +810,9 @@ func devirtLECall(v *Value, sym *obj.LSym) *Value { copy(v.Args[0:], v.Args[1:]) v.Args[len(v.Args)-1] = nil // aid GC v.Args = v.Args[:len(v.Args)-1] + if f := v.Block.Func; f.pass.debug > 0 { + f.Warnl(v.Pos, "de-virtualizing call") + } return v } @@ -1743,6 +1729,73 @@ func symIsROZero(sym Sym) bool { return true } +// isFixed32 returns true if the int32 at offset off in symbol sym +// is known and constant. +func isFixed32(c *Config, sym Sym, off int64) bool { + return isFixed(c, sym, off, 4) +} + +// isFixed returns true if the range [off,off+size] of the symbol sym +// is known and constant. +func isFixed(c *Config, sym Sym, off, size int64) bool { + lsym := sym.(*obj.LSym) + if lsym.Extra == nil { + return false + } + if _, ok := (*lsym.Extra).(*obj.TypeInfo); ok { + if off == 2*c.PtrSize && size == 4 { + return true // type hash field + } + } + return false +} +func fixed32(c *Config, sym Sym, off int64) int32 { + lsym := sym.(*obj.LSym) + if ti, ok := (*lsym.Extra).(*obj.TypeInfo); ok { + if off == 2*c.PtrSize { + return int32(types.TypeHash(ti.Type.(*types.Type))) + } + } + base.Fatalf("fixed32 data not known for %s:%d", sym, off) + return 0 +} + +// isFixedSym returns true if the contents of sym at the given offset +// is known and is the constant address of another symbol. +func isFixedSym(sym Sym, off int64) bool { + lsym := sym.(*obj.LSym) + switch { + case lsym.Type == objabi.SRODATA: + // itabs, dictionaries + default: + return false + } + for _, r := range lsym.R { + if (r.Type == objabi.R_ADDR || r.Type == objabi.R_WEAKADDR) && int64(r.Off) == off && r.Add == 0 { + return true + } + } + return false +} +func fixedSym(f *Func, sym Sym, off int64) Sym { + lsym := sym.(*obj.LSym) + for _, r := range lsym.R { + if (r.Type == objabi.R_ADDR || r.Type == objabi.R_WEAKADDR) && int64(r.Off) == off { + if strings.HasPrefix(r.Sym.Name, "type:") { + // In case we're loading a type out of a dictionary, we need to record + // that the containing function might put that type in an interface. + // That information is currently recorded in relocations in the dictionary, + // but if we perform this load at compile time then the dictionary + // might be dead. + reflectdata.MarkTypeSymUsedInInterface(r.Sym, f.fe.Func().Linksym()) + } + return r.Sym + } + } + base.Fatalf("fixedSym data not known for %s:%d", sym, off) + return nil +} + // read8 reads one byte from the read-only global sym at offset off. func read8(sym interface{}, off int64) uint8 { lsym := sym.(*obj.LSym) diff --git a/src/cmd/compile/internal/ssa/rewritegeneric.go b/src/cmd/compile/internal/ssa/rewritegeneric.go index 6026eac279..b9da2a6135 100644 --- a/src/cmd/compile/internal/ssa/rewritegeneric.go +++ b/src/cmd/compile/internal/ssa/rewritegeneric.go @@ -3,6 +3,7 @@ package ssa import "math" +import "cmd/internal/obj" import "cmd/compile/internal/types" import "cmd/compile/internal/ir" @@ -10117,6 +10118,28 @@ func rewriteValuegeneric_OpEqPtr(v *Value) bool { } break } + // match: (EqPtr (Convert (Addr {x} _) _) (Addr {y} _)) + // result: (ConstBool [x==y]) + for { + for _i0 := 0; _i0 <= 1; _i0, v_0, v_1 = _i0+1, v_1, v_0 { + if v_0.Op != OpConvert { + continue + } + v_0_0 := v_0.Args[0] + if v_0_0.Op != OpAddr { + continue + } + x := auxToSym(v_0_0.Aux) + if v_1.Op != OpAddr { + continue + } + y := auxToSym(v_1.Aux) + v.reset(OpConstBool) + v.AuxInt = boolToAuxInt(x == y) + return true + } + break + } // match: (EqPtr (LocalAddr _ _) (Addr _)) // result: (ConstBool [false]) for { @@ -10321,41 +10344,22 @@ func rewriteValuegeneric_OpIMake(v *Value) bool { return false } func rewriteValuegeneric_OpInterLECall(v *Value) bool { - // match: (InterLECall [argsize] {auxCall} (Load (OffPtr [off] (ITab (IMake (Addr {itab} (SB)) _))) _) ___) - // cond: devirtLESym(v, auxCall, itab, off) != nil - // result: devirtLECall(v, devirtLESym(v, auxCall, itab, off)) + // match: (InterLECall [argsize] {auxCall} (Addr {fn} (SB)) ___) + // result: devirtLECall(v, fn.(*obj.LSym)) for { if len(v.Args) < 1 { break } - auxCall := auxToCall(v.Aux) v_0 := v.Args[0] - if v_0.Op != OpLoad { + if v_0.Op != OpAddr { break } + fn := auxToSym(v_0.Aux) v_0_0 := v_0.Args[0] - if v_0_0.Op != OpOffPtr { + if v_0_0.Op != OpSB { break } - off := auxIntToInt64(v_0_0.AuxInt) - v_0_0_0 := v_0_0.Args[0] - if v_0_0_0.Op != OpITab { - break - } - v_0_0_0_0 := v_0_0_0.Args[0] - if v_0_0_0_0.Op != OpIMake { - break - } - v_0_0_0_0_0 := v_0_0_0_0.Args[0] - if v_0_0_0_0_0.Op != OpAddr { - break - } - itab := auxToSym(v_0_0_0_0_0.Aux) - v_0_0_0_0_0_0 := v_0_0_0_0_0.Args[0] - if v_0_0_0_0_0_0.Op != OpSB || !(devirtLESym(v, auxCall, itab, off) != nil) { - break - } - v.copyOf(devirtLECall(v, devirtLESym(v, auxCall, itab, off))) + v.copyOf(devirtLECall(v, fn.(*obj.LSym))) return true } return false @@ -11086,7 +11090,7 @@ func rewriteValuegeneric_OpIsNonNil(v *Value) bool { v.AuxInt = boolToAuxInt(c != 0) return true } - // match: (IsNonNil (Addr _)) + // match: (IsNonNil (Addr _) ) // result: (ConstBool [true]) for { if v_0.Op != OpAddr { @@ -11096,6 +11100,30 @@ func rewriteValuegeneric_OpIsNonNil(v *Value) bool { v.AuxInt = boolToAuxInt(true) return true } + // match: (IsNonNil (Convert (Addr _) _)) + // result: (ConstBool [true]) + for { + if v_0.Op != OpConvert { + break + } + v_0_0 := v_0.Args[0] + if v_0_0.Op != OpAddr { + break + } + v.reset(OpConstBool) + v.AuxInt = boolToAuxInt(true) + return true + } + // match: (IsNonNil (LocalAddr _ _)) + // result: (ConstBool [true]) + for { + if v_0.Op != OpLocalAddr { + break + } + v.reset(OpConstBool) + v.AuxInt = boolToAuxInt(true) + return true + } return false } func rewriteValuegeneric_OpIsSliceInBounds(v *Value) bool { @@ -12472,6 +12500,7 @@ func rewriteValuegeneric_OpLoad(v *Value) bool { v_1 := v.Args[1] v_0 := v.Args[0] b := v.Block + config := b.Func.Config fe := b.Func.fe // match: (Load p1 (Store {t2} p2 x _)) // cond: isSamePtr(p1, p2) && t1.Compare(x.Type) == types.CMPeq && t1.Size() == t2.Size() @@ -13163,6 +13192,230 @@ func rewriteValuegeneric_OpLoad(v *Value) bool { v.AddArg(v0) return true } + // match: (Load (OffPtr [off] (Addr {s} sb) ) _) + // cond: t.IsUintptr() && isFixedSym(s, off) + // result: (Addr {fixedSym(b.Func, s, off)} sb) + for { + t := v.Type + if v_0.Op != OpOffPtr { + break + } + off := auxIntToInt64(v_0.AuxInt) + v_0_0 := v_0.Args[0] + if v_0_0.Op != OpAddr { + break + } + s := auxToSym(v_0_0.Aux) + sb := v_0_0.Args[0] + if !(t.IsUintptr() && isFixedSym(s, off)) { + break + } + v.reset(OpAddr) + v.Aux = symToAux(fixedSym(b.Func, s, off)) + v.AddArg(sb) + return true + } + // match: (Load (OffPtr [off] (Convert (Addr {s} sb) _) ) _) + // cond: t.IsUintptr() && isFixedSym(s, off) + // result: (Addr {fixedSym(b.Func, s, off)} sb) + for { + t := v.Type + if v_0.Op != OpOffPtr { + break + } + off := auxIntToInt64(v_0.AuxInt) + v_0_0 := v_0.Args[0] + if v_0_0.Op != OpConvert { + break + } + v_0_0_0 := v_0_0.Args[0] + if v_0_0_0.Op != OpAddr { + break + } + s := auxToSym(v_0_0_0.Aux) + sb := v_0_0_0.Args[0] + if !(t.IsUintptr() && isFixedSym(s, off)) { + break + } + v.reset(OpAddr) + v.Aux = symToAux(fixedSym(b.Func, s, off)) + v.AddArg(sb) + return true + } + // match: (Load (OffPtr [off] (ITab (IMake (Addr {s} sb) _))) _) + // cond: t.IsUintptr() && isFixedSym(s, off) + // result: (Addr {fixedSym(b.Func, s, off)} sb) + for { + t := v.Type + if v_0.Op != OpOffPtr { + break + } + off := auxIntToInt64(v_0.AuxInt) + v_0_0 := v_0.Args[0] + if v_0_0.Op != OpITab { + break + } + v_0_0_0 := v_0_0.Args[0] + if v_0_0_0.Op != OpIMake { + break + } + v_0_0_0_0 := v_0_0_0.Args[0] + if v_0_0_0_0.Op != OpAddr { + break + } + s := auxToSym(v_0_0_0_0.Aux) + sb := v_0_0_0_0.Args[0] + if !(t.IsUintptr() && isFixedSym(s, off)) { + break + } + v.reset(OpAddr) + v.Aux = symToAux(fixedSym(b.Func, s, off)) + v.AddArg(sb) + return true + } + // match: (Load (OffPtr [off] (ITab (IMake (Convert (Addr {s} sb) _) _))) _) + // cond: t.IsUintptr() && isFixedSym(s, off) + // result: (Addr {fixedSym(b.Func, s, off)} sb) + for { + t := v.Type + if v_0.Op != OpOffPtr { + break + } + off := auxIntToInt64(v_0.AuxInt) + v_0_0 := v_0.Args[0] + if v_0_0.Op != OpITab { + break + } + v_0_0_0 := v_0_0.Args[0] + if v_0_0_0.Op != OpIMake { + break + } + v_0_0_0_0 := v_0_0_0.Args[0] + if v_0_0_0_0.Op != OpConvert { + break + } + v_0_0_0_0_0 := v_0_0_0_0.Args[0] + if v_0_0_0_0_0.Op != OpAddr { + break + } + s := auxToSym(v_0_0_0_0_0.Aux) + sb := v_0_0_0_0_0.Args[0] + if !(t.IsUintptr() && isFixedSym(s, off)) { + break + } + v.reset(OpAddr) + v.Aux = symToAux(fixedSym(b.Func, s, off)) + v.AddArg(sb) + return true + } + // match: (Load (OffPtr [off] (Addr {sym} _) ) _) + // cond: t.IsInteger() && t.Size() == 4 && isFixed32(config, sym, off) + // result: (Const32 [fixed32(config, sym, off)]) + for { + t := v.Type + if v_0.Op != OpOffPtr { + break + } + off := auxIntToInt64(v_0.AuxInt) + v_0_0 := v_0.Args[0] + if v_0_0.Op != OpAddr { + break + } + sym := auxToSym(v_0_0.Aux) + if !(t.IsInteger() && t.Size() == 4 && isFixed32(config, sym, off)) { + break + } + v.reset(OpConst32) + v.AuxInt = int32ToAuxInt(fixed32(config, sym, off)) + return true + } + // match: (Load (OffPtr [off] (Convert (Addr {sym} _) _) ) _) + // cond: t.IsInteger() && t.Size() == 4 && isFixed32(config, sym, off) + // result: (Const32 [fixed32(config, sym, off)]) + for { + t := v.Type + if v_0.Op != OpOffPtr { + break + } + off := auxIntToInt64(v_0.AuxInt) + v_0_0 := v_0.Args[0] + if v_0_0.Op != OpConvert { + break + } + v_0_0_0 := v_0_0.Args[0] + if v_0_0_0.Op != OpAddr { + break + } + sym := auxToSym(v_0_0_0.Aux) + if !(t.IsInteger() && t.Size() == 4 && isFixed32(config, sym, off)) { + break + } + v.reset(OpConst32) + v.AuxInt = int32ToAuxInt(fixed32(config, sym, off)) + return true + } + // match: (Load (OffPtr [off] (ITab (IMake (Addr {sym} _) _))) _) + // cond: t.IsInteger() && t.Size() == 4 && isFixed32(config, sym, off) + // result: (Const32 [fixed32(config, sym, off)]) + for { + t := v.Type + if v_0.Op != OpOffPtr { + break + } + off := auxIntToInt64(v_0.AuxInt) + v_0_0 := v_0.Args[0] + if v_0_0.Op != OpITab { + break + } + v_0_0_0 := v_0_0.Args[0] + if v_0_0_0.Op != OpIMake { + break + } + v_0_0_0_0 := v_0_0_0.Args[0] + if v_0_0_0_0.Op != OpAddr { + break + } + sym := auxToSym(v_0_0_0_0.Aux) + if !(t.IsInteger() && t.Size() == 4 && isFixed32(config, sym, off)) { + break + } + v.reset(OpConst32) + v.AuxInt = int32ToAuxInt(fixed32(config, sym, off)) + return true + } + // match: (Load (OffPtr [off] (ITab (IMake (Convert (Addr {sym} _) _) _))) _) + // cond: t.IsInteger() && t.Size() == 4 && isFixed32(config, sym, off) + // result: (Const32 [fixed32(config, sym, off)]) + for { + t := v.Type + if v_0.Op != OpOffPtr { + break + } + off := auxIntToInt64(v_0.AuxInt) + v_0_0 := v_0.Args[0] + if v_0_0.Op != OpITab { + break + } + v_0_0_0 := v_0_0.Args[0] + if v_0_0_0.Op != OpIMake { + break + } + v_0_0_0_0 := v_0_0_0.Args[0] + if v_0_0_0_0.Op != OpConvert { + break + } + v_0_0_0_0_0 := v_0_0_0_0.Args[0] + if v_0_0_0_0_0.Op != OpAddr { + break + } + sym := auxToSym(v_0_0_0_0_0.Aux) + if !(t.IsInteger() && t.Size() == 4 && isFixed32(config, sym, off)) { + break + } + v.reset(OpConst32) + v.AuxInt = int32ToAuxInt(fixed32(config, sym, off)) + return true + } return false } func rewriteValuegeneric_OpLsh16x16(v *Value) bool { @@ -18452,6 +18705,28 @@ func rewriteValuegeneric_OpNeqPtr(v *Value) bool { } break } + // match: (NeqPtr (Convert (Addr {x} _) _) (Addr {y} _)) + // result: (ConstBool [x!=y]) + for { + for _i0 := 0; _i0 <= 1; _i0, v_0, v_1 = _i0+1, v_1, v_0 { + if v_0.Op != OpConvert { + continue + } + v_0_0 := v_0.Args[0] + if v_0_0.Op != OpAddr { + continue + } + x := auxToSym(v_0_0.Aux) + if v_1.Op != OpAddr { + continue + } + y := auxToSym(v_1.Aux) + v.reset(OpConstBool) + v.AuxInt = boolToAuxInt(x != y) + return true + } + break + } // match: (NeqPtr (LocalAddr _ _) (Addr _)) // result: (ConstBool [true]) for { @@ -18653,6 +18928,36 @@ func rewriteValuegeneric_OpNilCheck(v *Value) bool { v.reset(OpInvalid) return true } + // match: (NilCheck (Addr {_} (SB)) _) + // result: (Invalid) + for { + if v_0.Op != OpAddr { + break + } + v_0_0 := v_0.Args[0] + if v_0_0.Op != OpSB { + break + } + v.reset(OpInvalid) + return true + } + // match: (NilCheck (Convert (Addr {_} (SB)) _) _) + // result: (Invalid) + for { + if v_0.Op != OpConvert { + break + } + v_0_0 := v_0.Args[0] + if v_0_0.Op != OpAddr { + break + } + v_0_0_0 := v_0_0.Args[0] + if v_0_0_0.Op != OpSB { + break + } + v.reset(OpInvalid) + return true + } return false } func rewriteValuegeneric_OpNot(v *Value) bool { diff --git a/src/cmd/internal/obj/link.go b/src/cmd/internal/obj/link.go index b50305f85c..74eb98520a 100644 --- a/src/cmd/internal/obj/link.go +++ b/src/cmd/internal/obj/link.go @@ -465,7 +465,7 @@ type LSym struct { P []byte R []Reloc - Extra *interface{} // *FuncInfo or *FileInfo, if present + Extra *interface{} // *FuncInfo, *FileInfo, or *TypeInfo, if present Pkg string PkgIdx int32 @@ -564,6 +564,22 @@ func (s *LSym) File() *FileInfo { return f } +// A TypeInfo contains information for a symbol +// that contains a runtime._type. +type TypeInfo struct { + Type interface{} // a *cmd/compile/internal/types.Type +} + +func (s *LSym) NewTypeInfo() *TypeInfo { + if s.Extra != nil { + panic(fmt.Sprintf("invalid use of LSym - NewTypeInfo with Extra of type %T", *s.Extra)) + } + t := new(TypeInfo) + s.Extra = new(interface{}) + *s.Extra = t + return t +} + // WasmImport represents a WebAssembly (WASM) imported function with // parameters and results translated into WASM types based on the Go function // declaration. From afe2d2221969c4c7082afdd4791dd8f2540fc684 Mon Sep 17 00:00:00 2001 From: Ian Lance Taylor Date: Thu, 20 Apr 2023 16:55:13 -0700 Subject: [PATCH 015/299] net: rewrite and simplify resolver configuration The resulting code behaves mostly the same. There are some minor differences in error cases when the cgo resolver is not available: instead of just falling back we keep trying to work out the right nsswitch.conf order. Change-Id: I17fadc940528fa2397043ac8f8ed7da3bd7a95c0 Reviewed-on: https://go-review.googlesource.com/c/go/+/487196 Reviewed-by: Damien Neil Run-TryBot: Ian Lance Taylor TryBot-Result: Gopher Robot Auto-Submit: Ian Lance Taylor Reviewed-by: Ian Lance Taylor Run-TryBot: Ian Lance Taylor Reviewed-by: Mateusz Poliwczak --- .../script/list_cgo_compiled_importmap.txt | 5 +- src/net/cgo_stub.go | 16 +- src/net/cgo_unix.go | 7 + src/net/cgo_windows.go | 13 - src/net/conf.go | 367 ++++++++++++------ src/net/conf_netcgo.go | 18 - src/net/conf_test.go | 49 ++- src/net/error_test.go | 8 +- src/net/lookup_plan9.go | 5 + src/net/lookup_test.go | 2 +- src/net/lookup_unix.go | 10 +- src/net/lookup_windows.go | 5 + src/net/net.go | 8 - src/net/netcgo_off.go | 9 + src/net/netcgo_on.go | 9 + src/net/netgo.go | 13 - src/net/netgo_netcgo.go | 14 + src/net/netgo_off.go | 9 + src/net/netgo_on.go | 9 + 19 files changed, 385 insertions(+), 191 deletions(-) delete mode 100644 src/net/cgo_windows.go delete mode 100644 src/net/conf_netcgo.go create mode 100644 src/net/netcgo_off.go create mode 100644 src/net/netcgo_on.go delete mode 100644 src/net/netgo.go create mode 100644 src/net/netgo_netcgo.go create mode 100644 src/net/netgo_off.go create mode 100644 src/net/netgo_on.go diff --git a/src/cmd/go/testdata/script/list_cgo_compiled_importmap.txt b/src/cmd/go/testdata/script/list_cgo_compiled_importmap.txt index 30effb104b..869333986c 100644 --- a/src/cmd/go/testdata/script/list_cgo_compiled_importmap.txt +++ b/src/cmd/go/testdata/script/list_cgo_compiled_importmap.txt @@ -7,9 +7,12 @@ [short] skip # -compiled can be slow (because it compiles things) [!cgo] skip +[GOOS:darwin] skip # net package does not import "C" on Darwin +[GOOS:windows] skip # net package does not import "C" on Windows +[GOOS:plan9] skip # net package does not import "C" on Plan 9 env CGO_ENABLED=1 -env GOFLAGS=-tags=netcgo # Force net to use cgo even on Windows. +env GOFLAGS=-tags=netcgo # Force net to use cgo # "runtime/cgo [runtime.test]" appears in the test dependencies of "runtime", diff --git a/src/net/cgo_stub.go b/src/net/cgo_stub.go index c901d4bb80..96d5dc8e25 100644 --- a/src/net/cgo_stub.go +++ b/src/net/cgo_stub.go @@ -2,17 +2,21 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. -//go:build (!cgo && !darwin) || netgo +// This file holds stub versions of the cgo functions called on Unix systems. +// We build this file if using the netgo build tag, or if cgo is not +// enabled and we are using a Unix system other than Darwin. +// Darwin is exempted because it always provides the cgo routines, +// in cgo_unix_syscall.go. + +//go:build netgo || (!cgo && unix && !darwin) package net import "context" -type addrinfoErrno int - -func (eai addrinfoErrno) Error() string { return "" } -func (eai addrinfoErrno) Temporary() bool { return false } -func (eai addrinfoErrno) Timeout() bool { return false } +// cgoAvailable set to false to indicate that the cgo resolver +// is not available on this system. +const cgoAvailable = false func cgoLookupHost(ctx context.Context, name string) (addrs []string, err error, completed bool) { return nil, nil, false diff --git a/src/net/cgo_unix.go b/src/net/cgo_unix.go index de6a64b23b..62b4f23367 100644 --- a/src/net/cgo_unix.go +++ b/src/net/cgo_unix.go @@ -21,6 +21,10 @@ import ( "golang.org/x/net/dns/dnsmessage" ) +// cgoAvailable set to true to indicate that the cgo resolver +// is available on this system. +const cgoAvailable = true + // An addrinfoErrno represents a getaddrinfo, getnameinfo-specific // error number. It's a signed number and a zero value is a non-error // by convention. @@ -30,6 +34,9 @@ func (eai addrinfoErrno) Error() string { return _C_gai_strerror(_C_int(eai)) func (eai addrinfoErrno) Temporary() bool { return eai == _C_EAI_AGAIN } func (eai addrinfoErrno) Timeout() bool { return false } +// isAddrinfoErrno is just for testing purposes. +func (eai addrinfoErrno) isAddrinfoErrno() {} + // doBlockingWithCtx executes a blocking function in a separate goroutine when the provided // context is cancellable. It is intended for use with calls that don't support context // cancellation (cgo, syscalls). blocking func may still be running after this function finishes. diff --git a/src/net/cgo_windows.go b/src/net/cgo_windows.go deleted file mode 100644 index 6bb6cbbb2f..0000000000 --- a/src/net/cgo_windows.go +++ /dev/null @@ -1,13 +0,0 @@ -// Copyright 2015 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -//go:build cgo && !netgo - -package net - -type addrinfoErrno int - -func (eai addrinfoErrno) Error() string { return "" } -func (eai addrinfoErrno) Temporary() bool { return false } -func (eai addrinfoErrno) Timeout() bool { return false } diff --git a/src/net/conf.go b/src/net/conf.go index 8a4ee935c6..2540ac5261 100644 --- a/src/net/conf.go +++ b/src/net/conf.go @@ -7,29 +7,70 @@ package net import ( + "errors" "internal/bytealg" "internal/godebug" + "io/fs" "os" "runtime" "sync" "syscall" ) -// conf represents a system's network configuration. +// The net package's name resolution is rather complicated. +// There are two main approaches, go and cgo. +// The cgo resolver uses C functions like getaddrinfo. +// The go resolver reads system files directly and +// sends DNS packets directly to servers. +// +// The netgo build tag prefers the go resolver. +// The netcgo build tag prefers the cgo resolver. +// +// The netgo build tag also prohibits the use of the cgo tool. +// However, on Darwin, Plan 9, and Windows the cgo resolver is still available. +// On those systems the cgo resolver does not require the cgo tool. +// (The term "cgo resolver" was locked in by GODEBUG settings +// at a time when the cgo resolver did require the cgo tool.) +// +// Adding netdns=go to GODEBUG will prefer the go resolver. +// Adding netdns=cgo to GODEBUG will prefer the cgo resolver. +// +// The Resolver struct has a PreferGo field that user code +// may set to prefer the go resolver. It is documented as being +// equivalent to adding netdns=go to GODEBUG. +// +// When deciding which resolver to use, we first check the PreferGo field. +// If that is not set, we check the GODEBUG setting. +// If that is not set, we check the netgo or netcgo build tag. +// If none of those are set, we normally prefer the go resolver by default. +// However, if the cgo resolver is available, +// there is a complex set of conditions for which we prefer the cgo resolver. +// +// Other files define the netGoBuildTag, netCgoBuildTag, and cgoAvailable +// constants. + +// conf is used to determine name resolution configuration. type conf struct { - // forceCgoLookupHost forces CGO to always be used, if available. - forceCgoLookupHost bool + netGo bool // prefer go approach, based on build tag and GODEBUG + netCgo bool // prefer cgo approach, based on build tag and GODEBUG - netGo bool // go DNS resolution forced - netCgo bool // non-go DNS resolution forced (cgo, or win32) + dnsDebugLevel int // from GODEBUG - // machine has an /etc/mdns.allow file - hasMDNSAllow bool + preferCgo bool // if no explicit preference, use cgo - goos string // the runtime.GOOS, to ease testing - dnsDebugLevel int + goos string // copy of runtime.GOOS, used for testing + mdnsTest mdnsTest // assume /etc/mdns.allow exists, for testing } +// mdnsTest is for testing only. +type mdnsTest int + +const ( + mdnsFromSystem mdnsTest = iota + mdnsAssumeExists + mdnsAssumeDoesNotExist +) + var ( confOnce sync.Once // guards init of confVal via initConfVal confVal = &conf{goos: runtime.GOOS} @@ -41,22 +82,13 @@ func systemConf() *conf { return confVal } +// initConfVal initializes confVal based on the environment +// that will not change during program execution. func initConfVal() { dnsMode, debugLevel := goDebugNetDNS() + confVal.netGo = netGoBuildTag || dnsMode == "go" + confVal.netCgo = netCgoBuildTag || dnsMode == "cgo" confVal.dnsDebugLevel = debugLevel - confVal.netGo = netGo || dnsMode == "go" - confVal.netCgo = netCgo || dnsMode == "cgo" - if !confVal.netGo && !confVal.netCgo && (runtime.GOOS == "windows" || runtime.GOOS == "plan9") { - // Neither of these platforms actually use cgo. - // - // The meaning of "cgo" mode in the net package is - // really "the native OS way", which for libc meant - // cgo on the original platforms that motivated - // PreferGo support before Windows and Plan9 got support, - // at which time the GODEBUG=netdns=go and GODEBUG=netdns=cgo - // names were already kinda locked in. - confVal.netCgo = true - } if confVal.dnsDebugLevel > 0 { defer func() { @@ -65,12 +97,14 @@ func initConfVal() { } switch { case confVal.netGo: - if netGo { + if netGoBuildTag { println("go package net: built with netgo build tag; using Go's DNS resolver") } else { println("go package net: GODEBUG setting forcing use of Go's resolver") } - case confVal.forceCgoLookupHost: + case !cgoAvailable: + println("go package net: cgo resolver not supported; using Go's DNS resolver") + case confVal.netCgo || confVal.preferCgo: println("go package net: using cgo DNS resolver") default: println("go package net: dynamic selection of DNS resolver") @@ -78,60 +112,100 @@ func initConfVal() { }() } - // Darwin pops up annoying dialog boxes if programs try to do - // their own DNS requests. So always use cgo instead, which - // avoids that. - if runtime.GOOS == "darwin" || runtime.GOOS == "ios" { - confVal.forceCgoLookupHost = true + // The remainder of this function sets preferCgo based on + // conditions that will not change during program execution. + + // By default, prefer the go resolver. + confVal.preferCgo = false + + // If the cgo resolver is not available, we can't prefer it. + if !cgoAvailable { return } - if runtime.GOOS == "windows" || runtime.GOOS == "plan9" { + // Some operating systems always prefer the cgo resolver. + if goosPrefersCgo(runtime.GOOS) { + confVal.preferCgo = true + return + } + + // The remaining checks are specific to Unix systems. + switch runtime.GOOS { + case "plan9", "windows", "js", "wasip1": return } // If any environment-specified resolver options are specified, - // force cgo. Note that LOCALDOMAIN can change behavior merely - // by being specified with the empty string. + // prefer the cgo resolver. + // Note that LOCALDOMAIN can change behavior merely by being + // specified with the empty string. _, localDomainDefined := syscall.Getenv("LOCALDOMAIN") - if os.Getenv("RES_OPTIONS") != "" || - os.Getenv("HOSTALIASES") != "" || - confVal.netCgo || - localDomainDefined { - confVal.forceCgoLookupHost = true + if localDomainDefined || os.Getenv("RES_OPTIONS") != "" || os.Getenv("HOSTALIASES") != "" { + confVal.preferCgo = true return } // OpenBSD apparently lets you override the location of resolv.conf // with ASR_CONFIG. If we notice that, defer to libc. if runtime.GOOS == "openbsd" && os.Getenv("ASR_CONFIG") != "" { - confVal.forceCgoLookupHost = true + confVal.preferCgo = true return } - - if _, err := os.Stat("/etc/mdns.allow"); err == nil { - confVal.hasMDNSAllow = true - } } -// canUseCgo reports whether calling cgo functions is allowed -// for non-hostname lookups. -func (c *conf) canUseCgo() bool { - ret, _ := c.hostLookupOrder(nil, "") - return ret == hostLookupCgo +// goosPreferCgo reports whether the GOOS value passed in prefers +// the cgo resolver. +func goosPrefersCgo(goos string) bool { + switch goos { + // Historically on Windows and Plan 9 we prefer the + // cgo resolver (which doesn't use the cgo tool) rather than + // the go resolver. This is because originally these + // systems did not support the go resolver. + // Keep it this way for better compatibility. + // Perhaps we can revisit this some day. + case "windows", "plan9": + return true + + // Darwin pops up annoying dialog boxes if programs try to + // do their own DNS requests, so prefer cgo. + case "darwin", "ios": + return true + + // DNS requests don't work on Android, so prefer the cgo resolver. + // Issue #10714. + case "android": + return true + + default: + return false + } +} + +// mustUseGoResolver reports whether a DNS lookup of any sort is +// required to use the go resolver. The provided Resolver is optional. +// This will report true if the cgo resolver is not available. +func (c *conf) mustUseGoResolver(r *Resolver) bool { + return c.netGo || r.preferGo() || !cgoAvailable } // hostLookupOrder determines which strategy to use to resolve hostname. // The provided Resolver is optional. nil means to not consider its options. // It also returns dnsConfig when it was used to determine the lookup order. -func (c *conf) hostLookupOrder(r *Resolver, hostname string) (ret hostLookupOrder, dnsConfig *dnsConfig) { +func (c *conf) hostLookupOrder(r *Resolver, hostname string) (ret hostLookupOrder, dnsConf *dnsConfig) { if c.dnsDebugLevel > 1 { defer func() { print("go package net: hostLookupOrder(", hostname, ") = ", ret.String(), "\n") }() } - fallbackOrder := hostLookupCgo - if c.netGo || r.preferGo() { + + // fallbackOrder is the order we return if we can't figure it out. + var fallbackOrder hostLookupOrder + + var canUseCgo bool + if c.mustUseGoResolver(r) { + // Go resolver was explicitly requested + // or cgo resolver is not available. + // Figure out the order below. switch c.goos { case "windows": // TODO(bradfitz): implement files-based @@ -142,123 +216,161 @@ func (c *conf) hostLookupOrder(r *Resolver, hostname string) (ret hostLookupOrde default: fallbackOrder = hostLookupFilesDNS } - } - if c.forceCgoLookupHost || c.goos == "android" || c.goos == "windows" || c.goos == "plan9" { - return fallbackOrder, nil - } - if bytealg.IndexByteString(hostname, '\\') != -1 || bytealg.IndexByteString(hostname, '%') != -1 { - // Don't deal with special form hostnames with backslashes - // or '%'. - return fallbackOrder, nil + canUseCgo = false + } else if c.netCgo { + // Cgo resolver was explicitly requested. + return hostLookupCgo, nil + } else if c.preferCgo { + // Given a choice, we prefer the cgo resolver. + return hostLookupCgo, nil + } else { + // Neither resolver was explicitly requested + // and we have no preference. + + // For testing purposes only, recheck the GOOS. + // This lets TestConfHostLookupOrder test different + // GOOS values. + if c.goos != runtime.GOOS && goosPrefersCgo(c.goos) { + return hostLookupCgo, nil + } + + if bytealg.IndexByteString(hostname, '\\') != -1 || bytealg.IndexByteString(hostname, '%') != -1 { + // Don't deal with special form hostnames + // with backslashes or '%'. + return hostLookupCgo, nil + } + + // If something is unrecognized, use cgo. + fallbackOrder = hostLookupCgo + canUseCgo = true } - conf := getSystemDNSConfig() - if conf.err != nil && !os.IsNotExist(conf.err) && !os.IsPermission(conf.err) { - // If we can't read the resolv.conf file, assume it - // had something important in it and defer to cgo. - // libc's resolver might then fail too, but at least - // it wasn't our fault. - return fallbackOrder, conf + // Try to figure out the order to use for searches. + // If we don't recognize something, use fallbackOrder. + // That will use cgo unless the Go resolver was explicitly requested. + // If we do figure out the order, return something other + // than fallbackOrder to use the Go resolver with that order. + + dnsConf = getSystemDNSConfig() + + if canUseCgo && dnsConf.err != nil && !errors.Is(dnsConf.err, fs.ErrNotExist) && !errors.Is(dnsConf.err, fs.ErrPermission) { + // We can't read the resolv.conf file, so use cgo if we can. + return hostLookupCgo, dnsConf } - if conf.unknownOpt { - return fallbackOrder, conf + if canUseCgo && dnsConf.unknownOpt { + // We didn't recognize something in resolv.conf, + // so use cgo if we can. + return hostLookupCgo, dnsConf } // OpenBSD is unique and doesn't use nsswitch.conf. // It also doesn't support mDNS. if c.goos == "openbsd" { - // OpenBSD's resolv.conf manpage says that a non-existent - // resolv.conf means "lookup" defaults to only "files", - // without DNS lookups. - if os.IsNotExist(conf.err) { - return hostLookupFiles, conf + // OpenBSD's resolv.conf manpage says that a + // non-existent resolv.conf means "lookup" defaults + // to only "files", without DNS lookups. + if errors.Is(dnsConf.err, fs.ErrNotExist) { + return hostLookupFiles, dnsConf } - lookup := conf.lookup + lookup := dnsConf.lookup if len(lookup) == 0 { // https://www.openbsd.org/cgi-bin/man.cgi/OpenBSD-current/man5/resolv.conf.5 // "If the lookup keyword is not used in the // system's resolv.conf file then the assumed // order is 'bind file'" - return hostLookupDNSFiles, conf + return hostLookupDNSFiles, dnsConf } if len(lookup) < 1 || len(lookup) > 2 { - return fallbackOrder, conf + // We don't recognize this format. + return fallbackOrder, dnsConf } switch lookup[0] { case "bind": if len(lookup) == 2 { if lookup[1] == "file" { - return hostLookupDNSFiles, conf + return hostLookupDNSFiles, dnsConf } - return fallbackOrder, conf + // Unrecognized. + return fallbackOrder, dnsConf } - return hostLookupDNS, conf + return hostLookupDNS, dnsConf case "file": if len(lookup) == 2 { if lookup[1] == "bind" { - return hostLookupFilesDNS, conf + return hostLookupFilesDNS, dnsConf } - return fallbackOrder, conf + // Unrecognized. + return fallbackOrder, dnsConf } - return hostLookupFiles, conf + return hostLookupFiles, dnsConf default: - return fallbackOrder, conf + // Unrecognized. + return fallbackOrder, dnsConf } + + // We always return before this point. + // The code below is for non-OpenBSD. } // Canonicalize the hostname by removing any trailing dot. if stringsHasSuffix(hostname, ".") { hostname = hostname[:len(hostname)-1] } - if stringsHasSuffixFold(hostname, ".local") { + if canUseCgo && stringsHasSuffixFold(hostname, ".local") { // Per RFC 6762, the ".local" TLD is special. And // because Go's native resolver doesn't do mDNS or // similar local resolution mechanisms, assume that // libc might (via Avahi, etc) and use cgo. - return fallbackOrder, conf + return hostLookupCgo, dnsConf } nss := getSystemNSS() srcs := nss.sources["hosts"] // If /etc/nsswitch.conf doesn't exist or doesn't specify any // sources for "hosts", assume Go's DNS will work fine. - if os.IsNotExist(nss.err) || (nss.err == nil && len(srcs) == 0) { - if c.goos == "solaris" { - // illumos defaults to "nis [NOTFOUND=return] files" - return fallbackOrder, conf + if errors.Is(nss.err, fs.ErrNotExist) || (nss.err == nil && len(srcs) == 0) { + if canUseCgo && c.goos == "solaris" { + // illumos defaults to + // "nis [NOTFOUND=return] files", + // which the go resolver doesn't support. + return hostLookupCgo, dnsConf } - return hostLookupFilesDNS, conf + return hostLookupFilesDNS, dnsConf } if nss.err != nil { // We failed to parse or open nsswitch.conf, so - // conservatively assume we should use cgo if it's - // available. - return fallbackOrder, conf + // we have nothing to base an order on. + return fallbackOrder, dnsConf } - var mdnsSource, filesSource, dnsSource bool + var mdnsSource, filesSource, dnsSource, unknownSource bool var first string for _, src := range srcs { if src.source == "myhostname" { - if isLocalhost(hostname) || isGateway(hostname) || isOutbound(hostname) { - return fallbackOrder, conf - } - hn, err := getHostname() - if err != nil || stringsEqualFold(hostname, hn) { - return fallbackOrder, conf + // Let the cgo resolver handle myhostname + // if we are looking up the local hostname. + if canUseCgo { + if isLocalhost(hostname) || isGateway(hostname) || isOutbound(hostname) { + return hostLookupCgo, dnsConf + } + hn, err := getHostname() + if err != nil || stringsEqualFold(hostname, hn) { + return hostLookupCgo, dnsConf + } } continue } if src.source == "files" || src.source == "dns" { - if !src.standardCriteria() { - return fallbackOrder, conf // non-standard; let libc deal with it. + if canUseCgo && !src.standardCriteria() { + // non-standard; let libc deal with it. + return hostLookupCgo, dnsConf } if src.source == "files" { filesSource = true - } else if src.source == "dns" { + } else { dnsSource = true } if first == "" { @@ -274,33 +386,62 @@ func (c *conf) hostLookupOrder(r *Resolver, hostname string) (ret hostLookupOrde continue } // Some source we don't know how to deal with. - return fallbackOrder, conf + if canUseCgo { + return hostLookupCgo, dnsConf + } + + unknownSource = true + if first == "" { + first = src.source + } } // We don't parse mdns.allow files. They're rare. If one // exists, it might list other TLDs (besides .local) or even // '*', so just let libc deal with it. - if mdnsSource && c.hasMDNSAllow { - return fallbackOrder, conf + if canUseCgo && mdnsSource { + var haveMDNSAllow bool + switch c.mdnsTest { + case mdnsFromSystem: + _, err := os.Stat("/etc/mdns.allow") + if err != nil && !errors.Is(err, fs.ErrNotExist) { + // Let libc figure out what is going on. + return hostLookupCgo, dnsConf + } + haveMDNSAllow = err == nil + case mdnsAssumeExists: + haveMDNSAllow = true + case mdnsAssumeDoesNotExist: + haveMDNSAllow = false + } + if haveMDNSAllow { + return hostLookupCgo, dnsConf + } } - // Cases where Go can handle it without cgo and C thread - // overhead. + // If we saw a source we don't recognize, which can only + // happen if we can't use the cgo resolver, treat it as DNS. + if unknownSource { + dnsSource = true + } + + // Cases where Go can handle it without cgo and C thread overhead, + // or where the Go resolver has been forced. switch { case filesSource && dnsSource: if first == "files" { - return hostLookupFilesDNS, conf + return hostLookupFilesDNS, dnsConf } else { - return hostLookupDNSFiles, conf + return hostLookupDNSFiles, dnsConf } case filesSource: - return hostLookupFiles, conf + return hostLookupFiles, dnsConf case dnsSource: - return hostLookupDNS, conf + return hostLookupDNS, dnsConf } - // Something weird. Let libc deal with it. - return fallbackOrder, conf + // Something weird. Fallback to the default. + return fallbackOrder, dnsConf } var netdns = godebug.New("netdns") diff --git a/src/net/conf_netcgo.go b/src/net/conf_netcgo.go deleted file mode 100644 index 82d1bb643e..0000000000 --- a/src/net/conf_netcgo.go +++ /dev/null @@ -1,18 +0,0 @@ -// Copyright 2015 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -//go:build netcgo - -package net - -/* - -// Fail if cgo isn't available. - -*/ -import "C" - -// The build tag "netcgo" forces use of the cgo DNS resolver. -// It is the opposite of "netgo". -func init() { netCgo = true } diff --git a/src/net/conf_test.go b/src/net/conf_test.go index 3736709295..6c9d247713 100644 --- a/src/net/conf_test.go +++ b/src/net/conf_test.go @@ -43,6 +43,15 @@ var defaultResolvConf = &dnsConfig{ } func TestConfHostLookupOrder(t *testing.T) { + // These tests are written for a system with cgo available, + // without using the netgo tag. + if netGoBuildTag { + t.Skip("skipping test because net package built with netgo tag") + } + if !cgoAvailable { + t.Skip("skipping test because cgo resolver not available") + } + tests := []struct { name string c *conf @@ -54,7 +63,8 @@ func TestConfHostLookupOrder(t *testing.T) { { name: "force", c: &conf{ - forceCgoLookupHost: true, + preferCgo: true, + netCgo: true, }, resolv: defaultResolvConf, nss: nssStr(t, "foo: bar"), @@ -82,12 +92,14 @@ func TestConfHostLookupOrder(t *testing.T) { resolv: defaultResolvConf, nss: nssStr(t, "hosts: dns files something_custom"), hostTests: []nssHostTest{ - {"x.com", "myhostname", hostLookupFilesDNS}, + {"x.com", "myhostname", hostLookupDNSFiles}, }, }, { - name: "ubuntu_trusty_avahi", - c: &conf{}, + name: "ubuntu_trusty_avahi", + c: &conf{ + mdnsTest: mdnsAssumeDoesNotExist, + }, resolv: defaultResolvConf, nss: nssStr(t, "hosts: files mdns4_minimal [NOTFOUND=return] dns mdns4"), hostTests: []nssHostTest{ @@ -203,8 +215,10 @@ func TestConfHostLookupOrder(t *testing.T) { hostTests: []nssHostTest{{"google.com", "myhostname", hostLookupFilesDNS}}, }, { - name: "files_mdns_dns", - c: &conf{}, + name: "files_mdns_dns", + c: &conf{ + mdnsTest: mdnsAssumeDoesNotExist, + }, resolv: defaultResolvConf, nss: nssStr(t, "hosts: files mdns dns"), hostTests: []nssHostTest{ @@ -226,7 +240,7 @@ func TestConfHostLookupOrder(t *testing.T) { { name: "mdns_allow", c: &conf{ - hasMDNSAllow: true, + mdnsTest: mdnsAssumeExists, }, resolv: defaultResolvConf, nss: nssStr(t, "hosts: files mdns dns"), @@ -294,8 +308,10 @@ func TestConfHostLookupOrder(t *testing.T) { }, }, { - name: "ubuntu14.04.02", - c: &conf{}, + name: "ubuntu14.04.02", + c: &conf{ + mdnsTest: mdnsAssumeDoesNotExist, + }, resolv: defaultResolvConf, nss: nssStr(t, "hosts: files myhostname mdns4_minimal [NOTFOUND=return] dns mdns4"), hostTests: []nssHostTest{ @@ -342,9 +358,8 @@ func TestConfHostLookupOrder(t *testing.T) { name: "resolver-prefergo", resolver: &Resolver{PreferGo: true}, c: &conf{ - goos: "darwin", - forceCgoLookupHost: true, // always true for darwin - netCgo: true, + preferCgo: true, + netCgo: true, }, resolv: defaultResolvConf, nss: nssStr(t, ""), @@ -352,6 +367,16 @@ func TestConfHostLookupOrder(t *testing.T) { {"localhost", "myhostname", hostLookupFilesDNS}, }, }, + { + name: "unknown-source", + resolver: &Resolver{PreferGo: true}, + c: &conf{}, + resolv: defaultResolvConf, + nss: nssStr(t, "hosts: resolve files"), + hostTests: []nssHostTest{ + {"x.com", "myhostname", hostLookupDNSFiles}, + }, + }, } origGetHostname := getHostname diff --git a/src/net/error_test.go b/src/net/error_test.go index fe0d9f676d..4538765d48 100644 --- a/src/net/error_test.go +++ b/src/net/error_test.go @@ -92,7 +92,9 @@ second: return nil } switch err := nestedErr.(type) { - case *AddrError, addrinfoErrno, *timeoutError, *DNSError, InvalidAddrError, *ParseError, *poll.DeadlineExceededError, UnknownNetworkError: + case *AddrError, *timeoutError, *DNSError, InvalidAddrError, *ParseError, *poll.DeadlineExceededError, UnknownNetworkError: + return nil + case interface{ isAddrinfoErrno() }: return nil case *os.SyscallError: nestedErr = err.Err @@ -472,7 +474,9 @@ second: return nil } switch err := nestedErr.(type) { - case *AddrError, addrinfoErrno, *timeoutError, *DNSError, InvalidAddrError, *ParseError, *poll.DeadlineExceededError, UnknownNetworkError: + case *AddrError, *timeoutError, *DNSError, InvalidAddrError, *ParseError, *poll.DeadlineExceededError, UnknownNetworkError: + return nil + case interface{ isAddrinfoErrno() }: return nil case *os.SyscallError: nestedErr = err.Err diff --git a/src/net/lookup_plan9.go b/src/net/lookup_plan9.go index 1995742f8c..7c423bfff6 100644 --- a/src/net/lookup_plan9.go +++ b/src/net/lookup_plan9.go @@ -13,6 +13,11 @@ import ( "os" ) +// cgoAvailable set to true to indicate that the cgo resolver +// is available on Plan 9. Note that on Plan 9 the cgo resolver +// does not actually use cgo. +const cgoAvailable = true + func query(ctx context.Context, filename, query string, bufSize int) (addrs []string, err error) { queryAddrs := func() (addrs []string, err error) { file, err := os.OpenFile(filename, os.O_RDWR, 0) diff --git a/src/net/lookup_test.go b/src/net/lookup_test.go index e02c45f638..0ea681f834 100644 --- a/src/net/lookup_test.go +++ b/src/net/lookup_test.go @@ -792,7 +792,7 @@ func TestLookupPort(t *testing.T) { switch runtime.GOOS { case "android": - if netGo { + if netGoBuildTag { t.Skipf("not supported on %s without cgo; see golang.org/issues/14576", runtime.GOOS) } default: diff --git a/src/net/lookup_unix.go b/src/net/lookup_unix.go index 3c67b9ecc8..0189db09e4 100644 --- a/src/net/lookup_unix.go +++ b/src/net/lookup_unix.go @@ -55,7 +55,7 @@ func lookupProtocol(_ context.Context, name string) (int, error) { func (r *Resolver) lookupHost(ctx context.Context, host string) (addrs []string, err error) { order, conf := systemConf().hostLookupOrder(r, host) - if !r.preferGo() && order == hostLookupCgo { + if order == hostLookupCgo { if addrs, err, ok := cgoLookupHost(ctx, host); ok { return addrs, err } @@ -82,7 +82,9 @@ func (r *Resolver) lookupIP(ctx context.Context, network, host string) (addrs [] } func (r *Resolver) lookupPort(ctx context.Context, network, service string) (int, error) { - if !r.preferGo() && systemConf().canUseCgo() { + // Port lookup is not a DNS operation. + // Prefer the cgo resolver if possible. + if !systemConf().mustUseGoResolver(r) { if port, err, ok := cgoLookupPort(ctx, network, service); ok { if err != nil { // Issue 18213: if cgo fails, first check to see whether we @@ -99,7 +101,7 @@ func (r *Resolver) lookupPort(ctx context.Context, network, service string) (int func (r *Resolver) lookupCNAME(ctx context.Context, name string) (string, error) { order, conf := systemConf().hostLookupOrder(r, name) - if !r.preferGo() && order == hostLookupCgo { + if order == hostLookupCgo { if cname, err, ok := cgoLookupCNAME(ctx, name); ok { return cname, err } @@ -125,7 +127,7 @@ func (r *Resolver) lookupTXT(ctx context.Context, name string) ([]string, error) func (r *Resolver) lookupAddr(ctx context.Context, addr string) ([]string, error) { order, conf := systemConf().hostLookupOrder(r, "") - if !r.preferGo() && order == hostLookupCgo { + if order == hostLookupCgo { if ptrs, err, ok := cgoLookupPTR(ctx, addr); ok { return ptrs, err } diff --git a/src/net/lookup_windows.go b/src/net/lookup_windows.go index 11f43fe1c7..9f88d82854 100644 --- a/src/net/lookup_windows.go +++ b/src/net/lookup_windows.go @@ -14,6 +14,11 @@ import ( "unsafe" ) +// cgoAvailable set to true to indicate that the cgo resolver +// is available on Windows. Note that on Windows the cgo resolver +// does not actually use cgo. +const cgoAvailable = true + const ( _WSAHOST_NOT_FOUND = syscall.Errno(11001) _WSATRY_AGAIN = syscall.Errno(11002) diff --git a/src/net/net.go b/src/net/net.go index a9e9a6478a..5cfc25ffca 100644 --- a/src/net/net.go +++ b/src/net/net.go @@ -93,14 +93,6 @@ import ( "time" ) -// netGo and netCgo contain the state of the build tags used -// to build this binary, and whether cgo is available. -// conf.go mirrors these into conf for easier testing. -var ( - netGo bool // set true in cgo_stub.go for build tag "netgo" (or no cgo) - netCgo bool // set true in conf_netcgo.go for build tag "netcgo" -) - // Addr represents a network end point address. // // The two methods Network and String conventionally return strings diff --git a/src/net/netcgo_off.go b/src/net/netcgo_off.go new file mode 100644 index 0000000000..54677dcac6 --- /dev/null +++ b/src/net/netcgo_off.go @@ -0,0 +1,9 @@ +// Copyright 2023 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +//go:build !netcgo + +package net + +const netCgoBuildTag = false diff --git a/src/net/netcgo_on.go b/src/net/netcgo_on.go new file mode 100644 index 0000000000..25d4bdca72 --- /dev/null +++ b/src/net/netcgo_on.go @@ -0,0 +1,9 @@ +// Copyright 2023 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +//go:build netcgo + +package net + +const netCgoBuildTag = true diff --git a/src/net/netgo.go b/src/net/netgo.go deleted file mode 100644 index e478c88b30..0000000000 --- a/src/net/netgo.go +++ /dev/null @@ -1,13 +0,0 @@ -// Copyright 2022 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -// Default netGo to true if the netgo build tag is being used, or the -// C library DNS routines are not available. Note that the C library -// routines are always available on Darwin and Windows. - -//go:build netgo || (!cgo && !darwin && !windows) - -package net - -func init() { netGo = true } diff --git a/src/net/netgo_netcgo.go b/src/net/netgo_netcgo.go new file mode 100644 index 0000000000..7f3a5fd007 --- /dev/null +++ b/src/net/netgo_netcgo.go @@ -0,0 +1,14 @@ +// Copyright 2023 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +//go:build netgo && netcgo + +package net + +func init() { + // This will give a compile time error about the unused constant. + // The advantage of this approach is that the gc compiler + // actually prints the constant, making the problem obvious. + "Do not use both netgo and netcgo build tags." +} diff --git a/src/net/netgo_off.go b/src/net/netgo_off.go new file mode 100644 index 0000000000..e6bc2d7d06 --- /dev/null +++ b/src/net/netgo_off.go @@ -0,0 +1,9 @@ +// Copyright 2023 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +//go:build !netgo + +package net + +const netGoBuildTag = false diff --git a/src/net/netgo_on.go b/src/net/netgo_on.go new file mode 100644 index 0000000000..4f088de6e3 --- /dev/null +++ b/src/net/netgo_on.go @@ -0,0 +1,9 @@ +// Copyright 2023 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +//go:build netgo + +package net + +const netGoBuildTag = true From da3a184b182d5d0b18fd139df9e4259df4378095 Mon Sep 17 00:00:00 2001 From: Achille Roussel Date: Thu, 27 Apr 2023 14:37:55 -0700 Subject: [PATCH 016/299] syscall: remove wasip1 O_DIRECTORY workaround Wasmtime used to error when opening a directory without passing the O_DIRECTORY flag, which required doing a stat to determine whether to inject the flag prior to opening any file. The workaround was subject to races since the stat and open calls were not atomic. Wasmtime fixed the issue in v8.0.1. For details see: - https://github.com/bytecodealliance/wasmtime/pull/4967 - https://github.com/bytecodealliance/wasmtime/pull/6163 - https://github.com/bytecodealliance/wasmtime/releases/tag/v8.0.1 Change-Id: I0f9fe6a696024b70fffe63b83e8f61e48acd0c4a Reviewed-on: https://go-review.googlesource.com/c/go/+/489955 Run-TryBot: Ian Lance Taylor Reviewed-by: Ian Lance Taylor Auto-Submit: Ian Lance Taylor TryBot-Result: Gopher Robot Reviewed-by: Johan Brandhorst-Satzkorn Run-TryBot: Ian Lance Taylor Reviewed-by: Dmitri Shuralyov --- src/syscall/fs_wasip1.go | 23 ----------------------- 1 file changed, 23 deletions(-) diff --git a/src/syscall/fs_wasip1.go b/src/syscall/fs_wasip1.go index b331629d79..ef04af6966 100644 --- a/src/syscall/fs_wasip1.go +++ b/src/syscall/fs_wasip1.go @@ -432,29 +432,6 @@ func Open(path string, openmode int, perm uint32) (int, error) { oflags |= OFLAG_EXCL } - // Remove when https://github.com/bytecodealliance/wasmtime/pull/4967 is merged. - var fi Stat_t - if errno := path_filestat_get( - dirFd, - LOOKUP_SYMLINK_FOLLOW, - pathPtr, - pathLen, - unsafe.Pointer(&fi), - ); errno != 0 && errno != ENOENT { - return -1, errnoErr(errno) - } - if fi.Filetype == FILETYPE_DIRECTORY { - oflags |= OFLAG_DIRECTORY - // WASM runtimes appear to return EINVAL when passing invalid - // combination of flags to open directories; however, TestOpenError - // in the os package expects EISDIR, so we precheck this condition - // here to emulate the expected behavior. - const invalidFlags = O_WRONLY | O_RDWR | O_CREATE | O_APPEND | O_TRUNC | O_EXCL - if (openmode & invalidFlags) != 0 { - return 0, EISDIR - } - } - var rights rights switch openmode & (O_RDONLY | O_WRONLY | O_RDWR) { case O_RDONLY: From 7bc8aacec185ca80839ef0a4f58b85e482ab769e Mon Sep 17 00:00:00 2001 From: "Bryan C. Mills" Date: Tue, 28 Feb 2023 17:07:36 +0000 Subject: [PATCH 017/299] misc/android: rework GOROOT installation - Fall back to 'go env GOROOT' to locate GOROOT if runtime.GOROOT() is empty (as may be the case if the tool is built with -trimpath). - Copy all of $GOROOT/android_$GOARCH/bin, not just cmd/go, to $GOROOT/bin. - For consistency with CL 404134, place $GOROOT/bin at the beginning of $PATH, not the end. - Don't use the install target for the "runtime" package to locate pkg/tool. As of Go 1.20 "runtime" doesn't have an install directory anyway. Since the real reason we need pkg/tool is for commands in "cmd", use an arbitrary command (namely "cmd/compile") to locate it. - Use 'go list' to determine the package import path for the current directory, instead of assuming that it is within GOROOT or GOPATH. (That assumption does not hold in module mode.) Updates #58775. Change-Id: If76ff22bce76d05175c40678230f046a4aff0940 Reviewed-on: https://go-review.googlesource.com/c/go/+/472096 TryBot-Result: Gopher Robot Reviewed-by: Ian Lance Taylor Run-TryBot: Bryan Mills Reviewed-by: Changkun Ou Auto-Submit: Bryan Mills --- misc/android/go_android_exec.go | 252 ++++++++++++++++++++++---------- 1 file changed, 171 insertions(+), 81 deletions(-) diff --git a/misc/android/go_android_exec.go b/misc/android/go_android_exec.go index 308dacaf7c..445ac284be 100644 --- a/misc/android/go_android_exec.go +++ b/misc/android/go_android_exec.go @@ -10,18 +10,20 @@ package main import ( + "bytes" "errors" "fmt" - "go/build" "io" "log" "os" "os/exec" "os/signal" + "path" "path/filepath" "runtime" "strconv" "strings" + "sync" "syscall" ) @@ -115,18 +117,22 @@ func runMain() (int, error) { // "$GOROOT/src/mime/multipart" or "$GOPATH/src/golang.org/x/mobile". // We extract everything after the $GOROOT or $GOPATH to run on the // same relative directory on the target device. - subdir, inGoRoot, err := subdir() + importPath, isStd, err := pkgPath() if err != nil { return 0, err } - deviceCwd := filepath.Join(deviceGopath, subdir) - if inGoRoot { - deviceCwd = filepath.Join(deviceGoroot, subdir) + var deviceCwd string + if isStd { + // Note that we use path.Join here instead of filepath.Join: + // The device paths should be slash-separated even if the go_android_exec + // wrapper itself is compiled for Windows. + deviceCwd = path.Join(deviceGoroot, "src", importPath) } else { + deviceCwd = path.Join(deviceGopath, "src", importPath) if err := adb("exec-out", "mkdir", "-p", deviceCwd); err != nil { return 0, err } - if err := adbCopyTree(deviceCwd, subdir); err != nil { + if err := adbCopyTree(deviceCwd, importPath); err != nil { return 0, err } @@ -170,7 +176,7 @@ func runMain() (int, error) { `; export CGO_ENABLED=0` + `; export GOPROXY=` + os.Getenv("GOPROXY") + `; export GOCACHE="` + deviceRoot + `/gocache"` + - `; export PATH=$PATH:"` + deviceGoroot + `/bin"` + + `; export PATH="` + deviceGoroot + `/bin":$PATH` + `; cd "` + deviceCwd + `"` + "; '" + deviceBin + "' " + strings.Join(os.Args[2:], " ") + "; echo -n " + exitstr + "$?" @@ -192,40 +198,37 @@ func runMain() (int, error) { return code, nil } -// subdir determines the package based on the current working directory, +// pkgPath determines the package import path of the current working directory, +// and indicates whether it is // and returns the path to the package source relative to $GOROOT (or $GOPATH). -func subdir() (pkgpath string, underGoRoot bool, err error) { - cwd, err := os.Getwd() +func pkgPath() (importPath string, isStd bool, err error) { + goTool, err := goTool() if err != nil { return "", false, err } - cwd, err = filepath.EvalSymlinks(cwd) + cmd := exec.Command(goTool, "list", "-e", "-f", "{{.ImportPath}}:{{.Standard}}", ".") + out, err := cmd.Output() if err != nil { - return "", false, err - } - goroot, err := filepath.EvalSymlinks(runtime.GOROOT()) - if err != nil { - return "", false, err - } - if subdir, err := filepath.Rel(goroot, cwd); err == nil { - if !strings.Contains(subdir, "..") { - return subdir, true, nil + if ee, ok := err.(*exec.ExitError); ok && len(ee.Stderr) > 0 { + return "", false, fmt.Errorf("%v: %s", cmd, ee.Stderr) } + return "", false, fmt.Errorf("%v: %w", cmd, err) } - for _, p := range filepath.SplitList(build.Default.GOPATH) { - pabs, err := filepath.EvalSymlinks(p) - if err != nil { - return "", false, err - } - if subdir, err := filepath.Rel(pabs, cwd); err == nil { - if !strings.Contains(subdir, "..") { - return subdir, false, nil - } - } + s := string(bytes.TrimSpace(out)) + importPath, isStdStr, ok := strings.Cut(s, ":") + if !ok { + return "", false, fmt.Errorf("%v: missing ':' in output: %q", cmd, out) } - return "", false, fmt.Errorf("the current path %q is not in either GOROOT(%q) or GOPATH(%q)", - cwd, runtime.GOROOT(), build.Default.GOPATH) + if importPath == "" || importPath == "." { + return "", false, fmt.Errorf("current directory does not have a Go import path") + } + isStd, err = strconv.ParseBool(isStdStr) + if err != nil { + return "", false, fmt.Errorf("%v: non-boolean .Standard in output: %q", cmd, out) + } + + return importPath, isStd, nil } // adbCopyTree copies testdata, go.mod, go.sum files from subdir @@ -236,16 +239,16 @@ func subdir() (pkgpath string, underGoRoot bool, err error) { func adbCopyTree(deviceCwd, subdir string) error { dir := "" for { - for _, path := range []string{"testdata", "go.mod", "go.sum"} { - path := filepath.Join(dir, path) - if _, err := os.Stat(path); err != nil { + for _, name := range []string{"testdata", "go.mod", "go.sum"} { + hostPath := filepath.Join(dir, name) + if _, err := os.Stat(hostPath); err != nil { continue } - devicePath := filepath.Join(deviceCwd, dir) + devicePath := path.Join(deviceCwd, dir) if err := adb("exec-out", "mkdir", "-p", devicePath); err != nil { return err } - if err := adb("push", path, devicePath); err != nil { + if err := adb("push", hostPath, devicePath); err != nil { return err } } @@ -253,7 +256,7 @@ func adbCopyTree(deviceCwd, subdir string) error { break } subdir = filepath.Dir(subdir) - dir = filepath.Join(dir, "..") + dir = path.Join(dir, "..") } return nil } @@ -264,6 +267,18 @@ func adbCopyTree(deviceCwd, subdir string) error { // A lock file ensures this only happens once, even with concurrent exec // wrappers. func adbCopyGoroot() error { + goTool, err := goTool() + if err != nil { + return err + } + cmd := exec.Command(goTool, "version") + cmd.Stderr = os.Stderr + out, err := cmd.Output() + if err != nil { + return fmt.Errorf("%v: %w", cmd, err) + } + goVersion := string(out) + // Also known by cmd/dist. The bootstrap command deletes the file. statPath := filepath.Join(os.TempDir(), "go_android_exec-adb-sync-status") stat, err := os.OpenFile(statPath, os.O_CREATE|os.O_RDWR, 0666) @@ -279,62 +294,137 @@ func adbCopyGoroot() error { if err != nil { return err } - if string(s) == "done" { + if string(s) == goVersion { return nil } - // Delete GOROOT, GOPATH and any leftover test data. - if err := adb("exec-out", "rm", "-rf", deviceRoot); err != nil { - return err - } - deviceBin := filepath.Join(deviceGoroot, "bin") - if err := adb("exec-out", "mkdir", "-p", deviceBin); err != nil { - return err - } - goroot := runtime.GOROOT() - // Build go for android. - goCmd := filepath.Join(goroot, "bin", "go") - tmpGo, err := os.CreateTemp("", "go_android_exec-cmd-go-*") + + goroot, err := findGoroot() if err != nil { return err } - tmpGo.Close() - defer os.Remove(tmpGo.Name()) - if out, err := exec.Command(goCmd, "build", "-o", tmpGo.Name(), "cmd/go").CombinedOutput(); err != nil { - return fmt.Errorf("failed to build go tool for device: %s\n%v", out, err) - } - deviceGo := filepath.Join(deviceBin, "go") - if err := adb("push", tmpGo.Name(), deviceGo); err != nil { + // Delete the device's GOROOT, GOPATH and any leftover test data, + // and recreate GOROOT. + if err := adb("exec-out", "rm", "-rf", deviceRoot); err != nil { return err } - for _, dir := range []string{"src", "test", "lib", "api"} { - if err := adb("push", filepath.Join(goroot, dir), filepath.Join(deviceGoroot)); err != nil { + + // Build Go for Android. + cmd = exec.Command(goTool, "install", "cmd") + out, err = cmd.CombinedOutput() + if err != nil { + if len(bytes.TrimSpace(out)) > 0 { + log.Printf("\n%s", out) + } + return fmt.Errorf("%v: %w", cmd, err) + } + if err := adb("exec-out", "mkdir", "-p", deviceGoroot); err != nil { + return err + } + + // Copy the Android tools from the relevant bin subdirectory to GOROOT/bin. + cmd = exec.Command(goTool, "list", "-f", "{{.Target}}", "cmd/go") + cmd.Stderr = os.Stderr + out, err = cmd.Output() + if err != nil { + return fmt.Errorf("%v: %w", cmd, err) + } + platformBin := filepath.Dir(string(bytes.TrimSpace(out))) + if platformBin == "." { + return errors.New("failed to locate cmd/go for target platform") + } + if err := adb("push", platformBin, path.Join(deviceGoroot, "bin")); err != nil { + return err + } + + // Copy only the relevant subdirectories from pkg: pkg/include and the + // platform-native binaries in pkg/tool. + if err := adb("exec-out", "mkdir", "-p", path.Join(deviceGoroot, "pkg", "tool")); err != nil { + return err + } + if err := adb("push", filepath.Join(goroot, "pkg", "include"), path.Join(deviceGoroot, "pkg", "include")); err != nil { + return err + } + + cmd = exec.Command(goTool, "list", "-f", "{{.Target}}", "cmd/compile") + cmd.Stderr = os.Stderr + out, err = cmd.Output() + if err != nil { + return fmt.Errorf("%v: %w", cmd, err) + } + platformToolDir := filepath.Dir(string(bytes.TrimSpace(out))) + if platformToolDir == "." { + return errors.New("failed to locate cmd/compile for target platform") + } + relToolDir, err := filepath.Rel(filepath.Join(goroot), platformToolDir) + if err != nil { + return err + } + if err := adb("push", platformToolDir, path.Join(deviceGoroot, relToolDir)); err != nil { + return err + } + + // Copy all other files from GOROOT. + dirents, err := os.ReadDir(goroot) + if err != nil { + return err + } + for _, de := range dirents { + switch de.Name() { + case "bin", "pkg": + // We already created GOROOT/bin and GOROOT/pkg above; skip those. + continue + } + if err := adb("push", filepath.Join(goroot, de.Name()), path.Join(deviceGoroot, de.Name())); err != nil { return err } } - // Copy only the relevant from pkg. - if err := adb("exec-out", "mkdir", "-p", filepath.Join(deviceGoroot, "pkg", "tool")); err != nil { - return err - } - if err := adb("push", filepath.Join(goroot, "pkg", "include"), filepath.Join(deviceGoroot, "pkg")); err != nil { - return err - } - runtimea, err := exec.Command(goCmd, "list", "-f", "{{.Target}}", "runtime").Output() - pkgdir := filepath.Dir(string(runtimea)) - if pkgdir == "" { - return errors.New("could not find android pkg dir") - } - if err := adb("push", pkgdir, filepath.Join(deviceGoroot, "pkg")); err != nil { - return err - } - tooldir := filepath.Join(goroot, "pkg", "tool", filepath.Base(pkgdir)) - if err := adb("push", tooldir, filepath.Join(deviceGoroot, "pkg", "tool")); err != nil { - return err - } - - if _, err := stat.Write([]byte("done")); err != nil { + if _, err := stat.WriteString(goVersion); err != nil { return err } return nil } + +func findGoroot() (string, error) { + gorootOnce.Do(func() { + // If runtime.GOROOT reports a non-empty path, assume that it is valid. + // (It may be empty if this binary was built with -trimpath.) + gorootPath = runtime.GOROOT() + if gorootPath != "" { + return + } + + // runtime.GOROOT is empty — perhaps go_android_exec was built with + // -trimpath and GOROOT is unset. Try 'go env GOROOT' as a fallback, + // assuming that the 'go' command in $PATH is the correct one. + + cmd := exec.Command("go", "env", "GOROOT") + cmd.Stderr = os.Stderr + out, err := cmd.Output() + if err != nil { + gorootErr = fmt.Errorf("%v: %w", cmd, err) + } + + gorootPath = string(bytes.TrimSpace(out)) + if gorootPath == "" { + gorootErr = errors.New("GOROOT not found") + } + }) + + return gorootPath, gorootErr +} + +func goTool() (string, error) { + goroot, err := findGoroot() + if err != nil { + return "", err + } + return filepath.Join(goroot, "bin", "go"), nil +} + +var ( + gorootOnce sync.Once + gorootPath string + gorootErr error +) From 71ad46cd2a4eb34befbb21c0a37b5a8092c2b7f5 Mon Sep 17 00:00:00 2001 From: "Bryan C. Mills" Date: Fri, 10 Mar 2023 14:25:41 -0500 Subject: [PATCH 018/299] internal/testenv: allow 'go build' on android when supported As of CL 472096, it should work on android/arm64 always (because internal linking is supported on that platform), and on other android platforms when a C toolchain is present in the test environment. Updates #58775. Change-Id: Ifa38dc69b258b38dcc341979dcbf8cd61265c787 Reviewed-on: https://go-review.googlesource.com/c/go/+/475456 Run-TryBot: Bryan Mills TryBot-Result: Gopher Robot Reviewed-by: Ian Lance Taylor Auto-Submit: Bryan Mills Reviewed-by: Cherry Mui Reviewed-by: Changkun Ou --- src/internal/testenv/testenv.go | 63 ++++++++++++++++++++++++++++++--- 1 file changed, 58 insertions(+), 5 deletions(-) diff --git a/src/internal/testenv/testenv.go b/src/internal/testenv/testenv.go index 9a649e037c..aeda1f964f 100644 --- a/src/internal/testenv/testenv.go +++ b/src/internal/testenv/testenv.go @@ -45,13 +45,62 @@ func HasGoBuild() bool { // run go build. return false } - switch runtime.GOOS { - case "android", "js", "ios", "wasip1": + + if !HasExec() { + // If we can't exec anything at all, we certainly can't exec 'go build'. return false } + + if platform.MustLinkExternal(runtime.GOOS, runtime.GOARCH, false) { + // We can assume that we always have a complete Go toolchain available. + // However, this platform requires a C linker to build even pure Go + // programs, including tests. Do we have one in the test environment? + // (On Android, for example, the device running the test might not have a + // C toolchain installed.) + // + // If CC is set explicitly, assume that we do. Otherwise, use 'go env CC' + // to determine which toolchain it would use by default. + if os.Getenv("CC") == "" { + if _, err := findCC(); err != nil { + return false + } + } + } + return true } +func findCC() (string, error) { + ccOnce.Do(func() { + goTool, err := findGoTool() + if err != nil { + ccErr = err + return + } + + cmd := exec.Command(goTool, "env", "CC") + out, err := cmd.Output() + out = bytes.TrimSpace(out) + if err != nil { + ccErr = fmt.Errorf("%v: %w", cmd, err) + return + } else if len(out) == 0 { + ccErr = fmt.Errorf("%v: no CC reported", cmd) + return + } + + cc := string(out) + ccPath, ccErr = exec.LookPath(cc) + }) + return ccPath, ccErr +} + +var ( + ccOnce sync.Once + ccPath string + ccErr error +) + // MustHaveGoBuild checks that the current system can build programs with “go build” // and then run them with os.StartProcess or exec.Command. // If not, MustHaveGoBuild calls t.Skip with an explanation. @@ -212,11 +261,15 @@ func GOROOT(t testing.TB) string { // GoTool reports the path to the Go tool. func GoTool() (string, error) { + if !HasGoBuild() { + return "", errors.New("platform cannot run go tool") + } + return findGoTool() +} + +func findGoTool() (string, error) { goToolOnce.Do(func() { goToolPath, goToolErr = func() (string, error) { - if !HasGoBuild() { - return "", errors.New("platform cannot run go tool") - } var exeSuffix string if runtime.GOOS == "windows" { exeSuffix = ".exe" From f0461808903d8781757e006704821fd72fb1c79a Mon Sep 17 00:00:00 2001 From: Chressie Himpel Date: Fri, 28 Apr 2023 10:54:57 +0000 Subject: [PATCH 019/299] Revert "cmd/compile: constant-fold loads from constant dictionaries and types" This reverts CL 486895. Reason for revert: This breaks internal tests at Google, see b/280035614. Change-Id: I48772a44f5f6070a7f06b5704e9f9aa272b5f978 Reviewed-on: https://go-review.googlesource.com/c/go/+/490156 TryBot-Result: Gopher Robot Reviewed-by: Michael Stapelberg Reviewed-by: Cuong Manh Le Run-TryBot: Michael Stapelberg Reviewed-by: David Chase --- src/cmd/compile/internal/noder/reader.go | 2 +- .../compile/internal/reflectdata/reflect.go | 13 +- .../compile/internal/ssa/_gen/generic.rules | 27 +- src/cmd/compile/internal/ssa/rewrite.go | 91 +---- .../compile/internal/ssa/rewritegeneric.go | 357 ++---------------- src/cmd/internal/obj/link.go | 18 +- 6 files changed, 55 insertions(+), 453 deletions(-) diff --git a/src/cmd/compile/internal/noder/reader.go b/src/cmd/compile/internal/noder/reader.go index 9a07ff7666..6098c92ac9 100644 --- a/src/cmd/compile/internal/noder/reader.go +++ b/src/cmd/compile/internal/noder/reader.go @@ -3988,7 +3988,7 @@ func setBasePos(pos src.XPos) { // // N.B., this variable name is known to Delve: // https://github.com/go-delve/delve/blob/cb91509630529e6055be845688fd21eb89ae8714/pkg/proc/eval.go#L28 -const dictParamName = typecheck.LocalDictName +const dictParamName = ".dict" // shapeSig returns a copy of fn's signature, except adding a // dictionary parameter and promoting the receiver parameter (if any) diff --git a/src/cmd/compile/internal/reflectdata/reflect.go b/src/cmd/compile/internal/reflectdata/reflect.go index 73bbff94d8..cde8c68876 100644 --- a/src/cmd/compile/internal/reflectdata/reflect.go +++ b/src/cmd/compile/internal/reflectdata/reflect.go @@ -844,14 +844,7 @@ func TypeLinksymLookup(name string) *obj.LSym { } func TypeLinksym(t *types.Type) *obj.LSym { - lsym := TypeSym(t).Linksym() - signatmu.Lock() - if lsym.Extra == nil { - ti := lsym.NewTypeInfo() - ti.Type = t - } - signatmu.Unlock() - return lsym + return TypeSym(t).Linksym() } // Deprecated: Use TypePtrAt instead. @@ -1885,9 +1878,7 @@ func MarkTypeUsedInInterface(t *types.Type, from *obj.LSym) { // Shape types shouldn't be put in interfaces, so we shouldn't ever get here. base.Fatalf("shape types have no methods %+v", t) } - MarkTypeSymUsedInInterface(TypeLinksym(t), from) -} -func MarkTypeSymUsedInInterface(tsym *obj.LSym, from *obj.LSym) { + tsym := TypeLinksym(t) // Emit a marker relocation. The linker will know the type is converted // to an interface if "from" is reachable. r := obj.Addrel(from) diff --git a/src/cmd/compile/internal/ssa/_gen/generic.rules b/src/cmd/compile/internal/ssa/_gen/generic.rules index bea743e94a..c7a525abb7 100644 --- a/src/cmd/compile/internal/ssa/_gen/generic.rules +++ b/src/cmd/compile/internal/ssa/_gen/generic.rules @@ -2061,10 +2061,6 @@ && warnRule(fe.Debug_checknil(), v, "removed nil check") => (Invalid) -// Addresses of globals are always non-nil. -(NilCheck (Addr {_} (SB)) _) => (Invalid) -(NilCheck (Convert (Addr {_} (SB)) _) _) => (Invalid) - // for late-expanded calls, recognize memequal applied to a single constant byte // Support is limited by 1, 2, 4, 8 byte sizes (StaticLECall {callAux} sptr (Addr {scon} (SB)) (Const64 [1]) mem) @@ -2152,8 +2148,6 @@ (NeqPtr (OffPtr [o1] p1) (OffPtr [o2] p2)) && isSamePtr(p1, p2) => (ConstBool [o1 != o2]) (EqPtr (Const(32|64) [c]) (Const(32|64) [d])) => (ConstBool [c == d]) (NeqPtr (Const(32|64) [c]) (Const(32|64) [d])) => (ConstBool [c != d]) -(EqPtr (Convert (Addr {x} _) _) (Addr {y} _)) => (ConstBool [x==y]) -(NeqPtr (Convert (Addr {x} _) _) (Addr {y} _)) => (ConstBool [x!=y]) (EqPtr (LocalAddr _ _) (Addr _)) => (ConstBool [false]) (EqPtr (OffPtr (LocalAddr _ _)) (Addr _)) => (ConstBool [false]) @@ -2175,8 +2169,7 @@ // Evaluate constant user nil checks. (IsNonNil (ConstNil)) => (ConstBool [false]) (IsNonNil (Const(32|64) [c])) => (ConstBool [c != 0]) -(IsNonNil (Addr _) ) => (ConstBool [true]) -(IsNonNil (Convert (Addr _) _)) => (ConstBool [true]) +(IsNonNil (Addr _)) => (ConstBool [true]) (IsNonNil (LocalAddr _ _)) => (ConstBool [true]) // Inline small or disjoint runtime.memmove calls with constant length. @@ -2219,7 +2212,11 @@ => (Move {types.Types[types.TUINT8]} [int64(sz)] dst src mem) // De-virtualize late-expanded interface calls into late-expanded static calls. -(InterLECall [argsize] {auxCall} (Addr {fn} (SB)) ___) => devirtLECall(v, fn.(*obj.LSym)) +// Note that (ITab (IMake)) doesn't get rewritten until after the first opt pass, +// so this rule should trigger reliably. +// devirtLECall removes the first argument, adds the devirtualized symbol to the AuxCall, and changes the opcode +(InterLECall [argsize] {auxCall} (Load (OffPtr [off] (ITab (IMake (Addr {itab} (SB)) _))) _) ___) && devirtLESym(v, auxCall, itab, off) != + nil => devirtLECall(v, devirtLESym(v, auxCall, itab, off)) // Move and Zero optimizations. // Move source and destination may overlap. @@ -2733,15 +2730,3 @@ (RotateLeft(64|32|16|8) (RotateLeft(64|32|16|8) x c) d) && c.Type.Size() == 4 && d.Type.Size() == 4 => (RotateLeft(64|32|16|8) x (Add32 c d)) (RotateLeft(64|32|16|8) (RotateLeft(64|32|16|8) x c) d) && c.Type.Size() == 2 && d.Type.Size() == 2 => (RotateLeft(64|32|16|8) x (Add16 c d)) (RotateLeft(64|32|16|8) (RotateLeft(64|32|16|8) x c) d) && c.Type.Size() == 1 && d.Type.Size() == 1 => (RotateLeft(64|32|16|8) x (Add8 c d)) - -// Loading constant values from dictionaries and itabs. -(Load (OffPtr [off] (Addr {s} sb) ) _) && t.IsUintptr() && isFixedSym(s, off) => (Addr {fixedSym(b.Func, s, off)} sb) -(Load (OffPtr [off] (Convert (Addr {s} sb) _) ) _) && t.IsUintptr() && isFixedSym(s, off) => (Addr {fixedSym(b.Func, s, off)} sb) -(Load (OffPtr [off] (ITab (IMake (Addr {s} sb) _))) _) && t.IsUintptr() && isFixedSym(s, off) => (Addr {fixedSym(b.Func, s, off)} sb) -(Load (OffPtr [off] (ITab (IMake (Convert (Addr {s} sb) _) _))) _) && t.IsUintptr() && isFixedSym(s, off) => (Addr {fixedSym(b.Func, s, off)} sb) - -// Loading constant values from runtime._type.hash. -(Load (OffPtr [off] (Addr {sym} _) ) _) && t.IsInteger() && t.Size() == 4 && isFixed32(config, sym, off) => (Const32 [fixed32(config, sym, off)]) -(Load (OffPtr [off] (Convert (Addr {sym} _) _) ) _) && t.IsInteger() && t.Size() == 4 && isFixed32(config, sym, off) => (Const32 [fixed32(config, sym, off)]) -(Load (OffPtr [off] (ITab (IMake (Addr {sym} _) _))) _) && t.IsInteger() && t.Size() == 4 && isFixed32(config, sym, off) => (Const32 [fixed32(config, sym, off)]) -(Load (OffPtr [off] (ITab (IMake (Convert (Addr {sym} _) _) _))) _) && t.IsInteger() && t.Size() == 4 && isFixed32(config, sym, off) => (Const32 [fixed32(config, sym, off)]) diff --git a/src/cmd/compile/internal/ssa/rewrite.go b/src/cmd/compile/internal/ssa/rewrite.go index d16fee0ba2..58813d2fbe 100644 --- a/src/cmd/compile/internal/ssa/rewrite.go +++ b/src/cmd/compile/internal/ssa/rewrite.go @@ -7,7 +7,6 @@ package ssa import ( "cmd/compile/internal/base" "cmd/compile/internal/logopt" - "cmd/compile/internal/reflectdata" "cmd/compile/internal/types" "cmd/internal/obj" "cmd/internal/obj/s390x" @@ -20,7 +19,6 @@ import ( "math/bits" "os" "path/filepath" - "strings" ) type deadValueChoice bool @@ -801,6 +799,25 @@ func loadLSymOffset(lsym *obj.LSym, offset int64) *obj.LSym { return nil } +// de-virtualize an InterLECall +// 'sym' is the symbol for the itab. +func devirtLESym(v *Value, aux Aux, sym Sym, offset int64) *obj.LSym { + n, ok := sym.(*obj.LSym) + if !ok { + return nil + } + + lsym := loadLSymOffset(n, offset) + if f := v.Block.Func; f.pass.debug > 0 { + if lsym != nil { + f.Warnl(v.Pos, "de-virtualizing call") + } else { + f.Warnl(v.Pos, "couldn't de-virtualize call") + } + } + return lsym +} + func devirtLECall(v *Value, sym *obj.LSym) *Value { v.Op = OpStaticLECall auxcall := v.Aux.(*AuxCall) @@ -810,9 +827,6 @@ func devirtLECall(v *Value, sym *obj.LSym) *Value { copy(v.Args[0:], v.Args[1:]) v.Args[len(v.Args)-1] = nil // aid GC v.Args = v.Args[:len(v.Args)-1] - if f := v.Block.Func; f.pass.debug > 0 { - f.Warnl(v.Pos, "de-virtualizing call") - } return v } @@ -1729,73 +1743,6 @@ func symIsROZero(sym Sym) bool { return true } -// isFixed32 returns true if the int32 at offset off in symbol sym -// is known and constant. -func isFixed32(c *Config, sym Sym, off int64) bool { - return isFixed(c, sym, off, 4) -} - -// isFixed returns true if the range [off,off+size] of the symbol sym -// is known and constant. -func isFixed(c *Config, sym Sym, off, size int64) bool { - lsym := sym.(*obj.LSym) - if lsym.Extra == nil { - return false - } - if _, ok := (*lsym.Extra).(*obj.TypeInfo); ok { - if off == 2*c.PtrSize && size == 4 { - return true // type hash field - } - } - return false -} -func fixed32(c *Config, sym Sym, off int64) int32 { - lsym := sym.(*obj.LSym) - if ti, ok := (*lsym.Extra).(*obj.TypeInfo); ok { - if off == 2*c.PtrSize { - return int32(types.TypeHash(ti.Type.(*types.Type))) - } - } - base.Fatalf("fixed32 data not known for %s:%d", sym, off) - return 0 -} - -// isFixedSym returns true if the contents of sym at the given offset -// is known and is the constant address of another symbol. -func isFixedSym(sym Sym, off int64) bool { - lsym := sym.(*obj.LSym) - switch { - case lsym.Type == objabi.SRODATA: - // itabs, dictionaries - default: - return false - } - for _, r := range lsym.R { - if (r.Type == objabi.R_ADDR || r.Type == objabi.R_WEAKADDR) && int64(r.Off) == off && r.Add == 0 { - return true - } - } - return false -} -func fixedSym(f *Func, sym Sym, off int64) Sym { - lsym := sym.(*obj.LSym) - for _, r := range lsym.R { - if (r.Type == objabi.R_ADDR || r.Type == objabi.R_WEAKADDR) && int64(r.Off) == off { - if strings.HasPrefix(r.Sym.Name, "type:") { - // In case we're loading a type out of a dictionary, we need to record - // that the containing function might put that type in an interface. - // That information is currently recorded in relocations in the dictionary, - // but if we perform this load at compile time then the dictionary - // might be dead. - reflectdata.MarkTypeSymUsedInInterface(r.Sym, f.fe.Func().Linksym()) - } - return r.Sym - } - } - base.Fatalf("fixedSym data not known for %s:%d", sym, off) - return nil -} - // read8 reads one byte from the read-only global sym at offset off. func read8(sym interface{}, off int64) uint8 { lsym := sym.(*obj.LSym) diff --git a/src/cmd/compile/internal/ssa/rewritegeneric.go b/src/cmd/compile/internal/ssa/rewritegeneric.go index b9da2a6135..6026eac279 100644 --- a/src/cmd/compile/internal/ssa/rewritegeneric.go +++ b/src/cmd/compile/internal/ssa/rewritegeneric.go @@ -3,7 +3,6 @@ package ssa import "math" -import "cmd/internal/obj" import "cmd/compile/internal/types" import "cmd/compile/internal/ir" @@ -10118,28 +10117,6 @@ func rewriteValuegeneric_OpEqPtr(v *Value) bool { } break } - // match: (EqPtr (Convert (Addr {x} _) _) (Addr {y} _)) - // result: (ConstBool [x==y]) - for { - for _i0 := 0; _i0 <= 1; _i0, v_0, v_1 = _i0+1, v_1, v_0 { - if v_0.Op != OpConvert { - continue - } - v_0_0 := v_0.Args[0] - if v_0_0.Op != OpAddr { - continue - } - x := auxToSym(v_0_0.Aux) - if v_1.Op != OpAddr { - continue - } - y := auxToSym(v_1.Aux) - v.reset(OpConstBool) - v.AuxInt = boolToAuxInt(x == y) - return true - } - break - } // match: (EqPtr (LocalAddr _ _) (Addr _)) // result: (ConstBool [false]) for { @@ -10344,22 +10321,41 @@ func rewriteValuegeneric_OpIMake(v *Value) bool { return false } func rewriteValuegeneric_OpInterLECall(v *Value) bool { - // match: (InterLECall [argsize] {auxCall} (Addr {fn} (SB)) ___) - // result: devirtLECall(v, fn.(*obj.LSym)) + // match: (InterLECall [argsize] {auxCall} (Load (OffPtr [off] (ITab (IMake (Addr {itab} (SB)) _))) _) ___) + // cond: devirtLESym(v, auxCall, itab, off) != nil + // result: devirtLECall(v, devirtLESym(v, auxCall, itab, off)) for { if len(v.Args) < 1 { break } + auxCall := auxToCall(v.Aux) v_0 := v.Args[0] - if v_0.Op != OpAddr { + if v_0.Op != OpLoad { break } - fn := auxToSym(v_0.Aux) v_0_0 := v_0.Args[0] - if v_0_0.Op != OpSB { + if v_0_0.Op != OpOffPtr { break } - v.copyOf(devirtLECall(v, fn.(*obj.LSym))) + off := auxIntToInt64(v_0_0.AuxInt) + v_0_0_0 := v_0_0.Args[0] + if v_0_0_0.Op != OpITab { + break + } + v_0_0_0_0 := v_0_0_0.Args[0] + if v_0_0_0_0.Op != OpIMake { + break + } + v_0_0_0_0_0 := v_0_0_0_0.Args[0] + if v_0_0_0_0_0.Op != OpAddr { + break + } + itab := auxToSym(v_0_0_0_0_0.Aux) + v_0_0_0_0_0_0 := v_0_0_0_0_0.Args[0] + if v_0_0_0_0_0_0.Op != OpSB || !(devirtLESym(v, auxCall, itab, off) != nil) { + break + } + v.copyOf(devirtLECall(v, devirtLESym(v, auxCall, itab, off))) return true } return false @@ -11090,7 +11086,7 @@ func rewriteValuegeneric_OpIsNonNil(v *Value) bool { v.AuxInt = boolToAuxInt(c != 0) return true } - // match: (IsNonNil (Addr _) ) + // match: (IsNonNil (Addr _)) // result: (ConstBool [true]) for { if v_0.Op != OpAddr { @@ -11100,30 +11096,6 @@ func rewriteValuegeneric_OpIsNonNil(v *Value) bool { v.AuxInt = boolToAuxInt(true) return true } - // match: (IsNonNil (Convert (Addr _) _)) - // result: (ConstBool [true]) - for { - if v_0.Op != OpConvert { - break - } - v_0_0 := v_0.Args[0] - if v_0_0.Op != OpAddr { - break - } - v.reset(OpConstBool) - v.AuxInt = boolToAuxInt(true) - return true - } - // match: (IsNonNil (LocalAddr _ _)) - // result: (ConstBool [true]) - for { - if v_0.Op != OpLocalAddr { - break - } - v.reset(OpConstBool) - v.AuxInt = boolToAuxInt(true) - return true - } return false } func rewriteValuegeneric_OpIsSliceInBounds(v *Value) bool { @@ -12500,7 +12472,6 @@ func rewriteValuegeneric_OpLoad(v *Value) bool { v_1 := v.Args[1] v_0 := v.Args[0] b := v.Block - config := b.Func.Config fe := b.Func.fe // match: (Load p1 (Store {t2} p2 x _)) // cond: isSamePtr(p1, p2) && t1.Compare(x.Type) == types.CMPeq && t1.Size() == t2.Size() @@ -13192,230 +13163,6 @@ func rewriteValuegeneric_OpLoad(v *Value) bool { v.AddArg(v0) return true } - // match: (Load (OffPtr [off] (Addr {s} sb) ) _) - // cond: t.IsUintptr() && isFixedSym(s, off) - // result: (Addr {fixedSym(b.Func, s, off)} sb) - for { - t := v.Type - if v_0.Op != OpOffPtr { - break - } - off := auxIntToInt64(v_0.AuxInt) - v_0_0 := v_0.Args[0] - if v_0_0.Op != OpAddr { - break - } - s := auxToSym(v_0_0.Aux) - sb := v_0_0.Args[0] - if !(t.IsUintptr() && isFixedSym(s, off)) { - break - } - v.reset(OpAddr) - v.Aux = symToAux(fixedSym(b.Func, s, off)) - v.AddArg(sb) - return true - } - // match: (Load (OffPtr [off] (Convert (Addr {s} sb) _) ) _) - // cond: t.IsUintptr() && isFixedSym(s, off) - // result: (Addr {fixedSym(b.Func, s, off)} sb) - for { - t := v.Type - if v_0.Op != OpOffPtr { - break - } - off := auxIntToInt64(v_0.AuxInt) - v_0_0 := v_0.Args[0] - if v_0_0.Op != OpConvert { - break - } - v_0_0_0 := v_0_0.Args[0] - if v_0_0_0.Op != OpAddr { - break - } - s := auxToSym(v_0_0_0.Aux) - sb := v_0_0_0.Args[0] - if !(t.IsUintptr() && isFixedSym(s, off)) { - break - } - v.reset(OpAddr) - v.Aux = symToAux(fixedSym(b.Func, s, off)) - v.AddArg(sb) - return true - } - // match: (Load (OffPtr [off] (ITab (IMake (Addr {s} sb) _))) _) - // cond: t.IsUintptr() && isFixedSym(s, off) - // result: (Addr {fixedSym(b.Func, s, off)} sb) - for { - t := v.Type - if v_0.Op != OpOffPtr { - break - } - off := auxIntToInt64(v_0.AuxInt) - v_0_0 := v_0.Args[0] - if v_0_0.Op != OpITab { - break - } - v_0_0_0 := v_0_0.Args[0] - if v_0_0_0.Op != OpIMake { - break - } - v_0_0_0_0 := v_0_0_0.Args[0] - if v_0_0_0_0.Op != OpAddr { - break - } - s := auxToSym(v_0_0_0_0.Aux) - sb := v_0_0_0_0.Args[0] - if !(t.IsUintptr() && isFixedSym(s, off)) { - break - } - v.reset(OpAddr) - v.Aux = symToAux(fixedSym(b.Func, s, off)) - v.AddArg(sb) - return true - } - // match: (Load (OffPtr [off] (ITab (IMake (Convert (Addr {s} sb) _) _))) _) - // cond: t.IsUintptr() && isFixedSym(s, off) - // result: (Addr {fixedSym(b.Func, s, off)} sb) - for { - t := v.Type - if v_0.Op != OpOffPtr { - break - } - off := auxIntToInt64(v_0.AuxInt) - v_0_0 := v_0.Args[0] - if v_0_0.Op != OpITab { - break - } - v_0_0_0 := v_0_0.Args[0] - if v_0_0_0.Op != OpIMake { - break - } - v_0_0_0_0 := v_0_0_0.Args[0] - if v_0_0_0_0.Op != OpConvert { - break - } - v_0_0_0_0_0 := v_0_0_0_0.Args[0] - if v_0_0_0_0_0.Op != OpAddr { - break - } - s := auxToSym(v_0_0_0_0_0.Aux) - sb := v_0_0_0_0_0.Args[0] - if !(t.IsUintptr() && isFixedSym(s, off)) { - break - } - v.reset(OpAddr) - v.Aux = symToAux(fixedSym(b.Func, s, off)) - v.AddArg(sb) - return true - } - // match: (Load (OffPtr [off] (Addr {sym} _) ) _) - // cond: t.IsInteger() && t.Size() == 4 && isFixed32(config, sym, off) - // result: (Const32 [fixed32(config, sym, off)]) - for { - t := v.Type - if v_0.Op != OpOffPtr { - break - } - off := auxIntToInt64(v_0.AuxInt) - v_0_0 := v_0.Args[0] - if v_0_0.Op != OpAddr { - break - } - sym := auxToSym(v_0_0.Aux) - if !(t.IsInteger() && t.Size() == 4 && isFixed32(config, sym, off)) { - break - } - v.reset(OpConst32) - v.AuxInt = int32ToAuxInt(fixed32(config, sym, off)) - return true - } - // match: (Load (OffPtr [off] (Convert (Addr {sym} _) _) ) _) - // cond: t.IsInteger() && t.Size() == 4 && isFixed32(config, sym, off) - // result: (Const32 [fixed32(config, sym, off)]) - for { - t := v.Type - if v_0.Op != OpOffPtr { - break - } - off := auxIntToInt64(v_0.AuxInt) - v_0_0 := v_0.Args[0] - if v_0_0.Op != OpConvert { - break - } - v_0_0_0 := v_0_0.Args[0] - if v_0_0_0.Op != OpAddr { - break - } - sym := auxToSym(v_0_0_0.Aux) - if !(t.IsInteger() && t.Size() == 4 && isFixed32(config, sym, off)) { - break - } - v.reset(OpConst32) - v.AuxInt = int32ToAuxInt(fixed32(config, sym, off)) - return true - } - // match: (Load (OffPtr [off] (ITab (IMake (Addr {sym} _) _))) _) - // cond: t.IsInteger() && t.Size() == 4 && isFixed32(config, sym, off) - // result: (Const32 [fixed32(config, sym, off)]) - for { - t := v.Type - if v_0.Op != OpOffPtr { - break - } - off := auxIntToInt64(v_0.AuxInt) - v_0_0 := v_0.Args[0] - if v_0_0.Op != OpITab { - break - } - v_0_0_0 := v_0_0.Args[0] - if v_0_0_0.Op != OpIMake { - break - } - v_0_0_0_0 := v_0_0_0.Args[0] - if v_0_0_0_0.Op != OpAddr { - break - } - sym := auxToSym(v_0_0_0_0.Aux) - if !(t.IsInteger() && t.Size() == 4 && isFixed32(config, sym, off)) { - break - } - v.reset(OpConst32) - v.AuxInt = int32ToAuxInt(fixed32(config, sym, off)) - return true - } - // match: (Load (OffPtr [off] (ITab (IMake (Convert (Addr {sym} _) _) _))) _) - // cond: t.IsInteger() && t.Size() == 4 && isFixed32(config, sym, off) - // result: (Const32 [fixed32(config, sym, off)]) - for { - t := v.Type - if v_0.Op != OpOffPtr { - break - } - off := auxIntToInt64(v_0.AuxInt) - v_0_0 := v_0.Args[0] - if v_0_0.Op != OpITab { - break - } - v_0_0_0 := v_0_0.Args[0] - if v_0_0_0.Op != OpIMake { - break - } - v_0_0_0_0 := v_0_0_0.Args[0] - if v_0_0_0_0.Op != OpConvert { - break - } - v_0_0_0_0_0 := v_0_0_0_0.Args[0] - if v_0_0_0_0_0.Op != OpAddr { - break - } - sym := auxToSym(v_0_0_0_0_0.Aux) - if !(t.IsInteger() && t.Size() == 4 && isFixed32(config, sym, off)) { - break - } - v.reset(OpConst32) - v.AuxInt = int32ToAuxInt(fixed32(config, sym, off)) - return true - } return false } func rewriteValuegeneric_OpLsh16x16(v *Value) bool { @@ -18705,28 +18452,6 @@ func rewriteValuegeneric_OpNeqPtr(v *Value) bool { } break } - // match: (NeqPtr (Convert (Addr {x} _) _) (Addr {y} _)) - // result: (ConstBool [x!=y]) - for { - for _i0 := 0; _i0 <= 1; _i0, v_0, v_1 = _i0+1, v_1, v_0 { - if v_0.Op != OpConvert { - continue - } - v_0_0 := v_0.Args[0] - if v_0_0.Op != OpAddr { - continue - } - x := auxToSym(v_0_0.Aux) - if v_1.Op != OpAddr { - continue - } - y := auxToSym(v_1.Aux) - v.reset(OpConstBool) - v.AuxInt = boolToAuxInt(x != y) - return true - } - break - } // match: (NeqPtr (LocalAddr _ _) (Addr _)) // result: (ConstBool [true]) for { @@ -18928,36 +18653,6 @@ func rewriteValuegeneric_OpNilCheck(v *Value) bool { v.reset(OpInvalid) return true } - // match: (NilCheck (Addr {_} (SB)) _) - // result: (Invalid) - for { - if v_0.Op != OpAddr { - break - } - v_0_0 := v_0.Args[0] - if v_0_0.Op != OpSB { - break - } - v.reset(OpInvalid) - return true - } - // match: (NilCheck (Convert (Addr {_} (SB)) _) _) - // result: (Invalid) - for { - if v_0.Op != OpConvert { - break - } - v_0_0 := v_0.Args[0] - if v_0_0.Op != OpAddr { - break - } - v_0_0_0 := v_0_0.Args[0] - if v_0_0_0.Op != OpSB { - break - } - v.reset(OpInvalid) - return true - } return false } func rewriteValuegeneric_OpNot(v *Value) bool { diff --git a/src/cmd/internal/obj/link.go b/src/cmd/internal/obj/link.go index 74eb98520a..b50305f85c 100644 --- a/src/cmd/internal/obj/link.go +++ b/src/cmd/internal/obj/link.go @@ -465,7 +465,7 @@ type LSym struct { P []byte R []Reloc - Extra *interface{} // *FuncInfo, *FileInfo, or *TypeInfo, if present + Extra *interface{} // *FuncInfo or *FileInfo, if present Pkg string PkgIdx int32 @@ -564,22 +564,6 @@ func (s *LSym) File() *FileInfo { return f } -// A TypeInfo contains information for a symbol -// that contains a runtime._type. -type TypeInfo struct { - Type interface{} // a *cmd/compile/internal/types.Type -} - -func (s *LSym) NewTypeInfo() *TypeInfo { - if s.Extra != nil { - panic(fmt.Sprintf("invalid use of LSym - NewTypeInfo with Extra of type %T", *s.Extra)) - } - t := new(TypeInfo) - s.Extra = new(interface{}) - *s.Extra = t - return t -} - // WasmImport represents a WebAssembly (WASM) imported function with // parameters and results translated into WASM types based on the Go function // declaration. From 752ef81056691603c8b305a5f894215d3eb64b14 Mon Sep 17 00:00:00 2001 From: Ian Lance Taylor Date: Thu, 27 Apr 2023 15:56:34 -0700 Subject: [PATCH 020/299] net: re check conf.goos even if it equals runtime.GOOS This field is only for testing purposes, where we can't assume that the conf value was initialized as expected for that GOOS. This fixes the net tests on android. Change-Id: I8432587f219a05adbb4d234a813467f876a764b2 Reviewed-on: https://go-review.googlesource.com/c/go/+/489975 Run-TryBot: Ian Lance Taylor Reviewed-by: Bryan Mills TryBot-Result: Gopher Robot Auto-Submit: Ian Lance Taylor Run-TryBot: Ian Lance Taylor Reviewed-by: Ian Lance Taylor --- src/net/conf.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/net/conf.go b/src/net/conf.go index 2540ac5261..6386078132 100644 --- a/src/net/conf.go +++ b/src/net/conf.go @@ -230,7 +230,7 @@ func (c *conf) hostLookupOrder(r *Resolver, hostname string) (ret hostLookupOrde // For testing purposes only, recheck the GOOS. // This lets TestConfHostLookupOrder test different // GOOS values. - if c.goos != runtime.GOOS && goosPrefersCgo(c.goos) { + if goosPrefersCgo(c.goos) { return hostLookupCgo, nil } From ff059add10d71fe13239cf893c0cca113de1fc21 Mon Sep 17 00:00:00 2001 From: Lucien Coffe Date: Fri, 21 Apr 2023 13:44:35 +0200 Subject: [PATCH 021/299] runtime: resolve checkdead panic by refining `startm` lock handling in caller context This change addresses a `checkdead` panic caused by a race condition between `sysmon->startm` and `checkdead` callers, due to prematurely releasing the scheduler lock. The solution involves allowing a `startm` caller to acquire the scheduler lock and call `startm` in this context. A new `lockheld` bool argument is added to `startm`, which manages all lock and unlock calls within the function. The`startIdle` function variable in `injectglist` is updated to call `startm` with the lock held, ensuring proper lock handling in this specific case. This refined lock handling resolves the observed race condition issue. Fixes #59600 Change-Id: I11663a15536c10c773fc2fde291d959099aa71be Reviewed-on: https://go-review.googlesource.com/c/go/+/487316 Reviewed-by: Dmitri Shuralyov Reviewed-by: Michael Pratt TryBot-Result: Gopher Robot Run-TryBot: Michael Pratt --- src/runtime/proc.go | 45 +++++++++++++++++++++++++++++++-------------- 1 file changed, 31 insertions(+), 14 deletions(-) diff --git a/src/runtime/proc.go b/src/runtime/proc.go index cef9680db2..d2901e3aa0 100644 --- a/src/runtime/proc.go +++ b/src/runtime/proc.go @@ -2435,10 +2435,15 @@ func mspinning() { // Callers passing a non-nil P must call from a non-preemptible context. See // comment on acquirem below. // +// Argument lockheld indicates whether the caller already acquired the +// scheduler lock. Callers holding the lock when making the call must pass +// true. The lock might be temporarily dropped, but will be reacquired before +// returning. +// // Must not have write barriers because this may be called without a P. // //go:nowritebarrierrec -func startm(pp *p, spinning bool) { +func startm(pp *p, spinning, lockheld bool) { // Disable preemption. // // Every owned P must have an owner that will eventually stop it in the @@ -2456,7 +2461,9 @@ func startm(pp *p, spinning bool) { // startm. Callers passing a nil P may be preemptible, so we must // disable preemption before acquiring a P from pidleget below. mp := acquirem() - lock(&sched.lock) + if !lockheld { + lock(&sched.lock) + } if pp == nil { if spinning { // TODO(prattmic): All remaining calls to this function @@ -2466,7 +2473,9 @@ func startm(pp *p, spinning bool) { } pp, _ = pidleget(0) if pp == nil { - unlock(&sched.lock) + if !lockheld { + unlock(&sched.lock) + } releasem(mp) return } @@ -2480,6 +2489,8 @@ func startm(pp *p, spinning bool) { // could find no idle P while checkdead finds a runnable G but // no running M's because this new M hasn't started yet, thus // throwing in an apparent deadlock. + // This apparent deadlock is possible when startm is called + // from sysmon, which doesn't count as a running M. // // Avoid this situation by pre-allocating the ID for the new M, // thus marking it as 'running' before we drop sched.lock. This @@ -2494,12 +2505,18 @@ func startm(pp *p, spinning bool) { fn = mspinning } newm(fn, pp, id) + + if lockheld { + lock(&sched.lock) + } // Ownership transfer of pp committed by start in newm. // Preemption is now safe. releasem(mp) return } - unlock(&sched.lock) + if !lockheld { + unlock(&sched.lock) + } if nmp.spinning { throw("startm: m is spinning") } @@ -2528,24 +2545,24 @@ func handoffp(pp *p) { // if it has local work, start it straight away if !runqempty(pp) || sched.runqsize != 0 { - startm(pp, false) + startm(pp, false, false) return } // if there's trace work to do, start it straight away if (trace.enabled || trace.shutdown) && traceReaderAvailable() != nil { - startm(pp, false) + startm(pp, false, false) return } // if it has GC work, start it straight away if gcBlackenEnabled != 0 && gcMarkWorkAvailable(pp) { - startm(pp, false) + startm(pp, false, false) return } // no local work, check that there are no spinning/idle M's, // otherwise our help is not required if sched.nmspinning.Load()+sched.npidle.Load() == 0 && sched.nmspinning.CompareAndSwap(0, 1) { // TODO: fast atomic sched.needspinning.Store(0) - startm(pp, true) + startm(pp, true, false) return } lock(&sched.lock) @@ -2567,14 +2584,14 @@ func handoffp(pp *p) { } if sched.runqsize != 0 { unlock(&sched.lock) - startm(pp, false) + startm(pp, false, false) return } // If this is the last running P and nobody is polling network, // need to wakeup another M to poll network. if sched.npidle.Load() == gomaxprocs-1 && sched.lastpoll.Load() != 0 { unlock(&sched.lock) - startm(pp, false) + startm(pp, false, false) return } @@ -2623,7 +2640,7 @@ func wakep() { // see at least one running M (ours). unlock(&sched.lock) - startm(pp, true) + startm(pp, true, false) releasem(mp) } @@ -3376,8 +3393,8 @@ func injectglist(glist *gList) { break } + startm(pp, false, true) unlock(&sched.lock) - startm(pp, false) releasem(mp) } } @@ -5484,7 +5501,7 @@ func sysmon() { // See issue 42515 and // https://gnats.netbsd.org/cgi-bin/query-pr-single.pl?number=50094. if next := timeSleepUntil(); next < now { - startm(nil, false) + startm(nil, false, false) } } if scavenger.sysmonWake.Load() != 0 { @@ -5756,7 +5773,7 @@ func schedEnableUser(enable bool) { globrunqputbatch(&sched.disable.runnable, n) unlock(&sched.lock) for ; n != 0 && sched.npidle.Load() != 0; n-- { - startm(nil, false) + startm(nil, false, false) } } else { unlock(&sched.lock) From 6c8ea391cf7f84f6861e26f27eef3bd8938cfb98 Mon Sep 17 00:00:00 2001 From: Cherry Mui Date: Wed, 26 Apr 2023 17:29:33 -0400 Subject: [PATCH 022/299] cmd/link: write buildid to plugin Currently, in plugin build mode we don't write the build ID. This is disabled in CL 29394 since plugin is supported on Darwin. Maybe it caused some problem with the Darwin dynamic linker. But it seems no problem currently. Enabled it. Fixes #59845. Change-Id: I60589ffc7937e4d30055960d391cac1e7cd0cd42 Reviewed-on: https://go-review.googlesource.com/c/go/+/489457 TryBot-Result: Gopher Robot Reviewed-by: Than McIntosh Run-TryBot: Cherry Mui --- misc/cgo/testplugin/plugin_test.go | 16 ++++++++++++++-- src/cmd/link/internal/ld/data.go | 2 +- 2 files changed, 15 insertions(+), 3 deletions(-) diff --git a/misc/cgo/testplugin/plugin_test.go b/misc/cgo/testplugin/plugin_test.go index 285681018a..8960694351 100644 --- a/misc/cgo/testplugin/plugin_test.go +++ b/misc/cgo/testplugin/plugin_test.go @@ -116,11 +116,15 @@ func testMain(m *testing.M) int { return m.Run() } -func goCmd(t *testing.T, op string, args ...string) { +func goCmd(t *testing.T, op string, args ...string) string { if t != nil { t.Helper() } - run(t, filepath.Join(goroot, "bin", "go"), append([]string{op, "-gcflags", gcflags}, args...)...) + var flags []string + if op != "tool" { + flags = []string{"-gcflags", gcflags} + } + return run(t, filepath.Join(goroot, "bin", "go"), append(append([]string{op}, flags...), args...)...) } // escape converts a string to something suitable for a shell command line. @@ -190,6 +194,14 @@ func TestDWARFSections(t *testing.T) { goCmd(t, "run", "./checkdwarf/main.go", "./host.exe", "main.main") } +func TestBuildID(t *testing.T) { + // check that plugin has build ID. + b := goCmd(t, "tool", "buildid", "plugin1.so") + if len(b) == 0 { + t.Errorf("build id not found") + } +} + func TestRunHost(t *testing.T) { run(t, "./host.exe") } diff --git a/src/cmd/link/internal/ld/data.go b/src/cmd/link/internal/ld/data.go index c3550e59a5..7c135ae7e6 100644 --- a/src/cmd/link/internal/ld/data.go +++ b/src/cmd/link/internal/ld/data.go @@ -2248,7 +2248,7 @@ func (state *dodataState) dodataSect(ctxt *Link, symn sym.SymKind, syms []loader // at the very beginning of the text segment. // This “header” is read by cmd/go. func (ctxt *Link) textbuildid() { - if ctxt.IsELF || ctxt.BuildMode == BuildModePlugin || *flagBuildid == "" { + if ctxt.IsELF || *flagBuildid == "" { return } From 972774c4448f41826577304da0b5fd7d6936df4c Mon Sep 17 00:00:00 2001 From: Robert Griesemer Date: Thu, 27 Apr 2023 16:07:11 -0700 Subject: [PATCH 023/299] go/types, types2: call mustParse when using mustTypecheck Syntactically incorrect source files may produce valid (but unexpected) syntax trees, leading to difficult to understand test failures. Make sure to call mustParse when we call mustTypecheck. Change-Id: I9f5ba3fe57ad3bbc16caabf285d2e7aeb5b9de0c Reviewed-on: https://go-review.googlesource.com/c/go/+/489995 Run-TryBot: Robert Griesemer TryBot-Result: Gopher Robot Reviewed-by: Robert Findley Reviewed-by: Robert Griesemer Auto-Submit: Robert Griesemer --- src/cmd/compile/internal/types2/api_test.go | 8 +++++++- src/go/types/api_test.go | 9 ++++++++- 2 files changed, 15 insertions(+), 2 deletions(-) diff --git a/src/cmd/compile/internal/types2/api_test.go b/src/cmd/compile/internal/types2/api_test.go index e824f56fae..dcd4d72328 100644 --- a/src/cmd/compile/internal/types2/api_test.go +++ b/src/cmd/compile/internal/types2/api_test.go @@ -49,7 +49,13 @@ func typecheck(path, src string, conf *Config, info *Info) (*Package, error) { } func mustTypecheck(path, src string, conf *Config, info *Info) *Package { - pkg, err := typecheck(path, src, conf, info) + f := mustParse(path, src) + if conf == nil { + conf = &Config{ + Importer: defaultImporter(), + } + } + pkg, err := conf.Check(f.PkgName.Value, []*syntax.File{f}, info) if err != nil { panic(err) // so we don't need to pass *testing.T } diff --git a/src/go/types/api_test.go b/src/go/types/api_test.go index 7a8c63a43b..2d0df43263 100644 --- a/src/go/types/api_test.go +++ b/src/go/types/api_test.go @@ -52,7 +52,14 @@ func typecheck(path, src string, conf *Config, info *Info) (*Package, error) { } func mustTypecheck(path, src string, conf *Config, info *Info) *Package { - pkg, err := typecheck(path, src, conf, info) + fset := token.NewFileSet() + f := mustParse(fset, path, src) + if conf == nil { + conf = &Config{ + Importer: importer.Default(), + } + } + pkg, err := conf.Check(f.Name.Name, fset, []*ast.File{f}, info) if err != nil { panic(err) // so we don't need to pass *testing.T } From 08357012247db5c84002b7a4c1693411a1a9b295 Mon Sep 17 00:00:00 2001 From: Takeshi Yoneda Date: Thu, 27 Apr 2023 13:37:48 +0900 Subject: [PATCH 024/299] os: fix TestChdirAndGetwd/ReaddirnamesOneAtATime on wasip1 to run on Windows hosts TestReaddirnamesOneAtATime and TestChdirAndGetwd assumes the underlying file system has /usr/bin but it is not the case when running it on WASI runtime hosted on Windows. This change adds wasip1 in the special cased switch case to make them host OS agonstic. Change-Id: Idb667021b565f939c814b9cd9e637cd75f9a610d Reviewed-on: https://go-review.googlesource.com/c/go/+/489575 Auto-Submit: Ian Lance Taylor Reviewed-by: Johan Brandhorst-Satzkorn Reviewed-by: Dmitri Shuralyov Reviewed-by: Ian Lance Taylor TryBot-Result: Gopher Robot Run-TryBot: Ian Lance Taylor Run-TryBot: Johan Brandhorst-Satzkorn --- src/os/os_test.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/os/os_test.go b/src/os/os_test.go index 55651d8ace..a0d9411b6e 100644 --- a/src/os/os_test.go +++ b/src/os/os_test.go @@ -633,7 +633,7 @@ func TestReaddirnamesOneAtATime(t *testing.T) { switch runtime.GOOS { case "android": dir = "/system/bin" - case "ios": + case "ios", "wasip1": wd, err := Getwd() if err != nil { t.Fatal(err) @@ -1490,7 +1490,7 @@ func TestChdirAndGetwd(t *testing.T) { dirs = []string{"/system/bin"} case "plan9": dirs = []string{"/", "/usr"} - case "ios", "windows": + case "ios", "windows", "wasip1": dirs = nil for _, dir := range []string{t.TempDir(), t.TempDir()} { // Expand symlinks so path equality tests work. From c9b0d8b61ef222b151206546b5d56c1cbe0d3449 Mon Sep 17 00:00:00 2001 From: Robert Griesemer Date: Wed, 19 Apr 2023 15:34:49 -0700 Subject: [PATCH 025/299] go/types, types2: extract package name from test sources automatically This simplifies explicit tests and ensures that the error messages contain the package name instead of a generic file name like "p.go". Fixes #59736. Change-Id: I1b42e30f53ba88456e92f990d80ca68ffc987e20 Reviewed-on: https://go-review.googlesource.com/c/go/+/486617 TryBot-Result: Gopher Robot Auto-Submit: Robert Griesemer Reviewed-by: Robert Griesemer Reviewed-by: Robert Findley Run-TryBot: Robert Griesemer --- src/cmd/compile/internal/types2/api_test.go | 103 +++++++++-------- .../compile/internal/types2/builtins_test.go | 2 +- .../compile/internal/types2/example_test.go | 30 +++-- .../compile/internal/types2/hilbert_test.go | 2 +- .../internal/types2/instantiate_test.go | 10 +- .../compile/internal/types2/issues_test.go | 55 +++++---- src/cmd/compile/internal/types2/mono_test.go | 2 +- src/cmd/compile/internal/types2/named_test.go | 4 +- .../compile/internal/types2/object_test.go | 4 +- .../compile/internal/types2/resolver_test.go | 4 +- src/cmd/compile/internal/types2/sizes_test.go | 4 +- .../internal/types2/typestring_test.go | 6 +- src/go/types/api_test.go | 105 ++++++++++-------- src/go/types/builtins_test.go | 2 +- src/go/types/example_test.go | 30 +++-- src/go/types/hilbert_test.go | 2 +- src/go/types/instantiate_test.go | 10 +- src/go/types/issues_test.go | 51 ++++----- src/go/types/methodset_test.go | 2 +- src/go/types/mono_test.go | 2 +- src/go/types/named_test.go | 4 +- src/go/types/object_test.go | 4 +- src/go/types/resolver_test.go | 4 +- src/go/types/sizes_test.go | 4 +- src/go/types/typestring_test.go | 6 +- 25 files changed, 226 insertions(+), 226 deletions(-) diff --git a/src/cmd/compile/internal/types2/api_test.go b/src/cmd/compile/internal/types2/api_test.go index dcd4d72328..0e76a73699 100644 --- a/src/cmd/compile/internal/types2/api_test.go +++ b/src/cmd/compile/internal/types2/api_test.go @@ -21,21 +21,21 @@ import ( // nopos indicates an unknown position var nopos syntax.Pos -func parse(path, src string) (*syntax.File, error) { +func parse(src string) (*syntax.File, error) { errh := func(error) {} // dummy error handler so that parsing continues in presence of errors - return syntax.Parse(syntax.NewFileBase(path), strings.NewReader(src), errh, nil, 0) + return syntax.Parse(syntax.NewFileBase(pkgName(src)), strings.NewReader(src), errh, nil, 0) } -func mustParse(path, src string) *syntax.File { - f, err := parse(path, src) +func mustParse(src string) *syntax.File { + f, err := parse(src) if err != nil { panic(err) // so we don't need to pass *testing.T } return f } -func typecheck(path, src string, conf *Config, info *Info) (*Package, error) { - f, err := parse(path, src) +func typecheck(src string, conf *Config, info *Info) (*Package, error) { + f, err := parse(src) if f == nil { // ignore errors unless f is nil return nil, err } @@ -48,8 +48,8 @@ func typecheck(path, src string, conf *Config, info *Info) (*Package, error) { return conf.Check(f.PkgName.Value, []*syntax.File{f}, info) } -func mustTypecheck(path, src string, conf *Config, info *Info) *Package { - f := mustParse(path, src) +func mustTypecheck(src string, conf *Config, info *Info) *Package { + f := mustParse(src) if conf == nil { conf = &Config{ Importer: defaultImporter(), @@ -62,6 +62,20 @@ func mustTypecheck(path, src string, conf *Config, info *Info) *Package { return pkg } +// pkgName extracts the package name from src, which must contain a package header. +func pkgName(src string) string { + const kw = "package " + if i := strings.Index(src, kw); i >= 0 { + after := src[i+len(kw):] + n := len(after) + if i := strings.IndexAny(after, "\n\t ;/"); i >= 0 { + n = i + } + return after[:n] + } + panic("missing package header: " + src) +} + func TestValuesInfo(t *testing.T) { var tests = []struct { src string @@ -145,7 +159,7 @@ func TestValuesInfo(t *testing.T) { info := Info{ Types: make(map[syntax.Expr]TypeAndValue), } - name := mustTypecheck("ValuesInfo", test.src, nil, &info).Name() + name := mustTypecheck(test.src, nil, &info).Name() // look for expression var expr syntax.Expr @@ -387,7 +401,7 @@ func TestTypesInfo(t *testing.T) { info := Info{Types: make(map[syntax.Expr]TypeAndValue)} var name string if strings.HasPrefix(test.src, brokenPkg) { - pkg, err := typecheck("TypesInfo", test.src, nil, &info) + pkg, err := typecheck(test.src, nil, &info) if err == nil { t.Errorf("package %s: expected to fail but passed", pkg.Name()) continue @@ -396,7 +410,7 @@ func TestTypesInfo(t *testing.T) { name = pkg.Name() } } else { - name = mustTypecheck("TypesInfo", test.src, nil, &info).Name() + name = mustTypecheck(test.src, nil, &info).Name() } // look for expression type @@ -561,7 +575,7 @@ type T[P any] []P instMap := make(map[*syntax.Name]Instance) useMap := make(map[*syntax.Name]Object) makePkg := func(src string) *Package { - pkg, _ := typecheck("p.go", src, &conf, &Info{Instances: instMap, Uses: useMap}) + pkg, _ := typecheck(src, &conf, &Info{Instances: instMap, Uses: useMap}) imports[pkg.Name()] = pkg return pkg } @@ -657,7 +671,7 @@ func TestDefsInfo(t *testing.T) { info := Info{ Defs: make(map[*syntax.Name]Object), } - name := mustTypecheck("DefsInfo", test.src, nil, &info).Name() + name := mustTypecheck(test.src, nil, &info).Name() // find object var def Object @@ -722,7 +736,7 @@ func TestUsesInfo(t *testing.T) { info := Info{ Uses: make(map[*syntax.Name]Object), } - name := mustTypecheck("UsesInfo", test.src, nil, &info).Name() + name := mustTypecheck(test.src, nil, &info).Name() // find object var use Object @@ -754,7 +768,7 @@ func (r N[B]) m() { r.m(); r.n() } func (r *N[C]) n() { } ` - f := mustParse("p.go", src) + f := mustParse(src) info := Info{ Defs: make(map[*syntax.Name]Object), Uses: make(map[*syntax.Name]Object), @@ -862,7 +876,7 @@ func TestImplicitsInfo(t *testing.T) { info := Info{ Implicits: make(map[syntax.Node]Object), } - name := mustTypecheck("ImplicitsInfo", test.src, nil, &info).Name() + name := mustTypecheck(test.src, nil, &info).Name() // the test cases expect at most one Implicits entry if len(info.Implicits) > 1 { @@ -990,7 +1004,7 @@ func TestPredicatesInfo(t *testing.T) { for _, test := range tests { info := Info{Types: make(map[syntax.Expr]TypeAndValue)} - name := mustTypecheck("PredicatesInfo", test.src, nil, &info).Name() + name := mustTypecheck(test.src, nil, &info).Name() // look for expression predicates got := "" @@ -1082,7 +1096,7 @@ func TestScopesInfo(t *testing.T) { for _, test := range tests { info := Info{Scopes: make(map[syntax.Node]*Scope)} - name := mustTypecheck("ScopesInfo", test.src, nil, &info).Name() + name := mustTypecheck(test.src, nil, &info).Name() // number of scopes must match if len(info.Scopes) != len(test.scopes) { @@ -1270,7 +1284,7 @@ func TestInitOrderInfo(t *testing.T) { for _, test := range tests { info := Info{} - name := mustTypecheck("InitOrderInfo", test.src, nil, &info).Name() + name := mustTypecheck(test.src, nil, &info).Name() // number of initializers must match if len(info.InitOrder) != len(test.inits) { @@ -1290,8 +1304,8 @@ func TestInitOrderInfo(t *testing.T) { } func TestMultiFileInitOrder(t *testing.T) { - fileA := mustParse("", `package main; var a = 1`) - fileB := mustParse("", `package main; var b = 2`) + fileA := mustParse(`package main; var a = 1`) + fileB := mustParse(`package main; var b = 2`) // The initialization order must not depend on the parse // order of the files, only on the presentation order to @@ -1326,10 +1340,8 @@ func TestFiles(t *testing.T) { var info Info check := NewChecker(&conf, pkg, &info) - for i, src := range sources { - filename := fmt.Sprintf("sources%d", i) - f := mustParse(filename, src) - if err := check.Files([]*syntax.File{f}); err != nil { + for _, src := range sources { + if err := check.Files([]*syntax.File{mustParse(src)}); err != nil { t.Error(err) } } @@ -1361,7 +1373,7 @@ func TestSelection(t *testing.T) { imports := make(testImporter) conf := Config{Importer: imports} makePkg := func(path, src string) { - pkg := mustTypecheck(path, src, &conf, &Info{Selections: selections}) + pkg := mustTypecheck(src, &conf, &Info{Selections: selections}) imports[path] = pkg } @@ -1547,9 +1559,7 @@ func TestIssue8518(t *testing.T) { Importer: imports, } makePkg := func(path, src string) { - f := mustParse(path, src) - pkg, _ := conf.Check(path, []*syntax.File{f}, nil) // errors logged via conf.Error - imports[path] = pkg + imports[path], _ = conf.Check(path, []*syntax.File{mustParse(src)}, nil) // errors logged via conf.Error } const libSrc = ` @@ -1577,9 +1587,7 @@ func TestIssue59603(t *testing.T) { Importer: imports, } makePkg := func(path, src string) { - f := mustParse(path, src) - pkg, _ := conf.Check(path, []*syntax.File{f}, nil) // errors logged via conf.Error - imports[path] = pkg + imports[path], _ = conf.Check(path, []*syntax.File{mustParse(src)}, nil) // errors logged via conf.Error } const libSrc = ` @@ -1662,7 +1670,7 @@ func TestLookupFieldOrMethod(t *testing.T) { } for _, test := range tests { - pkg := mustTypecheck("test", "package p;"+test.src, nil, nil) + pkg := mustTypecheck("package p;"+test.src, nil, nil) obj := pkg.Scope().Lookup("a") if obj == nil { @@ -1707,7 +1715,7 @@ type Node[T any] struct { type Instance = *Tree[int] ` - f := mustParse("foo.go", src) + f := mustParse(src) pkg := NewPackage("pkg", f.PkgName.Value) if err := NewChecker(nil, pkg, nil).Files([]*syntax.File{f}); err != nil { panic(err) @@ -1736,9 +1744,8 @@ func TestScopeLookupParent(t *testing.T) { conf := Config{Importer: imports} var info Info makePkg := func(path, src string) { - f := mustParse(path, src) var err error - imports[path], err = conf.Check(path, []*syntax.File{f}, &info) + imports[path], err = conf.Check(path, []*syntax.File{mustParse(src)}, &info) if err != nil { t.Fatal(err) } @@ -1942,7 +1949,7 @@ func TestIdentical(t *testing.T) { } for _, test := range tests { - pkg := mustTypecheck("test", "package p;"+test.src, nil, nil) + pkg := mustTypecheck("package p;"+test.src, nil, nil) X := pkg.Scope().Lookup("X") Y := pkg.Scope().Lookup("Y") if X == nil || Y == nil { @@ -2015,7 +2022,7 @@ func TestIdenticalUnions(t *testing.T) { func TestIssue15305(t *testing.T) { const src = "package p; func f() int16; var _ = f(undef)" - f := mustParse("issue15305.go", src) + f := mustParse(src) conf := Config{ Error: func(err error) {}, // allow errors } @@ -2038,7 +2045,7 @@ func TestIssue15305(t *testing.T) { // types for composite literal expressions and composite literal type // expressions. func TestCompositeLitTypes(t *testing.T) { - for _, test := range []struct { + for i, test := range []struct { lit, typ string }{ {`[16]byte{}`, `[16]byte`}, @@ -2050,7 +2057,7 @@ func TestCompositeLitTypes(t *testing.T) { {`struct{}{}`, `struct{}`}, {`struct{x, y int; z complex128}{}`, `struct{x int; y int; z complex128}`}, } { - f := mustParse(test.lit, "package p; var _ = "+test.lit) + f := mustParse(fmt.Sprintf("package p%d; var _ = %s", i, test.lit)) types := make(map[syntax.Expr]TypeAndValue) if _, err := new(Config).Check("p", []*syntax.File{f}, &Info{Types: types}); err != nil { t.Fatalf("%s: %v", test.lit, err) @@ -2104,7 +2111,7 @@ func (*T1) m2() {} func f(x int) { y := x; print(y) } ` - f := mustParse("src", src) + f := mustParse(src) info := &Info{ Defs: make(map[*syntax.Name]Object), @@ -2162,7 +2169,7 @@ type T = foo.T var v T = c func f(x T) T { return foo.F(x) } ` - f := mustParse("src", src) + f := mustParse(src) files := []*syntax.File{f} // type-check using all possible importers @@ -2217,7 +2224,7 @@ func f(x T) T { return foo.F(x) } func TestInstantiate(t *testing.T) { // eventually we like more tests but this is a start const src = "package p; type T[P any] *T[P]" - pkg := mustTypecheck(".", src, nil, nil) + pkg := mustTypecheck(src, nil, nil) // type T should have one type parameter T := pkg.Scope().Lookup("T").Type().(*Named) @@ -2252,7 +2259,7 @@ func TestInstantiateErrors(t *testing.T) { for _, test := range tests { src := "package p; " + test.src - pkg := mustTypecheck(".", src, nil, nil) + pkg := mustTypecheck(src, nil, nil) T := pkg.Scope().Lookup("T").Type().(*Named) @@ -2290,7 +2297,7 @@ func TestInstanceIdentity(t *testing.T) { imports := make(testImporter) conf := Config{Importer: imports} makePkg := func(src string) { - f := mustParse("", src) + f := mustParse(src) name := f.PkgName.Value pkg, err := conf.Check(name, []*syntax.File{f}, nil) if err != nil { @@ -2346,7 +2353,7 @@ func fn() { info := &Info{ Defs: make(map[*syntax.Name]Object), } - f := mustParse("p.go", src) + f := mustParse(src) conf := Config{} pkg, err := conf.Check(f.PkgName.Value, []*syntax.File{f}, info) if err != nil { @@ -2477,7 +2484,7 @@ func (N4) m() type Bad Bad // invalid type ` - f := mustParse("p.go", src) + f := mustParse(src) conf := Config{Error: func(error) {}} pkg, _ := conf.Check(f.PkgName.Value, []*syntax.File{f}, nil) @@ -2572,7 +2579,7 @@ type V4 struct{} func (V4) M() ` - pkg := mustTypecheck("p.go", src, nil, nil) + pkg := mustTypecheck(src, nil, nil) T := pkg.Scope().Lookup("T").Type().Underlying().(*Interface) lookup := func(name string) (*Func, bool) { diff --git a/src/cmd/compile/internal/types2/builtins_test.go b/src/cmd/compile/internal/types2/builtins_test.go index 863aa95680..1066a91c61 100644 --- a/src/cmd/compile/internal/types2/builtins_test.go +++ b/src/cmd/compile/internal/types2/builtins_test.go @@ -174,7 +174,7 @@ func testBuiltinSignature(t *testing.T, name, src0, want string) { uses := make(map[*syntax.Name]Object) types := make(map[syntax.Expr]TypeAndValue) - mustTypecheck("p", src, nil, &Info{Uses: uses, Types: types}) + mustTypecheck(src, nil, &Info{Uses: uses, Types: types}) // find called function n := 0 diff --git a/src/cmd/compile/internal/types2/example_test.go b/src/cmd/compile/internal/types2/example_test.go index 3fcad04b77..7031fdb1ad 100644 --- a/src/cmd/compile/internal/types2/example_test.go +++ b/src/cmd/compile/internal/types2/example_test.go @@ -30,25 +30,23 @@ import ( func ExampleScope() { // Parse the source files for a package. var files []*syntax.File - for _, file := range []struct{ name, input string }{ - {"main.go", ` -package main + for _, src := range []string{ + `package main import "fmt" func main() { freezing := FToC(-18) fmt.Println(freezing, Boiling) } -`}, - {"celsius.go", ` -package main +`, + `package main import "fmt" type Celsius float64 func (c Celsius) String() string { return fmt.Sprintf("%g°C", c) } func FToC(f float64) Celsius { return Celsius(f - 32 / 9 * 5) } const Boiling Celsius = 100 func Unused() { {}; {{ var x int; _ = x }} } // make sure empty block scopes get printed -`}, +`, } { - files = append(files, mustParse(file.name, file.input)) + files = append(files, mustParse(src)) } // Type-check a package consisting of these files. @@ -74,13 +72,13 @@ func Unused() { {}; {{ var x int; _ = x }} } // make sure empty block scopes get // . func temperature.FToC(f float64) temperature.Celsius // . func temperature.Unused() // . func temperature.main() - // . main.go scope { + // . main scope { // . . package fmt // . . function scope { // . . . var freezing temperature.Celsius // . . } // . } - // . celsius.go scope { + // . main scope { // . . package fmt // . . function scope { // . . . var c temperature.Celsius @@ -127,7 +125,7 @@ func fib(x int) int { Defs: make(map[*syntax.Name]types2.Object), Uses: make(map[*syntax.Name]types2.Object), } - pkg := mustTypecheck("fib.go", input, nil, &info) + pkg := mustTypecheck(input, nil, &info) // Print package-level variables in initialization order. fmt.Printf("InitOrder: %v\n\n", info.InitOrder) @@ -181,10 +179,10 @@ func fib(x int) int { // defined at // used at 6:15 // func fib(x int) int: - // defined at fib.go:8:6 + // defined at fib:8:6 // used at 12:20, 12:9 // type S string: - // defined at fib.go:4:6 + // defined at fib:4:6 // used at 6:23 // type int: // defined at @@ -193,13 +191,13 @@ func fib(x int) int { // defined at // used at 4:8 // var b S: - // defined at fib.go:6:8 + // defined at fib:6:8 // used at 6:19 // var c string: - // defined at fib.go:6:11 + // defined at fib:6:11 // used at 6:25 // var x int: - // defined at fib.go:8:10 + // defined at fib:8:10 // used at 10:10, 12:13, 12:24, 9:5 } diff --git a/src/cmd/compile/internal/types2/hilbert_test.go b/src/cmd/compile/internal/types2/hilbert_test.go index 8b7ceb3c97..5b2b087820 100644 --- a/src/cmd/compile/internal/types2/hilbert_test.go +++ b/src/cmd/compile/internal/types2/hilbert_test.go @@ -25,7 +25,7 @@ func TestHilbert(t *testing.T) { return } - mustTypecheck("hilbert.go", string(src), nil, nil) + mustTypecheck(string(src), nil, nil) } func program(n int, out string) []byte { diff --git a/src/cmd/compile/internal/types2/instantiate_test.go b/src/cmd/compile/internal/types2/instantiate_test.go index e809d17de1..af772b993c 100644 --- a/src/cmd/compile/internal/types2/instantiate_test.go +++ b/src/cmd/compile/internal/types2/instantiate_test.go @@ -107,7 +107,7 @@ func TestInstantiateEquality(t *testing.T) { } for _, test := range tests { - pkg := mustTypecheck(".", test.src, nil, nil) + pkg := mustTypecheck(test.src, nil, nil) t.Run(pkg.Name(), func(t *testing.T) { ctxt := NewContext() @@ -133,8 +133,8 @@ func TestInstantiateEquality(t *testing.T) { func TestInstantiateNonEquality(t *testing.T) { const src = "package p; type T[P any] int" - pkg1 := mustTypecheck(".", src, nil, nil) - pkg2 := mustTypecheck(".", src, nil, nil) + pkg1 := mustTypecheck(src, nil, nil) + pkg2 := mustTypecheck(src, nil, nil) // We consider T1 and T2 to be distinct types, so their instances should not // be deduplicated by the context. T1 := pkg1.Scope().Lookup("T").Type().(*Named) @@ -179,7 +179,7 @@ var X T[int] for _, test := range tests { src := prefix + test.decl - pkg := mustTypecheck(".", src, nil, nil) + pkg := mustTypecheck(src, nil, nil) typ := NewPointer(pkg.Scope().Lookup("X").Type()) obj, _, _ := LookupFieldOrMethod(typ, false, pkg, "m") m, _ := obj.(*Func) @@ -201,7 +201,7 @@ func (T[P]) m() {} var _ T[int] ` - pkg := mustTypecheck(".", src, nil, nil) + pkg := mustTypecheck(src, nil, nil) typ := pkg.Scope().Lookup("T").Type().(*Named) obj, _, _ := LookupFieldOrMethod(typ, false, pkg, "m") if obj == nil { diff --git a/src/cmd/compile/internal/types2/issues_test.go b/src/cmd/compile/internal/types2/issues_test.go index c7b63a1e68..e3e295e079 100644 --- a/src/cmd/compile/internal/types2/issues_test.go +++ b/src/cmd/compile/internal/types2/issues_test.go @@ -19,7 +19,7 @@ import ( ) func TestIssue5770(t *testing.T) { - _, err := typecheck("p", `package p; type S struct{T}`, nil, nil) + _, err := typecheck(`package p; type S struct{T}`, nil, nil) const want = "undefined: T" if err == nil || !strings.Contains(err.Error(), want) { t.Errorf("got: %v; want: %s", err, want) @@ -39,7 +39,7 @@ var ( _ = (interface{})(nil) )` types := make(map[syntax.Expr]TypeAndValue) - mustTypecheck("p", src, nil, &Info{Types: types}) + mustTypecheck(src, nil, &Info{Types: types}) for x, tv := range types { var want Type @@ -78,7 +78,7 @@ func f() int { } ` types := make(map[syntax.Expr]TypeAndValue) - mustTypecheck("p", src, nil, &Info{Types: types}) + mustTypecheck(src, nil, &Info{Types: types}) want := Typ[Int] n := 0 @@ -102,7 +102,7 @@ package p func (T) m() (res bool) { return } type T struct{} // receiver type after method declaration ` - f := mustParse("", src) + f := mustParse(src) var conf Config defs := make(map[*syntax.Name]Object) @@ -148,7 +148,7 @@ L7 uses var z int` conf := Config{Error: func(err error) { t.Log(err) }} defs := make(map[*syntax.Name]Object) uses := make(map[*syntax.Name]Object) - _, err := typecheck("p", src, &conf, &Info{Defs: defs, Uses: uses}) + _, err := typecheck(src, &conf, &Info{Defs: defs, Uses: uses}) if s := err.Error(); !strings.HasSuffix(s, "cannot assign to w") { t.Errorf("Check: unexpected error: %s", s) } @@ -228,7 +228,7 @@ func main() { ` f := func(test, src string) { info := &Info{Uses: make(map[*syntax.Name]Object)} - mustTypecheck("main", src, nil, info) + mustTypecheck(src, nil, info) var pkg *Package count := 0 @@ -256,13 +256,13 @@ func TestIssue22525(t *testing.T) { got := "\n" conf := Config{Error: func(err error) { got += err.Error() + "\n" }} - typecheck("", src, &conf, nil) // do not crash + typecheck(src, &conf, nil) // do not crash want := ` -:1:27: a declared and not used -:1:30: b declared and not used -:1:33: c declared and not used -:1:36: d declared and not used -:1:39: e declared and not used +p:1:27: a declared and not used +p:1:30: b declared and not used +p:1:33: c declared and not used +p:1:36: d declared and not used +p:1:39: e declared and not used ` if got != want { t.Errorf("got: %swant: %s", got, want) @@ -282,7 +282,7 @@ func TestIssue25627(t *testing.T) { `struct { *I }`, `struct { a int; b Missing; *Missing }`, } { - f := mustParse("", prefix+src) + f := mustParse(prefix + src) conf := Config{Importer: defaultImporter(), Error: func(err error) {}} info := &Info{Types: make(map[syntax.Expr]TypeAndValue)} @@ -319,7 +319,7 @@ func TestIssue28005(t *testing.T) { // compute original file ASTs var orig [len(sources)]*syntax.File for i, src := range sources { - orig[i] = mustParse("", src) + orig[i] = mustParse(src) } // run the test for all order permutations of the incoming files @@ -394,8 +394,8 @@ func TestIssue28282(t *testing.T) { } func TestIssue29029(t *testing.T) { - f1 := mustParse("", `package p; type A interface { M() }`) - f2 := mustParse("", `package p; var B interface { A }`) + f1 := mustParse(`package p; type A interface { M() }`) + f2 := mustParse(`package p; var B interface { A }`) // printInfo prints the *Func definitions recorded in info, one *Func per line. printInfo := func(info *Info) string { @@ -441,10 +441,10 @@ func TestIssue34151(t *testing.T) { const asrc = `package a; type I interface{ M() }; type T struct { F interface { I } }` const bsrc = `package b; import "a"; type T struct { F interface { a.I } }; var _ = a.T(T{})` - a := mustTypecheck("a", asrc, nil, nil) + a := mustTypecheck(asrc, nil, nil) conf := Config{Importer: importHelper{pkg: a}} - mustTypecheck("b", bsrc, &conf, nil) + mustTypecheck(bsrc, &conf, nil) } type importHelper struct { @@ -482,13 +482,8 @@ func TestIssue34921(t *testing.T) { var pkg *Package for _, src := range sources { - f := mustParse("", src) conf := Config{Importer: importHelper{pkg: pkg}} - res, err := conf.Check(f.PkgName.Value, []*syntax.File{f}, nil) - if err != nil { - t.Errorf("%q failed to typecheck: %v", src, err) - } - pkg = res // res is imported by the next package in this test + pkg = mustTypecheck(src, &conf, nil) // pkg imported by the next package in this test } } @@ -551,12 +546,12 @@ func TestIssue43124(t *testing.T) { csrc = `package c; import ("a"; "html/template"); func _() { a.G(template.Template{}) }` ) - a := mustTypecheck("a", asrc, nil, nil) + a := mustTypecheck(asrc, nil, nil) conf := Config{Importer: importHelper{pkg: a, fallback: defaultImporter()}} // Packages should be fully qualified when there is ambiguity within the // error string itself. - _, err := typecheck("b", bsrc, &conf, nil) + _, err := typecheck(bsrc, &conf, nil) if err == nil { t.Fatal("package b had no errors") } @@ -565,7 +560,7 @@ func TestIssue43124(t *testing.T) { } // ...and also when there is any ambiguity in reachable packages. - _, err = typecheck("c", csrc, &conf, nil) + _, err = typecheck(csrc, &conf, nil) if err == nil { t.Fatal("package c had no errors") } @@ -663,7 +658,7 @@ func TestIssue51093(t *testing.T) { for _, test := range tests { src := fmt.Sprintf("package p; func _[P %s]() { _ = P(%s) }", test.typ, test.val) types := make(map[syntax.Expr]TypeAndValue) - mustTypecheck("p", src, nil, &Info{Types: types}) + mustTypecheck(src, nil, &Info{Types: types}) var n int for x, tv := range types { @@ -793,8 +788,8 @@ func (S) M5(struct {S;t}) {} test := func(main, b, want string) { re := regexp.MustCompile(want) - bpkg := mustTypecheck("b", b, nil, nil) - mast := mustParse("main.go", main) + bpkg := mustTypecheck(b, nil, nil) + mast := mustParse(main) conf := Config{Importer: importHelper{pkg: bpkg}} _, err := conf.Check(mast.PkgName.Value, []*syntax.File{mast}, nil) if err == nil { diff --git a/src/cmd/compile/internal/types2/mono_test.go b/src/cmd/compile/internal/types2/mono_test.go index 17450aa04b..c2955a2828 100644 --- a/src/cmd/compile/internal/types2/mono_test.go +++ b/src/cmd/compile/internal/types2/mono_test.go @@ -20,7 +20,7 @@ func checkMono(t *testing.T, body string) error { Error: func(err error) { fmt.Fprintln(&buf, err) }, Importer: defaultImporter(), } - typecheck("x", src, &conf, nil) + typecheck(src, &conf, nil) if buf.Len() == 0 { return nil } diff --git a/src/cmd/compile/internal/types2/named_test.go b/src/cmd/compile/internal/types2/named_test.go index 0b1819bbf9..705dcaee27 100644 --- a/src/cmd/compile/internal/types2/named_test.go +++ b/src/cmd/compile/internal/types2/named_test.go @@ -31,7 +31,7 @@ func (G[P]) N() (p P) { return } type Inst = G[int] ` - pkg := mustTypecheck("p", src, nil, nil) + pkg := mustTypecheck(src, nil, nil) var ( T = pkg.Scope().Lookup("T").Type() @@ -92,7 +92,7 @@ func (Node[Q]) M(Q) {} type Inst = *Tree[int] ` - f := mustParse("foo.go", src) + f := mustParse(src) pkg := NewPackage("p", f.PkgName.Value) if err := NewChecker(nil, pkg, nil).Files([]*syntax.File{f}); err != nil { t.Fatal(err) diff --git a/src/cmd/compile/internal/types2/object_test.go b/src/cmd/compile/internal/types2/object_test.go index 85b9e697ee..ef1a864ec9 100644 --- a/src/cmd/compile/internal/types2/object_test.go +++ b/src/cmd/compile/internal/types2/object_test.go @@ -56,7 +56,7 @@ func TestIsAlias(t *testing.T) { // the same Func Object as the original method. See also go.dev/issue/34421. func TestEmbeddedMethod(t *testing.T) { const src = `package p; type I interface { error }` - pkg := mustTypecheck("p", src, nil, nil) + pkg := mustTypecheck(src, nil, nil) // get original error.Error method eface := Universe.Lookup("error") @@ -110,7 +110,7 @@ func TestObjectString(t *testing.T) { for _, test := range testObjects { src := "package p; " + test.src - pkg, err := typecheck(filename, src, nil, nil) + pkg, err := typecheck(src, nil, nil) if err != nil { t.Errorf("%s: %s", src, err) continue diff --git a/src/cmd/compile/internal/types2/resolver_test.go b/src/cmd/compile/internal/types2/resolver_test.go index cafbfc9af6..923712b268 100644 --- a/src/cmd/compile/internal/types2/resolver_test.go +++ b/src/cmd/compile/internal/types2/resolver_test.go @@ -116,8 +116,8 @@ func TestResolveIdents(t *testing.T) { // parse package files var files []*syntax.File - for i, src := range sources { - files = append(files, mustParse(fmt.Sprintf("sources[%d]", i), src)) + for _, src := range sources { + files = append(files, mustParse(src)) } // resolve and type-check package AST diff --git a/src/cmd/compile/internal/types2/sizes_test.go b/src/cmd/compile/internal/types2/sizes_test.go index f2e4a87c71..7af89583f2 100644 --- a/src/cmd/compile/internal/types2/sizes_test.go +++ b/src/cmd/compile/internal/types2/sizes_test.go @@ -20,7 +20,7 @@ func findStructType(t *testing.T, src string) *types2.Struct { func findStructTypeConfig(t *testing.T, src string, conf *types2.Config) *types2.Struct { types := make(map[syntax.Expr]types2.TypeAndValue) - mustTypecheck("x", src, nil, &types2.Info{Types: types}) + mustTypecheck(src, nil, &types2.Info{Types: types}) for _, tv := range types { if ts, ok := tv.Type.(*types2.Struct); ok { return ts @@ -89,7 +89,7 @@ const _ = unsafe.Offsetof(struct{ x int64 }{}.x) Importer: defaultImporter(), Sizes: &types2.StdSizes{WordSize: 8, MaxAlign: 8}, } - mustTypecheck("x", src, &conf, &info) + mustTypecheck(src, &conf, &info) for _, tv := range info.Types { _ = conf.Sizes.Sizeof(tv.Type) _ = conf.Sizes.Alignof(tv.Type) diff --git a/src/cmd/compile/internal/types2/typestring_test.go b/src/cmd/compile/internal/types2/typestring_test.go index 193ee251f0..c2be40da29 100644 --- a/src/cmd/compile/internal/types2/typestring_test.go +++ b/src/cmd/compile/internal/types2/typestring_test.go @@ -118,7 +118,7 @@ func TestTypeString(t *testing.T) { for _, test := range tests { src := `package generic_p; import "io"; type _ io.Writer; type T ` + test.src - pkg, err := typecheck(filename, src, nil, nil) + pkg, err := typecheck(src, nil, nil) if err != nil { t.Errorf("%s: %s", src, err) continue @@ -136,8 +136,8 @@ func TestTypeString(t *testing.T) { } func TestQualifiedTypeString(t *testing.T) { - p := mustTypecheck("p.go", "package p; type T int", nil, nil) - q := mustTypecheck("q.go", "package q", nil, nil) + p := mustTypecheck("package p; type T int", nil, nil) + q := mustTypecheck("package q", nil, nil) pT := p.Scope().Lookup("T").Type() for _, test := range []struct { diff --git a/src/go/types/api_test.go b/src/go/types/api_test.go index 2d0df43263..825f30585b 100644 --- a/src/go/types/api_test.go +++ b/src/go/types/api_test.go @@ -24,21 +24,21 @@ import ( // nopos indicates an unknown position var nopos token.Pos -func parse(fset *token.FileSet, filename, src string) (*ast.File, error) { - return parser.ParseFile(fset, filename, src, 0) +func parse(fset *token.FileSet, src string) (*ast.File, error) { + return parser.ParseFile(fset, pkgName(src), src, 0) } -func mustParse(fset *token.FileSet, filename, src string) *ast.File { - f, err := parse(fset, filename, src) +func mustParse(fset *token.FileSet, src string) *ast.File { + f, err := parse(fset, src) if err != nil { panic(err) // so we don't need to pass *testing.T } return f } -func typecheck(path, src string, conf *Config, info *Info) (*Package, error) { +func typecheck(src string, conf *Config, info *Info) (*Package, error) { fset := token.NewFileSet() - f, err := parse(fset, path, src) + f, err := parse(fset, src) if f == nil { // ignore errors unless f is nil return nil, err } @@ -51,9 +51,9 @@ func typecheck(path, src string, conf *Config, info *Info) (*Package, error) { return conf.Check(f.Name.Name, fset, []*ast.File{f}, info) } -func mustTypecheck(path, src string, conf *Config, info *Info) *Package { +func mustTypecheck(src string, conf *Config, info *Info) *Package { fset := token.NewFileSet() - f := mustParse(fset, path, src) + f := mustParse(fset, src) if conf == nil { conf = &Config{ Importer: importer.Default(), @@ -66,6 +66,20 @@ func mustTypecheck(path, src string, conf *Config, info *Info) *Package { return pkg } +// pkgName extracts the package name from src, which must contain a package header. +func pkgName(src string) string { + const kw = "package " + if i := strings.Index(src, kw); i >= 0 { + after := src[i+len(kw):] + n := len(after) + if i := strings.IndexAny(after, "\n\t ;/"); i >= 0 { + n = i + } + return after[:n] + } + panic("missing package header: " + src) +} + func TestValuesInfo(t *testing.T) { var tests = []struct { src string @@ -149,7 +163,7 @@ func TestValuesInfo(t *testing.T) { info := Info{ Types: make(map[ast.Expr]TypeAndValue), } - name := mustTypecheck("ValuesInfo", test.src, nil, &info).Name() + name := mustTypecheck(test.src, nil, &info).Name() // look for expression var expr ast.Expr @@ -387,7 +401,7 @@ func TestTypesInfo(t *testing.T) { info := Info{Types: make(map[ast.Expr]TypeAndValue)} var name string if strings.HasPrefix(test.src, broken) { - pkg, err := typecheck("TypesInfo", test.src, nil, &info) + pkg, err := typecheck(test.src, nil, &info) if err == nil { t.Errorf("package %s: expected to fail but passed", pkg.Name()) continue @@ -396,7 +410,7 @@ func TestTypesInfo(t *testing.T) { name = pkg.Name() } } else { - name = mustTypecheck("TypesInfo", test.src, nil, &info).Name() + name = mustTypecheck(test.src, nil, &info).Name() } // look for expression type @@ -561,7 +575,7 @@ type T[P any] []P instMap := make(map[*ast.Ident]Instance) useMap := make(map[*ast.Ident]Object) makePkg := func(src string) *Package { - pkg, _ := typecheck("p.go", src, &conf, &Info{Instances: instMap, Uses: useMap}) + pkg, _ := typecheck(src, &conf, &Info{Instances: instMap, Uses: useMap}) imports[pkg.Name()] = pkg return pkg } @@ -656,7 +670,7 @@ func TestDefsInfo(t *testing.T) { info := Info{ Defs: make(map[*ast.Ident]Object), } - name := mustTypecheck("DefsInfo", test.src, nil, &info).Name() + name := mustTypecheck(test.src, nil, &info).Name() // find object var def Object @@ -723,7 +737,7 @@ func TestUsesInfo(t *testing.T) { info := Info{ Uses: make(map[*ast.Ident]Object), } - name := mustTypecheck("UsesInfo", test.src, nil, &info).Name() + name := mustTypecheck(test.src, nil, &info).Name() // find object var use Object @@ -756,7 +770,7 @@ func (r N[B]) m() { r.m(); r.n() } func (r *N[C]) n() { } ` fset := token.NewFileSet() - f := mustParse(fset, "p.go", src) + f := mustParse(fset, src) info := Info{ Defs: make(map[*ast.Ident]Object), Uses: make(map[*ast.Ident]Object), @@ -864,7 +878,7 @@ func TestImplicitsInfo(t *testing.T) { info := Info{ Implicits: make(map[ast.Node]Object), } - name := mustTypecheck("ImplicitsInfo", test.src, nil, &info).Name() + name := mustTypecheck(test.src, nil, &info).Name() // the test cases expect at most one Implicits entry if len(info.Implicits) > 1 { @@ -992,7 +1006,7 @@ func TestPredicatesInfo(t *testing.T) { for _, test := range tests { info := Info{Types: make(map[ast.Expr]TypeAndValue)} - name := mustTypecheck("PredicatesInfo", test.src, nil, &info).Name() + name := mustTypecheck(test.src, nil, &info).Name() // look for expression predicates got := "" @@ -1084,7 +1098,7 @@ func TestScopesInfo(t *testing.T) { for _, test := range tests { info := Info{Scopes: make(map[ast.Node]*Scope)} - name := mustTypecheck("ScopesInfo", test.src, nil, &info).Name() + name := mustTypecheck(test.src, nil, &info).Name() // number of scopes must match if len(info.Scopes) != len(test.scopes) { @@ -1272,7 +1286,7 @@ func TestInitOrderInfo(t *testing.T) { for _, test := range tests { info := Info{} - name := mustTypecheck("InitOrderInfo", test.src, nil, &info).Name() + name := mustTypecheck(test.src, nil, &info).Name() // number of initializers must match if len(info.InitOrder) != len(test.inits) { @@ -1293,8 +1307,8 @@ func TestInitOrderInfo(t *testing.T) { func TestMultiFileInitOrder(t *testing.T) { fset := token.NewFileSet() - fileA := mustParse(fset, "", `package main; var a = 1`) - fileB := mustParse(fset, "", `package main; var b = 2`) + fileA := mustParse(fset, `package main; var a = 1`) + fileB := mustParse(fset, `package main; var b = 2`) // The initialization order must not depend on the parse // order of the files, only on the presentation order to @@ -1330,10 +1344,8 @@ func TestFiles(t *testing.T) { var info Info check := NewChecker(&conf, fset, pkg, &info) - for i, src := range sources { - filename := fmt.Sprintf("sources%d", i) - f := mustParse(fset, filename, src) - if err := check.Files([]*ast.File{f}); err != nil { + for _, src := range sources { + if err := check.Files([]*ast.File{mustParse(fset, src)}); err != nil { t.Error(err) } } @@ -1368,8 +1380,7 @@ func TestSelection(t *testing.T) { imports := make(testImporter) conf := Config{Importer: imports} makePkg := func(path, src string) { - f := mustParse(fset, path+".go", src) - pkg, err := conf.Check(path, fset, []*ast.File{f}, &Info{Selections: selections}) + pkg, err := conf.Check(path, fset, []*ast.File{mustParse(fset, src)}, &Info{Selections: selections}) if err != nil { t.Fatal(err) } @@ -1546,9 +1557,7 @@ func TestIssue8518(t *testing.T) { Importer: imports, } makePkg := func(path, src string) { - f := mustParse(fset, path, src) - pkg, _ := conf.Check(path, fset, []*ast.File{f}, nil) // errors logged via conf.Error - imports[path] = pkg + imports[path], _ = conf.Check(path, fset, []*ast.File{mustParse(fset, src)}, nil) // errors logged via conf.Error } const libSrc = ` @@ -1577,9 +1586,7 @@ func TestIssue59603(t *testing.T) { Importer: imports, } makePkg := func(path, src string) { - f := mustParse(fset, path, src) - pkg, _ := conf.Check(path, fset, []*ast.File{f}, nil) // errors logged via conf.Error - imports[path] = pkg + imports[path], _ = conf.Check(path, fset, []*ast.File{mustParse(fset, src)}, nil) // errors logged via conf.Error } const libSrc = ` @@ -1664,7 +1671,7 @@ func TestLookupFieldOrMethod(t *testing.T) { } for _, test := range tests { - pkg := mustTypecheck("test", "package p;"+test.src, nil, nil) + pkg := mustTypecheck("package p;"+test.src, nil, nil) obj := pkg.Scope().Lookup("a") if obj == nil { @@ -1710,7 +1717,7 @@ type Instance = *Tree[int] ` fset := token.NewFileSet() - f := mustParse(fset, "foo.go", src) + f := mustParse(fset, src) pkg := NewPackage("pkg", f.Name.Name) if err := NewChecker(nil, fset, pkg, nil).Files([]*ast.File{f}); err != nil { panic(err) @@ -1747,7 +1754,7 @@ func TestScopeLookupParent(t *testing.T) { } } - makePkg("lib", mustParse(fset, "", "package lib; var X int")) + makePkg("lib", mustParse(fset, "package lib; var X int")) // Each /*name=kind:line*/ comment makes the test look up the // name at that point and checks that it resolves to a decl of // the specified kind and line number. "undef" means undefined. @@ -1791,7 +1798,7 @@ func F(){ ` info.Uses = make(map[*ast.Ident]Object) - f := mustParse(fset, "", mainSrc) + f := mustParse(fset, mainSrc) makePkg("main", f) mainScope := imports["main"].Scope() rx := regexp.MustCompile(`^/\*(\w*)=([\w:]*)\*/$`) @@ -1943,7 +1950,7 @@ func TestIdentical(t *testing.T) { } for _, test := range tests { - pkg := mustTypecheck("test", "package p;"+test.src, nil, nil) + pkg := mustTypecheck("package p;"+test.src, nil, nil) X := pkg.Scope().Lookup("X") Y := pkg.Scope().Lookup("Y") if X == nil || Y == nil { @@ -2017,7 +2024,7 @@ func TestIdenticalUnions(t *testing.T) { func TestIssue15305(t *testing.T) { const src = "package p; func f() int16; var _ = f(undef)" fset := token.NewFileSet() - f := mustParse(fset, "issue15305.go", src) + f := mustParse(fset, src) conf := Config{ Error: func(err error) {}, // allow errors } @@ -2040,7 +2047,7 @@ func TestIssue15305(t *testing.T) { // types for composite literal expressions and composite literal type // expressions. func TestCompositeLitTypes(t *testing.T) { - for _, test := range []struct { + for i, test := range []struct { lit, typ string }{ {`[16]byte{}`, `[16]byte`}, @@ -2053,7 +2060,7 @@ func TestCompositeLitTypes(t *testing.T) { {`struct{x, y int; z complex128}{}`, `struct{x int; y int; z complex128}`}, } { fset := token.NewFileSet() - f := mustParse(fset, test.lit, "package p; var _ = "+test.lit) + f := mustParse(fset, fmt.Sprintf("package p%d; var _ = %s", i, test.lit)) types := make(map[ast.Expr]TypeAndValue) if _, err := new(Config).Check("p", fset, []*ast.File{f}, &Info{Types: types}); err != nil { t.Fatalf("%s: %v", test.lit, err) @@ -2108,7 +2115,7 @@ func f(x int) { y := x; print(y) } ` fset := token.NewFileSet() - f := mustParse(fset, "src", src) + f := mustParse(fset, src) info := &Info{ Defs: make(map[*ast.Ident]Object), @@ -2167,7 +2174,7 @@ var v T = c func f(x T) T { return foo.F(x) } ` fset := token.NewFileSet() - f := mustParse(fset, "src", src) + f := mustParse(fset, src) files := []*ast.File{f} // type-check using all possible importers @@ -2222,7 +2229,7 @@ func f(x T) T { return foo.F(x) } func TestInstantiate(t *testing.T) { // eventually we like more tests but this is a start const src = "package p; type T[P any] *T[P]" - pkg := mustTypecheck(".", src, nil, nil) + pkg := mustTypecheck(src, nil, nil) // type T should have one type parameter T := pkg.Scope().Lookup("T").Type().(*Named) @@ -2257,7 +2264,7 @@ func TestInstantiateErrors(t *testing.T) { for _, test := range tests { src := "package p; " + test.src - pkg := mustTypecheck(".", src, nil, nil) + pkg := mustTypecheck(src, nil, nil) T := pkg.Scope().Lookup("T").Type().(*Named) @@ -2296,7 +2303,7 @@ func TestInstanceIdentity(t *testing.T) { conf := Config{Importer: imports} makePkg := func(src string) { fset := token.NewFileSet() - f := mustParse(fset, "", src) + f := mustParse(fset, src) name := f.Name.Name pkg, err := conf.Check(name, fset, []*ast.File{f}, nil) if err != nil { @@ -2353,7 +2360,7 @@ func fn() { Defs: make(map[*ast.Ident]Object), } fset := token.NewFileSet() - f := mustParse(fset, "p.go", src) + f := mustParse(fset, src) conf := Config{} pkg, err := conf.Check(f.Name.Name, fset, []*ast.File{f}, info) if err != nil { @@ -2485,7 +2492,7 @@ type Bad Bad // invalid type ` fset := token.NewFileSet() - f := mustParse(fset, "p.go", src) + f := mustParse(fset, src) conf := Config{Error: func(error) {}} pkg, _ := conf.Check(f.Name.Name, fset, []*ast.File{f}, nil) @@ -2580,7 +2587,7 @@ type V4 struct{} func (V4) M() ` - pkg := mustTypecheck("p.go", src, nil, nil) + pkg := mustTypecheck(src, nil, nil) T := pkg.Scope().Lookup("T").Type().Underlying().(*Interface) lookup := func(name string) (*Func, bool) { diff --git a/src/go/types/builtins_test.go b/src/go/types/builtins_test.go index 5591fecf02..6238464f58 100644 --- a/src/go/types/builtins_test.go +++ b/src/go/types/builtins_test.go @@ -174,7 +174,7 @@ func testBuiltinSignature(t *testing.T, name, src0, want string) { uses := make(map[*ast.Ident]Object) types := make(map[ast.Expr]TypeAndValue) - mustTypecheck("p", src, nil, &Info{Uses: uses, Types: types}) + mustTypecheck(src, nil, &Info{Uses: uses, Types: types}) // find called function n := 0 diff --git a/src/go/types/example_test.go b/src/go/types/example_test.go index 37b5ea4511..1ee47bc123 100644 --- a/src/go/types/example_test.go +++ b/src/go/types/example_test.go @@ -35,25 +35,23 @@ func ExampleScope() { // Parse the source files for a package. fset := token.NewFileSet() var files []*ast.File - for _, file := range []struct{ name, input string }{ - {"main.go", ` -package main + for _, src := range []string{ + `package main import "fmt" func main() { freezing := FToC(-18) fmt.Println(freezing, Boiling) } -`}, - {"celsius.go", ` -package main +`, + `package main import "fmt" type Celsius float64 func (c Celsius) String() string { return fmt.Sprintf("%g°C", c) } func FToC(f float64) Celsius { return Celsius(f - 32 / 9 * 5) } const Boiling Celsius = 100 func Unused() { {}; {{ var x int; _ = x }} } // make sure empty block scopes get printed -`}, +`, } { - files = append(files, mustParse(fset, file.name, file.input)) + files = append(files, mustParse(fset, src)) } // Type-check a package consisting of these files. @@ -79,13 +77,13 @@ func Unused() { {}; {{ var x int; _ = x }} } // make sure empty block scopes get // . func temperature.FToC(f float64) temperature.Celsius // . func temperature.Unused() // . func temperature.main() - // . main.go scope { + // . main scope { // . . package fmt // . . function scope { // . . . var freezing temperature.Celsius // . . } // . } - // . celsius.go scope { + // . main scope { // . . package fmt // . . function scope { // . . . var c temperature.Celsius @@ -183,7 +181,7 @@ func fib(x int) int { // We need a specific fileset in this test below for positions. // Cannot use typecheck helper. fset := token.NewFileSet() - f := mustParse(fset, "fib.go", input) + f := mustParse(fset, input) // Type-check the package. // We create an empty map for each kind of input @@ -250,10 +248,10 @@ func fib(x int) int { // defined at - // used at 6:15 // func fib(x int) int: - // defined at fib.go:8:6 + // defined at fib:8:6 // used at 12:20, 12:9 // type S string: - // defined at fib.go:4:6 + // defined at fib:4:6 // used at 6:23 // type int: // defined at - @@ -262,13 +260,13 @@ func fib(x int) int { // defined at - // used at 4:8 // var b S: - // defined at fib.go:6:8 + // defined at fib:6:8 // used at 6:19 // var c string: - // defined at fib.go:6:11 + // defined at fib:6:11 // used at 6:25 // var x int: - // defined at fib.go:8:10 + // defined at fib:8:10 // used at 10:10, 12:13, 12:24, 9:5 // // Types and Values of each expression: diff --git a/src/go/types/hilbert_test.go b/src/go/types/hilbert_test.go index 7da2a7ded1..f530422492 100644 --- a/src/go/types/hilbert_test.go +++ b/src/go/types/hilbert_test.go @@ -27,7 +27,7 @@ func TestHilbert(t *testing.T) { return } - mustTypecheck("hilbert.go", string(src), nil, nil) + mustTypecheck(string(src), nil, nil) } func program(n int, out string) []byte { diff --git a/src/go/types/instantiate_test.go b/src/go/types/instantiate_test.go index 574f3aeb86..58dfa70131 100644 --- a/src/go/types/instantiate_test.go +++ b/src/go/types/instantiate_test.go @@ -109,7 +109,7 @@ func TestInstantiateEquality(t *testing.T) { } for _, test := range tests { - pkg := mustTypecheck(".", test.src, nil, nil) + pkg := mustTypecheck(test.src, nil, nil) t.Run(pkg.Name(), func(t *testing.T) { ctxt := NewContext() @@ -135,8 +135,8 @@ func TestInstantiateEquality(t *testing.T) { func TestInstantiateNonEquality(t *testing.T) { const src = "package p; type T[P any] int" - pkg1 := mustTypecheck(".", src, nil, nil) - pkg2 := mustTypecheck(".", src, nil, nil) + pkg1 := mustTypecheck(src, nil, nil) + pkg2 := mustTypecheck(src, nil, nil) // We consider T1 and T2 to be distinct types, so their instances should not // be deduplicated by the context. T1 := pkg1.Scope().Lookup("T").Type().(*Named) @@ -181,7 +181,7 @@ var X T[int] for _, test := range tests { src := prefix + test.decl - pkg := mustTypecheck(".", src, nil, nil) + pkg := mustTypecheck(src, nil, nil) typ := NewPointer(pkg.Scope().Lookup("X").Type()) obj, _, _ := LookupFieldOrMethod(typ, false, pkg, "m") m, _ := obj.(*Func) @@ -203,7 +203,7 @@ func (T[P]) m() {} var _ T[int] ` - pkg := mustTypecheck(".", src, nil, nil) + pkg := mustTypecheck(src, nil, nil) typ := pkg.Scope().Lookup("T").Type().(*Named) obj, _, _ := LookupFieldOrMethod(typ, false, pkg, "m") if obj == nil { diff --git a/src/go/types/issues_test.go b/src/go/types/issues_test.go index 888ca3cc70..a464659aaf 100644 --- a/src/go/types/issues_test.go +++ b/src/go/types/issues_test.go @@ -21,7 +21,7 @@ import ( ) func TestIssue5770(t *testing.T) { - _, err := typecheck("p", `package p; type S struct{T}`, nil, nil) + _, err := typecheck(`package p; type S struct{T}`, nil, nil) const want = "undefined: T" if err == nil || !strings.Contains(err.Error(), want) { t.Errorf("got: %v; want: %s", err, want) @@ -41,7 +41,7 @@ var ( _ = (interface{})(nil) )` types := make(map[ast.Expr]TypeAndValue) - mustTypecheck("p", src, nil, &Info{Types: types}) + mustTypecheck(src, nil, &Info{Types: types}) for x, tv := range types { var want Type @@ -80,7 +80,7 @@ func f() int { } ` types := make(map[ast.Expr]TypeAndValue) - mustTypecheck("p", src, nil, &Info{Types: types}) + mustTypecheck(src, nil, &Info{Types: types}) want := Typ[Int] n := 0 @@ -104,7 +104,7 @@ package p func (T) m() (res bool) { return } type T struct{} // receiver type after method declaration ` - f := mustParse(fset, "", src) + f := mustParse(fset, src) var conf Config defs := make(map[*ast.Ident]Object) @@ -138,7 +138,7 @@ func _() { // We need a specific fileset in this test below for positions. // Cannot use typecheck helper. fset := token.NewFileSet() - f := mustParse(fset, "", src) + f := mustParse(fset, src) const want = `L3 defs func p._() L4 defs const w untyped int @@ -235,7 +235,7 @@ func main() { ` f := func(test, src string) { info := &Info{Uses: make(map[*ast.Ident]Object)} - mustTypecheck("main", src, nil, info) + mustTypecheck(src, nil, info) var pkg *Package count := 0 @@ -263,13 +263,13 @@ func TestIssue22525(t *testing.T) { got := "\n" conf := Config{Error: func(err error) { got += err.Error() + "\n" }} - typecheck("", src, &conf, nil) // do not crash + typecheck(src, &conf, nil) // do not crash want := ` -1:27: a declared and not used -1:30: b declared and not used -1:33: c declared and not used -1:36: d declared and not used -1:39: e declared and not used +p:1:27: a declared and not used +p:1:30: b declared and not used +p:1:33: c declared and not used +p:1:36: d declared and not used +p:1:39: e declared and not used ` if got != want { t.Errorf("got: %swant: %s", got, want) @@ -289,7 +289,7 @@ func TestIssue25627(t *testing.T) { `struct { *I }`, `struct { a int; b Missing; *Missing }`, } { - f := mustParse(fset, "", prefix+src) + f := mustParse(fset, prefix+src) cfg := Config{Importer: importer.Default(), Error: func(err error) {}} info := &Info{Types: make(map[ast.Expr]TypeAndValue)} @@ -326,7 +326,7 @@ func TestIssue28005(t *testing.T) { // compute original file ASTs var orig [len(sources)]*ast.File for i, src := range sources { - orig[i] = mustParse(fset, "", src) + orig[i] = mustParse(fset, src) } // run the test for all order permutations of the incoming files @@ -400,8 +400,8 @@ func TestIssue28282(t *testing.T) { } func TestIssue29029(t *testing.T) { - f1 := mustParse(fset, "", `package p; type A interface { M() }`) - f2 := mustParse(fset, "", `package p; var B interface { A }`) + f1 := mustParse(fset, `package p; type A interface { M() }`) + f2 := mustParse(fset, `package p; var B interface { A }`) // printInfo prints the *Func definitions recorded in info, one *Func per line. printInfo := func(info *Info) string { @@ -447,10 +447,10 @@ func TestIssue34151(t *testing.T) { const asrc = `package a; type I interface{ M() }; type T struct { F interface { I } }` const bsrc = `package b; import "a"; type T struct { F interface { a.I } }; var _ = a.T(T{})` - a := mustTypecheck("a", asrc, nil, nil) + a := mustTypecheck(asrc, nil, nil) conf := Config{Importer: importHelper{pkg: a}} - mustTypecheck("b", bsrc, &conf, nil) + mustTypecheck(bsrc, &conf, nil) } type importHelper struct { @@ -488,13 +488,8 @@ func TestIssue34921(t *testing.T) { var pkg *Package for _, src := range sources { - f := mustParse(fset, "", src) conf := Config{Importer: importHelper{pkg: pkg}} - res, err := conf.Check(f.Name.Name, fset, []*ast.File{f}, nil) - if err != nil { - t.Errorf("%q failed to typecheck: %v", src, err) - } - pkg = res // res is imported by the next package in this test + pkg = mustTypecheck(src, &conf, nil) // pkg imported by the next package in this test } } @@ -599,7 +594,7 @@ var _ T = template /* ERRORx "cannot use.*text/template.* as T value" */.Templat ` ) - a := mustTypecheck("a", asrc, nil, nil) + a := mustTypecheck(asrc, nil, nil) imp := importHelper{pkg: a, fallback: importer.Default()} testFiles(t, nil, []string{"b.go"}, [][]byte{[]byte(bsrc)}, false, imp) @@ -696,7 +691,7 @@ func TestIssue51093(t *testing.T) { for _, test := range tests { src := fmt.Sprintf("package p; func _[P %s]() { _ = P(%s) }", test.typ, test.val) types := make(map[ast.Expr]TypeAndValue) - mustTypecheck("p", src, nil, &Info{Types: types}) + mustTypecheck(src, nil, &Info{Types: types}) var n int for x, tv := range types { @@ -828,8 +823,8 @@ func (S) M5(struct {S;t}) {} fset := token.NewFileSet() test := func(main, b, want string) { re := regexp.MustCompile(want) - bpkg := mustTypecheck("b", b, nil, nil) - mast := mustParse(fset, "main.go", main) + bpkg := mustTypecheck(b, nil, nil) + mast := mustParse(fset, main) conf := Config{Importer: importHelper{pkg: bpkg}} _, err := conf.Check(mast.Name.Name, fset, []*ast.File{mast}, nil) if err == nil { diff --git a/src/go/types/methodset_test.go b/src/go/types/methodset_test.go index 3f8a0b1a10..918b51d93b 100644 --- a/src/go/types/methodset_test.go +++ b/src/go/types/methodset_test.go @@ -84,7 +84,7 @@ func TestNewMethodSet(t *testing.T) { } check := func(src string, methods []method, generic bool) { - pkg := mustTypecheck("test", "package p;"+src, nil, nil) + pkg := mustTypecheck("package p;"+src, nil, nil) scope := pkg.Scope() if generic { diff --git a/src/go/types/mono_test.go b/src/go/types/mono_test.go index a8d2acfd66..ccab846c6d 100644 --- a/src/go/types/mono_test.go +++ b/src/go/types/mono_test.go @@ -21,7 +21,7 @@ func checkMono(t *testing.T, body string) error { Error: func(err error) { fmt.Fprintln(&buf, err) }, Importer: importer.Default(), } - typecheck("x", src, &conf, nil) + typecheck(src, &conf, nil) if buf.Len() == 0 { return nil } diff --git a/src/go/types/named_test.go b/src/go/types/named_test.go index 55c0021398..8e00f6e0f9 100644 --- a/src/go/types/named_test.go +++ b/src/go/types/named_test.go @@ -32,7 +32,7 @@ func (G[P]) N() (p P) { return } type Inst = G[int] ` - pkg := mustTypecheck("p", src, nil, nil) + pkg := mustTypecheck(src, nil, nil) var ( T = pkg.Scope().Lookup("T").Type() @@ -107,7 +107,7 @@ type Inst = *Tree[int] ` fset := token.NewFileSet() - f := mustParse(fset, "foo.go", src) + f := mustParse(fset, src) pkg := NewPackage("p", f.Name.Name) if err := NewChecker(nil, fset, pkg, nil).Files([]*ast.File{f}); err != nil { t.Fatal(err) diff --git a/src/go/types/object_test.go b/src/go/types/object_test.go index bed8de3637..74acdaeeeb 100644 --- a/src/go/types/object_test.go +++ b/src/go/types/object_test.go @@ -58,7 +58,7 @@ func TestIsAlias(t *testing.T) { // the same Func Object as the original method. See also go.dev/issue/34421. func TestEmbeddedMethod(t *testing.T) { const src = `package p; type I interface { error }` - pkg := mustTypecheck("p", src, nil, nil) + pkg := mustTypecheck(src, nil, nil) // get original error.Error method eface := Universe.Lookup("error") @@ -112,7 +112,7 @@ func TestObjectString(t *testing.T) { for _, test := range testObjects { src := "package p; " + test.src - pkg, err := typecheck(filename, src, nil, nil) + pkg, err := typecheck(src, nil, nil) if err != nil { t.Errorf("%s: %s", src, err) continue diff --git a/src/go/types/resolver_test.go b/src/go/types/resolver_test.go index 284ad8e998..e95af80585 100644 --- a/src/go/types/resolver_test.go +++ b/src/go/types/resolver_test.go @@ -119,8 +119,8 @@ func TestResolveIdents(t *testing.T) { // parse package files fset := token.NewFileSet() var files []*ast.File - for i, src := range sources { - files = append(files, mustParse(fset, fmt.Sprintf("sources[%d]", i), src)) + for _, src := range sources { + files = append(files, mustParse(fset, src)) } // resolve and type-check package AST diff --git a/src/go/types/sizes_test.go b/src/go/types/sizes_test.go index 4964bf2cf9..f2e7e8ab2e 100644 --- a/src/go/types/sizes_test.go +++ b/src/go/types/sizes_test.go @@ -21,7 +21,7 @@ func findStructType(t *testing.T, src string) *types.Struct { func findStructTypeConfig(t *testing.T, src string, conf *types.Config) *types.Struct { types_ := make(map[ast.Expr]types.TypeAndValue) - mustTypecheck("x", src, nil, &types.Info{Types: types_}) + mustTypecheck(src, nil, &types.Info{Types: types_}) for _, tv := range types_ { if ts, ok := tv.Type.(*types.Struct); ok { return ts @@ -90,7 +90,7 @@ const _ = unsafe.Offsetof(struct{ x int64 }{}.x) Importer: importer.Default(), Sizes: &types.StdSizes{WordSize: 8, MaxAlign: 8}, } - mustTypecheck("x", src, &conf, &info) + mustTypecheck(src, &conf, &info) for _, tv := range info.Types { _ = conf.Sizes.Sizeof(tv.Type) _ = conf.Sizes.Alignof(tv.Type) diff --git a/src/go/types/typestring_test.go b/src/go/types/typestring_test.go index d3172d6bb9..45670b7e15 100644 --- a/src/go/types/typestring_test.go +++ b/src/go/types/typestring_test.go @@ -119,7 +119,7 @@ func TestTypeString(t *testing.T) { for _, test := range tests { src := `package p; import "io"; type _ io.Writer; type T ` + test.src - pkg, err := typecheck(filename, src, nil, nil) + pkg, err := typecheck(src, nil, nil) if err != nil { t.Errorf("%s: %s", src, err) continue @@ -137,8 +137,8 @@ func TestTypeString(t *testing.T) { } func TestQualifiedTypeString(t *testing.T) { - p := mustTypecheck("p.go", "package p; type T int", nil, nil) - q := mustTypecheck("q.go", "package q", nil, nil) + p := mustTypecheck("package p; type T int", nil, nil) + q := mustTypecheck("package q", nil, nil) pT := p.Scope().Lookup("T").Type() for _, test := range []struct { From 69e66a1626eb585cd3e7261f8192590a931a4874 Mon Sep 17 00:00:00 2001 From: Robert Griesemer Date: Thu, 27 Apr 2023 16:42:26 -0700 Subject: [PATCH 026/299] go/types, types2: remove parse (we only need mustParse for tests) While at it, also simplify mustTypecheck again as it can just use typecheck. Change-Id: I6cb07b1078d9a39e0f22851028fdd4442127f2f1 Reviewed-on: https://go-review.googlesource.com/c/go/+/490015 Run-TryBot: Robert Griesemer Reviewed-by: Robert Griesemer Reviewed-by: Robert Findley Auto-Submit: Robert Griesemer TryBot-Result: Gopher Robot --- src/cmd/compile/internal/types2/api_test.go | 22 ++++--------------- src/go/types/api_test.go | 24 +++++---------------- 2 files changed, 9 insertions(+), 37 deletions(-) diff --git a/src/cmd/compile/internal/types2/api_test.go b/src/cmd/compile/internal/types2/api_test.go index 0e76a73699..a13f43111c 100644 --- a/src/cmd/compile/internal/types2/api_test.go +++ b/src/cmd/compile/internal/types2/api_test.go @@ -21,13 +21,8 @@ import ( // nopos indicates an unknown position var nopos syntax.Pos -func parse(src string) (*syntax.File, error) { - errh := func(error) {} // dummy error handler so that parsing continues in presence of errors - return syntax.Parse(syntax.NewFileBase(pkgName(src)), strings.NewReader(src), errh, nil, 0) -} - func mustParse(src string) *syntax.File { - f, err := parse(src) + f, err := syntax.Parse(syntax.NewFileBase(pkgName(src)), strings.NewReader(src), nil, nil, 0) if err != nil { panic(err) // so we don't need to pass *testing.T } @@ -35,10 +30,7 @@ func mustParse(src string) *syntax.File { } func typecheck(src string, conf *Config, info *Info) (*Package, error) { - f, err := parse(src) - if f == nil { // ignore errors unless f is nil - return nil, err - } + f := mustParse(src) if conf == nil { conf = &Config{ Error: func(err error) {}, // collect all errors @@ -49,13 +41,7 @@ func typecheck(src string, conf *Config, info *Info) (*Package, error) { } func mustTypecheck(src string, conf *Config, info *Info) *Package { - f := mustParse(src) - if conf == nil { - conf = &Config{ - Importer: defaultImporter(), - } - } - pkg, err := conf.Check(f.PkgName.Value, []*syntax.File{f}, info) + pkg, err := typecheck(src, conf, info) if err != nil { panic(err) // so we don't need to pass *testing.T } @@ -339,7 +325,7 @@ func TestTypesInfo(t *testing.T) { {`package issue47243_i; var x int32; var _ = 1 << (2 << x)`, `(2 << x)`, `untyped int`}, {`package issue47243_j; var x int32; var _ = 1 << (2 << x)`, `2`, `untyped int`}, - // tests for broken code that doesn't parse or type-check + // tests for broken code that doesn't type-check {brokenPkg + `x0; func _() { var x struct {f string}; x.f := 0 }`, `x.f`, `string`}, {brokenPkg + `x1; func _() { var z string; type x struct {f string}; y := &x{q: z}}`, `z`, `string`}, {brokenPkg + `x2; func _() { var a, b string; type x struct {f string}; z := &x{f: a, f: b,}}`, `b`, `string`}, diff --git a/src/go/types/api_test.go b/src/go/types/api_test.go index 825f30585b..ae1a7e50a7 100644 --- a/src/go/types/api_test.go +++ b/src/go/types/api_test.go @@ -24,12 +24,8 @@ import ( // nopos indicates an unknown position var nopos token.Pos -func parse(fset *token.FileSet, src string) (*ast.File, error) { - return parser.ParseFile(fset, pkgName(src), src, 0) -} - func mustParse(fset *token.FileSet, src string) *ast.File { - f, err := parse(fset, src) + f, err := parser.ParseFile(fset, pkgName(src), src, 0) if err != nil { panic(err) // so we don't need to pass *testing.T } @@ -38,10 +34,7 @@ func mustParse(fset *token.FileSet, src string) *ast.File { func typecheck(src string, conf *Config, info *Info) (*Package, error) { fset := token.NewFileSet() - f, err := parse(fset, src) - if f == nil { // ignore errors unless f is nil - return nil, err - } + f := mustParse(fset, src) if conf == nil { conf = &Config{ Error: func(err error) {}, // collect all errors @@ -52,14 +45,7 @@ func typecheck(src string, conf *Config, info *Info) (*Package, error) { } func mustTypecheck(src string, conf *Config, info *Info) *Package { - fset := token.NewFileSet() - f := mustParse(fset, src) - if conf == nil { - conf = &Config{ - Importer: importer.Default(), - } - } - pkg, err := conf.Check(f.Name.Name, fset, []*ast.File{f}, info) + pkg, err := typecheck(src, conf, info) if err != nil { panic(err) // so we don't need to pass *testing.T } @@ -339,10 +325,10 @@ func TestTypesInfo(t *testing.T) { {`package issue47243_i; var x int32; var _ = 1 << (2 << x)`, `(2 << x)`, `untyped int`}, {`package issue47243_j; var x int32; var _ = 1 << (2 << x)`, `2`, `untyped int`}, - // tests for broken code that doesn't parse or type-check + // tests for broken code that doesn't type-check {broken + `x0; func _() { var x struct {f string}; x.f := 0 }`, `x.f`, `string`}, {broken + `x1; func _() { var z string; type x struct {f string}; y := &x{q: z}}`, `z`, `string`}, - {broken + `x2; func _() { var a, b string; type x struct {f string}; z := &x{f: a; f: b;}}`, `b`, `string`}, + {broken + `x2; func _() { var a, b string; type x struct {f string}; z := &x{f: a, f: b,}}`, `b`, `string`}, {broken + `x3; var x = panic("");`, `panic`, `func(interface{})`}, {`package x4; func _() { panic("") }`, `panic`, `func(interface{})`}, {broken + `x5; func _() { var x map[string][...]int; x = map[string][...]int{"": {1,2,3}} }`, `x`, `map[string]invalid type`}, From e7af0e0cac6c65043335da6b6329f7141d2f68c5 Mon Sep 17 00:00:00 2001 From: Joel Sing Date: Sat, 29 Apr 2023 02:00:15 +1000 Subject: [PATCH 027/299] cmd/internal/obj/arm64: use appropriate return type for regoff All of the callers of regoff cast the return value from uint32 to int32. Instead, simply return int32 in the first place. Change-Id: I43a672bb3143a71f4a37779ed8ae9adcda623ba4 Reviewed-on: https://go-review.googlesource.com/c/go/+/490355 Run-TryBot: Joel Sing TryBot-Result: Gopher Robot Reviewed-by: Dmitri Shuralyov Reviewed-by: Cherry Mui --- src/cmd/internal/obj/arm64/asm7.go | 32 +++++++++++++++--------------- 1 file changed, 16 insertions(+), 16 deletions(-) diff --git a/src/cmd/internal/obj/arm64/asm7.go b/src/cmd/internal/obj/arm64/asm7.go index 4906839cf7..60add054ce 100644 --- a/src/cmd/internal/obj/arm64/asm7.go +++ b/src/cmd/internal/obj/arm64/asm7.go @@ -1388,10 +1388,10 @@ func roundUp(x, to uint32) uint32 { return (x + to - 1) &^ (to - 1) } -func (c *ctxt7) regoff(a *obj.Addr) uint32 { +func (c *ctxt7) regoff(a *obj.Addr) int32 { c.instoffset = 0 c.aclass(a) - return uint32(c.instoffset) + return int32(c.instoffset) } func isSTLXRop(op obj.As) bool { @@ -3371,7 +3371,7 @@ func (c *ctxt7) asmout(p *obj.Prog, o *Optab, out []uint32) { if r == obj.REG_NONE { r = rt } - v := int32(c.regoff(&p.From)) + v := c.regoff(&p.From) o1 = c.oaddi(p, int32(o1), v, r, rt) case 3: /* op R< strT */ - v := int32(c.regoff(&p.To)) + v := c.regoff(&p.To) sz := int32(1 << uint(movesize(p.As))) rt, rf := p.To.Reg, p.From.Reg @@ -3721,7 +3721,7 @@ func (c *ctxt7) asmout(p *obj.Prog, o *Optab, out []uint32) { } case 21: /* movT O(R),R -> ldrT */ - v := int32(c.regoff(&p.From)) + v := c.regoff(&p.From) sz := int32(1 << uint(movesize(p.As))) rt, rf := p.To.Reg, p.From.Reg @@ -3894,7 +3894,7 @@ func (c *ctxt7) asmout(p *obj.Prog, o *Optab, out []uint32) { r = int(o.param) } - v := int32(c.regoff(&p.To)) + v := c.regoff(&p.To) var hi int32 if v < 0 || (v&((1< adrp + add + reloc */ @@ -4670,7 +4670,7 @@ func (c *ctxt7) asmout(p *obj.Prog, o *Optab, out []uint32) { if rf == obj.REG_NONE { c.ctxt.Diag("invalid ldp source: %v", p) } - v := int32(c.regoff(&p.From)) + v := c.regoff(&p.From) o1 = c.oaddi12(p, v, REGTMP, rf) o2 = c.opldpstp(p, o, 0, REGTMP, rt1, rt2, 1) @@ -4708,7 +4708,7 @@ func (c *ctxt7) asmout(p *obj.Prog, o *Optab, out []uint32) { if rt == obj.REG_NONE { c.ctxt.Diag("invalid stp destination: %v", p) } - v := int32(c.regoff(&p.To)) + v := c.regoff(&p.To) o1 = c.oaddi12(p, v, REGTMP, rt) o2 = c.opldpstp(p, o, 0, REGTMP, rf1, rf2, 0) @@ -5265,7 +5265,7 @@ func (c *ctxt7) asmout(p *obj.Prog, o *Optab, out []uint32) { rf := int((p.To.Reg) & 31) r := int(p.To.Index & 31) index := int(p.From.Index) - offset := int32(c.regoff(&p.To)) + offset := c.regoff(&p.To) if o.scond == C_XPOST { if (p.To.Index != 0) && (offset != 0) { @@ -5337,7 +5337,7 @@ func (c *ctxt7) asmout(p *obj.Prog, o *Optab, out []uint32) { rf := int((p.From.Reg) & 31) r := int(p.From.Index & 31) index := int(p.To.Index) - offset := int32(c.regoff(&p.From)) + offset := c.regoff(&p.From) if o.scond == C_XPOST { if (p.From.Index != 0) && (offset != 0) { From 19fd96512c4ff96415cd4dacb5fac1854422e1fa Mon Sep 17 00:00:00 2001 From: Cherry Mui Date: Fri, 28 Apr 2023 13:46:38 -0400 Subject: [PATCH 028/299] cmd/link: put zero-sized data symbols at same address as runtime.zerobase Put zero-sized data symbols at same address as runtime.zerobase, so zero-sized global variables have the same address as zero-sized allocations. Change-Id: Ib3145dc1b663a9794dfabc0e6abd2384960f2c49 Reviewed-on: https://go-review.googlesource.com/c/go/+/490435 Run-TryBot: Cherry Mui TryBot-Result: Gopher Robot Reviewed-by: Russ Cox --- src/cmd/link/internal/ld/data.go | 12 +++++++++--- test/zerosize.go | 33 ++++++++++++++++++++++++++++++++ 2 files changed, 42 insertions(+), 3 deletions(-) create mode 100644 test/zerosize.go diff --git a/src/cmd/link/internal/ld/data.go b/src/cmd/link/internal/ld/data.go index 7c135ae7e6..d0efcdc052 100644 --- a/src/cmd/link/internal/ld/data.go +++ b/src/cmd/link/internal/ld/data.go @@ -2156,7 +2156,7 @@ type symNameSize struct { } func (state *dodataState) dodataSect(ctxt *Link, symn sym.SymKind, syms []loader.Sym) (result []loader.Sym, maxAlign int32) { - var head, tail loader.Sym + var head, tail, zerobase loader.Sym ldr := ctxt.loader sl := make([]symNameSize, len(syms)) @@ -2196,20 +2196,26 @@ func (state *dodataState) dodataSect(ctxt *Link, symn sym.SymKind, syms []loader } } } + zerobase = ldr.Lookup("runtime.zerobase", 0) // Perform the sort. if symn != sym.SPCLNTAB { sort.Slice(sl, func(i, j int) bool { si, sj := sl[i].sym, sl[j].sym + isz, jsz := sl[i].sz, sl[j].sz switch { case si == head, sj == tail: return true case sj == head, si == tail: return false + // put zerobase right after all the zero-sized symbols, + // so zero-sized symbols have the same address as zerobase. + case si == zerobase: + return jsz != 0 // zerobase < nonzero-sized + case sj == zerobase: + return isz == 0 // 0-sized < zerobase } if checkSize { - isz := sl[i].sz - jsz := sl[j].sz if isz != jsz { return isz < jsz } diff --git a/test/zerosize.go b/test/zerosize.go new file mode 100644 index 0000000000..53a29f7927 --- /dev/null +++ b/test/zerosize.go @@ -0,0 +1,33 @@ +// run + +// Copyright 2023 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Test that zero-sized variables get same address as +// runtime.zerobase. + +package main + +var x, y [0]int +var p, q = new([0]int), new([0]int) // should get &runtime.zerobase + +func main() { + if &x != &y { + // Failing for now. x and y are at same address, but compiler optimizes &x==&y to false. Skip. + // print("&x=", &x, " &y=", &y, " &x==&y = ", &x==&y, "\n") + // panic("FAIL") + } + if p != q { + print("p=", p, " q=", q, " p==q = ", p==q, "\n") + panic("FAIL") + } + if &x != p { + print("&x=", &x, " p=", p, " &x==p = ", &x==p, "\n") + panic("FAIL") + } + if &y != p { + print("&y=", &y, " p=", p, " &y==p = ", &y==p, "\n") + panic("FAIL") + } +} From 8b67cf0bc6ad657fddcbaaa10729d0086f08f9a9 Mon Sep 17 00:00:00 2001 From: Joel Sing Date: Mon, 9 Jan 2023 05:14:13 +1100 Subject: [PATCH 029/299] cmd/internal/obj/arm64: pass obj.As to oaddi rather than an instruction This simplifies callers, as they do not need to call opirr before calling oaddi. Additionally, use appropriate types (int16) for registers, which avoids the need to continually cast. Change-Id: I8ca3807a97867ac49d63792f6922a18f35824448 Reviewed-on: https://go-review.googlesource.com/c/go/+/471520 Reviewed-by: Cherry Mui Run-TryBot: Joel Sing Reviewed-by: Dmitri Shuralyov TryBot-Result: Gopher Robot --- src/cmd/internal/obj/arm64/asm7.go | 62 ++++++++++++++---------------- 1 file changed, 29 insertions(+), 33 deletions(-) diff --git a/src/cmd/internal/obj/arm64/asm7.go b/src/cmd/internal/obj/arm64/asm7.go index 60add054ce..1a10c48bd8 100644 --- a/src/cmd/internal/obj/arm64/asm7.go +++ b/src/cmd/internal/obj/arm64/asm7.go @@ -3359,20 +3359,18 @@ func (c *ctxt7) asmout(p *obj.Prog, o *Optab, out []uint32) { } o1 = c.opirr(p, p.As) - rt := int(p.To.Reg) + rt, r := p.To.Reg, p.Reg if p.To.Type == obj.TYPE_NONE { if (o1 & Sbit) == 0 { c.ctxt.Diag("ineffective ZR destination\n%v", p) } rt = REGZERO } - - r := int(p.Reg) if r == obj.REG_NONE { r = rt } v := c.regoff(&p.From) - o1 = c.oaddi(p, int32(o1), v, r, rt) + o1 = c.oaddi(p, p.As, v, rt, r) case 3: /* op R<>uint(s))&0xFFF, REGTMP, p.From.Reg) break @@ -3919,7 +3913,7 @@ func (c *ctxt7) asmout(p *obj.Prog, o *Optab, out []uint32) { c.ctxt.Diag("REGTMP used in large offset store: %v", p) } o1 = c.omovlit(AMOVD, p, &p.To, REGTMP) - o2 = c.olsxrr(p, int32(c.opstrr(p, p.As, false)), int(p.From.Reg), r, REGTMP) + o2 = c.olsxrr(p, int32(c.opstrr(p, p.As, false)), int(p.From.Reg), int(r), REGTMP) case 31: /* movT L(R), R -> ldrT */ // if offset L can be split into hi+lo, and both fit into instructions, do @@ -3933,9 +3927,9 @@ func (c *ctxt7) asmout(p *obj.Prog, o *Optab, out []uint32) { c.ctxt.Diag("unexpected long move, op %v tab %v\n%v", p.As, o.as, p) } - r := int(p.From.Reg) + r := p.From.Reg if r == obj.REG_NONE { - r = int(o.param) + r = o.param } v := c.regoff(&p.From) @@ -3954,7 +3948,7 @@ func (c *ctxt7) asmout(p *obj.Prog, o *Optab, out []uint32) { goto loadusepool } - o1 = c.oaddi(p, int32(c.opirr(p, AADD)), hi, r, REGTMP) + o1 = c.oaddi(p, AADD, hi, REGTMP, r) o2 = c.olsr12u(p, c.opldr(p, p.As), ((v-hi)>>uint(s))&0xFFF, REGTMP, p.To.Reg) break @@ -3963,7 +3957,7 @@ func (c *ctxt7) asmout(p *obj.Prog, o *Optab, out []uint32) { c.ctxt.Diag("REGTMP used in large offset load: %v", p) } o1 = c.omovlit(AMOVD, p, &p.From, REGTMP) - o2 = c.olsxrr(p, int32(c.opldrr(p, p.As, false)), int(p.To.Reg), r, REGTMP) + o2 = c.olsxrr(p, int32(c.opldrr(p, p.As, false)), int(p.To.Reg), int(r), REGTMP) case 32: /* mov $con, R -> movz/movn */ o1 = c.omovconst(p.As, p, &p.From, int(p.To.Reg)) @@ -4234,13 +4228,12 @@ func (c *ctxt7) asmout(p *obj.Prog, o *Optab, out []uint32) { if op&Sbit != 0 { c.ctxt.Diag("can not break addition/subtraction when S bit is set", p) } - rt := int(p.To.Reg) - r := int(p.Reg) + rt, r := p.To.Reg, p.Reg if r == obj.REG_NONE { r = rt } - o1 = c.oaddi(p, int32(op), c.regoff(&p.From)&0x000fff, r, rt) - o2 = c.oaddi(p, int32(op), c.regoff(&p.From)&0xfff000, rt, rt) + o1 = c.oaddi(p, p.As, c.regoff(&p.From)&0x000fff, rt, r) + o2 = c.oaddi(p, p.As, c.regoff(&p.From)&0xfff000, rt, rt) case 50: /* sys/sysl */ o1 = c.opirr(p, p.As) @@ -7097,17 +7090,20 @@ func (c *ctxt7) opstrr(p *obj.Prog, a obj.As, extension bool) uint32 { return 0 } -func (c *ctxt7) oaddi(p *obj.Prog, o1 int32, v int32, r int, rt int) uint32 { +func (c *ctxt7) oaddi(p *obj.Prog, a obj.As, v int32, rd, rn int16) uint32 { + op := c.opirr(p, a) + if (v & 0xFFF000) != 0 { if v&0xFFF != 0 { c.ctxt.Diag("%v misuses oaddi", p) } v >>= 12 - o1 |= 1 << 22 + op |= 1 << 22 } - o1 |= ((v & 0xFFF) << 10) | (int32(r&31) << 5) | int32(rt&31) - return uint32(o1) + op |= (uint32(v&0xFFF) << 10) | (uint32(rn&31) << 5) | uint32(rd&31) + + return op } func (c *ctxt7) oaddi12(p *obj.Prog, v int32, rd, rn int16) uint32 { @@ -7120,7 +7116,7 @@ func (c *ctxt7) oaddi12(p *obj.Prog, v int32, rd, rn int16) uint32 { a = ASUB v = -v } - return c.oaddi(p, int32(c.opirr(p, a)), v, int(rn), int(rd)) + return c.oaddi(p, a, v, rd, rn) } /* From 0fd6ae548f550bdbee4a434285ff052fb9dc7417 Mon Sep 17 00:00:00 2001 From: Michael Pratt Date: Fri, 28 Apr 2023 15:52:16 -0400 Subject: [PATCH 030/299] cmd/compile: escape package path for PGO symbol matching Symbol names in the final executable apply escaping to the final component of a package path (main in example.com becomes example%2ecom.main). ir.PkgFuncName does not perform this escaping, meaning we'd fail to match functions that are escaped in the profile. Add ir.LinkFuncName which does perform escaping and use it for PGO. Fixes #59887. Change-Id: I10634d63d99d0a6fd2f72b929ab35ea227e1336f Reviewed-on: https://go-review.googlesource.com/c/go/+/490555 Reviewed-by: Cherry Mui Auto-Submit: Michael Pratt Run-TryBot: Michael Pratt TryBot-Result: Gopher Robot --- src/cmd/compile/internal/inline/inl.go | 4 ++-- src/cmd/compile/internal/ir/func.go | 25 +++++++++++++++++++----- src/cmd/compile/internal/pgo/irgraph.go | 26 ++++++++++++------------- 3 files changed, 35 insertions(+), 20 deletions(-) diff --git a/src/cmd/compile/internal/inline/inl.go b/src/cmd/compile/internal/inline/inl.go index 1a65e16f51..df12f9a625 100644 --- a/src/cmd/compile/internal/inline/inl.go +++ b/src/cmd/compile/internal/inline/inl.go @@ -163,7 +163,7 @@ func pgoInlineEpilogue(p *pgo.Profile, decls []ir.Node) { if base.Debug.PGOInline >= 2 { ir.VisitFuncsBottomUp(decls, func(list []*ir.Func, recursive bool) { for _, f := range list { - name := ir.PkgFuncName(f) + name := ir.LinkFuncName(f) if n, ok := p.WeightedCG.IRNodes[name]; ok { p.RedirectEdges(n, inlinedCallSites) } @@ -352,7 +352,7 @@ func CanInline(fn *ir.Func, profile *pgo.Profile) { // Update the budget for profile-guided inlining. budget := int32(inlineMaxBudget) if profile != nil { - if n, ok := profile.WeightedCG.IRNodes[ir.PkgFuncName(fn)]; ok { + if n, ok := profile.WeightedCG.IRNodes[ir.LinkFuncName(fn)]; ok { if _, ok := candHotCalleeMap[n]; ok { budget = int32(inlineHotMaxBudget) if base.Debug.PGOInline > 0 { diff --git a/src/cmd/compile/internal/ir/func.go b/src/cmd/compile/internal/ir/func.go index 2886185f0a..b36b1fa494 100644 --- a/src/cmd/compile/internal/ir/func.go +++ b/src/cmd/compile/internal/ir/func.go @@ -8,6 +8,7 @@ import ( "cmd/compile/internal/base" "cmd/compile/internal/types" "cmd/internal/obj" + "cmd/internal/objabi" "cmd/internal/src" "fmt" ) @@ -263,7 +264,7 @@ func (f *Func) SetWBPos(pos src.XPos) { } } -// FuncName returns the name (without the package) of the function n. +// FuncName returns the name (without the package) of the function f. func FuncName(f *Func) string { if f == nil || f.Nname == nil { return "" @@ -271,10 +272,12 @@ func FuncName(f *Func) string { return f.Sym().Name } -// PkgFuncName returns the name of the function referenced by n, with package prepended. -// This differs from the compiler's internal convention where local functions lack a package -// because the ultimate consumer of this is a human looking at an IDE; package is only empty -// if the compilation package is actually the empty string. +// PkgFuncName returns the name of the function referenced by f, with package +// prepended. +// +// This differs from the compiler's internal convention where local functions +// lack a package. This is primarily useful when the ultimate consumer of this +// is a human looking at message. func PkgFuncName(f *Func) string { if f == nil || f.Nname == nil { return "" @@ -285,6 +288,18 @@ func PkgFuncName(f *Func) string { return pkg.Path + "." + s.Name } +// LinkFuncName returns the name of the function f, as it will appear in the +// symbol table of the final linked binary. +func LinkFuncName(f *Func) string { + if f == nil || f.Nname == nil { + return "" + } + s := f.Sym() + pkg := s.Pkg + + return objabi.PathToPrefix(pkg.Path) + "." + s.Name +} + // IsEqOrHashFunc reports whether f is type eq/hash function. func IsEqOrHashFunc(f *Func) bool { if f == nil || f.Nname == nil { diff --git a/src/cmd/compile/internal/pgo/irgraph.go b/src/cmd/compile/internal/pgo/irgraph.go index ff0995eaea..72ffc8ce78 100644 --- a/src/cmd/compile/internal/pgo/irgraph.go +++ b/src/cmd/compile/internal/pgo/irgraph.go @@ -270,7 +270,7 @@ func (p *Profile) VisitIR(fn *ir.Func) { if g.InEdges == nil { g.InEdges = make(map[*IRNode][]*IREdge) } - name := ir.PkgFuncName(fn) + name := ir.LinkFuncName(fn) node := new(IRNode) node.AST = fn if g.IRNodes[name] == nil { @@ -308,7 +308,7 @@ func (p *Profile) addIREdge(caller *IRNode, callername string, call ir.Node, cal // Create an IRNode for the callee. calleenode := new(IRNode) calleenode.AST = callee - calleename := ir.PkgFuncName(callee) + calleename := ir.LinkFuncName(callee) // Create key for NodeMapKey. nodeinfo := NodeMapKey{ @@ -395,7 +395,7 @@ func (p *Profile) PrintWeightedCallGraphDOT(edgeThreshold float64) { funcs := make(map[string]struct{}) ir.VisitFuncsBottomUp(typecheck.Target.Decls, func(list []*ir.Func, recursive bool) { for _, f := range list { - name := ir.PkgFuncName(f) + name := ir.LinkFuncName(f) funcs[name] = struct{}{} } }) @@ -405,15 +405,15 @@ func (p *Profile) PrintWeightedCallGraphDOT(edgeThreshold float64) { for name := range funcs { if n, ok := p.WeightedCG.IRNodes[name]; ok { for _, e := range p.WeightedCG.OutEdges[n] { - if _, ok := nodes[ir.PkgFuncName(e.Src.AST)]; !ok { - nodes[ir.PkgFuncName(e.Src.AST)] = e.Src.AST + if _, ok := nodes[ir.LinkFuncName(e.Src.AST)]; !ok { + nodes[ir.LinkFuncName(e.Src.AST)] = e.Src.AST } - if _, ok := nodes[ir.PkgFuncName(e.Dst.AST)]; !ok { - nodes[ir.PkgFuncName(e.Dst.AST)] = e.Dst.AST + if _, ok := nodes[ir.LinkFuncName(e.Dst.AST)]; !ok { + nodes[ir.LinkFuncName(e.Dst.AST)] = e.Dst.AST } } - if _, ok := nodes[ir.PkgFuncName(n.AST)]; !ok { - nodes[ir.PkgFuncName(n.AST)] = n.AST + if _, ok := nodes[ir.LinkFuncName(n.AST)]; !ok { + nodes[ir.LinkFuncName(n.AST)] = n.AST } } } @@ -424,16 +424,16 @@ func (p *Profile) PrintWeightedCallGraphDOT(edgeThreshold float64) { nodeweight := WeightInPercentage(n.Flat, p.TotalNodeWeight) color := "black" if ast.Inl != nil { - fmt.Printf("\"%v\" [color=%v,label=\"%v,freq=%.2f,inl_cost=%d\"];\n", ir.PkgFuncName(ast), color, ir.PkgFuncName(ast), nodeweight, ast.Inl.Cost) + fmt.Printf("\"%v\" [color=%v,label=\"%v,freq=%.2f,inl_cost=%d\"];\n", ir.LinkFuncName(ast), color, ir.LinkFuncName(ast), nodeweight, ast.Inl.Cost) } else { - fmt.Printf("\"%v\" [color=%v, label=\"%v,freq=%.2f\"];\n", ir.PkgFuncName(ast), color, ir.PkgFuncName(ast), nodeweight) + fmt.Printf("\"%v\" [color=%v, label=\"%v,freq=%.2f\"];\n", ir.LinkFuncName(ast), color, ir.LinkFuncName(ast), nodeweight) } } } // Print edges. ir.VisitFuncsBottomUp(typecheck.Target.Decls, func(list []*ir.Func, recursive bool) { for _, f := range list { - name := ir.PkgFuncName(f) + name := ir.LinkFuncName(f) if n, ok := p.WeightedCG.IRNodes[name]; ok { for _, e := range p.WeightedCG.OutEdges[n] { edgepercent := WeightInPercentage(e.Weight, p.TotalEdgeWeight) @@ -443,7 +443,7 @@ func (p *Profile) PrintWeightedCallGraphDOT(edgeThreshold float64) { fmt.Printf("edge [color=black, style=solid];\n") } - fmt.Printf("\"%v\" -> \"%v\" [label=\"%.2f\"];\n", ir.PkgFuncName(n.AST), ir.PkgFuncName(e.Dst.AST), edgepercent) + fmt.Printf("\"%v\" -> \"%v\" [label=\"%.2f\"];\n", ir.LinkFuncName(n.AST), ir.LinkFuncName(e.Dst.AST), edgepercent) } } } From 265d19ed526b6d6a01a20150918b362c1e6befba Mon Sep 17 00:00:00 2001 From: Nick Ripley Date: Tue, 25 Apr 2023 14:48:00 -0400 Subject: [PATCH 031/299] runtime/trace: avoid frame pointer unwinding for events during cgocallbackg MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The current mp.incgocallback() logic allows for trace events to be recorded using frame pointer unwinding during cgocallbackg when they shouldn't be. Specifically, mp.incgo will be false during the reentersyscall call at the end. It's possible to crash with tracing enabled because of this, if C code which uses the frame pointer register for other purposes calls into Go. This can be seen, for example, by forcing testprogcgo/trace_unix.c to write a garbage value to RBP prior to calling into Go. We can drop the mp.incgo check, and instead conservatively avoid doing frame pointer unwinding if there is any C on the stack. This is the case if mp.ncgo > 0, or if mp.isextra is true (meaning we're coming from a thread created by C). Rename incgocallback to reflect that we're checking if there's any C on the stack. We can also move the ncgo increment in cgocall closer to where the transition to C happens, which lets us use frame pointer unwinding for the entersyscall event during the first Go-to-C call on a stack, when there isn't yet any C on the stack. Fixes #59830. Change-Id: If178a705a9d38d0d2fb19589a9e669cd982d32cd Reviewed-on: https://go-review.googlesource.com/c/go/+/488755 Reviewed-by: Michael Knyszek Reviewed-by: Dmitri Shuralyov Reviewed-by: Felix Geisendörfer Auto-Submit: Michael Knyszek Run-TryBot: Nick Ripley TryBot-Result: Gopher Robot --- src/runtime/cgocall.go | 9 ++++++++- src/runtime/proc.go | 4 ++-- src/runtime/trace.go | 4 ++-- 3 files changed, 12 insertions(+), 5 deletions(-) diff --git a/src/runtime/cgocall.go b/src/runtime/cgocall.go index 7f1a02fb4b..c2552cbdf5 100644 --- a/src/runtime/cgocall.go +++ b/src/runtime/cgocall.go @@ -136,7 +136,6 @@ func cgocall(fn, arg unsafe.Pointer) int32 { mp := getg().m mp.ncgocall++ - mp.ncgo++ // Reset traceback. mp.cgoCallers[0] = 0 @@ -165,6 +164,14 @@ func cgocall(fn, arg unsafe.Pointer) int32 { osPreemptExtEnter(mp) mp.incgo = true + // We use ncgo as a check during execution tracing for whether there is + // any C on the call stack, which there will be after this point. If + // there isn't, we can use frame pointer unwinding to collect call + // stacks efficiently. This will be the case for the first Go-to-C call + // on a stack, so it's prefereable to update it here, after we emit a + // trace event in entersyscall above. + mp.ncgo++ + errno := asmcgocall(fn, arg) // Update accounting before exitsyscall because exitsyscall may diff --git a/src/runtime/proc.go b/src/runtime/proc.go index d2901e3aa0..0b9df169b2 100644 --- a/src/runtime/proc.go +++ b/src/runtime/proc.go @@ -868,8 +868,8 @@ func (mp *m) becomeSpinning() { sched.needspinning.Store(0) } -func (mp *m) incgocallback() bool { - return (!mp.incgo && mp.ncgo > 0) || mp.isextra +func (mp *m) hasCgoOnStack() bool { + return mp.ncgo > 0 || mp.isextra } var fastrandseed uintptr diff --git a/src/runtime/trace.go b/src/runtime/trace.go index c382068e2f..79ccebb4b3 100644 --- a/src/runtime/trace.go +++ b/src/runtime/trace.go @@ -889,10 +889,10 @@ func traceStackID(mp *m, pcBuf []uintptr, skip int) uint64 { gp := getg() curgp := mp.curg nstk := 1 - if tracefpunwindoff() || mp.incgocallback() { + if tracefpunwindoff() || mp.hasCgoOnStack() { // Slow path: Unwind using default unwinder. Used when frame pointer // unwinding is unavailable or disabled (tracefpunwindoff), or might - // produce incomplete results or crashes (incgocallback). Note that no + // produce incomplete results or crashes (hasCgoOnStack). Note that no // cgo callback related crashes have been observed yet. The main // motivation is to take advantage of a potentially registered cgo // symbolizer. From 3c46d8f5119475adbadb6141acd463b910c1747b Mon Sep 17 00:00:00 2001 From: "Paul E. Murphy" Date: Tue, 17 Jan 2023 11:33:28 -0600 Subject: [PATCH 032/299] cmd/link: load external ELF PPC64 objects which set st_other=1 This indicates the symbol does not use or preserve the TOC pointer in R2. Likewise, it does not have a distinct local entry point. This happens when gcc compiles an object with -mcpu=power10. Recycle the SymLocalentry field of a text symbol to pass through this hint as the bogus value 1 (A valid offset must be a multiple of 4 bytes), and update the usage to check and generate errors further into the linking process. This matches the behavior of st_other as used by ELFv2. Change-Id: Ic89ce17b57f400ab44213b21a3730a98c7cdf842 Reviewed-on: https://go-review.googlesource.com/c/go/+/490295 Run-TryBot: Paul Murphy Reviewed-by: Cherry Mui Reviewed-by: Dmitri Shuralyov TryBot-Result: Gopher Robot --- src/cmd/link/internal/loadelf/ldelf.go | 12 ++++++++++-- src/cmd/link/internal/loader/loader.go | 3 +++ src/cmd/link/internal/ppc64/asm.go | 12 ++++++++++-- 3 files changed, 23 insertions(+), 4 deletions(-) diff --git a/src/cmd/link/internal/loadelf/ldelf.go b/src/cmd/link/internal/loadelf/ldelf.go index c1bfec059d..77247b47f4 100644 --- a/src/cmd/link/internal/loadelf/ldelf.go +++ b/src/cmd/link/internal/loadelf/ldelf.go @@ -639,11 +639,15 @@ func Load(l *loader.Loader, arch *sys.Arch, localSymVersion int, f *bio.Reader, case 0: // No local entry. R2 is preserved. case 1: - // These require R2 be saved and restored by the caller. This isn't supported today. - return errorf("%s: unable to handle local entry type 1", sb.Name()) + // This is kind of a hack, but pass the hint about this symbol's + // usage of R2 (R2 is a caller-save register not a TOC pointer, and + // this function does not have a distinct local entry) by setting + // its SymLocalentry to 1. + l.SetSymLocalentry(s, 1) case 7: return errorf("%s: invalid sym.other 0x%x", sb.Name(), elfsym.other) default: + // Convert the word sized offset into bytes. l.SetSymLocalentry(s, 4< Date: Fri, 28 Apr 2023 15:05:36 -0700 Subject: [PATCH 033/299] cmd/distpack: remove internal/platform/zosarch.go cmd/dist started generating that file in CL 483695. Also rearrange the list of files to remove slightly to explain better where they come from. Fixes #59889 Change-Id: I062c858596d801157e0d943e4ba2761c0547ac3a Reviewed-on: https://go-review.googlesource.com/c/go/+/490655 Auto-Submit: Ian Lance Taylor TryBot-Result: Gopher Robot Reviewed-by: Ian Lance Taylor Run-TryBot: Ian Lance Taylor Run-TryBot: Ian Lance Taylor Reviewed-by: Heschi Kreinick Auto-Submit: Ian Lance Taylor --- src/cmd/distpack/pack.go | 23 ++++++++++++++--------- 1 file changed, 14 insertions(+), 9 deletions(-) diff --git a/src/cmd/distpack/pack.go b/src/cmd/distpack/pack.go index ffeb4a1611..55e07f88c3 100644 --- a/src/cmd/distpack/pack.go +++ b/src/cmd/distpack/pack.go @@ -5,16 +5,15 @@ // Distpack creates the tgz and zip files for a Go distribution. // It writes into GOROOT/pkg/distpack: // -// - a binary distribution (tgz or zip) for the current GOOS and GOARCH -// - a source distribution that is independent of GOOS/GOARCH -// - the module mod, info, and zip files for a distribution in module form -// (as used by GOTOOLCHAIN support in the go command). +// - a binary distribution (tgz or zip) for the current GOOS and GOARCH +// - a source distribution that is independent of GOOS/GOARCH +// - the module mod, info, and zip files for a distribution in module form +// (as used by GOTOOLCHAIN support in the go command). // // Distpack is typically invoked by the -distpack flag to make.bash. // A cross-compiled distribution for goos/goarch can be built using: // // GOOS=goos GOARCH=goarch ./make.bash -distpack -// package main import ( @@ -113,15 +112,21 @@ func main() { srcArch.Remove( "bin/**", "pkg/**", + // Generated during cmd/dist. See ../dist/build.go:/deptab. - "src/cmd/cgo/zdefaultcc.go", "src/cmd/go/internal/cfg/zdefaultcc.go", - "src/cmd/go/internal/cfg/zosarch.go", - "src/cmd/internal/objabi/zbootstrap.go", "src/go/build/zcgo.go", - "src/internal/buildcfg/zbootstrap.go", + "src/internal/platform/zosarch.go", "src/runtime/internal/sys/zversion.go", "src/time/tzdata/zzipdata.go", + + // Generated during cmd/dist by bootstrapBuildTools. + "src/cmd/cgo/zdefaultcc.go", + "src/cmd/internal/objabi/zbootstrap.go", + "src/internal/buildcfg/zbootstrap.go", + + // Generated by earlier versions of cmd/dist . + "src/cmd/go/internal/cfg/zosarch.go", ) srcArch.AddPrefix("go") testSrc(srcArch) From 4b66502ddaa264257343aae58395ef8cd4176cfd Mon Sep 17 00:00:00 2001 From: Achille Roussel Date: Sat, 29 Apr 2023 11:17:25 -0700 Subject: [PATCH 034/299] syscall: fix opening of directories on wasip1 Go programs targeting GOOS=wasip1 were failing to open directories when executed with runtimes like wasmtime or wasmedge due to requesting rights for operations that are not supported on directories such as fd_read, fd_write, etc... This change addresses the issue by performing a second path_open when observing EISDIR, and masking the requested rights to only ask for permissions to perform operations supported by a directory. Change-Id: Ibf65acf4a38bc848a649f41dbd026507d8b63c82 Reviewed-on: https://go-review.googlesource.com/c/go/+/490755 Auto-Submit: Ian Lance Taylor Run-TryBot: Ian Lance Taylor Reviewed-by: Dmitri Shuralyov Reviewed-by: Ian Lance Taylor TryBot-Result: Gopher Robot --- src/syscall/fs_wasip1.go | 87 ++++++++++++++++++++++++++++++++++++++-- 1 file changed, 83 insertions(+), 4 deletions(-) diff --git a/src/syscall/fs_wasip1.go b/src/syscall/fs_wasip1.go index ef04af6966..ab00e5ba22 100644 --- a/src/syscall/fs_wasip1.go +++ b/src/syscall/fs_wasip1.go @@ -100,6 +100,63 @@ const ( fullRights = rights(^uint32(0)) readRights = rights(RIGHT_FD_READ | RIGHT_FD_READDIR) writeRights = rights(RIGHT_FD_DATASYNC | RIGHT_FD_WRITE | RIGHT_FD_ALLOCATE | RIGHT_PATH_FILESTAT_SET_SIZE) + + // Some runtimes have very strict expectations when it comes to which + // rights can be enabled on files opened by path_open. The fileRights + // constant is used as a mask to retain only bits for operations that + // are supported on files. + fileRights rights = RIGHT_FD_DATASYNC | + RIGHT_FD_READ | + RIGHT_FD_SEEK | + RIGHT_FDSTAT_SET_FLAGS | + RIGHT_FD_SYNC | + RIGHT_FD_TELL | + RIGHT_FD_WRITE | + RIGHT_FD_ADVISE | + RIGHT_FD_ALLOCATE | + RIGHT_PATH_CREATE_DIRECTORY | + RIGHT_PATH_CREATE_FILE | + RIGHT_PATH_LINK_SOURCE | + RIGHT_PATH_LINK_TARGET | + RIGHT_PATH_OPEN | + RIGHT_FD_READDIR | + RIGHT_PATH_READLINK | + RIGHT_PATH_RENAME_SOURCE | + RIGHT_PATH_RENAME_TARGET | + RIGHT_PATH_FILESTAT_GET | + RIGHT_PATH_FILESTAT_SET_SIZE | + RIGHT_PATH_FILESTAT_SET_TIMES | + RIGHT_FD_FILESTAT_GET | + RIGHT_FD_FILESTAT_SET_SIZE | + RIGHT_FD_FILESTAT_SET_TIMES | + RIGHT_PATH_SYMLINK | + RIGHT_PATH_REMOVE_DIRECTORY | + RIGHT_PATH_UNLINK_FILE | + RIGHT_POLL_FD_READWRITE + + // Runtimes like wasmtime and wasmedge will refuse to open directories + // if the rights requested by the application exceed the operations that + // can be performed on a directory. + dirRights rights = RIGHT_FD_SEEK | + RIGHT_FDSTAT_SET_FLAGS | + RIGHT_FD_SYNC | + RIGHT_PATH_CREATE_DIRECTORY | + RIGHT_PATH_CREATE_FILE | + RIGHT_PATH_LINK_SOURCE | + RIGHT_PATH_LINK_TARGET | + RIGHT_PATH_OPEN | + RIGHT_FD_READDIR | + RIGHT_PATH_READLINK | + RIGHT_PATH_RENAME_SOURCE | + RIGHT_PATH_RENAME_TARGET | + RIGHT_PATH_FILESTAT_GET | + RIGHT_PATH_FILESTAT_SET_SIZE | + RIGHT_PATH_FILESTAT_SET_TIMES | + RIGHT_FD_FILESTAT_GET | + RIGHT_FD_FILESTAT_SET_TIMES | + RIGHT_PATH_SYMLINK | + RIGHT_PATH_REMOVE_DIRECTORY | + RIGHT_PATH_UNLINK_FILE ) // https://github.com/WebAssembly/WASI/blob/a2b96e81c0586125cc4dc79a5be0b78d9a059925/legacy/preview1/docs.md#-fd_closefd-fd---result-errno @@ -435,11 +492,11 @@ func Open(path string, openmode int, perm uint32) (int, error) { var rights rights switch openmode & (O_RDONLY | O_WRONLY | O_RDWR) { case O_RDONLY: - rights = fullRights & ^writeRights + rights = fileRights & ^writeRights case O_WRONLY: - rights = fullRights & ^readRights + rights = fileRights & ^readRights case O_RDWR: - rights = fullRights + rights = fileRights } var fdflags fdflags @@ -458,10 +515,32 @@ func Open(path string, openmode int, perm uint32) (int, error) { pathLen, oflags, rights, - fullRights, + fileRights, fdflags, unsafe.Pointer(&fd), ) + if errno == EISDIR && oflags == 0 && fdflags == 0 && ((rights & writeRights) == 0) { + // wasmtime and wasmedge will error if attempting to open a directory + // because we are asking for too many rights. However, we cannot + // determine ahread of time if the path we are about to open is a + // directory, so instead we fallback to a second call to path_open with + // a more limited set of rights. + // + // This approach is subject to a race if the file system is modified + // concurrently, so we also inject OFLAG_DIRECTORY to ensure that we do + // not accidentally open a file which is not a directory. + errno = path_open( + dirFd, + LOOKUP_SYMLINK_FOLLOW, + pathPtr, + pathLen, + oflags|OFLAG_DIRECTORY, + rights&dirRights, + fileRights, + fdflags, + unsafe.Pointer(&fd), + ) + } return int(fd), errnoErr(errno) } From 2380d17aea2f1b5d6f379b76152b20f7a820ed9f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Felix=20Geisend=C3=B6rfer?= Date: Sat, 22 Apr 2023 09:54:46 +0300 Subject: [PATCH 035/299] runtime: fix systemstack frame pointer adjustment MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Change adjustframe to adjust the frame pointer of systemstack (aka FuncID_systemstack_switch) before returning early. Without this fix it is possible for traceEvent() to crash when using frame pointer unwinding. The issue occurs when a goroutine calls systemstack in order to call shrinkstack. While returning, systemstack will restore the unadjusted frame pointer from its frame as part of its epilogue. If the callee of systemstack then triggers a traceEvent, it will try to unwind into the old stack. This can lead to a crash if the memory of the old stack has been reused or freed in the meantime. The most common situation in which this will manifest is when when gcAssistAlloc() invokes gcAssistAlloc1() on systemstack() and performs a shrinkstack() followed by a traceGCMarkAssistDone() or Gosched() triggering traceEvent(). See CL 489115 for a deterministic test case that triggers the issue. Meanwhile the problem can frequently be observed using the command below: $ GODEBUG=tracefpunwindoff=0 ../bin/go test -trace /dev/null -run TestDeferHeapAndStack ./runtime SIGSEGV: segmentation violation PC=0x45f977 m=14 sigcode=128 goroutine 0 [idle]: runtime.fpTracebackPCs(...) .../go/src/runtime/trace.go:945 runtime.traceStackID(0xcdab904677a?, {0x7f1584346018, 0x0?, 0x80}, 0x0?) .../go/src/runtime/trace.go:917 +0x217 fp=0x7f1565ffab00 sp=0x7f1565ffaab8 pc=0x45f977 runtime.traceEventLocked(0x0?, 0x0?, 0x0?, 0xc00003dbd0, 0x12, 0x0, 0x1, {0x0, 0x0, 0x0}) .../go/src/runtime/trace.go:760 +0x285 fp=0x7f1565ffab78 sp=0x7f1565ffab00 pc=0x45ef45 runtime.traceEvent(0xf5?, 0x1, {0x0, 0x0, 0x0}) .../go/src/runtime/trace.go:692 +0xa9 fp=0x7f1565ffabe0 sp=0x7f1565ffab78 pc=0x45ec49 runtime.traceGoPreempt(...) .../go/src/runtime/trace.go:1535 runtime.gopreempt_m(0xc000328340?) .../go/src/runtime/proc.go:3551 +0x45 fp=0x7f1565ffac20 sp=0x7f1565ffabe0 pc=0x4449a5 runtime.newstack() .../go/src/runtime/stack.go:1077 +0x3cb fp=0x7f1565ffadd0 sp=0x7f1565ffac20 pc=0x455feb runtime.morestack() .../go/src/runtime/asm_amd64.s:593 +0x8f fp=0x7f1565ffadd8 sp=0x7f1565ffadd0 pc=0x47644f goroutine 19 [running]: runtime.traceEvent(0x2c?, 0xffffffffffffffff, {0x0, 0x0, 0x0}) .../go/src/runtime/trace.go:669 +0xe8 fp=0xc0006e6c28 sp=0xc0006e6c20 pc=0x45ec88 runtime.traceGCMarkAssistDone(...) .../go/src/runtime/trace.go:1497 runtime.gcAssistAlloc(0xc0003281a0) .../go/src/runtime/mgcmark.go:517 +0x27d fp=0xc0006e6c88 sp=0xc0006e6c28 pc=0x421a1d runtime.deductAssistCredit(0x0?) .../go/src/runtime/malloc.go:1287 +0x54 fp=0xc0006e6cb0 sp=0xc0006e6c88 pc=0x40fed4 runtime.mallocgc(0x400, 0x7a9420, 0x1) .../go/src/runtime/malloc.go:1002 +0xc9 fp=0xc0006e6d18 sp=0xc0006e6cb0 pc=0x40f709 runtime.newobject(0xb3?) .../go/src/runtime/malloc.go:1324 +0x25 fp=0xc0006e6d40 sp=0xc0006e6d18 pc=0x40ffc5 runtime_test.deferHeapAndStack(0xb4) .../go/src/runtime/stack_test.go:924 +0x165 fp=0xc0006e6e20 sp=0xc0006e6d40 pc=0x75c2a5 Fixes #59692 Co-Authored-By: Cherry Mui Co-Authored-By: Michael Knyszek Co-Authored-By: Nick Ripley Change-Id: I1c0c28327fc2fac0b8cfdbaa72e25584331be31e Reviewed-on: https://go-review.googlesource.com/c/go/+/489015 TryBot-Result: Gopher Robot Reviewed-by: Cherry Mui Reviewed-by: Michael Knyszek Run-TryBot: Felix Geisendörfer --- src/runtime/stack.go | 29 +++++++++++++++-------------- 1 file changed, 15 insertions(+), 14 deletions(-) diff --git a/src/runtime/stack.go b/src/runtime/stack.go index 3f1e5ff919..54513eba65 100644 --- a/src/runtime/stack.go +++ b/src/runtime/stack.go @@ -650,20 +650,6 @@ func adjustframe(frame *stkframe, adjinfo *adjustinfo) { if stackDebug >= 2 { print(" adjusting ", funcname(f), " frame=[", hex(frame.sp), ",", hex(frame.fp), "] pc=", hex(frame.pc), " continpc=", hex(frame.continpc), "\n") } - if f.funcID == abi.FuncID_systemstack_switch { - // A special routine at the bottom of stack of a goroutine that does a systemstack call. - // We will allow it to be copied even though we don't - // have full GC info for it (because it is written in asm). - return - } - - locals, args, objs := frame.getStackMap(&adjinfo.cache, true) - - // Adjust local variables if stack frame has been allocated. - if locals.n > 0 { - size := uintptr(locals.n) * goarch.PtrSize - adjustpointers(unsafe.Pointer(frame.varp-size), &locals, adjinfo, f) - } // Adjust saved frame pointer if there is one. if (goarch.ArchFamily == goarch.AMD64 || goarch.ArchFamily == goarch.ARM64) && frame.argp-frame.varp == 2*goarch.PtrSize { @@ -687,6 +673,21 @@ func adjustframe(frame *stkframe, adjinfo *adjustinfo) { adjustpointer(adjinfo, unsafe.Pointer(frame.varp)) } + if f.funcID == abi.FuncID_systemstack_switch { + // A special routine at the bottom of stack of a goroutine that does a systemstack call. + // We will allow it to be copied even though we don't + // have full GC info for it (because it is written in asm). + return + } + + locals, args, objs := frame.getStackMap(&adjinfo.cache, true) + + // Adjust local variables if stack frame has been allocated. + if locals.n > 0 { + size := uintptr(locals.n) * goarch.PtrSize + adjustpointers(unsafe.Pointer(frame.varp-size), &locals, adjinfo, f) + } + // Adjust arguments. if args.n > 0 { if stackDebug >= 3 { From 069f9fb20548e904cb94f165f4cbf716a0fc108e Mon Sep 17 00:00:00 2001 From: Ian Lance Taylor Date: Mon, 24 Apr 2023 17:27:33 -0700 Subject: [PATCH 036/299] debug/pe: return error on reading from section with uninitialized data A section with uninitialized data contains no bytes and occupies no space in the file. This change makes it return an error on reading from this section so that it will force the caller to check for a section with uninitialized data. This is the debug/pe version of CL 429601. This will break programs that expect a byte slice with the length described by the SizeOfRawData field. There are two reasons to introduce this breaking change: 1) uninitialized data is uninitialized and there is no reason to allocate memory for it; 2) it could result in an OOM if the file is corrupted and has a large invalid SizeOfRawData. No test case because the problem can only happen for invalid data. Let the fuzzer find cases like this. For #47653 Fixes #59817 Change-Id: I1ae94e9508f803b37926275d9a571f724a09af9f Reviewed-on: https://go-review.googlesource.com/c/go/+/488475 Reviewed-by: Bryan Mills Reviewed-by: kortschak Reviewed-by: Alex Brainman Reviewed-by: Ian Lance Taylor TryBot-Result: Gopher Robot Run-TryBot: Ian Lance Taylor Run-TryBot: Ian Lance Taylor Auto-Submit: Ian Lance Taylor --- src/debug/pe/file.go | 17 ++++++++--------- src/debug/pe/file_test.go | 14 +++----------- src/debug/pe/section.go | 6 ++++++ 3 files changed, 17 insertions(+), 20 deletions(-) diff --git a/src/debug/pe/file.go b/src/debug/pe/file.go index de4bb9b736..06c160105f 100644 --- a/src/debug/pe/file.go +++ b/src/debug/pe/file.go @@ -20,6 +20,7 @@ import ( "compress/zlib" "debug/dwarf" "encoding/binary" + "errors" "fmt" "io" "os" @@ -165,7 +166,7 @@ func NewFile(r io.ReaderAt) (*File, error) { } r2 := r if sh.PointerToRawData == 0 { // .bss must have all 0s - r2 = zeroReaderAt{} + r2 = &nobitsSectionReader{} } s.sr = io.NewSectionReader(r2, int64(s.SectionHeader.Offset), int64(s.SectionHeader.Size)) s.ReaderAt = s.sr @@ -182,15 +183,10 @@ func NewFile(r io.ReaderAt) (*File, error) { return f, nil } -// zeroReaderAt is ReaderAt that reads 0s. -type zeroReaderAt struct{} +type nobitsSectionReader struct{} -// ReadAt writes len(p) 0s into p. -func (w zeroReaderAt) ReadAt(p []byte, off int64) (n int, err error) { - for i := range p { - p[i] = 0 - } - return len(p), nil +func (*nobitsSectionReader) ReadAt(p []byte, off int64) (n int, err error) { + return 0, errors.New("unexpected read from section with uninitialized data") } // getString extracts a string from symbol string table. @@ -363,6 +359,9 @@ func (f *File) ImportedSymbols() ([]string, error) { var ds *Section ds = nil for _, s := range f.Sections { + if s.Offset == 0 { + continue + } // We are using distance between s.VirtualAddress and idd.VirtualAddress // to avoid potential overflow of uint32 caused by addition of s.VirtualSize // to s.VirtualAddress. diff --git a/src/debug/pe/file_test.go b/src/debug/pe/file_test.go index 5368e08ad7..3d960ab7f3 100644 --- a/src/debug/pe/file_test.go +++ b/src/debug/pe/file_test.go @@ -511,17 +511,9 @@ main(void) if bss == nil { t.Fatal("could not find .bss section") } - data, err := bss.Data() - if err != nil { - t.Fatal(err) - } - if len(data) == 0 { - t.Fatalf("%s file .bss section cannot be empty", objpath) - } - for _, b := range data { - if b != 0 { - t.Fatalf(".bss section has non zero bytes: %v", data) - } + // We expect an error from bss.Data, as there are no contents. + if _, err := bss.Data(); err == nil { + t.Error("bss.Data succeeded, expected error") } } diff --git a/src/debug/pe/section.go b/src/debug/pe/section.go index fabb47af2e..70d0c220ce 100644 --- a/src/debug/pe/section.go +++ b/src/debug/pe/section.go @@ -97,11 +97,17 @@ type Section struct { } // Data reads and returns the contents of the PE section s. +// +// If s.Offset is 0, the section has no contents, +// and Data will always return a non-nil error. func (s *Section) Data() ([]byte, error) { return saferio.ReadDataAt(s.sr, uint64(s.Size), 0) } // Open returns a new ReadSeeker reading the PE section s. +// +// If s.Offset is 0, the section has no contents, and all calls +// to the returned reader will return a non-nil error. func (s *Section) Open() io.ReadSeeker { return io.NewSectionReader(s.sr, 0, 1<<63-1) } From d1f0552892b555ed6edeb58d4793396dd2b7ce62 Mon Sep 17 00:00:00 2001 From: Austin Clements Date: Fri, 28 Apr 2023 14:51:18 -0400 Subject: [PATCH 037/299] misc/android: rename to misc/go_android_exec, make go build work This makes it reasonable to "go build" from this directory by changing the name of the directory to a more reasonable name for the binary and dropping the unnecessary "ignore" build tag. The resulting binary doesn't *quite* have the necessary name for a Go exec wrapper because that needs to have the form, go_android_$GOARCH_exec, but it's close. Change-Id: I036cb1af9c034462a952b176a794526fa3ffd1ab Reviewed-on: https://go-review.googlesource.com/c/go/+/490495 Reviewed-by: Bryan Mills Run-TryBot: Austin Clements Auto-Submit: Austin Clements TryBot-Result: Gopher Robot --- misc/{android => go_android_exec}/README | 0 misc/{android/go_android_exec.go => go_android_exec/main.go} | 3 --- src/cmd/dist/build.go | 2 +- 3 files changed, 1 insertion(+), 4 deletions(-) rename misc/{android => go_android_exec}/README (100%) rename misc/{android/go_android_exec.go => go_android_exec/main.go} (99%) diff --git a/misc/android/README b/misc/go_android_exec/README similarity index 100% rename from misc/android/README rename to misc/go_android_exec/README diff --git a/misc/android/go_android_exec.go b/misc/go_android_exec/main.go similarity index 99% rename from misc/android/go_android_exec.go rename to misc/go_android_exec/main.go index 445ac284be..639b744610 100644 --- a/misc/android/go_android_exec.go +++ b/misc/go_android_exec/main.go @@ -2,9 +2,6 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. -//go:build ignore -// +build ignore - // This program can be used as go_android_GOARCH_exec by the Go tool. // It executes binaries on an android device using adb. package main diff --git a/src/cmd/dist/build.go b/src/cmd/dist/build.go index 1d329ab9f1..11fb5f0753 100644 --- a/src/cmd/dist/build.go +++ b/src/cmd/dist/build.go @@ -1661,7 +1661,7 @@ func wrapperPathFor(goos, goarch string) string { switch { case goos == "android": if gohostos != "android" { - return pathf("%s/misc/android/go_android_exec.go", goroot) + return pathf("%s/misc/go_android_exec/main.go", goroot) } case goos == "ios": if gohostos != "ios" { From 5901ab80b9c9ac2daa7c30f0c0ede3b7adfc7007 Mon Sep 17 00:00:00 2001 From: qmuntal Date: Sun, 30 Apr 2023 11:38:00 +0200 Subject: [PATCH 038/299] runtime/cgo: use atomic.Uintptr instead of atomic.AddUintptr. cgo.NewHandle atomically increments a global uintptr index using atomic.AddUintptr. Use atomic.Uintptr instead, which is cleaner and clearer. Change-Id: I845b3e4cb8c461e787a9b9bb2a9ceaaef1d21d8e Reviewed-on: https://go-review.googlesource.com/c/go/+/490775 Run-TryBot: Quim Muntal TryBot-Result: Gopher Robot Reviewed-by: Bryan Mills Auto-Submit: Ian Lance Taylor Run-TryBot: Ian Lance Taylor Reviewed-by: Ian Lance Taylor --- src/runtime/cgo/handle.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/runtime/cgo/handle.go b/src/runtime/cgo/handle.go index d711900d79..061dfb0e2e 100644 --- a/src/runtime/cgo/handle.go +++ b/src/runtime/cgo/handle.go @@ -106,7 +106,7 @@ type Handle uintptr // The intended use is to pass the returned handle to C code, which // passes it back to Go, which calls Value. func NewHandle(v any) Handle { - h := atomic.AddUintptr(&handleIdx, 1) + h := handleIdx.Add(1) if h == 0 { panic("runtime/cgo: ran out of handle space") } @@ -140,5 +140,5 @@ func (h Handle) Delete() { var ( handles = sync.Map{} // map[Handle]interface{} - handleIdx uintptr // atomic + handleIdx atomic.Uintptr ) From 8f763b55a58855b88f0afe28f21e3870fff42ee8 Mon Sep 17 00:00:00 2001 From: "Bryan C. Mills" Date: Tue, 25 Apr 2023 11:31:49 -0400 Subject: [PATCH 039/299] misc/android: copy entire modules in module mode Change-Id: I8a8aea4d4b9824b53f17bff160055c0d9d2960ff Reviewed-on: https://go-review.googlesource.com/c/go/+/488655 Reviewed-by: Ian Lance Taylor Auto-Submit: Bryan Mills Run-TryBot: Bryan Mills TryBot-Result: Gopher Robot --- misc/go_android_exec/main.go | 83 +++++++++++++++++++++++------------- 1 file changed, 54 insertions(+), 29 deletions(-) diff --git a/misc/go_android_exec/main.go b/misc/go_android_exec/main.go index 639b744610..d88d4da1f2 100644 --- a/misc/go_android_exec/main.go +++ b/misc/go_android_exec/main.go @@ -114,7 +114,7 @@ func runMain() (int, error) { // "$GOROOT/src/mime/multipart" or "$GOPATH/src/golang.org/x/mobile". // We extract everything after the $GOROOT or $GOPATH to run on the // same relative directory on the target device. - importPath, isStd, err := pkgPath() + importPath, isStd, modPath, modDir, err := pkgPath() if err != nil { return 0, err } @@ -126,23 +126,41 @@ func runMain() (int, error) { deviceCwd = path.Join(deviceGoroot, "src", importPath) } else { deviceCwd = path.Join(deviceGopath, "src", importPath) - if err := adb("exec-out", "mkdir", "-p", deviceCwd); err != nil { - return 0, err - } - if err := adbCopyTree(deviceCwd, importPath); err != nil { - return 0, err - } - - // Copy .go files from the package. - goFiles, err := filepath.Glob("*.go") - if err != nil { - return 0, err - } - if len(goFiles) > 0 { - args := append(append([]string{"push"}, goFiles...), deviceCwd) - if err := adb(args...); err != nil { + if modDir != "" { + // In module mode, the user may reasonably expect the entire module + // to be present. Copy it over. + deviceModDir := path.Join(deviceGopath, "src", modPath) + if err := adb("exec-out", "mkdir", "-p", path.Dir(deviceModDir)); err != nil { return 0, err } + // We use a single recursive 'adb push' of the module root instead of + // walking the tree and copying it piecewise. If the directory tree + // contains nested modules this could push a lot of unnecessary contents, + // but for the golang.org/x repos it seems to be significantly (~2x) + // faster than copying one file at a time (via filepath.WalkDir), + // apparently due to high latency in 'adb' commands. + if err := adb("push", modDir, deviceModDir); err != nil { + return 0, err + } + } else { + if err := adb("exec-out", "mkdir", "-p", deviceCwd); err != nil { + return 0, err + } + if err := adbCopyTree(deviceCwd, importPath); err != nil { + return 0, err + } + + // Copy .go files from the package. + goFiles, err := filepath.Glob("*.go") + if err != nil { + return 0, err + } + if len(goFiles) > 0 { + args := append(append([]string{"push"}, goFiles...), deviceCwd) + if err := adb(args...); err != nil { + return 0, err + } + } } } @@ -198,34 +216,41 @@ func runMain() (int, error) { // pkgPath determines the package import path of the current working directory, // and indicates whether it is // and returns the path to the package source relative to $GOROOT (or $GOPATH). -func pkgPath() (importPath string, isStd bool, err error) { +func pkgPath() (importPath string, isStd bool, modPath, modDir string, err error) { + errorf := func(format string, args ...any) (string, bool, string, string, error) { + return "", false, "", "", fmt.Errorf(format, args...) + } goTool, err := goTool() if err != nil { - return "", false, err + return errorf("%w", err) } - cmd := exec.Command(goTool, "list", "-e", "-f", "{{.ImportPath}}:{{.Standard}}", ".") + cmd := exec.Command(goTool, "list", "-e", "-f", "{{.ImportPath}}:{{.Standard}}{{with .Module}}:{{.Path}}:{{.Dir}}{{end}}", ".") out, err := cmd.Output() if err != nil { if ee, ok := err.(*exec.ExitError); ok && len(ee.Stderr) > 0 { - return "", false, fmt.Errorf("%v: %s", cmd, ee.Stderr) + return errorf("%v: %s", cmd, ee.Stderr) } - return "", false, fmt.Errorf("%v: %w", cmd, err) + return errorf("%v: %w", cmd, err) } - s := string(bytes.TrimSpace(out)) - importPath, isStdStr, ok := strings.Cut(s, ":") - if !ok { - return "", false, fmt.Errorf("%v: missing ':' in output: %q", cmd, out) + parts := strings.SplitN(string(bytes.TrimSpace(out)), ":", 4) + if len(parts) < 2 { + return errorf("%v: missing ':' in output: %q", cmd, out) } + importPath = parts[0] if importPath == "" || importPath == "." { - return "", false, fmt.Errorf("current directory does not have a Go import path") + return errorf("current directory does not have a Go import path") } - isStd, err = strconv.ParseBool(isStdStr) + isStd, err = strconv.ParseBool(parts[1]) if err != nil { - return "", false, fmt.Errorf("%v: non-boolean .Standard in output: %q", cmd, out) + return errorf("%v: non-boolean .Standard in output: %q", cmd, out) + } + if len(parts) >= 4 { + modPath = parts[2] + modDir = parts[3] } - return importPath, isStd, nil + return importPath, isStd, modPath, modDir, nil } // adbCopyTree copies testdata, go.mod, go.sum files from subdir From 6b859d9d5816bd3eaeff40eece7bd797b2824e8f Mon Sep 17 00:00:00 2001 From: Will Faught Date: Fri, 28 Apr 2023 23:44:50 +0000 Subject: [PATCH 040/299] text/template: reword uncover to unwrap Matches the preceding "wrap" terminology. Change-Id: Ia783de578c2942fe1474281c3d6056b1074d41b0 GitHub-Last-Rev: 4fcff4e9b2836d428ba668186441089a9618c028 GitHub-Pull-Request: golang/go#59891 Reviewed-on: https://go-review.googlesource.com/c/go/+/490675 TryBot-Result: Gopher Robot Run-TryBot: Ian Lance Taylor Reviewed-by: Ian Lance Taylor Auto-Submit: Ian Lance Taylor Reviewed-by: Carlos Amedee --- src/text/template/funcs.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/text/template/funcs.go b/src/text/template/funcs.go index dbea6e705a..b5a8c9ec50 100644 --- a/src/text/template/funcs.go +++ b/src/text/template/funcs.go @@ -23,7 +23,7 @@ import ( // Execute returns that error. // // Errors returned by Execute wrap the underlying error; call errors.As to -// uncover them. +// unwrap them. // // When template execution invokes a function with an argument list, that list // must be assignable to the function's parameter types. Functions meant to From 63edd418b6684c0eed6d19d818eaef42e813a2ae Mon Sep 17 00:00:00 2001 From: Michael Pratt Date: Mon, 1 May 2023 13:09:47 -0400 Subject: [PATCH 041/299] cmd/compile: drop unused arg to mkinlcall Change-Id: I3cd8d81cc434257d78b34dfaae09a77ab3211121 Reviewed-on: https://go-review.googlesource.com/c/go/+/490896 TryBot-Result: Gopher Robot Run-TryBot: Michael Pratt Auto-Submit: Michael Pratt Reviewed-by: Than McIntosh --- src/cmd/compile/internal/inline/inl.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/cmd/compile/internal/inline/inl.go b/src/cmd/compile/internal/inline/inl.go index df12f9a625..d5bd1a50bf 100644 --- a/src/cmd/compile/internal/inline/inl.go +++ b/src/cmd/compile/internal/inline/inl.go @@ -892,7 +892,7 @@ func inlnode(n ir.Node, bigCaller bool, inlCalls *[]*ir.InlinedCallExpr, edit fu break } if fn := inlCallee(call.X, profile); fn != nil && typecheck.HaveInlineBody(fn) { - n = mkinlcall(call, fn, bigCaller, inlCalls, edit) + n = mkinlcall(call, fn, bigCaller, inlCalls) if fn.IsHiddenClosure() { // Visit function to pick out any contained hidden // closures to mark them as dead, since they will no @@ -1023,7 +1023,7 @@ func inlineCostOK(n *ir.CallExpr, caller, callee *ir.Func, bigCaller bool) (bool // The result of mkinlcall MUST be assigned back to n, e.g. // // n.Left = mkinlcall(n.Left, fn, isddd) -func mkinlcall(n *ir.CallExpr, fn *ir.Func, bigCaller bool, inlCalls *[]*ir.InlinedCallExpr, edit func(ir.Node) ir.Node) ir.Node { +func mkinlcall(n *ir.CallExpr, fn *ir.Func, bigCaller bool, inlCalls *[]*ir.InlinedCallExpr) ir.Node { if fn.Inl == nil { if logopt.Enabled() { logopt.LogOpt(n.Pos(), "cannotInlineCall", "inline", ir.FuncName(ir.CurFunc), From 73a4684caa1567c06e239dc657b82ede77777e3b Mon Sep 17 00:00:00 2001 From: Robert Griesemer Date: Fri, 28 Apr 2023 17:46:00 -0700 Subject: [PATCH 042/299] go/types, types2: isParameterized must be able to handle tuples CL 484615 rewrote isParameterized by handling tuple types only where they occur (function signatures). However, isParameterized is also called from Checker.callExpr, with a result parameter list which is a tuple. This CL handles tuples again. Fixes #59890. Change-Id: I35159ff65f23322432557e6abcab939933933d40 Reviewed-on: https://go-review.googlesource.com/c/go/+/490695 Reviewed-by: Robert Griesemer Auto-Submit: Robert Griesemer TryBot-Result: Gopher Robot Reviewed-by: Robert Findley Run-TryBot: Robert Griesemer --- src/cmd/compile/internal/types2/call.go | 4 ++-- src/cmd/compile/internal/types2/infer.go | 10 +++++++--- src/go/types/call.go | 4 ++-- src/go/types/infer.go | 10 +++++++--- .../types/testdata/fixedbugs/issue59890.go | 17 +++++++++++++++++ 5 files changed, 35 insertions(+), 10 deletions(-) create mode 100644 src/internal/types/testdata/fixedbugs/issue59890.go diff --git a/src/cmd/compile/internal/types2/call.go b/src/cmd/compile/internal/types2/call.go index 7e8fce4350..20cde9f44e 100644 --- a/src/cmd/compile/internal/types2/call.go +++ b/src/cmd/compile/internal/types2/call.go @@ -329,8 +329,8 @@ func (check *Checker) callExpr(x *operand, call *syntax.CallExpr) exprKind { x.expr = call check.hasCallOrRecv = true - // if type inference failed, a parametrized result must be invalidated - // (operands cannot have a parametrized type) + // if type inference failed, a parameterized result must be invalidated + // (operands cannot have a parameterized type) if x.mode == value && sig.TypeParams().Len() > 0 && isParameterized(sig.TypeParams().list(), x.typ) { x.mode = invalid } diff --git a/src/cmd/compile/internal/types2/infer.go b/src/cmd/compile/internal/types2/infer.go index 46f461ea09..dbe621cded 100644 --- a/src/cmd/compile/internal/types2/infer.go +++ b/src/cmd/compile/internal/types2/infer.go @@ -498,9 +498,13 @@ func (w *tpWalker) isParameterized(typ Type) (res bool) { case *Pointer: return w.isParameterized(t.base) - // case *Tuple: - // This case should not occur because tuples only appear - // in signatures where they are handled explicitly. + case *Tuple: + // This case does not occur from within isParameterized + // because tuples only appear in signatures where they + // are handled explicitly. But isParameterized is also + // called by Checker.callExpr with a function result tuple + // if instantiation failed (go.dev/issue/59890). + return t != nil && w.varList(t.vars) case *Signature: // t.tparams may not be nil if we are looking at a signature diff --git a/src/go/types/call.go b/src/go/types/call.go index 418de06e76..979de2338f 100644 --- a/src/go/types/call.go +++ b/src/go/types/call.go @@ -334,8 +334,8 @@ func (check *Checker) callExpr(x *operand, call *ast.CallExpr) exprKind { x.expr = call check.hasCallOrRecv = true - // if type inference failed, a parametrized result must be invalidated - // (operands cannot have a parametrized type) + // if type inference failed, a parameterized result must be invalidated + // (operands cannot have a parameterized type) if x.mode == value && sig.TypeParams().Len() > 0 && isParameterized(sig.TypeParams().list(), x.typ) { x.mode = invalid } diff --git a/src/go/types/infer.go b/src/go/types/infer.go index f24c729d7a..3aa66105c4 100644 --- a/src/go/types/infer.go +++ b/src/go/types/infer.go @@ -500,9 +500,13 @@ func (w *tpWalker) isParameterized(typ Type) (res bool) { case *Pointer: return w.isParameterized(t.base) - // case *Tuple: - // This case should not occur because tuples only appear - // in signatures where they are handled explicitly. + case *Tuple: + // This case does not occur from within isParameterized + // because tuples only appear in signatures where they + // are handled explicitly. But isParameterized is also + // called by Checker.callExpr with a function result tuple + // if instantiation failed (go.dev/issue/59890). + return t != nil && w.varList(t.vars) case *Signature: // t.tparams may not be nil if we are looking at a signature diff --git a/src/internal/types/testdata/fixedbugs/issue59890.go b/src/internal/types/testdata/fixedbugs/issue59890.go new file mode 100644 index 0000000000..ed7afd939a --- /dev/null +++ b/src/internal/types/testdata/fixedbugs/issue59890.go @@ -0,0 +1,17 @@ +// Copyright 2023 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package p + +func _() { g /* ERROR "cannot infer T" */ () } + +func g[T any]() (_ /* ERROR "cannot use _ as value or type" */, int) { panic(0) } + +// test case from issue + +var _ = append(f /* ERROR "cannot infer T" */ ()()) + +func f[T any]() (_ /* ERROR "cannot use _" */, _ /* ERROR "cannot use _" */, int) { + panic("not implemented") +} From 80d64adb7950d09371a351bbd3a23facc2cfe5cf Mon Sep 17 00:00:00 2001 From: cui fliter Date: Mon, 1 May 2023 12:41:56 +0800 Subject: [PATCH 043/299] runtime: fix comment typo in page allocator A commit looks to have some minor bug that makes comments look confusing. The commit in question: https://go-review.googlesource.com/c/go/+/250517 Change-Id: I7859587be15a22f8330d6ad476058f74ca2ca6ab Reviewed-on: https://go-review.googlesource.com/c/go/+/490795 Run-TryBot: shuang cui Reviewed-by: Michael Knyszek Auto-Submit: Ian Lance Taylor Reviewed-by: Michael Pratt TryBot-Result: Gopher Robot --- src/runtime/mpagealloc_64bit.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/runtime/mpagealloc_64bit.go b/src/runtime/mpagealloc_64bit.go index 0ebeafad61..1418831a50 100644 --- a/src/runtime/mpagealloc_64bit.go +++ b/src/runtime/mpagealloc_64bit.go @@ -90,7 +90,7 @@ func (p *pageAlloc) sysInit(test bool) { // sysGrow performs architecture-dependent operations on heap // growth for the page allocator, such as mapping in new memory // for summaries. It also updates the length of the slices in -// [.summary. +// p.summary. // // base is the base of the newly-added heap memory and limit is // the first address past the end of the newly-added heap memory. From 04f059f9efbb2a202ddb6a3c6f87efa8073e562f Mon Sep 17 00:00:00 2001 From: Ian Lance Taylor Date: Fri, 28 Apr 2023 10:29:58 -0700 Subject: [PATCH 044/299] net: don't recheck goosPrefersCgo in hostLookupOrder We only did it for testing. Remove the single test that required it. Change-Id: Ib6c3a2debfd3f48e95af37f23fdfde847ff87a41 Reviewed-on: https://go-review.googlesource.com/c/go/+/490395 Run-TryBot: Ian Lance Taylor TryBot-Result: Gopher Robot Reviewed-by: Bryan Mills Reviewed-by: Damien Neil Auto-Submit: Ian Lance Taylor --- src/net/conf.go | 13 +++---------- src/net/conf_test.go | 12 ------------ 2 files changed, 3 insertions(+), 22 deletions(-) diff --git a/src/net/conf.go b/src/net/conf.go index 6386078132..d11a568502 100644 --- a/src/net/conf.go +++ b/src/net/conf.go @@ -124,7 +124,7 @@ func initConfVal() { } // Some operating systems always prefer the cgo resolver. - if goosPrefersCgo(runtime.GOOS) { + if goosPrefersCgo() { confVal.preferCgo = true return } @@ -155,8 +155,8 @@ func initConfVal() { // goosPreferCgo reports whether the GOOS value passed in prefers // the cgo resolver. -func goosPrefersCgo(goos string) bool { - switch goos { +func goosPrefersCgo() bool { + switch runtime.GOOS { // Historically on Windows and Plan 9 we prefer the // cgo resolver (which doesn't use the cgo tool) rather than // the go resolver. This is because originally these @@ -227,13 +227,6 @@ func (c *conf) hostLookupOrder(r *Resolver, hostname string) (ret hostLookupOrde // Neither resolver was explicitly requested // and we have no preference. - // For testing purposes only, recheck the GOOS. - // This lets TestConfHostLookupOrder test different - // GOOS values. - if goosPrefersCgo(c.goos) { - return hostLookupCgo, nil - } - if bytealg.IndexByteString(hostname, '\\') != -1 || bytealg.IndexByteString(hostname, '%') != -1 { // Don't deal with special form hostnames // with backslashes or '%'. diff --git a/src/net/conf_test.go b/src/net/conf_test.go index 6c9d247713..08d774bfe2 100644 --- a/src/net/conf_test.go +++ b/src/net/conf_test.go @@ -341,18 +341,6 @@ func TestConfHostLookupOrder(t *testing.T) { nss: nssStr(t, "foo: bar"), hostTests: []nssHostTest{{"google.com", "myhostname", hostLookupCgo}}, }, - // Android should always use cgo. - { - name: "android", - c: &conf{ - goos: "android", - }, - resolv: defaultResolvConf, - nss: nssStr(t, ""), - hostTests: []nssHostTest{ - {"x.com", "myhostname", hostLookupCgo}, - }, - }, // Issue 24393: make sure "Resolver.PreferGo = true" acts like netgo. { name: "resolver-prefergo", From 4e8c6af239b6a941dafc1288bb5e275add530873 Mon Sep 17 00:00:00 2001 From: Cherry Mui Date: Fri, 28 Apr 2023 21:35:31 -0400 Subject: [PATCH 045/299] cmd/link, cmd/internal/obj: use aux symbol for global variable DWARF info Currently, for a global variable, its debug info symbol is a named symbol with the variable's name with a special prefix. And the linker looks it up by name. This CL makes the debug info symbol an aux symbol of the variable symbol. Change-Id: I55614d0ef2af9c53eb40144ad80e09339bf3cbee Reviewed-on: https://go-review.googlesource.com/c/go/+/490816 Run-TryBot: Cherry Mui TryBot-Result: Gopher Robot Reviewed-by: Than McIntosh --- src/cmd/internal/obj/dwarf.go | 11 +++++------ src/cmd/internal/obj/link.go | 26 +++++++++++++++++++++++++- src/cmd/internal/obj/objfile.go | 21 ++++++++++++++++----- src/cmd/internal/obj/sym.go | 9 ++++++--- src/cmd/link/internal/ld/dwarf.go | 10 +--------- src/cmd/link/internal/loader/loader.go | 9 +++++++++ 6 files changed, 62 insertions(+), 24 deletions(-) diff --git a/src/cmd/internal/obj/dwarf.go b/src/cmd/internal/obj/dwarf.go index 3f4c6e8ef3..f1330c9258 100644 --- a/src/cmd/internal/obj/dwarf.go +++ b/src/cmd/internal/obj/dwarf.go @@ -412,12 +412,11 @@ func (ctxt *Link) DwarfGlobal(myimportpath, typename string, varSym *LSym) { return } varname := varSym.Name - dieSymName := dwarf.InfoPrefix + varname - dieSym := ctxt.LookupInit(dieSymName, func(s *LSym) { - s.Type = objabi.SDWARFVAR - s.Set(AttrDuplicateOK, true) // needed for shared linkage - ctxt.Data = append(ctxt.Data, s) - }) + dieSym := &LSym{ + Type: objabi.SDWARFVAR, + } + varSym.NewVarInfo().dwarfInfoSym = dieSym + ctxt.Data = append(ctxt.Data, dieSym) typeSym := ctxt.Lookup(dwarf.InfoPrefix + typename) dwarf.PutGlobal(dwCtxt{ctxt}, dieSym, typeSym, varSym, varname) } diff --git a/src/cmd/internal/obj/link.go b/src/cmd/internal/obj/link.go index b50305f85c..def92e103b 100644 --- a/src/cmd/internal/obj/link.go +++ b/src/cmd/internal/obj/link.go @@ -465,7 +465,7 @@ type LSym struct { P []byte R []Reloc - Extra *interface{} // *FuncInfo or *FileInfo, if present + Extra *interface{} // *FuncInfo, *VarInfo, or *FileInfo, if present Pkg string PkgIdx int32 @@ -537,6 +537,30 @@ func (s *LSym) Func() *FuncInfo { return f } +type VarInfo struct { + dwarfInfoSym *LSym +} + +// NewVarInfo allocates and returns a VarInfo for LSym. +func (s *LSym) NewVarInfo() *VarInfo { + if s.Extra != nil { + panic(fmt.Sprintf("invalid use of LSym - NewVarInfo with Extra of type %T", *s.Extra)) + } + f := new(VarInfo) + s.Extra = new(interface{}) + *s.Extra = f + return f +} + +// VarInfo returns the *VarInfo associated with s, or else nil. +func (s *LSym) VarInfo() *VarInfo { + if s.Extra == nil { + return nil + } + f, _ := (*s.Extra).(*VarInfo) + return f +} + // A FileInfo contains extra fields for SDATA symbols backed by files. // (If LSym.Extra is a *FileInfo, LSym.P == nil.) type FileInfo struct { diff --git a/src/cmd/internal/obj/objfile.go b/src/cmd/internal/obj/objfile.go index a9ddf0edf1..aa99855565 100644 --- a/src/cmd/internal/obj/objfile.go +++ b/src/cmd/internal/obj/objfile.go @@ -615,6 +615,10 @@ func (w *writer) Aux(s *LSym) { } w.aux1(goobj.AuxWasmImport, fn.WasmImportSym) } + } else if v := s.VarInfo(); v != nil { + if v.dwarfInfoSym != nil && v.dwarfInfoSym.Size != 0 { + w.aux1(goobj.AuxDwarfInfo, v.dwarfInfoSym) + } } } @@ -721,6 +725,10 @@ func nAuxSym(s *LSym) int { } n++ } + } else if v := s.VarInfo(); v != nil { + if v.dwarfInfoSym != nil && v.dwarfInfoSym.Size != 0 { + n++ + } } return n } @@ -795,11 +803,14 @@ func genFuncInfoSyms(ctxt *Link) { func writeAuxSymDebug(ctxt *Link, par *LSym, aux *LSym) { // Most aux symbols (ex: funcdata) are not interesting-- // pick out just the DWARF ones for now. - if aux.Type != objabi.SDWARFLOC && - aux.Type != objabi.SDWARFFCN && - aux.Type != objabi.SDWARFABSFCN && - aux.Type != objabi.SDWARFLINES && - aux.Type != objabi.SDWARFRANGE { + switch aux.Type { + case objabi.SDWARFLOC, + objabi.SDWARFFCN, + objabi.SDWARFABSFCN, + objabi.SDWARFLINES, + objabi.SDWARFRANGE, + objabi.SDWARFVAR: + default: return } ctxt.writeSymDebugNamed(aux, "aux for "+par.Name) diff --git a/src/cmd/internal/obj/sym.go b/src/cmd/internal/obj/sym.go index 49968d3177..6a5ab6c349 100644 --- a/src/cmd/internal/obj/sym.go +++ b/src/cmd/internal/obj/sym.go @@ -367,6 +367,8 @@ func (ctxt *Link) traverseSyms(flag traverseFlag, fn func(*LSym)) { fn(aux) } ctxt.traverseFuncAux(flag, s, f, files) + } else if v := s.VarInfo(); v != nil { + fnNoNil(v.dwarfInfoSym) } } if flag&traversePcdata != 0 && s.Type == objabi.STEXT { @@ -443,10 +445,11 @@ func (ctxt *Link) traverseAuxSyms(flag traverseFlag, fn func(parent *LSym, aux * fn(s, s.Gotype) } } - if s.Type != objabi.STEXT { - continue + if s.Type == objabi.STEXT { + ctxt.traverseFuncAux(flag, s, fn, files) + } else if v := s.VarInfo(); v != nil && v.dwarfInfoSym != nil { + fn(s, v.dwarfInfoSym) } - ctxt.traverseFuncAux(flag, s, fn, files) } } } diff --git a/src/cmd/link/internal/ld/dwarf.go b/src/cmd/link/internal/ld/dwarf.go index 41da25805f..4eb0baf63c 100644 --- a/src/cmd/link/internal/ld/dwarf.go +++ b/src/cmd/link/internal/ld/dwarf.go @@ -1936,21 +1936,13 @@ func dwarfGenerateDebugInfo(ctxt *Link) { if d.ldr.IsFileLocal(idx) { continue } - sn := d.ldr.SymName(idx) - if sn == "" { - // skip aux symbols - continue - } // Find compiler-generated DWARF info sym for global in question, // and tack it onto the appropriate unit. Note that there are // circumstances under which we can't find the compiler-generated // symbol-- this typically happens as a result of compiler options // (e.g. compile package X with "-dwarf=0"). - - // FIXME: use an aux sym or a relocation here instead of a - // name lookup. - varDIE := d.ldr.Lookup(dwarf.InfoPrefix+sn, 0) + varDIE := d.ldr.GetVarDwarfAuxSym(idx) if varDIE != 0 { unit := d.ldr.SymUnit(idx) d.defgotype(gt) diff --git a/src/cmd/link/internal/loader/loader.go b/src/cmd/link/internal/loader/loader.go index 4bccce047b..a989d14362 100644 --- a/src/cmd/link/internal/loader/loader.go +++ b/src/cmd/link/internal/loader/loader.go @@ -1685,6 +1685,15 @@ func (l *Loader) GetFuncDwarfAuxSyms(fnSymIdx Sym) (auxDwarfInfo, auxDwarfLoc, a return } +func (l *Loader) GetVarDwarfAuxSym(i Sym) Sym { + aux := l.aux1(i, goobj.AuxDwarfInfo) + if aux != 0 && l.SymType(aux) != sym.SDWARFVAR { + fmt.Println(l.SymName(i), l.SymType(i), l.SymType(aux), sym.SDWARFVAR) + panic("aux dwarf info sym with wrong type") + } + return aux +} + // AddInteriorSym sets up 'interior' as an interior symbol of // container/payload symbol 'container'. An interior symbol does not // itself have data, but gives a name to a subrange of the data in its From 53279a6af372e3708afe8eaf618d56ee98edf045 Mon Sep 17 00:00:00 2001 From: Johan Brandhorst-Satzkorn Date: Thu, 27 Apr 2023 21:39:57 -0700 Subject: [PATCH 046/299] internal/testenv: probe for symlink on wasip1 Certain WASI runtimes do not support generic symlinks, and instead return permission errors when they are attempted. Perform a runtime probe of symlink support in hasSymlink on wasip1 to determine whether the runtime supports generic symlinks. Also perform the same probe on android. For #59583 Change-Id: Iae5b704e670650d38ee350a5a98f99dcce8b5b28 Reviewed-on: https://go-review.googlesource.com/c/go/+/490115 Auto-Submit: Johan Brandhorst-Satzkorn Reviewed-by: Achille Roussel TryBot-Bypass: Johan Brandhorst-Satzkorn Reviewed-by: Dmitri Shuralyov Run-TryBot: Johan Brandhorst-Satzkorn Reviewed-by: Bryan Mills --- src/internal/testenv/testenv.go | 2 +- src/internal/testenv/testenv_notunix.go | 3 ++- src/internal/testenv/testenv_notwin.go | 28 ++++++++++++++++++++++++- 3 files changed, 30 insertions(+), 3 deletions(-) diff --git a/src/internal/testenv/testenv.go b/src/internal/testenv/testenv.go index aeda1f964f..d03bb0550a 100644 --- a/src/internal/testenv/testenv.go +++ b/src/internal/testenv/testenv.go @@ -387,7 +387,7 @@ func HasSymlink() bool { func MustHaveSymlink(t testing.TB) { ok, reason := hasSymlink() if !ok { - t.Skipf("skipping test: cannot make symlinks on %s/%s%s", runtime.GOOS, runtime.GOARCH, reason) + t.Skipf("skipping test: cannot make symlinks on %s/%s: %s", runtime.GOOS, runtime.GOARCH, reason) } } diff --git a/src/internal/testenv/testenv_notunix.go b/src/internal/testenv/testenv_notunix.go index 31abe8d092..a7df5f5ddc 100644 --- a/src/internal/testenv/testenv_notunix.go +++ b/src/internal/testenv/testenv_notunix.go @@ -8,6 +8,7 @@ package testenv import ( "errors" + "io/fs" "os" ) @@ -16,5 +17,5 @@ import ( var Sigquit = os.Kill func syscallIsNotSupported(err error) bool { - return errors.Is(err, errors.ErrUnsupported) + return errors.Is(err, fs.ErrPermission) || errors.Is(err, errors.ErrUnsupported) } diff --git a/src/internal/testenv/testenv_notwin.go b/src/internal/testenv/testenv_notwin.go index 81171fd193..30e159a6ec 100644 --- a/src/internal/testenv/testenv_notwin.go +++ b/src/internal/testenv/testenv_notwin.go @@ -7,13 +7,39 @@ package testenv import ( + "fmt" + "os" + "path/filepath" "runtime" ) func hasSymlink() (ok bool, reason string) { switch runtime.GOOS { - case "android", "plan9": + case "plan9": return false, "" + case "android", "wasip1": + // For wasip1, some runtimes forbid absolute symlinks, + // or symlinks that escape the current working directory. + // Perform a simple test to see whether the runtime + // supports symlinks or not. If we get a permission + // error, the runtime does not support symlinks. + dir, err := os.MkdirTemp("", "") + if err != nil { + return false, "" + } + defer func() { + _ = os.RemoveAll(dir) + }() + fpath := filepath.Join(dir, "testfile.txt") + if err := os.WriteFile(fpath, nil, 0644); err != nil { + return false, "" + } + if err := os.Symlink(fpath, filepath.Join(dir, "testlink")); err != nil { + if SyscallIsNotSupported(err) { + return false, fmt.Sprintf("symlinks unsupported: %s", err.Error()) + } + return false, "" + } } return true, "" From 14cf82aa37ec33012ca48febd83fb16e1178beee Mon Sep 17 00:00:00 2001 From: qmuntal Date: Tue, 17 Jan 2023 08:15:33 +0100 Subject: [PATCH 047/299] cmd/link: generate .pdata PE section This CL adds a .pdata section to the PE file generated by the Go linker. The .pdata section is a standard section [1] that contains an array of function table entries that are used for stack unwinding. The table entries layout is taken from [2]. This CL just generates the table entries without any unwinding information, which is enough to start doing some E2E tests between the Go linker and the Win32 APIs. The goal of the .pdata table is to allow Windows retrieve unwind information for a function at a given PC. It does so by doing a binary search on the table, looking for an entry that meets BeginAddress >= PC < EndAddress. Each table entry takes 12 bytes and only non-leaf functions with frame pointer needs an entry on the .pdata table. The result is that PE binaries will be ~0.7% bigger due to the unwind information, a reasonable amount considering the benefits in debuggability. Updates #57302 [1] https://learn.microsoft.com/en-us/windows/win32/debug/pe-format#the-pdata-section [2] https://learn.microsoft.com/en-us/cpp/build/exception-handling-x64#struct-runtime_function Change-Id: If675d10c64452946dbab76709da20569651e3e9f Reviewed-on: https://go-review.googlesource.com/c/go/+/461738 TryBot-Result: Gopher Robot Reviewed-by: Alex Brainman Reviewed-by: Than McIntosh Run-TryBot: Quim Muntal Reviewed-by: Cherry Mui --- src/cmd/link/internal/ld/asmb.go | 4 ++ src/cmd/link/internal/ld/data.go | 39 ++++++++++++ src/cmd/link/internal/ld/lib.go | 3 +- src/cmd/link/internal/ld/pe.go | 30 ++++++++- src/cmd/link/internal/ld/seh.go | 54 ++++++++++++++++ src/cmd/link/internal/loader/loader.go | 10 +++ src/cmd/link/internal/sym/symkind.go | 1 + src/cmd/link/internal/sym/symkind_string.go | 5 +- .../syscall/windows/syscall_windows.go | 2 + .../syscall/windows/zsyscall_windows.go | 7 +++ src/runtime/defs_windows_386.go | 3 + src/runtime/defs_windows_amd64.go | 1 + src/runtime/defs_windows_arm.go | 3 + src/runtime/defs_windows_arm64.go | 1 + src/runtime/export_windows_test.go | 16 +++++ src/runtime/runtime-seh_windows_test.go | 63 +++++++++++++++++++ 16 files changed, 237 insertions(+), 5 deletions(-) create mode 100644 src/cmd/link/internal/ld/seh.go create mode 100644 src/runtime/runtime-seh_windows_test.go diff --git a/src/cmd/link/internal/ld/asmb.go b/src/cmd/link/internal/ld/asmb.go index cd8927b087..fc088be51e 100644 --- a/src/cmd/link/internal/ld/asmb.go +++ b/src/cmd/link/internal/ld/asmb.go @@ -60,6 +60,10 @@ func asmb(ctxt *Link) { writeParallel(&wg, dwarfblk, ctxt, Segdwarf.Fileoff, Segdwarf.Vaddr, Segdwarf.Filelen) + if Segpdata.Filelen > 0 { + writeParallel(&wg, pdatablk, ctxt, Segpdata.Fileoff, Segpdata.Vaddr, Segpdata.Filelen) + } + wg.Wait() } diff --git a/src/cmd/link/internal/ld/data.go b/src/cmd/link/internal/ld/data.go index d0efcdc052..849629ebe3 100644 --- a/src/cmd/link/internal/ld/data.go +++ b/src/cmd/link/internal/ld/data.go @@ -1154,6 +1154,10 @@ func dwarfblk(ctxt *Link, out *OutBuf, addr int64, size int64) { writeBlocks(ctxt, out, ctxt.outSem, ctxt.loader, syms, addr, size, zeros[:]) } +func pdatablk(ctxt *Link, out *OutBuf, addr int64, size int64) { + writeBlocks(ctxt, out, ctxt.outSem, ctxt.loader, []loader.Sym{sehp.pdata}, addr, size, zeros[:]) +} + var covCounterDataStartOff, covCounterDataLen uint64 var zeros [512]byte @@ -1649,6 +1653,8 @@ func (ctxt *Link) dodata(symGroupType []sym.SymKind) { // data/rodata (and related) symbols. state.allocateDataSections(ctxt) + state.allocateSEHSections(ctxt) + // Create *sym.Section objects and assign symbols to sections for // DWARF symbols. state.allocateDwarfSections(ctxt) @@ -1676,6 +1682,10 @@ func (ctxt *Link) dodata(symGroupType []sym.SymKind) { sect.Extnum = n n++ } + for _, sect := range Segpdata.Sections { + sect.Extnum = n + n++ + } } // allocateDataSectionForSym creates a new sym.Section into which a @@ -2148,6 +2158,16 @@ func (state *dodataState) allocateDwarfSections(ctxt *Link) { } } +// allocateSEHSections allocate a sym.Section object for SEH +// symbols, and assigns symbols to sections. +func (state *dodataState) allocateSEHSections(ctxt *Link) { + if sehp.pdata > 0 { + sect := state.allocateDataSectionForSym(&Segpdata, sehp.pdata, 04) + state.assignDsymsToSection(sect, []loader.Sym{sehp.pdata}, sym.SRODATA, aligndatsize) + state.checkdatsize(sym.SPDATASECT) + } +} + type symNameSize struct { name string sz int64 @@ -2684,6 +2704,21 @@ func (ctxt *Link) address() []*sym.Segment { // simply because right now we know where the BSS starts. Segdata.Filelen = bss.Vaddr - Segdata.Vaddr + if len(Segpdata.Sections) > 0 { + va = uint64(Rnd(int64(va), int64(*FlagRound))) + order = append(order, &Segpdata) + Segpdata.Rwx = 04 + Segpdata.Vaddr = va + // Segpdata.Sections is intended to contain just one section. + // Loop through the slice anyway for consistency. + for _, s := range Segpdata.Sections { + va = uint64(Rnd(int64(va), int64(s.Align))) + s.Vaddr = va + va += s.Length + } + Segpdata.Length = va - Segpdata.Vaddr + } + va = uint64(Rnd(int64(va), int64(*FlagRound))) order = append(order, &Segdwarf) Segdwarf.Rwx = 06 @@ -2735,6 +2770,10 @@ func (ctxt *Link) address() []*sym.Segment { } } + if sect := ldr.SymSect(sehp.pdata); sect != nil { + ldr.AddToSymValue(sehp.pdata, int64(sect.Vaddr)) + } + if ctxt.BuildMode == BuildModeShared { s := ldr.LookupOrCreateSym("go:link.abihashbytes", 0) sect := ldr.SymSect(ldr.LookupOrCreateSym(".note.go.abihash", 0)) diff --git a/src/cmd/link/internal/ld/lib.go b/src/cmd/link/internal/ld/lib.go index c88a955a0c..f1eff33c6e 100644 --- a/src/cmd/link/internal/ld/lib.go +++ b/src/cmd/link/internal/ld/lib.go @@ -328,8 +328,9 @@ var ( Segrelrodata sym.Segment Segdata sym.Segment Segdwarf sym.Segment + Segpdata sym.Segment // windows-only - Segments = []*sym.Segment{&Segtext, &Segrodata, &Segrelrodata, &Segdata, &Segdwarf} + Segments = []*sym.Segment{&Segtext, &Segrodata, &Segrelrodata, &Segdata, &Segdwarf, &Segpdata} ) const pkgdef = "__.PKGDEF" diff --git a/src/cmd/link/internal/ld/pe.go b/src/cmd/link/internal/ld/pe.go index a3bb47d232..b07f2763eb 100644 --- a/src/cmd/link/internal/ld/pe.go +++ b/src/cmd/link/internal/ld/pe.go @@ -433,6 +433,7 @@ type peFile struct { dataSect *peSection bssSect *peSection ctorsSect *peSection + pdataSect *peSection nextSectOffset uint32 nextFileOffset uint32 symtabOffset int64 // offset to the start of symbol table @@ -498,6 +499,25 @@ func (f *peFile) addDWARF() { } } +// addSEH adds SEH information to the COFF file f. +func (f *peFile) addSEH(ctxt *Link) { + if Segpdata.Length == 0 { + return + } + d := pefile.addSection(".pdata", int(Segpdata.Length), int(Segpdata.Length)) + d.characteristics = IMAGE_SCN_CNT_INITIALIZED_DATA | IMAGE_SCN_MEM_READ + if ctxt.LinkMode == LinkExternal { + // Some gcc versions don't honor the default alignment for the .pdata section. + d.characteristics |= IMAGE_SCN_ALIGN_4BYTES + } + pefile.pdataSect = d + d.checkSegment(&Segpdata) + // TODO: remove extraSize once the dummy unwind info is removed from the .pdata section. + const extraSize = 12 + pefile.dataDirectory[pe.IMAGE_DIRECTORY_ENTRY_EXCEPTION].VirtualAddress = d.virtualAddress + extraSize + pefile.dataDirectory[pe.IMAGE_DIRECTORY_ENTRY_EXCEPTION].Size = d.virtualSize - extraSize +} + // addInitArray adds .ctors COFF section to the file f. func (f *peFile) addInitArray(ctxt *Link) *peSection { // The size below was determined by the specification for array relocations, @@ -593,15 +613,19 @@ func (f *peFile) emitRelocations(ctxt *Link) { return int(sect.Rellen / relocLen) } - sects := []struct { + type relsect struct { peSect *peSection seg *sym.Segment syms []loader.Sym - }{ + } + sects := []relsect{ {f.textSect, &Segtext, ctxt.Textp}, {f.rdataSect, &Segrodata, ctxt.datap}, {f.dataSect, &Segdata, ctxt.datap}, } + if sehp.pdata != 0 { + sects = append(sects, relsect{f.pdataSect, &Segpdata, []loader.Sym{sehp.pdata}}) + } for _, s := range sects { s.peSect.emitRelocations(ctxt.Out, func() int { var n int @@ -1595,6 +1619,7 @@ func addPEBaseReloc(ctxt *Link) { func (ctxt *Link) dope() { initdynimport(ctxt) initdynexport(ctxt) + writeSEH(ctxt) } func setpersrc(ctxt *Link, syms []loader.Sym) { @@ -1689,6 +1714,7 @@ func asmbPe(ctxt *Link) { pefile.bssSect = b } + pefile.addSEH(ctxt) pefile.addDWARF() if ctxt.LinkMode == LinkExternal { diff --git a/src/cmd/link/internal/ld/seh.go b/src/cmd/link/internal/ld/seh.go new file mode 100644 index 0000000000..b95084751c --- /dev/null +++ b/src/cmd/link/internal/ld/seh.go @@ -0,0 +1,54 @@ +// Copyright 2023 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package ld + +import ( + "cmd/internal/sys" + "cmd/link/internal/loader" + "cmd/link/internal/sym" +) + +var sehp struct { + pdata loader.Sym +} + +func writeSEH(ctxt *Link) { + switch ctxt.Arch.Family { + case sys.AMD64: + writeSEHAMD64(ctxt) + } +} + +func writeSEHAMD64(ctxt *Link) { + ldr := ctxt.loader + mkSecSym := func(name string, kind sym.SymKind) *loader.SymbolBuilder { + s := ldr.CreateSymForUpdate(name, 0) + s.SetType(kind) + s.SetAlign(4) + return s + } + pdata := mkSecSym(".pdata", sym.SPDATASECT) + // TODO: the following 12 bytes represent a dummy unwind info, + // remove once unwind infos are encoded in the .xdata section. + pdata.AddUint64(ctxt.Arch, 0) + pdata.AddUint32(ctxt.Arch, 0) + for _, s := range ctxt.Textp { + if fi := ldr.FuncInfo(s); !fi.Valid() || fi.TopFrame() { + continue + } + uw := ldr.SEHUnwindSym(s) + if uw == 0 { + continue + } + + // Reference: + // https://learn.microsoft.com/en-us/cpp/build/exception-handling-x64#struct-runtime_function + pdata.AddPEImageRelativeAddrPlus(ctxt.Arch, s, 0) + pdata.AddPEImageRelativeAddrPlus(ctxt.Arch, s, ldr.SymSize(s)) + // TODO: reference the .xdata symbol. + pdata.AddPEImageRelativeAddrPlus(ctxt.Arch, pdata.Sym(), 0) + } + sehp.pdata = pdata.Sym() +} diff --git a/src/cmd/link/internal/loader/loader.go b/src/cmd/link/internal/loader/loader.go index a989d14362..5fcbf160e0 100644 --- a/src/cmd/link/internal/loader/loader.go +++ b/src/cmd/link/internal/loader/loader.go @@ -1646,6 +1646,16 @@ func (l *Loader) WasmImportSym(fnSymIdx Sym) (Sym, bool) { return 0, false } +// SEHUnwindSym returns the auxiliary SEH unwind symbol associated with +// a given function symbol. +func (l *Loader) SEHUnwindSym(fnSymIdx Sym) Sym { + if l.SymType(fnSymIdx) != sym.STEXT { + log.Fatalf("error: non-function sym %d/%s t=%s passed to SEHUnwindSym", fnSymIdx, l.SymName(fnSymIdx), l.SymType(fnSymIdx).String()) + } + + return l.aux1(fnSymIdx, goobj.AuxSehUnwindInfo) +} + // GetFuncDwarfAuxSyms collects and returns the auxiliary DWARF // symbols associated with a given function symbol. Prior to the // introduction of the loader, this was done purely using name diff --git a/src/cmd/link/internal/sym/symkind.go b/src/cmd/link/internal/sym/symkind.go index db87212a17..acb96ad0ad 100644 --- a/src/cmd/link/internal/sym/symkind.go +++ b/src/cmd/link/internal/sym/symkind.go @@ -126,6 +126,7 @@ const ( // SEH symbol types SSEHUNWINDINFO + SPDATASECT ) // AbiSymKindToSymKind maps values read from object files (which are diff --git a/src/cmd/link/internal/sym/symkind_string.go b/src/cmd/link/internal/sym/symkind_string.go index 09508ce766..30de0a812f 100644 --- a/src/cmd/link/internal/sym/symkind_string.go +++ b/src/cmd/link/internal/sym/symkind_string.go @@ -68,11 +68,12 @@ func _() { _ = x[SDWARFLOC-57] _ = x[SDWARFLINES-58] _ = x[SSEHUNWINDINFO-59] + _ = x[SPDATASECT-60] } -const _SymKind_name = "SxxxSTEXTSELFRXSECTSMACHOPLTSTYPESSTRINGSGOSTRINGSGOFUNCSGCBITSSRODATASFUNCTABSELFROSECTSTYPERELROSSTRINGRELROSGOSTRINGRELROSGOFUNCRELROSGCBITSRELROSRODATARELROSFUNCTABRELROSTYPELINKSITABLINKSSYMTABSPCLNTABSFirstWritableSBUILDINFOSELFSECTSMACHOSMACHOGOTSWINDOWSSELFGOTSNOPTRDATASINITARRSDATASXCOFFTOCSBSSSNOPTRBSSSLIBFUZZER_8BIT_COUNTERSCOVERAGE_COUNTERSCOVERAGE_AUXVARSTLSBSSSXREFSMACHOSYMSTRSMACHOSYMTABSMACHOINDIRECTPLTSMACHOINDIRECTGOTSFILEPATHSDYNIMPORTSHOSTOBJSUNDEFEXTSDWARFSECTSDWARFCUINFOSDWARFCONSTSDWARFFCNSDWARFABSFCNSDWARFTYPESDWARFVARSDWARFRANGESDWARFLOCSDWARFLINESSSEHUNWINDINFO" +const _SymKind_name = "SxxxSTEXTSELFRXSECTSMACHOPLTSTYPESSTRINGSGOSTRINGSGOFUNCSGCBITSSRODATASFUNCTABSELFROSECTSTYPERELROSSTRINGRELROSGOSTRINGRELROSGOFUNCRELROSGCBITSRELROSRODATARELROSFUNCTABRELROSTYPELINKSITABLINKSSYMTABSPCLNTABSFirstWritableSBUILDINFOSELFSECTSMACHOSMACHOGOTSWINDOWSSELFGOTSNOPTRDATASINITARRSDATASXCOFFTOCSBSSSNOPTRBSSSLIBFUZZER_8BIT_COUNTERSCOVERAGE_COUNTERSCOVERAGE_AUXVARSTLSBSSSXREFSMACHOSYMSTRSMACHOSYMTABSMACHOINDIRECTPLTSMACHOINDIRECTGOTSFILEPATHSDYNIMPORTSHOSTOBJSUNDEFEXTSDWARFSECTSDWARFCUINFOSDWARFCONSTSDWARFFCNSDWARFABSFCNSDWARFTYPESDWARFVARSDWARFRANGESDWARFLOCSDWARFLINESSSEHUNWINDINFOSPDATASECT" -var _SymKind_index = [...]uint16{0, 4, 9, 19, 28, 33, 40, 49, 56, 63, 70, 78, 88, 98, 110, 124, 136, 148, 160, 173, 182, 191, 198, 206, 220, 230, 238, 244, 253, 261, 268, 278, 286, 291, 300, 304, 313, 336, 353, 369, 376, 381, 393, 405, 422, 439, 448, 458, 466, 475, 485, 497, 508, 517, 529, 539, 548, 559, 568, 579, 593} +var _SymKind_index = [...]uint16{0, 4, 9, 19, 28, 33, 40, 49, 56, 63, 70, 78, 88, 98, 110, 124, 136, 148, 160, 173, 182, 191, 198, 206, 220, 230, 238, 244, 253, 261, 268, 278, 286, 291, 300, 304, 313, 336, 353, 369, 376, 381, 393, 405, 422, 439, 448, 458, 466, 475, 485, 497, 508, 517, 529, 539, 548, 559, 568, 579, 593, 603} func (i SymKind) String() string { if i >= SymKind(len(_SymKind_index)-1) { diff --git a/src/internal/syscall/windows/syscall_windows.go b/src/internal/syscall/windows/syscall_windows.go index 4ae9e4f1b2..409b334bcb 100644 --- a/src/internal/syscall/windows/syscall_windows.go +++ b/src/internal/syscall/windows/syscall_windows.go @@ -399,3 +399,5 @@ type FILE_ID_BOTH_DIR_INFO struct { } //sys GetVolumeInformationByHandle(file syscall.Handle, volumeNameBuffer *uint16, volumeNameSize uint32, volumeNameSerialNumber *uint32, maximumComponentLength *uint32, fileSystemFlags *uint32, fileSystemNameBuffer *uint16, fileSystemNameSize uint32) (err error) = GetVolumeInformationByHandleW + +//sys RtlLookupFunctionEntry(pc uintptr, baseAddress *uintptr, table *byte) (ret uintptr) = kernel32.RtlLookupFunctionEntry diff --git a/src/internal/syscall/windows/zsyscall_windows.go b/src/internal/syscall/windows/zsyscall_windows.go index 3a7423a304..4a6ca406d1 100644 --- a/src/internal/syscall/windows/zsyscall_windows.go +++ b/src/internal/syscall/windows/zsyscall_windows.go @@ -69,6 +69,7 @@ var ( procModule32NextW = modkernel32.NewProc("Module32NextW") procMoveFileExW = modkernel32.NewProc("MoveFileExW") procMultiByteToWideChar = modkernel32.NewProc("MultiByteToWideChar") + procRtlLookupFunctionEntry = modkernel32.NewProc("RtlLookupFunctionEntry") procSetFileInformationByHandle = modkernel32.NewProc("SetFileInformationByHandle") procUnlockFileEx = modkernel32.NewProc("UnlockFileEx") procVirtualQuery = modkernel32.NewProc("VirtualQuery") @@ -289,6 +290,12 @@ func MultiByteToWideChar(codePage uint32, dwFlags uint32, str *byte, nstr int32, return } +func RtlLookupFunctionEntry(pc uintptr, baseAddress *uintptr, table *byte) (ret uintptr) { + r0, _, _ := syscall.Syscall(procRtlLookupFunctionEntry.Addr(), 3, uintptr(pc), uintptr(unsafe.Pointer(baseAddress)), uintptr(unsafe.Pointer(table))) + ret = uintptr(r0) + return +} + func SetFileInformationByHandle(handle syscall.Handle, fileInformationClass uint32, buf uintptr, bufsize uint32) (err error) { r1, _, e1 := syscall.Syscall6(procSetFileInformationByHandle.Addr(), 4, uintptr(handle), uintptr(fileInformationClass), uintptr(buf), uintptr(bufsize), 0, 0) if r1 == 0 { diff --git a/src/runtime/defs_windows_386.go b/src/runtime/defs_windows_386.go index 8d6c443a14..b11b15554e 100644 --- a/src/runtime/defs_windows_386.go +++ b/src/runtime/defs_windows_386.go @@ -56,6 +56,9 @@ func (c *context) set_lr(x uintptr) {} func (c *context) set_ip(x uintptr) { c.eip = uint32(x) } func (c *context) set_sp(x uintptr) { c.esp = uint32(x) } +// 386 does not have frame pointer register. +func (c *context) set_fp(x uintptr) {} + func prepareContextForSigResume(c *context) { c.edx = c.esp c.ecx = c.eip diff --git a/src/runtime/defs_windows_amd64.go b/src/runtime/defs_windows_amd64.go index afa8a657b8..20c9c4d932 100644 --- a/src/runtime/defs_windows_amd64.go +++ b/src/runtime/defs_windows_amd64.go @@ -69,6 +69,7 @@ func (c *context) set_lr(x uintptr) {} func (c *context) set_ip(x uintptr) { c.rip = uint64(x) } func (c *context) set_sp(x uintptr) { c.rsp = uint64(x) } +func (c *context) set_fp(x uintptr) { c.rbp = uint64(x) } func prepareContextForSigResume(c *context) { c.r8 = c.rsp diff --git a/src/runtime/defs_windows_arm.go b/src/runtime/defs_windows_arm.go index 21c7991519..7a18c95cf1 100644 --- a/src/runtime/defs_windows_arm.go +++ b/src/runtime/defs_windows_arm.go @@ -58,6 +58,9 @@ func (c *context) set_ip(x uintptr) { c.pc = uint32(x) } func (c *context) set_sp(x uintptr) { c.spr = uint32(x) } func (c *context) set_lr(x uintptr) { c.lrr = uint32(x) } +// arm does not have frame pointer register. +func (c *context) set_fp(x uintptr) {} + func prepareContextForSigResume(c *context) { c.r0 = c.spr c.r1 = c.pc diff --git a/src/runtime/defs_windows_arm64.go b/src/runtime/defs_windows_arm64.go index 6c71133b43..ef2efb1bb3 100644 --- a/src/runtime/defs_windows_arm64.go +++ b/src/runtime/defs_windows_arm64.go @@ -40,6 +40,7 @@ func (c *context) lr() uintptr { return uintptr(c.x[30]) } func (c *context) set_ip(x uintptr) { c.pc = uint64(x) } func (c *context) set_sp(x uintptr) { c.xsp = uint64(x) } func (c *context) set_lr(x uintptr) { c.x[30] = uint64(x) } +func (c *context) set_fp(x uintptr) { c.x[29] = uint64(x) } func prepareContextForSigResume(c *context) { c.x[0] = c.xsp diff --git a/src/runtime/export_windows_test.go b/src/runtime/export_windows_test.go index 332136b586..5b9f08fb79 100644 --- a/src/runtime/export_windows_test.go +++ b/src/runtime/export_windows_test.go @@ -20,3 +20,19 @@ func NumberOfProcessors() int32 { stdcall1(_GetSystemInfo, uintptr(unsafe.Pointer(&info))) return int32(info.dwnumberofprocessors) } + +type ContextStub struct { + context +} + +func (c ContextStub) GetPC() uintptr { + return c.ip() +} + +func NewContextStub() ContextStub { + var ctx context + ctx.set_ip(getcallerpc()) + ctx.set_sp(getcallersp()) + ctx.set_fp(getcallerfp()) + return ContextStub{ctx} +} diff --git a/src/runtime/runtime-seh_windows_test.go b/src/runtime/runtime-seh_windows_test.go new file mode 100644 index 0000000000..23f5b87bf5 --- /dev/null +++ b/src/runtime/runtime-seh_windows_test.go @@ -0,0 +1,63 @@ +// Copyright 2023 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package runtime_test + +import ( + "internal/abi" + "internal/syscall/windows" + "runtime" + "testing" +) + +func sehf1() int { + return sehf1() +} + +func sehf2() {} + +func TestSehLookupFunctionEntry(t *testing.T) { + if runtime.GOARCH != "amd64" { + t.Skip("skipping amd64-only test") + } + // This test checks that Win32 is able to retrieve + // function metadata stored in the .pdata section + // by the Go linker. + // Win32 unwinding will fail if this test fails, + // as RtlUnwindEx uses RtlLookupFunctionEntry internally. + // If that's the case, don't bother investigating further, + // first fix the .pdata generation. + sehf1pc := abi.FuncPCABIInternal(sehf1) + var fnwithframe func() + fnwithframe = func() { + fnwithframe() + } + fnwithoutframe := func() {} + tests := []struct { + name string + pc uintptr + hasframe bool + }{ + {"no frame func", abi.FuncPCABIInternal(sehf2), false}, + {"no func", sehf1pc - 1, false}, + {"func at entry", sehf1pc, true}, + {"func in prologue", sehf1pc + 1, true}, + {"anonymous func with frame", abi.FuncPCABIInternal(fnwithframe), true}, + {"anonymous func without frame", abi.FuncPCABIInternal(fnwithoutframe), false}, + {"pc at func body", runtime.NewContextStub().GetPC(), true}, + } + for _, tt := range tests { + var base uintptr + fn := windows.RtlLookupFunctionEntry(tt.pc, &base, nil) + if !tt.hasframe { + if fn != 0 { + t.Errorf("%s: unexpected frame", tt.name) + } + continue + } + if fn == 0 { + t.Errorf("%s: missing frame", tt.name) + } + } +} From 39ca989b883b913287d282365510a9152a3f80e6 Mon Sep 17 00:00:00 2001 From: qmuntal Date: Thu, 12 Jan 2023 15:47:19 +0100 Subject: [PATCH 048/299] cmd/link: generate .xdata PE section This CL adds a .xdata section to the PE file generated by the Go linker. It is also the first CL of the SEH chain that adds effective support for unwinding the Go stack, as demonstrated by the newly added tests. The .xdata section is a standard PE section that contains an array of unwind data info structures. This structures are used to record the effects a function has on the stack pointer, and where the nonvolatile registers are saved on the stack [1]. Note that this CL still does not support unwinding the cgo stack. Updates #57302 [1] https://learn.microsoft.com/en-us/cpp/build/exception-handling-x64#struct-unwind_info Change-Id: I6f305a51ed130b758ff9ca7b90c091e50a109a6f Reviewed-on: https://go-review.googlesource.com/c/go/+/457455 Reviewed-by: Cherry Mui Reviewed-by: Davis Goodin Run-TryBot: Quim Muntal TryBot-Result: Gopher Robot Reviewed-by: Than McIntosh --- src/cmd/link/internal/ld/asmb.go | 3 + src/cmd/link/internal/ld/data.go | 36 ++++- src/cmd/link/internal/ld/lib.go | 3 +- src/cmd/link/internal/ld/pe.go | 23 +++- src/cmd/link/internal/ld/seh.go | 26 +++- src/cmd/link/internal/sym/symkind.go | 2 +- src/cmd/link/internal/sym/symkind_string.go | 6 +- .../syscall/windows/syscall_windows.go | 1 + .../syscall/windows/zsyscall_windows.go | 7 + src/runtime/runtime-seh_windows_test.go | 128 ++++++++++++++++++ 10 files changed, 216 insertions(+), 19 deletions(-) diff --git a/src/cmd/link/internal/ld/asmb.go b/src/cmd/link/internal/ld/asmb.go index fc088be51e..ca9a57741c 100644 --- a/src/cmd/link/internal/ld/asmb.go +++ b/src/cmd/link/internal/ld/asmb.go @@ -63,6 +63,9 @@ func asmb(ctxt *Link) { if Segpdata.Filelen > 0 { writeParallel(&wg, pdatablk, ctxt, Segpdata.Fileoff, Segpdata.Vaddr, Segpdata.Filelen) } + if Segxdata.Filelen > 0 { + writeParallel(&wg, xdatablk, ctxt, Segxdata.Fileoff, Segxdata.Vaddr, Segxdata.Filelen) + } wg.Wait() } diff --git a/src/cmd/link/internal/ld/data.go b/src/cmd/link/internal/ld/data.go index 849629ebe3..0a0c17e928 100644 --- a/src/cmd/link/internal/ld/data.go +++ b/src/cmd/link/internal/ld/data.go @@ -1158,6 +1158,10 @@ func pdatablk(ctxt *Link, out *OutBuf, addr int64, size int64) { writeBlocks(ctxt, out, ctxt.outSem, ctxt.loader, []loader.Sym{sehp.pdata}, addr, size, zeros[:]) } +func xdatablk(ctxt *Link, out *OutBuf, addr int64, size int64) { + writeBlocks(ctxt, out, ctxt.outSem, ctxt.loader, []loader.Sym{sehp.xdata}, addr, size, zeros[:]) +} + var covCounterDataStartOff, covCounterDataLen uint64 var zeros [512]byte @@ -1686,6 +1690,10 @@ func (ctxt *Link) dodata(symGroupType []sym.SymKind) { sect.Extnum = n n++ } + for _, sect := range Segxdata.Sections { + sect.Extnum = n + n++ + } } // allocateDataSectionForSym creates a new sym.Section into which a @@ -2164,7 +2172,12 @@ func (state *dodataState) allocateSEHSections(ctxt *Link) { if sehp.pdata > 0 { sect := state.allocateDataSectionForSym(&Segpdata, sehp.pdata, 04) state.assignDsymsToSection(sect, []loader.Sym{sehp.pdata}, sym.SRODATA, aligndatsize) - state.checkdatsize(sym.SPDATASECT) + state.checkdatsize(sym.SSEHSECT) + } + if sehp.xdata > 0 { + sect := state.allocateNamedDataSection(&Segxdata, ".xdata", []sym.SymKind{}, 04) + state.assignDsymsToSection(sect, []loader.Sym{sehp.xdata}, sym.SRODATA, aligndatsize) + state.checkdatsize(sym.SSEHSECT) } } @@ -2719,6 +2732,21 @@ func (ctxt *Link) address() []*sym.Segment { Segpdata.Length = va - Segpdata.Vaddr } + if len(Segxdata.Sections) > 0 { + va = uint64(Rnd(int64(va), int64(*FlagRound))) + order = append(order, &Segxdata) + Segxdata.Rwx = 04 + Segxdata.Vaddr = va + // Segxdata.Sections is intended to contain just one section. + // Loop through the slice anyway for consistency. + for _, s := range Segxdata.Sections { + va = uint64(Rnd(int64(va), int64(s.Align))) + s.Vaddr = va + va += s.Length + } + Segxdata.Length = va - Segxdata.Vaddr + } + va = uint64(Rnd(int64(va), int64(*FlagRound))) order = append(order, &Segdwarf) Segdwarf.Rwx = 06 @@ -2770,8 +2798,10 @@ func (ctxt *Link) address() []*sym.Segment { } } - if sect := ldr.SymSect(sehp.pdata); sect != nil { - ldr.AddToSymValue(sehp.pdata, int64(sect.Vaddr)) + for _, s := range []loader.Sym{sehp.pdata, sehp.xdata} { + if sect := ldr.SymSect(s); sect != nil { + ldr.AddToSymValue(s, int64(sect.Vaddr)) + } } if ctxt.BuildMode == BuildModeShared { diff --git a/src/cmd/link/internal/ld/lib.go b/src/cmd/link/internal/ld/lib.go index f1eff33c6e..5b6575b3fb 100644 --- a/src/cmd/link/internal/ld/lib.go +++ b/src/cmd/link/internal/ld/lib.go @@ -329,8 +329,9 @@ var ( Segdata sym.Segment Segdwarf sym.Segment Segpdata sym.Segment // windows-only + Segxdata sym.Segment // windows-only - Segments = []*sym.Segment{&Segtext, &Segrodata, &Segrelrodata, &Segdata, &Segdwarf, &Segpdata} + Segments = []*sym.Segment{&Segtext, &Segrodata, &Segrelrodata, &Segdata, &Segdwarf, &Segpdata, &Segxdata} ) const pkgdef = "__.PKGDEF" diff --git a/src/cmd/link/internal/ld/pe.go b/src/cmd/link/internal/ld/pe.go index b07f2763eb..9f4b00371a 100644 --- a/src/cmd/link/internal/ld/pe.go +++ b/src/cmd/link/internal/ld/pe.go @@ -434,6 +434,7 @@ type peFile struct { bssSect *peSection ctorsSect *peSection pdataSect *peSection + xdataSect *peSection nextSectOffset uint32 nextFileOffset uint32 symtabOffset int64 // offset to the start of symbol table @@ -501,6 +502,8 @@ func (f *peFile) addDWARF() { // addSEH adds SEH information to the COFF file f. func (f *peFile) addSEH(ctxt *Link) { + // .pdata section can exist without the .xdata section. + // .xdata section depends on the .pdata section. if Segpdata.Length == 0 { return } @@ -512,10 +515,19 @@ func (f *peFile) addSEH(ctxt *Link) { } pefile.pdataSect = d d.checkSegment(&Segpdata) - // TODO: remove extraSize once the dummy unwind info is removed from the .pdata section. - const extraSize = 12 - pefile.dataDirectory[pe.IMAGE_DIRECTORY_ENTRY_EXCEPTION].VirtualAddress = d.virtualAddress + extraSize - pefile.dataDirectory[pe.IMAGE_DIRECTORY_ENTRY_EXCEPTION].Size = d.virtualSize - extraSize + pefile.dataDirectory[pe.IMAGE_DIRECTORY_ENTRY_EXCEPTION].VirtualAddress = d.virtualAddress + pefile.dataDirectory[pe.IMAGE_DIRECTORY_ENTRY_EXCEPTION].Size = d.virtualSize + + if Segxdata.Length > 0 { + d = pefile.addSection(".xdata", int(Segxdata.Length), int(Segxdata.Length)) + d.characteristics = IMAGE_SCN_CNT_INITIALIZED_DATA | IMAGE_SCN_MEM_READ + if ctxt.LinkMode == LinkExternal { + // Some gcc versions don't honor the default alignment for the .xdata section. + d.characteristics |= IMAGE_SCN_ALIGN_4BYTES + } + pefile.xdataSect = d + d.checkSegment(&Segxdata) + } } // addInitArray adds .ctors COFF section to the file f. @@ -626,6 +638,9 @@ func (f *peFile) emitRelocations(ctxt *Link) { if sehp.pdata != 0 { sects = append(sects, relsect{f.pdataSect, &Segpdata, []loader.Sym{sehp.pdata}}) } + if sehp.xdata != 0 { + sects = append(sects, relsect{f.xdataSect, &Segxdata, []loader.Sym{sehp.xdata}}) + } for _, s := range sects { s.peSect.emitRelocations(ctxt.Out, func() int { var n int diff --git a/src/cmd/link/internal/ld/seh.go b/src/cmd/link/internal/ld/seh.go index b95084751c..5379528c30 100644 --- a/src/cmd/link/internal/ld/seh.go +++ b/src/cmd/link/internal/ld/seh.go @@ -12,6 +12,7 @@ import ( var sehp struct { pdata loader.Sym + xdata loader.Sym } func writeSEH(ctxt *Link) { @@ -29,11 +30,15 @@ func writeSEHAMD64(ctxt *Link) { s.SetAlign(4) return s } - pdata := mkSecSym(".pdata", sym.SPDATASECT) - // TODO: the following 12 bytes represent a dummy unwind info, - // remove once unwind infos are encoded in the .xdata section. - pdata.AddUint64(ctxt.Arch, 0) - pdata.AddUint32(ctxt.Arch, 0) + pdata := mkSecSym(".pdata", sym.SSEHSECT) + xdata := mkSecSym(".xdata", sym.SSEHSECT) + // The .xdata entries have very low cardinality + // as it only contains frame pointer operations, + // which are very similar across functions. + // These are referenced by .pdata entries using + // an RVA, so it is possible, and binary-size wise, + // to deduplicate .xdata entries. + uwcache := make(map[string]int64) // aux symbol name --> .xdata offset for _, s := range ctxt.Textp { if fi := ldr.FuncInfo(s); !fi.Valid() || fi.TopFrame() { continue @@ -42,13 +47,20 @@ func writeSEHAMD64(ctxt *Link) { if uw == 0 { continue } + name := ctxt.SymName(uw) + off, cached := uwcache[name] + if !cached { + off = xdata.Size() + uwcache[name] = off + xdata.AddBytes(ldr.Data(uw)) + } // Reference: // https://learn.microsoft.com/en-us/cpp/build/exception-handling-x64#struct-runtime_function pdata.AddPEImageRelativeAddrPlus(ctxt.Arch, s, 0) pdata.AddPEImageRelativeAddrPlus(ctxt.Arch, s, ldr.SymSize(s)) - // TODO: reference the .xdata symbol. - pdata.AddPEImageRelativeAddrPlus(ctxt.Arch, pdata.Sym(), 0) + pdata.AddPEImageRelativeAddrPlus(ctxt.Arch, xdata.Sym(), off) } sehp.pdata = pdata.Sym() + sehp.xdata = xdata.Sym() } diff --git a/src/cmd/link/internal/sym/symkind.go b/src/cmd/link/internal/sym/symkind.go index acb96ad0ad..77dbf75a51 100644 --- a/src/cmd/link/internal/sym/symkind.go +++ b/src/cmd/link/internal/sym/symkind.go @@ -126,7 +126,7 @@ const ( // SEH symbol types SSEHUNWINDINFO - SPDATASECT + SSEHSECT ) // AbiSymKindToSymKind maps values read from object files (which are diff --git a/src/cmd/link/internal/sym/symkind_string.go b/src/cmd/link/internal/sym/symkind_string.go index 30de0a812f..62b4fd92e5 100644 --- a/src/cmd/link/internal/sym/symkind_string.go +++ b/src/cmd/link/internal/sym/symkind_string.go @@ -68,12 +68,12 @@ func _() { _ = x[SDWARFLOC-57] _ = x[SDWARFLINES-58] _ = x[SSEHUNWINDINFO-59] - _ = x[SPDATASECT-60] + _ = x[SSEHSECT-60] } -const _SymKind_name = "SxxxSTEXTSELFRXSECTSMACHOPLTSTYPESSTRINGSGOSTRINGSGOFUNCSGCBITSSRODATASFUNCTABSELFROSECTSTYPERELROSSTRINGRELROSGOSTRINGRELROSGOFUNCRELROSGCBITSRELROSRODATARELROSFUNCTABRELROSTYPELINKSITABLINKSSYMTABSPCLNTABSFirstWritableSBUILDINFOSELFSECTSMACHOSMACHOGOTSWINDOWSSELFGOTSNOPTRDATASINITARRSDATASXCOFFTOCSBSSSNOPTRBSSSLIBFUZZER_8BIT_COUNTERSCOVERAGE_COUNTERSCOVERAGE_AUXVARSTLSBSSSXREFSMACHOSYMSTRSMACHOSYMTABSMACHOINDIRECTPLTSMACHOINDIRECTGOTSFILEPATHSDYNIMPORTSHOSTOBJSUNDEFEXTSDWARFSECTSDWARFCUINFOSDWARFCONSTSDWARFFCNSDWARFABSFCNSDWARFTYPESDWARFVARSDWARFRANGESDWARFLOCSDWARFLINESSSEHUNWINDINFOSPDATASECT" +const _SymKind_name = "SxxxSTEXTSELFRXSECTSMACHOPLTSTYPESSTRINGSGOSTRINGSGOFUNCSGCBITSSRODATASFUNCTABSELFROSECTSTYPERELROSSTRINGRELROSGOSTRINGRELROSGOFUNCRELROSGCBITSRELROSRODATARELROSFUNCTABRELROSTYPELINKSITABLINKSSYMTABSPCLNTABSFirstWritableSBUILDINFOSELFSECTSMACHOSMACHOGOTSWINDOWSSELFGOTSNOPTRDATASINITARRSDATASXCOFFTOCSBSSSNOPTRBSSSLIBFUZZER_8BIT_COUNTERSCOVERAGE_COUNTERSCOVERAGE_AUXVARSTLSBSSSXREFSMACHOSYMSTRSMACHOSYMTABSMACHOINDIRECTPLTSMACHOINDIRECTGOTSFILEPATHSDYNIMPORTSHOSTOBJSUNDEFEXTSDWARFSECTSDWARFCUINFOSDWARFCONSTSDWARFFCNSDWARFABSFCNSDWARFTYPESDWARFVARSDWARFRANGESDWARFLOCSDWARFLINESSSEHUNWINDINFOSSEHSECT" -var _SymKind_index = [...]uint16{0, 4, 9, 19, 28, 33, 40, 49, 56, 63, 70, 78, 88, 98, 110, 124, 136, 148, 160, 173, 182, 191, 198, 206, 220, 230, 238, 244, 253, 261, 268, 278, 286, 291, 300, 304, 313, 336, 353, 369, 376, 381, 393, 405, 422, 439, 448, 458, 466, 475, 485, 497, 508, 517, 529, 539, 548, 559, 568, 579, 593, 603} +var _SymKind_index = [...]uint16{0, 4, 9, 19, 28, 33, 40, 49, 56, 63, 70, 78, 88, 98, 110, 124, 136, 148, 160, 173, 182, 191, 198, 206, 220, 230, 238, 244, 253, 261, 268, 278, 286, 291, 300, 304, 313, 336, 353, 369, 376, 381, 393, 405, 422, 439, 448, 458, 466, 475, 485, 497, 508, 517, 529, 539, 548, 559, 568, 579, 593, 601} func (i SymKind) String() string { if i >= SymKind(len(_SymKind_index)-1) { diff --git a/src/internal/syscall/windows/syscall_windows.go b/src/internal/syscall/windows/syscall_windows.go index 409b334bcb..cfe4695258 100644 --- a/src/internal/syscall/windows/syscall_windows.go +++ b/src/internal/syscall/windows/syscall_windows.go @@ -401,3 +401,4 @@ type FILE_ID_BOTH_DIR_INFO struct { //sys GetVolumeInformationByHandle(file syscall.Handle, volumeNameBuffer *uint16, volumeNameSize uint32, volumeNameSerialNumber *uint32, maximumComponentLength *uint32, fileSystemFlags *uint32, fileSystemNameBuffer *uint16, fileSystemNameSize uint32) (err error) = GetVolumeInformationByHandleW //sys RtlLookupFunctionEntry(pc uintptr, baseAddress *uintptr, table *byte) (ret uintptr) = kernel32.RtlLookupFunctionEntry +//sys RtlVirtualUnwind(handlerType uint32, baseAddress uintptr, pc uintptr, entry uintptr, ctxt uintptr, data *uintptr, frame *uintptr, ctxptrs *byte) (ret uintptr) = kernel32.RtlVirtualUnwind diff --git a/src/internal/syscall/windows/zsyscall_windows.go b/src/internal/syscall/windows/zsyscall_windows.go index 4a6ca406d1..32744b00fc 100644 --- a/src/internal/syscall/windows/zsyscall_windows.go +++ b/src/internal/syscall/windows/zsyscall_windows.go @@ -70,6 +70,7 @@ var ( procMoveFileExW = modkernel32.NewProc("MoveFileExW") procMultiByteToWideChar = modkernel32.NewProc("MultiByteToWideChar") procRtlLookupFunctionEntry = modkernel32.NewProc("RtlLookupFunctionEntry") + procRtlVirtualUnwind = modkernel32.NewProc("RtlVirtualUnwind") procSetFileInformationByHandle = modkernel32.NewProc("SetFileInformationByHandle") procUnlockFileEx = modkernel32.NewProc("UnlockFileEx") procVirtualQuery = modkernel32.NewProc("VirtualQuery") @@ -296,6 +297,12 @@ func RtlLookupFunctionEntry(pc uintptr, baseAddress *uintptr, table *byte) (ret return } +func RtlVirtualUnwind(handlerType uint32, baseAddress uintptr, pc uintptr, entry uintptr, ctxt uintptr, data *uintptr, frame *uintptr, ctxptrs *byte) (ret uintptr) { + r0, _, _ := syscall.Syscall9(procRtlVirtualUnwind.Addr(), 8, uintptr(handlerType), uintptr(baseAddress), uintptr(pc), uintptr(entry), uintptr(ctxt), uintptr(unsafe.Pointer(data)), uintptr(unsafe.Pointer(frame)), uintptr(unsafe.Pointer(ctxptrs)), 0) + ret = uintptr(r0) + return +} + func SetFileInformationByHandle(handle syscall.Handle, fileInformationClass uint32, buf uintptr, bufsize uint32) (err error) { r1, _, e1 := syscall.Syscall6(procSetFileInformationByHandle.Addr(), 4, uintptr(handle), uintptr(fileInformationClass), uintptr(buf), uintptr(bufsize), 0, 0) if r1 == 0 { diff --git a/src/runtime/runtime-seh_windows_test.go b/src/runtime/runtime-seh_windows_test.go index 23f5b87bf5..c8a4a593b9 100644 --- a/src/runtime/runtime-seh_windows_test.go +++ b/src/runtime/runtime-seh_windows_test.go @@ -8,7 +8,9 @@ import ( "internal/abi" "internal/syscall/windows" "runtime" + "slices" "testing" + "unsafe" ) func sehf1() int { @@ -61,3 +63,129 @@ func TestSehLookupFunctionEntry(t *testing.T) { } } } + +func sehCallers() []uintptr { + // We don't need a real context, + // RtlVirtualUnwind just needs a context with + // valid a pc, sp and fp (aka bp). + ctx := runtime.NewContextStub() + + pcs := make([]uintptr, 15) + var base, frame uintptr + var n int + for i := 0; i < len(pcs); i++ { + fn := windows.RtlLookupFunctionEntry(ctx.GetPC(), &base, nil) + if fn == 0 { + break + } + windows.RtlVirtualUnwind(0, base, ctx.GetPC(), fn, uintptr(unsafe.Pointer(&ctx)), nil, &frame, nil) + n++ + pcs[i] = ctx.GetPC() + } + return pcs[:n] +} + +// SEH unwinding does not report inlined frames. +// +//go:noinline +func sehf3(pan bool) []uintptr { + return sehf4(pan) +} + +//go:noinline +func sehf4(pan bool) []uintptr { + var pcs []uintptr + if pan { + panic("sehf4") + } + pcs = sehCallers() + return pcs +} + +func testSehCallersEqual(t *testing.T, pcs []uintptr, want []string) { + t.Helper() + got := make([]string, 0, len(want)) + for _, pc := range pcs { + fn := runtime.FuncForPC(pc) + if fn == nil || len(got) >= len(want) { + break + } + name := fn.Name() + switch name { + case "runtime.deferCallSave", "runtime.runOpenDeferFrame", "runtime.panicmem": + // These functions are skipped as they appear inconsistently depending + // whether inlining is on or off. + continue + } + got = append(got, name) + } + if !slices.Equal(want, got) { + t.Fatalf("wanted %v, got %v", want, got) + } +} + +func TestSehUnwind(t *testing.T) { + if runtime.GOARCH != "amd64" { + t.Skip("skipping amd64-only test") + } + pcs := sehf3(false) + testSehCallersEqual(t, pcs, []string{"runtime_test.sehCallers", "runtime_test.sehf4", + "runtime_test.sehf3", "runtime_test.TestSehUnwind"}) +} + +func TestSehUnwindPanic(t *testing.T) { + if runtime.GOARCH != "amd64" { + t.Skip("skipping amd64-only test") + } + want := []string{"runtime_test.sehCallers", "runtime_test.TestSehUnwindPanic.func1", "runtime.gopanic", + "runtime_test.sehf4", "runtime_test.sehf3", "runtime_test.TestSehUnwindPanic"} + defer func() { + if r := recover(); r == nil { + t.Fatal("did not panic") + } + pcs := sehCallers() + testSehCallersEqual(t, pcs, want) + }() + sehf3(true) +} + +func TestSehUnwindDoublePanic(t *testing.T) { + if runtime.GOARCH != "amd64" { + t.Skip("skipping amd64-only test") + } + want := []string{"runtime_test.sehCallers", "runtime_test.TestSehUnwindDoublePanic.func1.1", "runtime.gopanic", + "runtime_test.TestSehUnwindDoublePanic.func1", "runtime.gopanic", "runtime_test.TestSehUnwindDoublePanic"} + defer func() { + defer func() { + if recover() == nil { + t.Fatal("did not panic") + } + pcs := sehCallers() + testSehCallersEqual(t, pcs, want) + }() + if recover() == nil { + t.Fatal("did not panic") + } + panic(2) + }() + panic(1) +} + +func TestSehUnwindNilPointerPanic(t *testing.T) { + if runtime.GOARCH != "amd64" { + t.Skip("skipping amd64-only test") + } + want := []string{"runtime_test.sehCallers", "runtime_test.TestSehUnwindNilPointerPanic.func1", "runtime.gopanic", + "runtime.sigpanic", "runtime_test.TestSehUnwindNilPointerPanic"} + defer func() { + if r := recover(); r == nil { + t.Fatal("did not panic") + } + pcs := sehCallers() + testSehCallersEqual(t, pcs, want) + }() + var p *int + if *p == 3 { + t.Fatal("did not see nil pointer panic") + } +} From 3494a726009d049887bb3fb95181f71e07b21da3 Mon Sep 17 00:00:00 2001 From: Cherry Mui Date: Fri, 28 Apr 2023 22:15:48 -0400 Subject: [PATCH 049/299] cmd/compile: don't generate DWARF info for static vars Static data symbols are compiler generated, not user symbols. The linker already does not include them in the final DWARF section. Don't generate the DWARF info in the first place. Change-Id: Id2ae36683bfc1ed60b9924b7305eae5e8aa14d80 Reviewed-on: https://go-review.googlesource.com/c/go/+/490817 Run-TryBot: Cherry Mui Reviewed-by: Than McIntosh TryBot-Result: Gopher Robot --- src/cmd/compile/internal/gc/obj.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/cmd/compile/internal/gc/obj.go b/src/cmd/compile/internal/gc/obj.go index 504072bb17..e895c452f2 100644 --- a/src/cmd/compile/internal/gc/obj.go +++ b/src/cmd/compile/internal/gc/obj.go @@ -195,7 +195,7 @@ func dumpGlobal(n *ir.Name) { } types.CalcSize(n.Type()) ggloblnod(n) - if n.CoverageCounter() || n.CoverageAuxVar() { + if n.CoverageCounter() || n.CoverageAuxVar() || n.Linksym().Static() { return } base.Ctxt.DwarfGlobal(base.Ctxt.Pkgpath, types.TypeSymName(n.Type()), n.Linksym()) From a9a01ea280892e69c3722eebbc60d14c17a77e8d Mon Sep 17 00:00:00 2001 From: Cherry Mui Date: Mon, 1 May 2023 12:47:15 -0400 Subject: [PATCH 050/299] cmd/link: work around dsymutils not cleaning temp file Some versions of dsymutils, notably the one in clang 14.0.3, which is shipped in some versions of Xcode, have a bug that it creates a temporary directory but doesn't clean it up at exit. The temporary directory is created in DSYMUTIL_REPRODUCER_PATH (if set, otherwise TMPDIR). Work around the issue by setting DSYMUTIL_REPRODUCER_PATH to the linker's temporary directory, so the linker will clean it up at exit anyway. Fixes #59026. Change-Id: Ie3e90a2d6a01f90040dc2eac91e8e536ccdda5a2 Reviewed-on: https://go-review.googlesource.com/c/go/+/490818 Reviewed-by: Than McIntosh TryBot-Result: Gopher Robot Run-TryBot: Cherry Mui --- src/cmd/link/internal/ld/lib.go | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/cmd/link/internal/ld/lib.go b/src/cmd/link/internal/ld/lib.go index 5b6575b3fb..03b13da37a 100644 --- a/src/cmd/link/internal/ld/lib.go +++ b/src/cmd/link/internal/ld/lib.go @@ -1905,7 +1905,11 @@ func (ctxt *Link) hostlink() { stripCmd := strings.TrimSuffix(string(out), "\n") dsym := filepath.Join(*flagTmpdir, "go.dwarf") - if out, err := exec.Command(dsymutilCmd, "-f", *flagOutfile, "-o", dsym).CombinedOutput(); err != nil { + cmd := exec.Command(dsymutilCmd, "-f", *flagOutfile, "-o", dsym) + // dsymutil may not clean up its temp directory at exit. + // Set DSYMUTIL_REPRODUCER_PATH to work around. see issue 59026. + cmd.Env = append(os.Environ(), "DSYMUTIL_REPRODUCER_PATH="+*flagTmpdir) + if out, err := cmd.CombinedOutput(); err != nil { Exitf("%s: running dsymutil failed: %v\n%s", os.Args[0], err, out) } // Remove STAB (symbolic debugging) symbols after we are done with them (by dsymutil). From 46f60d650616b192c74bf8960df6e0bb99be2a9b Mon Sep 17 00:00:00 2001 From: Alan Donovan Date: Mon, 1 May 2023 13:02:04 -0400 Subject: [PATCH 051/299] cmd/cgo: reject attempts to declare methods on C types This change causes cgo to emit an error (with the same message as the compiler) when it encounters a declaration of a method whose receiver type is C.T or *C.T. Conceptually, C is another package, but unfortunately the desugaring of C.T is a type within the same package, causing the previous behavior to accept invalid input. It is likely that at least some users are intentionally exploiting this behavior, so this may break their build. We should mention it in the release notes. Fixes #57926 Change-Id: I513cffb7e13bc93d08a07b7e61301ac1762fd42d Reviewed-on: https://go-review.googlesource.com/c/go/+/490819 Reviewed-by: Ian Lance Taylor Run-TryBot: Bryan Mills TryBot-Result: Gopher Robot Reviewed-by: Bryan Mills --- src/cmd/cgo/ast.go | 68 +++++++++++++------ .../script/cgo_badmethod_issue57926.txt | 31 +++++++++ 2 files changed, 79 insertions(+), 20 deletions(-) create mode 100644 src/cmd/go/testdata/script/cgo_badmethod_issue57926.txt diff --git a/src/cmd/cgo/ast.go b/src/cmd/cgo/ast.go index 81060c67ed..6a1cf38720 100644 --- a/src/cmd/cgo/ast.go +++ b/src/cmd/cgo/ast.go @@ -9,6 +9,7 @@ package main import ( "fmt" "go/ast" + "go/format" "go/parser" "go/scanner" "go/token" @@ -62,29 +63,48 @@ func (f *File) ParseGo(abspath string, src []byte) { // In ast1, find the import "C" line and get any extra C preamble. sawC := false for _, decl := range ast1.Decls { - d, ok := decl.(*ast.GenDecl) - if !ok { - continue - } - for _, spec := range d.Specs { - s, ok := spec.(*ast.ImportSpec) - if !ok || s.Path.Value != `"C"` { - continue + switch decl := decl.(type) { + case *ast.GenDecl: + for _, spec := range decl.Specs { + s, ok := spec.(*ast.ImportSpec) + if !ok || s.Path.Value != `"C"` { + continue + } + sawC = true + if s.Name != nil { + error_(s.Path.Pos(), `cannot rename import "C"`) + } + cg := s.Doc + if cg == nil && len(decl.Specs) == 1 { + cg = decl.Doc + } + if cg != nil { + f.Preamble += fmt.Sprintf("#line %d %q\n", sourceLine(cg), abspath) + f.Preamble += commentText(cg) + "\n" + f.Preamble += "#line 1 \"cgo-generated-wrapper\"\n" + } } - sawC = true - if s.Name != nil { - error_(s.Path.Pos(), `cannot rename import "C"`) - } - cg := s.Doc - if cg == nil && len(d.Specs) == 1 { - cg = d.Doc - } - if cg != nil { - f.Preamble += fmt.Sprintf("#line %d %q\n", sourceLine(cg), abspath) - f.Preamble += commentText(cg) + "\n" - f.Preamble += "#line 1 \"cgo-generated-wrapper\"\n" + + case *ast.FuncDecl: + // Also, reject attempts to declare methods on C.T or *C.T. + // (The generated code would otherwise accept this + // invalid input; see issue #57926.) + if decl.Recv != nil && len(decl.Recv.List) > 0 { + recvType := decl.Recv.List[0].Type + if recvType != nil { + t := recvType + if star, ok := unparen(t).(*ast.StarExpr); ok { + t = star.X + } + if sel, ok := unparen(t).(*ast.SelectorExpr); ok { + var buf strings.Builder + format.Node(&buf, fset, recvType) + error_(sel.Pos(), `cannot define new methods on non-local type %s`, &buf) + } + } } } + } if !sawC { error_(ast1.Package, `cannot find import "C"`) @@ -542,3 +562,11 @@ func (f *File) walk(x interface{}, context astContext, visit func(*File, interfa } } } + +// If x is of the form (T), unparen returns unparen(T), otherwise it returns x. +func unparen(x ast.Expr) ast.Expr { + if p, isParen := x.(*ast.ParenExpr); isParen { + x = unparen(p.X) + } + return x +} diff --git a/src/cmd/go/testdata/script/cgo_badmethod_issue57926.txt b/src/cmd/go/testdata/script/cgo_badmethod_issue57926.txt new file mode 100644 index 0000000000..81ef850cb9 --- /dev/null +++ b/src/cmd/go/testdata/script/cgo_badmethod_issue57926.txt @@ -0,0 +1,31 @@ +[short] skip +[!cgo] skip + +# Test that cgo rejects attempts to declare methods +# on the types C.T or *C.T; see issue #57926. + +! go build +stderr 'cannot define new methods on non-local type C.T' +stderr 'cannot define new methods on non-local type \*C.T' +! stderr 'Alias' + +-- go.mod -- +module example.com +go 1.12 + +-- a.go -- +package a + +/* +typedef int T; +*/ +import "C" + +func (C.T) f() {} +func (recv *C.T) g() {} + +// The check is more education than enforcement, +// and is easily defeated using a type alias. +type Alias = C.T +func (Alias) h() {} +func (*Alias) i() {} From c63066123bb5c2ef7a6d26d1a6e3e5f1012a1e23 Mon Sep 17 00:00:00 2001 From: Michael Fraenkel Date: Fri, 28 Apr 2023 07:14:08 -0600 Subject: [PATCH 052/299] net/http: avoid leaking the writing goroutine The test will wait for all goroutines. A race can occur if the writing goroutine uses the Log after the test exits. For #58264 For #59883 For #59884 Change-Id: I9b8ec7c9d024ff74b922b69efa438be5a4fa3483 Reviewed-on: https://go-review.googlesource.com/c/go/+/490255 Reviewed-by: Damien Neil Reviewed-by: Bryan Mills Run-TryBot: Damien Neil TryBot-Result: Gopher Robot Auto-Submit: Bryan Mills --- src/net/http/serve_test.go | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/net/http/serve_test.go b/src/net/http/serve_test.go index 9b8496e7ad..819152658b 100644 --- a/src/net/http/serve_test.go +++ b/src/net/http/serve_test.go @@ -1744,7 +1744,13 @@ func testServerExpect(t *testing.T, mode testMode) { // that doesn't send 100-continue expectations. writeBody := test.contentLength != 0 && strings.ToLower(test.expectation) != "100-continue" + wg := sync.WaitGroup{} + wg.Add(1) + defer wg.Wait() + go func() { + defer wg.Done() + contentLen := fmt.Sprintf("Content-Length: %d", test.contentLength) if test.chunked { contentLen = "Transfer-Encoding: chunked" From 2d83b646d677ceb3df2ea592b7938efe8214c52a Mon Sep 17 00:00:00 2001 From: Nick Ripley Date: Mon, 1 May 2023 09:08:01 -0400 Subject: [PATCH 053/299] runtime/trace: enable frame pointer unwinding by default for amd64 and arm64 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Re-enable frame pointer unwinding for execution tracing on amd64 by default, now that CL 489015 and CL 488755 have fixed recently-discovered crashes. This reverts CL 486382. These fixes, together with CL 241158 to fix up frame pointers when copying stacks on arm64, also make frame pointer unwinding for tracing safe to enable for arm64. This should significantly reduce the CPU and latency overhead of execution tracing on arm64, as it has for amd64. Co-Authored-By: Felix Geisendörfer Change-Id: I64a88bd69dfd8cb13956ec46f8b1203dbeaa26a6 Reviewed-on: https://go-review.googlesource.com/c/go/+/490815 Reviewed-by: Felix Geisendörfer Run-TryBot: Nick Ripley Reviewed-by: Michael Knyszek TryBot-Result: Gopher Robot Reviewed-by: Michael Pratt Auto-Submit: Michael Pratt --- src/runtime/runtime1.go | 1 - src/runtime/trace.go | 6 ++---- 2 files changed, 2 insertions(+), 5 deletions(-) diff --git a/src/runtime/runtime1.go b/src/runtime/runtime1.go index 7dc65bdcc1..65bed433c3 100644 --- a/src/runtime/runtime1.go +++ b/src/runtime/runtime1.go @@ -369,7 +369,6 @@ func parsedebugvars() { debug.cgocheck = 1 debug.invalidptr = 1 debug.adaptivestackstart = 1 // set this to 0 to turn larger initial goroutine stacks off - debug.tracefpunwindoff = 1 // Frame pointer unwinding sometimes crashes on amd64. See issue 59692. if GOOS == "linux" { // On Linux, MADV_FREE is faster than MADV_DONTNEED, // but doesn't affect many of the statistics that diff --git a/src/runtime/trace.go b/src/runtime/trace.go index 79ccebb4b3..5eb68271d1 100644 --- a/src/runtime/trace.go +++ b/src/runtime/trace.go @@ -927,12 +927,10 @@ func traceStackID(mp *m, pcBuf []uintptr, skip int) uint64 { return uint64(id) } -// tracefpunwindoff returns false if frame pointer unwinding for the tracer is +// tracefpunwindoff returns true if frame pointer unwinding for the tracer is // disabled via GODEBUG or not supported by the architecture. func tracefpunwindoff() bool { - // compiler emits frame pointers for amd64 and arm64, but issue 58432 blocks - // arm64 support for now. - return debug.tracefpunwindoff != 0 || goarch.ArchFamily != goarch.AMD64 + return debug.tracefpunwindoff != 0 || (goarch.ArchFamily != goarch.AMD64 && goarch.ArchFamily != goarch.ARM64) } // fpTracebackPCs populates pcBuf with the return addresses for each frame and From fa4781a41502b283b270f7d83e2678152fd01682 Mon Sep 17 00:00:00 2001 From: "Bryan C. Mills" Date: Mon, 24 Apr 2023 13:52:28 -0400 Subject: [PATCH 054/299] cmd/go/internal/script: log the stop message instead of the state when stopping Change-Id: I3c887b5f3716e78dfffad6869f986c4381fae3e7 Reviewed-on: https://go-review.googlesource.com/c/go/+/488235 Run-TryBot: Bryan Mills Reviewed-by: Michael Matloob Auto-Submit: Bryan Mills TryBot-Result: Gopher Robot --- src/cmd/go/internal/script/engine.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/cmd/go/internal/script/engine.go b/src/cmd/go/internal/script/engine.go index dfce755522..43054a279b 100644 --- a/src/cmd/go/internal/script/engine.go +++ b/src/cmd/go/internal/script/engine.go @@ -295,7 +295,7 @@ func (e *Engine) Execute(s *State, file string, script *bufio.Reader, log io.Wri // Since the 'stop' command halts execution of the entire script, // log its message separately from the section in which it appears. err = endSection(true) - s.Logf("%v\n", s) + s.Logf("%v\n", stop) if err == nil { return nil } From 630ef2edc2a7f6138c9eb0ed8eb8216a32ed0339 Mon Sep 17 00:00:00 2001 From: Cherry Mui Date: Mon, 1 May 2023 12:54:27 -0400 Subject: [PATCH 055/299] cmd/link: remove allocation in decoding type name MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The type name symbol is always from a Go object file and we never change it. Convert the data to string using unsafe conversion without allocation. Linking cmd/go (on macOS/amd64), name old alloc/op new alloc/op delta Deadcode_GC 1.25MB ± 0% 1.17MB ± 0% -6.29% (p=0.000 n=20+20) name old allocs/op new allocs/op delta Deadcode_GC 8.98k ± 0% 0.10k ± 3% -98.91% (p=0.000 n=20+20) Change-Id: I33117ad1f991e4f14ce0b38cceec50b041e3c0a4 Reviewed-on: https://go-review.googlesource.com/c/go/+/490915 TryBot-Result: Gopher Robot Run-TryBot: Cherry Mui Reviewed-by: Than McIntosh --- src/cmd/internal/goobj/objfile.go | 9 +++++++++ src/cmd/link/internal/ld/deadcode.go | 2 +- src/cmd/link/internal/ld/decodesym.go | 10 +++++++--- src/cmd/link/internal/loader/loader.go | 10 ++++++++++ 4 files changed, 27 insertions(+), 4 deletions(-) diff --git a/src/cmd/internal/goobj/objfile.go b/src/cmd/internal/goobj/objfile.go index 64d453abdc..c9d7ca434c 100644 --- a/src/cmd/internal/goobj/objfile.go +++ b/src/cmd/internal/goobj/objfile.go @@ -853,6 +853,15 @@ func (r *Reader) Data(i uint32) []byte { return r.BytesAt(base+off, int(end-off)) } +// DataString returns the i-th symbol's data as a string. +func (r *Reader) DataString(i uint32) string { + dataIdxOff := r.h.Offsets[BlkDataIdx] + i*4 + base := r.h.Offsets[BlkData] + off := r.uint32At(dataIdxOff) + end := r.uint32At(dataIdxOff + 4) + return r.StringAt(base+off, end-off) +} + // NRefName returns the number of referenced symbol names. func (r *Reader) NRefName() int { return int(r.h.Offsets[BlkRefName+1]-r.h.Offsets[BlkRefName]) / RefNameSize diff --git a/src/cmd/link/internal/ld/deadcode.go b/src/cmd/link/internal/ld/deadcode.go index c80bacd92c..e7028d3b54 100644 --- a/src/cmd/link/internal/ld/deadcode.go +++ b/src/cmd/link/internal/ld/deadcode.go @@ -487,7 +487,7 @@ func (d *deadcodePass) decodeIfaceMethod(ldr *loader.Loader, arch *sys.Arch, sym // Decode the method name stored in symbol symIdx. The symbol should contain just the bytes of a method name. func (d *deadcodePass) decodeGenericIfaceMethod(ldr *loader.Loader, symIdx loader.Sym) string { - return string(ldr.Data(symIdx)) + return ldr.DataString(symIdx) } func (d *deadcodePass) decodetypeMethods(ldr *loader.Loader, arch *sys.Arch, symIdx loader.Sym, relocs *loader.Relocs) []methodsig { diff --git a/src/cmd/link/internal/ld/decodesym.go b/src/cmd/link/internal/ld/decodesym.go index b0f4b87563..53ed6ccc86 100644 --- a/src/cmd/link/internal/ld/decodesym.go +++ b/src/cmd/link/internal/ld/decodesym.go @@ -127,9 +127,13 @@ func decodetypeName(ldr *loader.Loader, symIdx loader.Sym, relocs *loader.Relocs return "" } - data := ldr.Data(r) - nameLen, nameLenLen := binary.Uvarint(data[1:]) - return string(data[1+nameLenLen : 1+nameLenLen+int(nameLen)]) + data := ldr.DataString(r) + n := 1 + binary.MaxVarintLen64 + if len(data) < n { + n = len(data) + } + nameLen, nameLenLen := binary.Uvarint([]byte(data[1:n])) + return data[1+nameLenLen : 1+nameLenLen+int(nameLen)] } func decodetypeNameEmbedded(ldr *loader.Loader, symIdx loader.Sym, relocs *loader.Relocs, off int) bool { diff --git a/src/cmd/link/internal/loader/loader.go b/src/cmd/link/internal/loader/loader.go index 5fcbf160e0..fa74dcede4 100644 --- a/src/cmd/link/internal/loader/loader.go +++ b/src/cmd/link/internal/loader/loader.go @@ -1247,6 +1247,16 @@ func (l *Loader) Data(i Sym) []byte { return r.Data(li) } +// Returns the symbol content of the i-th symbol as a string. i is global index. +func (l *Loader) DataString(i Sym) string { + if l.IsExternal(i) { + pp := l.getPayload(i) + return string(pp.data) + } + r, li := l.toLocal(i) + return r.DataString(li) +} + // FreeData clears the symbol data of an external symbol, allowing the memory // to be freed earlier. No-op for non-external symbols. // i is global index. From 72ba91902a39abb47ee9681319d517d4413e3b65 Mon Sep 17 00:00:00 2001 From: Ian Lance Taylor Date: Wed, 26 Apr 2023 17:44:24 -0700 Subject: [PATCH 056/299] io/fs: add FormatFileInfo and FormatDirEntry functions For #54451 Change-Id: I3214066f77b1398ac1f2786ea035c83f32f0a826 Reviewed-on: https://go-review.googlesource.com/c/go/+/489555 Run-TryBot: Ian Lance Taylor TryBot-Result: Gopher Robot Reviewed-by: Ian Lance Taylor Run-TryBot: Ian Lance Taylor Reviewed-by: Joseph Tsai Reviewed-by: Dmitri Shuralyov Auto-Submit: Ian Lance Taylor --- api/next/54451.txt | 2 + src/io/fs/format.go | 76 ++++++++++++++++++++++++ src/io/fs/format_test.go | 123 +++++++++++++++++++++++++++++++++++++++ 3 files changed, 201 insertions(+) create mode 100644 api/next/54451.txt create mode 100644 src/io/fs/format.go create mode 100644 src/io/fs/format_test.go diff --git a/api/next/54451.txt b/api/next/54451.txt new file mode 100644 index 0000000000..a1d4fd5280 --- /dev/null +++ b/api/next/54451.txt @@ -0,0 +1,2 @@ +pkg io/fs, func FormatDirEntry(DirEntry) string #54451 +pkg io/fs, func FormatFileInfo(FileInfo) string #54451 diff --git a/src/io/fs/format.go b/src/io/fs/format.go new file mode 100644 index 0000000000..f490341f6c --- /dev/null +++ b/src/io/fs/format.go @@ -0,0 +1,76 @@ +// Copyright 2023 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package fs + +import ( + "time" +) + +// FormatFileInfo returns a formatted version of info for human readability. +// Implementations of FileInfo can call this from a String method. +// The output for a file named "hello.go", 100 bytes, mode 0o644, created +// January 1, 1970 at noon is +// +// -rw-r--r-- 100 1970-01-01 12:00:00 hello.go +func FormatFileInfo(info FileInfo) string { + name := info.Name() + b := make([]byte, 0, 40+len(name)) + b = append(b, info.Mode().String()...) + b = append(b, ' ') + + size := info.Size() + var usize uint64 + if size >= 0 { + usize = uint64(size) + } else { + b = append(b, '-') + usize = uint64(-size) + } + var buf [20]byte + i := len(buf) - 1 + for usize >= 10 { + q := usize / 10 + buf[i] = byte('0' + usize - q*10) + i-- + usize = q + } + buf[i] = byte('0' + usize) + b = append(b, buf[i:]...) + b = append(b, ' ') + + b = append(b, info.ModTime().Format(time.DateTime)...) + b = append(b, ' ') + + b = append(b, name...) + if info.IsDir() { + b = append(b, '/') + } + + return string(b) +} + +// FormatDirEntry returns a formatted version of dir for human readability. +// Implementations of DirEntry can call this from a String method. +// The outputs for a directory named subdir and a file named hello.go are: +// +// d subdir/ +// - hello.go +func FormatDirEntry(dir DirEntry) string { + name := dir.Name() + b := make([]byte, 0, 5+len(name)) + + // The Type method does not return any permission bits, + // so strip them from the string. + mode := dir.Type().String() + mode = mode[:len(mode)-9] + + b = append(b, mode...) + b = append(b, ' ') + b = append(b, name...) + if dir.IsDir() { + b = append(b, '/') + } + return string(b) +} diff --git a/src/io/fs/format_test.go b/src/io/fs/format_test.go new file mode 100644 index 0000000000..a5f5066f36 --- /dev/null +++ b/src/io/fs/format_test.go @@ -0,0 +1,123 @@ +// Copyright 2023 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package fs_test + +import ( + . "io/fs" + "testing" + "time" +) + +// formatTest implements FileInfo to test FormatFileInfo, +// and implements DirEntry to test FormatDirEntry. +type formatTest struct { + name string + size int64 + mode FileMode + modTime time.Time + isDir bool +} + +func (fs *formatTest) Name() string { + return fs.name +} + +func (fs *formatTest) Size() int64 { + return fs.size +} + +func (fs *formatTest) Mode() FileMode { + return fs.mode +} + +func (fs *formatTest) ModTime() time.Time { + return fs.modTime +} + +func (fs *formatTest) IsDir() bool { + return fs.isDir +} + +func (fs *formatTest) Sys() any { + return nil +} + +func (fs *formatTest) Type() FileMode { + return fs.mode.Type() +} + +func (fs *formatTest) Info() (FileInfo, error) { + return fs, nil +} + +var formatTests = []struct { + input formatTest + wantFileInfo string + wantDirEntry string +}{ + { + formatTest{ + name: "hello.go", + size: 100, + mode: 0o644, + modTime: time.Date(1970, time.January, 1, 12, 0, 0, 0, time.UTC), + isDir: false, + }, + "-rw-r--r-- 100 1970-01-01 12:00:00 hello.go", + "- hello.go", + }, + { + formatTest{ + name: "home/gopher", + size: 0, + mode: ModeDir | 0o755, + modTime: time.Date(1970, time.January, 1, 12, 0, 0, 0, time.UTC), + isDir: true, + }, + "drwxr-xr-x 0 1970-01-01 12:00:00 home/gopher/", + "d home/gopher/", + }, + { + formatTest{ + name: "big", + size: 0x7fffffffffffffff, + mode: ModeIrregular | 0o644, + modTime: time.Date(1970, time.January, 1, 12, 0, 0, 0, time.UTC), + isDir: false, + }, + "?rw-r--r-- 9223372036854775807 1970-01-01 12:00:00 big", + "? big", + }, + { + formatTest{ + name: "small", + size: -0x8000000000000000, + mode: ModeSocket | ModeSetuid | 0o644, + modTime: time.Date(1970, time.January, 1, 12, 0, 0, 0, time.UTC), + isDir: false, + }, + "Surw-r--r-- -9223372036854775808 1970-01-01 12:00:00 small", + "S small", + }, +} + +func TestFormatFileInfo(t *testing.T) { + for i, test := range formatTests { + got := FormatFileInfo(&test.input) + if got != test.wantFileInfo { + t.Errorf("%d: FormatFileInfo(%#v) = %q, want %q", i, test.input, got, test.wantFileInfo) + } + } +} + +func TestFormatDirEntry(t *testing.T) { + for i, test := range formatTests { + got := FormatDirEntry(&test.input) + if got != test.wantDirEntry { + t.Errorf("%d: FormatDirEntry(%#v) = %q, want %q", i, test.input, got, test.wantDirEntry) + } + } + +} From 8673ca81e5340b87709db2d9749c92a3bf925df1 Mon Sep 17 00:00:00 2001 From: Roland Shoemaker Date: Thu, 13 Apr 2023 15:40:44 -0700 Subject: [PATCH 057/299] html/template: disallow angle brackets in CSS values Angle brackets should not appear in CSS contexts, as they may affect token boundaries (such as closing a