diff --git a/api/next/50860.txt b/api/next/50860.txt new file mode 100644 index 0000000000..9ff0feca24 --- /dev/null +++ b/api/next/50860.txt @@ -0,0 +1,40 @@ +pkg sync/atomic, method (*Bool) CompareAndSwap(bool, bool) bool #50860 +pkg sync/atomic, method (*Bool) Load() bool #50860 +pkg sync/atomic, method (*Bool) Store(bool) #50860 +pkg sync/atomic, method (*Bool) Swap(bool) bool #50860 +pkg sync/atomic, method (*Int32) Add(int32) int32 #50860 +pkg sync/atomic, method (*Int32) CompareAndSwap(int32, int32) bool #50860 +pkg sync/atomic, method (*Int32) Load() int32 #50860 +pkg sync/atomic, method (*Int32) Store(int32) #50860 +pkg sync/atomic, method (*Int32) Swap(int32) int32 #50860 +pkg sync/atomic, method (*Int64) Add(int64) int64 #50860 +pkg sync/atomic, method (*Int64) CompareAndSwap(int64, int64) bool #50860 +pkg sync/atomic, method (*Int64) Load() int64 #50860 +pkg sync/atomic, method (*Int64) Store(int64) #50860 +pkg sync/atomic, method (*Int64) Swap(int64) int64 #50860 +pkg sync/atomic, method (*Pointer[$0]) CompareAndSwap(*$0, *$0) bool #50860 +pkg sync/atomic, method (*Pointer[$0]) Load() *$0 #50860 +pkg sync/atomic, method (*Pointer[$0]) Store(*$0) #50860 +pkg sync/atomic, method (*Pointer[$0]) Swap(*$0) *$0 #50860 +pkg sync/atomic, method (*Uint32) Add(uint32) uint32 #50860 +pkg sync/atomic, method (*Uint32) CompareAndSwap(uint32, uint32) bool #50860 +pkg sync/atomic, method (*Uint32) Load() uint32 #50860 +pkg sync/atomic, method (*Uint32) Store(uint32) #50860 +pkg sync/atomic, method (*Uint32) Swap(uint32) uint32 #50860 +pkg sync/atomic, method (*Uint64) Add(uint64) uint64 #50860 +pkg sync/atomic, method (*Uint64) CompareAndSwap(uint64, uint64) bool #50860 +pkg sync/atomic, method (*Uint64) Load() uint64 #50860 +pkg sync/atomic, method (*Uint64) Store(uint64) #50860 +pkg sync/atomic, method (*Uint64) Swap(uint64) uint64 #50860 +pkg sync/atomic, method (*Uintptr) Add(uintptr) uintptr #50860 +pkg sync/atomic, method (*Uintptr) CompareAndSwap(uintptr, uintptr) bool #50860 +pkg sync/atomic, method (*Uintptr) Load() uintptr #50860 +pkg sync/atomic, method (*Uintptr) Store(uintptr) #50860 +pkg sync/atomic, method (*Uintptr) Swap(uintptr) uintptr #50860 +pkg sync/atomic, type Bool struct #50860 +pkg sync/atomic, type Int32 struct #50860 +pkg sync/atomic, type Int64 struct #50860 +pkg sync/atomic, type Pointer[$0 interface{}] struct #50860 +pkg sync/atomic, type Uint32 struct #50860 +pkg sync/atomic, type Uint64 struct #50860 +pkg sync/atomic, type Uintptr struct #50860 diff --git a/src/cmd/compile/internal/test/inl_test.go b/src/cmd/compile/internal/test/inl_test.go index af66a32085..0e52a572ab 100644 --- a/src/cmd/compile/internal/test/inl_test.go +++ b/src/cmd/compile/internal/test/inl_test.go @@ -180,6 +180,42 @@ func TestIntendedInlining(t *testing.T) { "net": { "(*UDPConn).ReadFromUDP", }, + "sync/atomic": { + // (*Bool).CompareAndSwap handled below. + "(*Bool).Load", + "(*Bool).Store", + "(*Bool).Swap", + "(*Int32).Add", + "(*Int32).CompareAndSwap", + "(*Int32).Load", + "(*Int32).Store", + "(*Int32).Swap", + "(*Int64).Add", + "(*Int64).CompareAndSwap", + "(*Int64).Load", + "(*Int64).Store", + "(*Int64).Swap", + "(*Uint32).Add", + "(*Uint32).CompareAndSwap", + "(*Uint32).Load", + "(*Uint32).Store", + "(*Uint32).Swap", + "(*Uint64).Add", + "(*Uint64).CompareAndSwap", + "(*Uint64).Load", + "(*Uint64).Store", + "(*Uint64).Swap", + "(*Uintptr).Add", + "(*Uintptr).CompareAndSwap", + "(*Uintptr).Load", + "(*Uintptr).Store", + "(*Uintptr).Swap", + // TODO(rsc): Why are these not reported as inlined? + // "(*Pointer[T]).CompareAndSwap", + // "(*Pointer[T]).Load", + // "(*Pointer[T]).Store", + // "(*Pointer[T]).Swap", + }, } if runtime.GOARCH != "386" && runtime.GOARCH != "mips64" && runtime.GOARCH != "mips64le" && runtime.GOARCH != "riscv64" { @@ -199,6 +235,8 @@ func TestIntendedInlining(t *testing.T) { if bits.UintSize == 64 { // mix is only defined on 64-bit architectures want["runtime"] = append(want["runtime"], "mix") + // (*Bool).CompareAndSwap is just over budget on 32-bit systems (386, arm). + want["sync/atomic"] = append(want["sync/atomic"], "(*Bool).CompareAndSwap") } switch runtime.GOARCH { diff --git a/src/cmd/compile/internal/types/size.go b/src/cmd/compile/internal/types/size.go index 9fa3e49e34..a6e43c8a75 100644 --- a/src/cmd/compile/internal/types/size.go +++ b/src/cmd/compile/internal/types/size.go @@ -167,6 +167,11 @@ func calcStructOffset(errtype *Type, t *Type, o int64, flag int) int64 { if maxalign < 1 { maxalign = 1 } + // Special case: sync/atomic.align64 is an empty struct we recognize + // as a signal that the struct it contains must be 64-bit-aligned. + if isStruct && t.NumFields() == 0 && t.Sym() != nil && t.Sym().Name == "align64" && isSyncAtomic(t.Sym().Pkg) { + maxalign = 8 + } lastzero := int64(0) for _, f := range t.Fields().Slice() { if f.Type == nil { @@ -226,6 +231,10 @@ func calcStructOffset(errtype *Type, t *Type, o int64, flag int) int64 { return o } +func isSyncAtomic(p *Pkg) bool { + return p.Prefix == "sync/atomic" || p.Prefix == `""` && base.Ctxt.Pkgpath == "sync/atomic" +} + // CalcSize calculates and stores the size and alignment for t. // If CalcSizeDisabled is set, and the size/alignment // have not already been calculated, it calls Fatal. diff --git a/src/sync/atomic/atomic_test.go b/src/sync/atomic/atomic_test.go index 09f93a4fe3..ef0a5d990e 100644 --- a/src/sync/atomic/atomic_test.go +++ b/src/sync/atomic/atomic_test.go @@ -6,6 +6,7 @@ package atomic_test import ( "fmt" + "reflect" "runtime" "runtime/debug" "strings" @@ -62,6 +63,27 @@ func TestSwapInt32(t *testing.T) { } } +func TestSwapInt32Method(t *testing.T) { + var x struct { + before int32 + i Int32 + after int32 + } + x.before = magic32 + x.after = magic32 + var j int32 + for delta := int32(1); delta+delta > delta; delta += delta { + k := x.i.Swap(delta) + if x.i.Load() != delta || k != j { + t.Fatalf("delta=%d i=%d j=%d k=%d", delta, x.i.Load(), j, k) + } + j = delta + } + if x.before != magic32 || x.after != magic32 { + t.Fatalf("wrong magic: %#x _ %#x != %#x _ %#x", x.before, x.after, magic32, magic32) + } +} + func TestSwapUint32(t *testing.T) { var x struct { before uint32 @@ -83,6 +105,27 @@ func TestSwapUint32(t *testing.T) { } } +func TestSwapUint32Method(t *testing.T) { + var x struct { + before uint32 + i Uint32 + after uint32 + } + x.before = magic32 + x.after = magic32 + var j uint32 + for delta := uint32(1); delta+delta > delta; delta += delta { + k := x.i.Swap(delta) + if x.i.Load() != delta || k != j { + t.Fatalf("delta=%d i=%d j=%d k=%d", delta, x.i.Load(), j, k) + } + j = delta + } + if x.before != magic32 || x.after != magic32 { + t.Fatalf("wrong magic: %#x _ %#x != %#x _ %#x", x.before, x.after, magic32, magic32) + } +} + func TestSwapInt64(t *testing.T) { if test64err != nil { t.Skipf("Skipping 64-bit tests: %v", test64err) @@ -92,6 +135,7 @@ func TestSwapInt64(t *testing.T) { i int64 after int64 } + magic64 := int64(magic64) x.before = magic64 x.after = magic64 var j int64 @@ -103,7 +147,32 @@ func TestSwapInt64(t *testing.T) { j = delta } if x.before != magic64 || x.after != magic64 { - t.Fatalf("wrong magic: %#x _ %#x != %#x _ %#x", x.before, x.after, uint64(magic64), uint64(magic64)) + t.Fatalf("wrong magic: %#x _ %#x != %#x _ %#x", x.before, x.after, magic64, magic64) + } +} + +func TestSwapInt64Method(t *testing.T) { + if test64err != nil { + t.Skipf("Skipping 64-bit tests: %v", test64err) + } + var x struct { + before int64 + i Int64 + after int64 + } + magic64 := int64(magic64) + x.before = magic64 + x.after = magic64 + var j int64 + for delta := int64(1); delta+delta > delta; delta += delta { + k := x.i.Swap(delta) + if x.i.Load() != delta || k != j { + t.Fatalf("delta=%d i=%d j=%d k=%d", delta, x.i.Load(), j, k) + } + j = delta + } + if x.before != magic64 || x.after != magic64 { + t.Fatalf("wrong magic: %#x _ %#x != %#x _ %#x", x.before, x.after, magic64, magic64) } } @@ -116,6 +185,7 @@ func TestSwapUint64(t *testing.T) { i uint64 after uint64 } + magic64 := uint64(magic64) x.before = magic64 x.after = magic64 var j uint64 @@ -127,7 +197,32 @@ func TestSwapUint64(t *testing.T) { j = delta } if x.before != magic64 || x.after != magic64 { - t.Fatalf("wrong magic: %#x _ %#x != %#x _ %#x", x.before, x.after, uint64(magic64), uint64(magic64)) + t.Fatalf("wrong magic: %#x _ %#x != %#x _ %#x", x.before, x.after, magic64, magic64) + } +} + +func TestSwapUint64Method(t *testing.T) { + if test64err != nil { + t.Skipf("Skipping 64-bit tests: %v", test64err) + } + var x struct { + before uint64 + i Uint64 + after uint64 + } + magic64 := uint64(magic64) + x.before = magic64 + x.after = magic64 + var j uint64 + for delta := uint64(1); delta+delta > delta; delta += delta { + k := x.i.Swap(delta) + if x.i.Load() != delta || k != j { + t.Fatalf("delta=%d i=%d j=%d k=%d", delta, x.i.Load(), j, k) + } + j = delta + } + if x.before != magic64 || x.after != magic64 { + t.Fatalf("wrong magic: %#x _ %#x != %#x _ %#x", x.before, x.after, magic64, magic64) } } @@ -154,6 +249,29 @@ func TestSwapUintptr(t *testing.T) { } } +func TestSwapUintptrMethod(t *testing.T) { + var x struct { + before uintptr + i Uintptr + after uintptr + } + var m uint64 = magic64 + magicptr := uintptr(m) + x.before = magicptr + x.after = magicptr + var j uintptr + for delta := uintptr(1); delta+delta > delta; delta += delta { + k := x.i.Swap(delta) + if x.i.Load() != delta || k != j { + t.Fatalf("delta=%d i=%d j=%d k=%d", delta, x.i.Load(), j, k) + } + j = delta + } + if x.before != magicptr || x.after != magicptr { + t.Fatalf("wrong magic: %#x _ %#x != %#x _ %#x", x.before, x.after, magicptr, magicptr) + } +} + var global [1024]byte func testPointers() []unsafe.Pointer { @@ -193,6 +311,30 @@ func TestSwapPointer(t *testing.T) { } } +func TestSwapPointerMethod(t *testing.T) { + var x struct { + before uintptr + i Pointer[byte] + after uintptr + } + var m uint64 = magic64 + magicptr := uintptr(m) + x.before = magicptr + x.after = magicptr + var j *byte + for _, p := range testPointers() { + p := (*byte)(p) + k := x.i.Swap(p) + if x.i.Load() != p || k != j { + t.Fatalf("p=%p i=%p j=%p k=%p", p, x.i.Load(), j, k) + } + j = p + } + if x.before != magicptr || x.after != magicptr { + t.Fatalf("wrong magic: %#x _ %#x != %#x _ %#x", x.before, x.after, magicptr, magicptr) + } +} + func TestAddInt32(t *testing.T) { var x struct { before int32 @@ -214,6 +356,27 @@ func TestAddInt32(t *testing.T) { } } +func TestAddInt32Method(t *testing.T) { + var x struct { + before int32 + i Int32 + after int32 + } + x.before = magic32 + x.after = magic32 + var j int32 + for delta := int32(1); delta+delta > delta; delta += delta { + k := x.i.Add(delta) + j += delta + if x.i.Load() != j || k != j { + t.Fatalf("delta=%d i=%d j=%d k=%d", delta, x.i.Load(), j, k) + } + } + if x.before != magic32 || x.after != magic32 { + t.Fatalf("wrong magic: %#x _ %#x != %#x _ %#x", x.before, x.after, magic32, magic32) + } +} + func TestAddUint32(t *testing.T) { var x struct { before uint32 @@ -235,6 +398,27 @@ func TestAddUint32(t *testing.T) { } } +func TestAddUint32Method(t *testing.T) { + var x struct { + before uint32 + i Uint32 + after uint32 + } + x.before = magic32 + x.after = magic32 + var j uint32 + for delta := uint32(1); delta+delta > delta; delta += delta { + k := x.i.Add(delta) + j += delta + if x.i.Load() != j || k != j { + t.Fatalf("delta=%d i=%d j=%d k=%d", delta, x.i.Load(), j, k) + } + } + if x.before != magic32 || x.after != magic32 { + t.Fatalf("wrong magic: %#x _ %#x != %#x _ %#x", x.before, x.after, magic32, magic32) + } +} + func TestAddInt64(t *testing.T) { if test64err != nil { t.Skipf("Skipping 64-bit tests: %v", test64err) @@ -244,6 +428,7 @@ func TestAddInt64(t *testing.T) { i int64 after int64 } + magic64 := int64(magic64) x.before = magic64 x.after = magic64 var j int64 @@ -255,7 +440,32 @@ func TestAddInt64(t *testing.T) { } } if x.before != magic64 || x.after != magic64 { - t.Fatalf("wrong magic: %#x _ %#x != %#x _ %#x", x.before, x.after, int64(magic64), int64(magic64)) + t.Fatalf("wrong magic: %#x _ %#x != %#x _ %#x", x.before, x.after, magic64, magic64) + } +} + +func TestAddInt64Method(t *testing.T) { + if test64err != nil { + t.Skipf("Skipping 64-bit tests: %v", test64err) + } + var x struct { + before int64 + i Int64 + after int64 + } + magic64 := int64(magic64) + x.before = magic64 + x.after = magic64 + var j int64 + for delta := int64(1); delta+delta > delta; delta += delta { + k := x.i.Add(delta) + j += delta + if x.i.Load() != j || k != j { + t.Fatalf("delta=%d i=%d j=%d k=%d", delta, x.i.Load(), j, k) + } + } + if x.before != magic64 || x.after != magic64 { + t.Fatalf("wrong magic: %#x _ %#x != %#x _ %#x", x.before, x.after, magic64, magic64) } } @@ -268,6 +478,7 @@ func TestAddUint64(t *testing.T) { i uint64 after uint64 } + magic64 := uint64(magic64) x.before = magic64 x.after = magic64 var j uint64 @@ -279,7 +490,32 @@ func TestAddUint64(t *testing.T) { } } if x.before != magic64 || x.after != magic64 { - t.Fatalf("wrong magic: %#x _ %#x != %#x _ %#x", x.before, x.after, uint64(magic64), uint64(magic64)) + t.Fatalf("wrong magic: %#x _ %#x != %#x _ %#x", x.before, x.after, magic64, magic64) + } +} + +func TestAddUint64Method(t *testing.T) { + if test64err != nil { + t.Skipf("Skipping 64-bit tests: %v", test64err) + } + var x struct { + before uint64 + i Uint64 + after uint64 + } + magic64 := uint64(magic64) + x.before = magic64 + x.after = magic64 + var j uint64 + for delta := uint64(1); delta+delta > delta; delta += delta { + k := x.i.Add(delta) + j += delta + if x.i.Load() != j || k != j { + t.Fatalf("delta=%d i=%d j=%d k=%d", delta, x.i.Load(), j, k) + } + } + if x.before != magic64 || x.after != magic64 { + t.Fatalf("wrong magic: %#x _ %#x != %#x _ %#x", x.before, x.after, magic64, magic64) } } @@ -306,6 +542,29 @@ func TestAddUintptr(t *testing.T) { } } +func TestAddUintptrMethod(t *testing.T) { + var x struct { + before uintptr + i Uintptr + after uintptr + } + var m uint64 = magic64 + magicptr := uintptr(m) + x.before = magicptr + x.after = magicptr + var j uintptr + for delta := uintptr(1); delta+delta > delta; delta += delta { + k := x.i.Add(delta) + j += delta + if x.i.Load() != j || k != j { + t.Fatalf("delta=%d i=%d j=%d k=%d", delta, x.i.Load(), j, k) + } + } + if x.before != magicptr || x.after != magicptr { + t.Fatalf("wrong magic: %#x _ %#x != %#x _ %#x", x.before, x.after, magicptr, magicptr) + } +} + func TestCompareAndSwapInt32(t *testing.T) { var x struct { before int32 @@ -335,6 +594,35 @@ func TestCompareAndSwapInt32(t *testing.T) { } } +func TestCompareAndSwapInt32Method(t *testing.T) { + var x struct { + before int32 + i Int32 + after int32 + } + x.before = magic32 + x.after = magic32 + for val := int32(1); val+val > val; val += val { + x.i.Store(val) + if !x.i.CompareAndSwap(val, val+1) { + t.Fatalf("should have swapped %#x %#x", val, val+1) + } + if x.i.Load() != val+1 { + t.Fatalf("wrong x.i after swap: x.i=%#x val+1=%#x", x.i.Load(), val+1) + } + x.i.Store(val + 1) + if x.i.CompareAndSwap(val, val+2) { + t.Fatalf("should not have swapped %#x %#x", val, val+2) + } + if x.i.Load() != val+1 { + t.Fatalf("wrong x.i after swap: x.i=%#x val+1=%#x", x.i.Load(), val+1) + } + } + if x.before != magic32 || x.after != magic32 { + t.Fatalf("wrong magic: %#x _ %#x != %#x _ %#x", x.before, x.after, magic32, magic32) + } +} + func TestCompareAndSwapUint32(t *testing.T) { var x struct { before uint32 @@ -364,6 +652,35 @@ func TestCompareAndSwapUint32(t *testing.T) { } } +func TestCompareAndSwapUint32Method(t *testing.T) { + var x struct { + before uint32 + i Uint32 + after uint32 + } + x.before = magic32 + x.after = magic32 + for val := uint32(1); val+val > val; val += val { + x.i.Store(val) + if !x.i.CompareAndSwap(val, val+1) { + t.Fatalf("should have swapped %#x %#x", val, val+1) + } + if x.i.Load() != val+1 { + t.Fatalf("wrong x.i after swap: x.i=%#x val+1=%#x", x.i.Load(), val+1) + } + x.i.Store(val + 1) + if x.i.CompareAndSwap(val, val+2) { + t.Fatalf("should not have swapped %#x %#x", val, val+2) + } + if x.i.Load() != val+1 { + t.Fatalf("wrong x.i after swap: x.i=%#x val+1=%#x", x.i.Load(), val+1) + } + } + if x.before != magic32 || x.after != magic32 { + t.Fatalf("wrong magic: %#x _ %#x != %#x _ %#x", x.before, x.after, magic32, magic32) + } +} + func TestCompareAndSwapInt64(t *testing.T) { if test64err != nil { t.Skipf("Skipping 64-bit tests: %v", test64err) @@ -373,6 +690,7 @@ func TestCompareAndSwapInt64(t *testing.T) { i int64 after int64 } + magic64 := int64(magic64) x.before = magic64 x.after = magic64 for val := int64(1); val+val > val; val += val { @@ -392,7 +710,40 @@ func TestCompareAndSwapInt64(t *testing.T) { } } if x.before != magic64 || x.after != magic64 { - t.Fatalf("wrong magic: %#x _ %#x != %#x _ %#x", x.before, x.after, uint64(magic64), uint64(magic64)) + t.Fatalf("wrong magic: %#x _ %#x != %#x _ %#x", x.before, x.after, magic64, magic64) + } +} + +func TestCompareAndSwapInt64Method(t *testing.T) { + if test64err != nil { + t.Skipf("Skipping 64-bit tests: %v", test64err) + } + var x struct { + before int64 + i Int64 + after int64 + } + magic64 := int64(magic64) + x.before = magic64 + x.after = magic64 + for val := int64(1); val+val > val; val += val { + x.i.Store(val) + if !x.i.CompareAndSwap(val, val+1) { + t.Fatalf("should have swapped %#x %#x", val, val+1) + } + if x.i.Load() != val+1 { + t.Fatalf("wrong x.i after swap: x.i=%#x val+1=%#x", x.i.Load(), val+1) + } + x.i.Store(val + 1) + if x.i.CompareAndSwap(val, val+2) { + t.Fatalf("should not have swapped %#x %#x", val, val+2) + } + if x.i.Load() != val+1 { + t.Fatalf("wrong x.i after swap: x.i=%#x val+1=%#x", x.i.Load(), val+1) + } + } + if x.before != magic64 || x.after != magic64 { + t.Fatalf("wrong magic: %#x _ %#x != %#x _ %#x", x.before, x.after, magic64, magic64) } } @@ -405,6 +756,7 @@ func testCompareAndSwapUint64(t *testing.T, cas func(*uint64, uint64, uint64) bo i uint64 after uint64 } + magic64 := uint64(magic64) x.before = magic64 x.after = magic64 for val := uint64(1); val+val > val; val += val { @@ -424,7 +776,7 @@ func testCompareAndSwapUint64(t *testing.T, cas func(*uint64, uint64, uint64) bo } } if x.before != magic64 || x.after != magic64 { - t.Fatalf("wrong magic: %#x _ %#x != %#x _ %#x", x.before, x.after, uint64(magic64), uint64(magic64)) + t.Fatalf("wrong magic: %#x _ %#x != %#x _ %#x", x.before, x.after, magic64, magic64) } } @@ -432,6 +784,39 @@ func TestCompareAndSwapUint64(t *testing.T) { testCompareAndSwapUint64(t, CompareAndSwapUint64) } +func TestCompareAndSwapUint64Method(t *testing.T) { + if test64err != nil { + t.Skipf("Skipping 64-bit tests: %v", test64err) + } + var x struct { + before uint64 + i Uint64 + after uint64 + } + magic64 := uint64(magic64) + x.before = magic64 + x.after = magic64 + for val := uint64(1); val+val > val; val += val { + x.i.Store(val) + if !x.i.CompareAndSwap(val, val+1) { + t.Fatalf("should have swapped %#x %#x", val, val+1) + } + if x.i.Load() != val+1 { + t.Fatalf("wrong x.i after swap: x.i=%#x val+1=%#x", x.i.Load(), val+1) + } + x.i.Store(val + 1) + if x.i.CompareAndSwap(val, val+2) { + t.Fatalf("should not have swapped %#x %#x", val, val+2) + } + if x.i.Load() != val+1 { + t.Fatalf("wrong x.i after swap: x.i=%#x val+1=%#x", x.i.Load(), val+1) + } + } + if x.before != magic64 || x.after != magic64 { + t.Fatalf("wrong magic: %#x _ %#x != %#x _ %#x", x.before, x.after, magic64, magic64) + } +} + func TestCompareAndSwapUintptr(t *testing.T) { var x struct { before uintptr @@ -463,6 +848,37 @@ func TestCompareAndSwapUintptr(t *testing.T) { } } +func TestCompareAndSwapUintptrMethod(t *testing.T) { + var x struct { + before uintptr + i Uintptr + after uintptr + } + var m uint64 = magic64 + magicptr := uintptr(m) + x.before = magicptr + x.after = magicptr + for val := uintptr(1); val+val > val; val += val { + x.i.Store(val) + if !x.i.CompareAndSwap(val, val+1) { + t.Fatalf("should have swapped %#x %#x", val, val+1) + } + if x.i.Load() != val+1 { + t.Fatalf("wrong x.i after swap: x.i=%#x val+1=%#x", x.i.Load(), val+1) + } + x.i.Store(val + 1) + if x.i.CompareAndSwap(val, val+2) { + t.Fatalf("should not have swapped %#x %#x", val, val+2) + } + if x.i.Load() != val+1 { + t.Fatalf("wrong x.i after swap: x.i=%#x val+1=%#x", x.i.Load(), val+1) + } + } + if x.before != magicptr || x.after != magicptr { + t.Fatalf("wrong magic: %#x _ %#x != %#x _ %#x", x.before, x.after, uintptr(magicptr), uintptr(magicptr)) + } +} + func TestCompareAndSwapPointer(t *testing.T) { var x struct { before uintptr @@ -494,6 +910,38 @@ func TestCompareAndSwapPointer(t *testing.T) { } } +func TestCompareAndSwapPointerMethod(t *testing.T) { + var x struct { + before uintptr + i Pointer[byte] + after uintptr + } + var m uint64 = magic64 + magicptr := uintptr(m) + x.before = magicptr + x.after = magicptr + q := new(byte) + for _, p := range testPointers() { + p := (*byte)(p) + x.i.Store(p) + if !x.i.CompareAndSwap(p, q) { + t.Fatalf("should have swapped %p %p", p, q) + } + if x.i.Load() != q { + t.Fatalf("wrong x.i after swap: x.i=%p want %p", x.i.Load(), q) + } + if x.i.CompareAndSwap(p, nil) { + t.Fatalf("should not have swapped %p nil", p) + } + if x.i.Load() != q { + t.Fatalf("wrong x.i after swap: x.i=%p want %p", x.i.Load(), q) + } + } + if x.before != magicptr || x.after != magicptr { + t.Fatalf("wrong magic: %#x _ %#x != %#x _ %#x", x.before, x.after, magicptr, magicptr) + } +} + func TestLoadInt32(t *testing.T) { var x struct { before int32 @@ -514,6 +962,28 @@ func TestLoadInt32(t *testing.T) { } } +func TestLoadInt32Method(t *testing.T) { + var x struct { + before int32 + i Int32 + after int32 + } + x.before = magic32 + x.after = magic32 + want := int32(0) + for delta := int32(1); delta+delta > delta; delta += delta { + k := x.i.Load() + if k != want { + t.Fatalf("delta=%d i=%d k=%d want=%d", delta, x.i.Load(), k, want) + } + x.i.Store(k + delta) + want = k + delta + } + if x.before != magic32 || x.after != magic32 { + t.Fatalf("wrong magic: %#x _ %#x != %#x _ %#x", x.before, x.after, magic32, magic32) + } +} + func TestLoadUint32(t *testing.T) { var x struct { before uint32 @@ -534,6 +1004,28 @@ func TestLoadUint32(t *testing.T) { } } +func TestLoadUint32Method(t *testing.T) { + var x struct { + before uint32 + i Uint32 + after uint32 + } + x.before = magic32 + x.after = magic32 + want := uint32(0) + for delta := uint32(1); delta+delta > delta; delta += delta { + k := x.i.Load() + if k != want { + t.Fatalf("delta=%d i=%d k=%d want=%d", delta, x.i.Load(), k, want) + } + x.i.Store(k + delta) + want = k + delta + } + if x.before != magic32 || x.after != magic32 { + t.Fatalf("wrong magic: %#x _ %#x != %#x _ %#x", x.before, x.after, magic32, magic32) + } +} + func TestLoadInt64(t *testing.T) { if test64err != nil { t.Skipf("Skipping 64-bit tests: %v", test64err) @@ -543,6 +1035,7 @@ func TestLoadInt64(t *testing.T) { i int64 after int64 } + magic64 := int64(magic64) x.before = magic64 x.after = magic64 for delta := int64(1); delta+delta > delta; delta += delta { @@ -553,7 +1046,33 @@ func TestLoadInt64(t *testing.T) { x.i += delta } if x.before != magic64 || x.after != magic64 { - t.Fatalf("wrong magic: %#x _ %#x != %#x _ %#x", x.before, x.after, uint64(magic64), uint64(magic64)) + t.Fatalf("wrong magic: %#x _ %#x != %#x _ %#x", x.before, x.after, magic64, magic64) + } +} + +func TestLoadInt64Method(t *testing.T) { + if test64err != nil { + t.Skipf("Skipping 64-bit tests: %v", test64err) + } + var x struct { + before int64 + i Int64 + after int64 + } + magic64 := int64(magic64) + x.before = magic64 + x.after = magic64 + want := int64(0) + for delta := int64(1); delta+delta > delta; delta += delta { + k := x.i.Load() + if k != want { + t.Fatalf("delta=%d i=%d k=%d want=%d", delta, x.i.Load(), k, want) + } + x.i.Store(k + delta) + want = k + delta + } + if x.before != magic64 || x.after != magic64 { + t.Fatalf("wrong magic: %#x _ %#x != %#x _ %#x", x.before, x.after, magic64, magic64) } } @@ -566,6 +1085,7 @@ func TestLoadUint64(t *testing.T) { i uint64 after uint64 } + magic64 := uint64(magic64) x.before = magic64 x.after = magic64 for delta := uint64(1); delta+delta > delta; delta += delta { @@ -576,7 +1096,33 @@ func TestLoadUint64(t *testing.T) { x.i += delta } if x.before != magic64 || x.after != magic64 { - t.Fatalf("wrong magic: %#x _ %#x != %#x _ %#x", x.before, x.after, uint64(magic64), uint64(magic64)) + t.Fatalf("wrong magic: %#x _ %#x != %#x _ %#x", x.before, x.after, magic64, magic64) + } +} + +func TestLoadUint64Method(t *testing.T) { + if test64err != nil { + t.Skipf("Skipping 64-bit tests: %v", test64err) + } + var x struct { + before uint64 + i Uint64 + after uint64 + } + magic64 := uint64(magic64) + x.before = magic64 + x.after = magic64 + want := uint64(0) + for delta := uint64(1); delta+delta > delta; delta += delta { + k := x.i.Load() + if k != want { + t.Fatalf("delta=%d i=%d k=%d want=%d", delta, x.i.Load(), k, want) + } + x.i.Store(k + delta) + want = k + delta + } + if x.before != magic64 || x.after != magic64 { + t.Fatalf("wrong magic: %#x _ %#x != %#x _ %#x", x.before, x.after, magic64, magic64) } } @@ -602,6 +1148,30 @@ func TestLoadUintptr(t *testing.T) { } } +func TestLoadUintptrMethod(t *testing.T) { + var x struct { + before uintptr + i Uintptr + after uintptr + } + var m uint64 = magic64 + magicptr := uintptr(m) + x.before = magicptr + x.after = magicptr + want := uintptr(0) + for delta := uintptr(1); delta+delta > delta; delta += delta { + k := x.i.Load() + if k != want { + t.Fatalf("delta=%d i=%d k=%d want=%d", delta, x.i.Load(), k, want) + } + x.i.Store(k + delta) + want = k + delta + } + if x.before != magicptr || x.after != magicptr { + t.Fatalf("wrong magic: %#x _ %#x != %#x _ %#x", x.before, x.after, magicptr, magicptr) + } +} + func TestLoadPointer(t *testing.T) { var x struct { before uintptr @@ -624,6 +1194,29 @@ func TestLoadPointer(t *testing.T) { } } +func TestLoadPointerMethod(t *testing.T) { + var x struct { + before uintptr + i Pointer[byte] + after uintptr + } + var m uint64 = magic64 + magicptr := uintptr(m) + x.before = magicptr + x.after = magicptr + for _, p := range testPointers() { + p := (*byte)(p) + x.i.Store(p) + k := x.i.Load() + if k != p { + t.Fatalf("p=%x k=%x", p, k) + } + } + if x.before != magicptr || x.after != magicptr { + t.Fatalf("wrong magic: %#x _ %#x != %#x _ %#x", x.before, x.after, magicptr, magicptr) + } +} + func TestStoreInt32(t *testing.T) { var x struct { before int32 @@ -645,6 +1238,27 @@ func TestStoreInt32(t *testing.T) { } } +func TestStoreInt32Method(t *testing.T) { + var x struct { + before int32 + i Int32 + after int32 + } + x.before = magic32 + x.after = magic32 + v := int32(0) + for delta := int32(1); delta+delta > delta; delta += delta { + x.i.Store(v) + if x.i.Load() != v { + t.Fatalf("delta=%d i=%d v=%d", delta, x.i.Load(), v) + } + v += delta + } + if x.before != magic32 || x.after != magic32 { + t.Fatalf("wrong magic: %#x _ %#x != %#x _ %#x", x.before, x.after, magic32, magic32) + } +} + func TestStoreUint32(t *testing.T) { var x struct { before uint32 @@ -666,6 +1280,27 @@ func TestStoreUint32(t *testing.T) { } } +func TestStoreUint32Method(t *testing.T) { + var x struct { + before uint32 + i Uint32 + after uint32 + } + x.before = magic32 + x.after = magic32 + v := uint32(0) + for delta := uint32(1); delta+delta > delta; delta += delta { + x.i.Store(v) + if x.i.Load() != v { + t.Fatalf("delta=%d i=%d v=%d", delta, x.i.Load(), v) + } + v += delta + } + if x.before != magic32 || x.after != magic32 { + t.Fatalf("wrong magic: %#x _ %#x != %#x _ %#x", x.before, x.after, magic32, magic32) + } +} + func TestStoreInt64(t *testing.T) { if test64err != nil { t.Skipf("Skipping 64-bit tests: %v", test64err) @@ -675,6 +1310,7 @@ func TestStoreInt64(t *testing.T) { i int64 after int64 } + magic64 := int64(magic64) x.before = magic64 x.after = magic64 v := int64(0) @@ -686,7 +1322,29 @@ func TestStoreInt64(t *testing.T) { v += delta } if x.before != magic64 || x.after != magic64 { - t.Fatalf("wrong magic: %#x _ %#x != %#x _ %#x", x.before, x.after, uint64(magic64), uint64(magic64)) + t.Fatalf("wrong magic: %#x _ %#x != %#x _ %#x", x.before, x.after, magic64, magic64) + } +} + +func TestStoreInt64Method(t *testing.T) { + var x struct { + before int64 + i Int64 + after int64 + } + magic64 := int64(magic64) + x.before = magic64 + x.after = magic64 + v := int64(0) + for delta := int64(1); delta+delta > delta; delta += delta { + x.i.Store(v) + if x.i.Load() != v { + t.Fatalf("delta=%d i=%d v=%d", delta, x.i.Load(), v) + } + v += delta + } + if x.before != magic64 || x.after != magic64 { + t.Fatalf("wrong magic: %#x _ %#x != %#x _ %#x", x.before, x.after, magic64, magic64) } } @@ -699,6 +1357,7 @@ func TestStoreUint64(t *testing.T) { i uint64 after uint64 } + magic64 := uint64(magic64) x.before = magic64 x.after = magic64 v := uint64(0) @@ -710,7 +1369,32 @@ func TestStoreUint64(t *testing.T) { v += delta } if x.before != magic64 || x.after != magic64 { - t.Fatalf("wrong magic: %#x _ %#x != %#x _ %#x", x.before, x.after, uint64(magic64), uint64(magic64)) + t.Fatalf("wrong magic: %#x _ %#x != %#x _ %#x", x.before, x.after, magic64, magic64) + } +} + +func TestStoreUint64Method(t *testing.T) { + if test64err != nil { + t.Skipf("Skipping 64-bit tests: %v", test64err) + } + var x struct { + before uint64 + i Uint64 + after uint64 + } + magic64 := uint64(magic64) + x.before = magic64 + x.after = magic64 + v := uint64(0) + for delta := uint64(1); delta+delta > delta; delta += delta { + x.i.Store(v) + if x.i.Load() != v { + t.Fatalf("delta=%d i=%d v=%d", delta, x.i.Load(), v) + } + v += delta + } + if x.before != magic64 || x.after != magic64 { + t.Fatalf("wrong magic: %#x _ %#x != %#x _ %#x", x.before, x.after, magic64, magic64) } } @@ -737,6 +1421,29 @@ func TestStoreUintptr(t *testing.T) { } } +func TestStoreUintptrMethod(t *testing.T) { + var x struct { + before uintptr + i Uintptr + after uintptr + } + var m uint64 = magic64 + magicptr := uintptr(m) + x.before = magicptr + x.after = magicptr + v := uintptr(0) + for delta := uintptr(1); delta+delta > delta; delta += delta { + x.i.Store(v) + if x.i.Load() != v { + t.Fatalf("delta=%d i=%d v=%d", delta, x.i.Load(), v) + } + v += delta + } + if x.before != magicptr || x.after != magicptr { + t.Fatalf("wrong magic: %#x _ %#x != %#x _ %#x", x.before, x.after, magicptr, magicptr) + } +} + func TestStorePointer(t *testing.T) { var x struct { before uintptr @@ -758,6 +1465,28 @@ func TestStorePointer(t *testing.T) { } } +func TestStorePointerMethod(t *testing.T) { + var x struct { + before uintptr + i Pointer[byte] + after uintptr + } + var m uint64 = magic64 + magicptr := uintptr(m) + x.before = magicptr + x.after = magicptr + for _, p := range testPointers() { + p := (*byte)(p) + x.i.Store(p) + if x.i.Load() != p { + t.Fatalf("x.i=%p p=%p", x.i.Load(), p) + } + } + if x.before != magicptr || x.after != magicptr { + t.Fatalf("wrong magic: %#x _ %#x != %#x _ %#x", x.before, x.after, magicptr, magicptr) + } +} + // Tests of correct behavior, with contention. // (Is the function atomic?) // @@ -780,6 +1509,16 @@ var hammer32 = map[string]func(*uint32, int){ "CompareAndSwapInt32": hammerCompareAndSwapInt32, "CompareAndSwapUint32": hammerCompareAndSwapUint32, "CompareAndSwapUintptr": hammerCompareAndSwapUintptr32, + + "SwapInt32Method": hammerSwapInt32Method, + "SwapUint32Method": hammerSwapUint32Method, + "SwapUintptrMethod": hammerSwapUintptr32Method, + "AddInt32Method": hammerAddInt32Method, + "AddUint32Method": hammerAddUint32Method, + "AddUintptrMethod": hammerAddUintptr32Method, + "CompareAndSwapInt32Method": hammerCompareAndSwapInt32Method, + "CompareAndSwapUint32Method": hammerCompareAndSwapUint32Method, + "CompareAndSwapUintptrMethod": hammerCompareAndSwapUintptr32Method, } func init() { @@ -789,6 +1528,9 @@ func init() { delete(hammer32, "SwapUintptr") delete(hammer32, "AddUintptr") delete(hammer32, "CompareAndSwapUintptr") + delete(hammer32, "SwapUintptrMethod") + delete(hammer32, "AddUintptrMethod") + delete(hammer32, "CompareAndSwapUintptrMethod") } } @@ -804,6 +1546,18 @@ func hammerSwapInt32(uaddr *uint32, count int) { } } +func hammerSwapInt32Method(uaddr *uint32, count int) { + addr := (*Int32)(unsafe.Pointer(uaddr)) + seed := int(uintptr(unsafe.Pointer(&count))) + for i := 0; i < count; i++ { + new := uint32(seed+i)<<16 | uint32(seed+i)<<16>>16 + old := uint32(addr.Swap(int32(new))) + if old>>16 != old<<16>>16 { + panic(fmt.Sprintf("SwapInt32 is not atomic: %v", old)) + } + } +} + func hammerSwapUint32(addr *uint32, count int) { seed := int(uintptr(unsafe.Pointer(&count))) for i := 0; i < count; i++ { @@ -815,6 +1569,18 @@ func hammerSwapUint32(addr *uint32, count int) { } } +func hammerSwapUint32Method(uaddr *uint32, count int) { + addr := (*Uint32)(unsafe.Pointer(uaddr)) + seed := int(uintptr(unsafe.Pointer(&count))) + for i := 0; i < count; i++ { + new := uint32(seed+i)<<16 | uint32(seed+i)<<16>>16 + old := addr.Swap(new) + if old>>16 != old<<16>>16 { + panic(fmt.Sprintf("SwapUint32 is not atomic: %v", old)) + } + } +} + func hammerSwapUintptr32(uaddr *uint32, count int) { // only safe when uintptr is 32-bit. // not called on 64-bit systems. @@ -829,6 +1595,20 @@ func hammerSwapUintptr32(uaddr *uint32, count int) { } } +func hammerSwapUintptr32Method(uaddr *uint32, count int) { + // only safe when uintptr is 32-bit. + // not called on 64-bit systems. + addr := (*Uintptr)(unsafe.Pointer(uaddr)) + seed := int(uintptr(unsafe.Pointer(&count))) + for i := 0; i < count; i++ { + new := uintptr(seed+i)<<16 | uintptr(seed+i)<<16>>16 + old := addr.Swap(new) + if old>>16 != old<<16>>16 { + panic(fmt.Sprintf("Uintptr.Swap is not atomic: %#08x", old)) + } + } +} + func hammerAddInt32(uaddr *uint32, count int) { addr := (*int32)(unsafe.Pointer(uaddr)) for i := 0; i < count; i++ { @@ -836,12 +1616,26 @@ func hammerAddInt32(uaddr *uint32, count int) { } } +func hammerAddInt32Method(uaddr *uint32, count int) { + addr := (*Int32)(unsafe.Pointer(uaddr)) + for i := 0; i < count; i++ { + addr.Add(1) + } +} + func hammerAddUint32(addr *uint32, count int) { for i := 0; i < count; i++ { AddUint32(addr, 1) } } +func hammerAddUint32Method(uaddr *uint32, count int) { + addr := (*Uint32)(unsafe.Pointer(uaddr)) + for i := 0; i < count; i++ { + addr.Add(1) + } +} + func hammerAddUintptr32(uaddr *uint32, count int) { // only safe when uintptr is 32-bit. // not called on 64-bit systems. @@ -851,6 +1645,15 @@ func hammerAddUintptr32(uaddr *uint32, count int) { } } +func hammerAddUintptr32Method(uaddr *uint32, count int) { + // only safe when uintptr is 32-bit. + // not called on 64-bit systems. + addr := (*Uintptr)(unsafe.Pointer(uaddr)) + for i := 0; i < count; i++ { + addr.Add(1) + } +} + func hammerCompareAndSwapInt32(uaddr *uint32, count int) { addr := (*int32)(unsafe.Pointer(uaddr)) for i := 0; i < count; i++ { @@ -863,6 +1666,18 @@ func hammerCompareAndSwapInt32(uaddr *uint32, count int) { } } +func hammerCompareAndSwapInt32Method(uaddr *uint32, count int) { + addr := (*Int32)(unsafe.Pointer(uaddr)) + for i := 0; i < count; i++ { + for { + v := addr.Load() + if addr.CompareAndSwap(v, v+1) { + break + } + } + } +} + func hammerCompareAndSwapUint32(addr *uint32, count int) { for i := 0; i < count; i++ { for { @@ -874,6 +1689,18 @@ func hammerCompareAndSwapUint32(addr *uint32, count int) { } } +func hammerCompareAndSwapUint32Method(uaddr *uint32, count int) { + addr := (*Uint32)(unsafe.Pointer(uaddr)) + for i := 0; i < count; i++ { + for { + v := addr.Load() + if addr.CompareAndSwap(v, v+1) { + break + } + } + } +} + func hammerCompareAndSwapUintptr32(uaddr *uint32, count int) { // only safe when uintptr is 32-bit. // not called on 64-bit systems. @@ -888,6 +1715,20 @@ func hammerCompareAndSwapUintptr32(uaddr *uint32, count int) { } } +func hammerCompareAndSwapUintptr32Method(uaddr *uint32, count int) { + // only safe when uintptr is 32-bit. + // not called on 64-bit systems. + addr := (*Uintptr)(unsafe.Pointer(uaddr)) + for i := 0; i < count; i++ { + for { + v := addr.Load() + if addr.CompareAndSwap(v, v+1) { + break + } + } + } +} + func TestHammer32(t *testing.T) { const p = 4 n := 100000 @@ -929,6 +1770,16 @@ var hammer64 = map[string]func(*uint64, int){ "CompareAndSwapInt64": hammerCompareAndSwapInt64, "CompareAndSwapUint64": hammerCompareAndSwapUint64, "CompareAndSwapUintptr": hammerCompareAndSwapUintptr64, + + "SwapInt64Method": hammerSwapInt64Method, + "SwapUint64Method": hammerSwapUint64Method, + "SwapUintptrMethod": hammerSwapUintptr64Method, + "AddInt64Method": hammerAddInt64Method, + "AddUint64Method": hammerAddUint64Method, + "AddUintptrMethod": hammerAddUintptr64Method, + "CompareAndSwapInt64Method": hammerCompareAndSwapInt64Method, + "CompareAndSwapUint64Method": hammerCompareAndSwapUint64Method, + "CompareAndSwapUintptrMethod": hammerCompareAndSwapUintptr64Method, } func init() { @@ -953,6 +1804,18 @@ func hammerSwapInt64(uaddr *uint64, count int) { } } +func hammerSwapInt64Method(uaddr *uint64, count int) { + addr := (*Int64)(unsafe.Pointer(uaddr)) + seed := int(uintptr(unsafe.Pointer(&count))) + for i := 0; i < count; i++ { + new := uint64(seed+i)<<32 | uint64(seed+i)<<32>>32 + old := uint64(addr.Swap(int64(new))) + if old>>32 != old<<32>>32 { + panic(fmt.Sprintf("SwapInt64 is not atomic: %v", old)) + } + } +} + func hammerSwapUint64(addr *uint64, count int) { seed := int(uintptr(unsafe.Pointer(&count))) for i := 0; i < count; i++ { @@ -964,6 +1827,18 @@ func hammerSwapUint64(addr *uint64, count int) { } } +func hammerSwapUint64Method(uaddr *uint64, count int) { + addr := (*Uint64)(unsafe.Pointer(uaddr)) + seed := int(uintptr(unsafe.Pointer(&count))) + for i := 0; i < count; i++ { + new := uint64(seed+i)<<32 | uint64(seed+i)<<32>>32 + old := addr.Swap(new) + if old>>32 != old<<32>>32 { + panic(fmt.Sprintf("SwapUint64 is not atomic: %v", old)) + } + } +} + const arch32 = unsafe.Sizeof(uintptr(0)) == 4 func hammerSwapUintptr64(uaddr *uint64, count int) { @@ -982,6 +1857,22 @@ func hammerSwapUintptr64(uaddr *uint64, count int) { } } +func hammerSwapUintptr64Method(uaddr *uint64, count int) { + // only safe when uintptr is 64-bit. + // not called on 32-bit systems. + if !arch32 { + addr := (*Uintptr)(unsafe.Pointer(uaddr)) + seed := int(uintptr(unsafe.Pointer(&count))) + for i := 0; i < count; i++ { + new := uintptr(seed+i)<<32 | uintptr(seed+i)<<32>>32 + old := addr.Swap(new) + if old>>32 != old<<32>>32 { + panic(fmt.Sprintf("SwapUintptr is not atomic: %v", old)) + } + } + } +} + func hammerAddInt64(uaddr *uint64, count int) { addr := (*int64)(unsafe.Pointer(uaddr)) for i := 0; i < count; i++ { @@ -989,12 +1880,26 @@ func hammerAddInt64(uaddr *uint64, count int) { } } +func hammerAddInt64Method(uaddr *uint64, count int) { + addr := (*Int64)(unsafe.Pointer(uaddr)) + for i := 0; i < count; i++ { + addr.Add(1) + } +} + func hammerAddUint64(addr *uint64, count int) { for i := 0; i < count; i++ { AddUint64(addr, 1) } } +func hammerAddUint64Method(uaddr *uint64, count int) { + addr := (*Uint64)(unsafe.Pointer(uaddr)) + for i := 0; i < count; i++ { + addr.Add(1) + } +} + func hammerAddUintptr64(uaddr *uint64, count int) { // only safe when uintptr is 64-bit. // not called on 32-bit systems. @@ -1004,6 +1909,15 @@ func hammerAddUintptr64(uaddr *uint64, count int) { } } +func hammerAddUintptr64Method(uaddr *uint64, count int) { + // only safe when uintptr is 64-bit. + // not called on 32-bit systems. + addr := (*Uintptr)(unsafe.Pointer(uaddr)) + for i := 0; i < count; i++ { + addr.Add(1) + } +} + func hammerCompareAndSwapInt64(uaddr *uint64, count int) { addr := (*int64)(unsafe.Pointer(uaddr)) for i := 0; i < count; i++ { @@ -1016,6 +1930,18 @@ func hammerCompareAndSwapInt64(uaddr *uint64, count int) { } } +func hammerCompareAndSwapInt64Method(uaddr *uint64, count int) { + addr := (*Int64)(unsafe.Pointer(uaddr)) + for i := 0; i < count; i++ { + for { + v := addr.Load() + if addr.CompareAndSwap(v, v+1) { + break + } + } + } +} + func hammerCompareAndSwapUint64(addr *uint64, count int) { for i := 0; i < count; i++ { for { @@ -1027,6 +1953,18 @@ func hammerCompareAndSwapUint64(addr *uint64, count int) { } } +func hammerCompareAndSwapUint64Method(uaddr *uint64, count int) { + addr := (*Uint64)(unsafe.Pointer(uaddr)) + for i := 0; i < count; i++ { + for { + v := addr.Load() + if addr.CompareAndSwap(v, v+1) { + break + } + } + } +} + func hammerCompareAndSwapUintptr64(uaddr *uint64, count int) { // only safe when uintptr is 64-bit. // not called on 32-bit systems. @@ -1041,6 +1979,20 @@ func hammerCompareAndSwapUintptr64(uaddr *uint64, count int) { } } +func hammerCompareAndSwapUintptr64Method(uaddr *uint64, count int) { + // only safe when uintptr is 64-bit. + // not called on 32-bit systems. + addr := (*Uintptr)(unsafe.Pointer(uaddr)) + for i := 0; i < count; i++ { + for { + v := addr.Load() + if addr.CompareAndSwap(v, v+1) { + break + } + } + } +} + func TestHammer64(t *testing.T) { if test64err != nil { t.Skipf("Skipping 64-bit tests: %v", test64err) @@ -1090,6 +2042,21 @@ func hammerStoreLoadInt32(t *testing.T, paddr unsafe.Pointer) { StoreInt32(addr, new) } +func hammerStoreLoadInt32Method(t *testing.T, paddr unsafe.Pointer) { + addr := (*int32)(paddr) + v := LoadInt32(addr) + vlo := v & ((1 << 16) - 1) + vhi := v >> 16 + if vlo != vhi { + t.Fatalf("Int32: %#x != %#x", vlo, vhi) + } + new := v + 1 + 1<<16 + if vlo == 1e4 { + new = 0 + } + StoreInt32(addr, new) +} + func hammerStoreLoadUint32(t *testing.T, paddr unsafe.Pointer) { addr := (*uint32)(paddr) v := LoadUint32(addr) @@ -1105,6 +2072,21 @@ func hammerStoreLoadUint32(t *testing.T, paddr unsafe.Pointer) { StoreUint32(addr, new) } +func hammerStoreLoadUint32Method(t *testing.T, paddr unsafe.Pointer) { + addr := (*Uint32)(paddr) + v := addr.Load() + vlo := v & ((1 << 16) - 1) + vhi := v >> 16 + if vlo != vhi { + t.Fatalf("Uint32: %#x != %#x", vlo, vhi) + } + new := v + 1 + 1<<16 + if vlo == 1e4 { + new = 0 + } + addr.Store(new) +} + func hammerStoreLoadInt64(t *testing.T, paddr unsafe.Pointer) { addr := (*int64)(paddr) v := LoadInt64(addr) @@ -1117,6 +2099,18 @@ func hammerStoreLoadInt64(t *testing.T, paddr unsafe.Pointer) { StoreInt64(addr, new) } +func hammerStoreLoadInt64Method(t *testing.T, paddr unsafe.Pointer) { + addr := (*Int64)(paddr) + v := addr.Load() + vlo := v & ((1 << 32) - 1) + vhi := v >> 32 + if vlo != vhi { + t.Fatalf("Int64: %#x != %#x", vlo, vhi) + } + new := v + 1 + 1<<32 + addr.Store(new) +} + func hammerStoreLoadUint64(t *testing.T, paddr unsafe.Pointer) { addr := (*uint64)(paddr) v := LoadUint64(addr) @@ -1129,6 +2123,18 @@ func hammerStoreLoadUint64(t *testing.T, paddr unsafe.Pointer) { StoreUint64(addr, new) } +func hammerStoreLoadUint64Method(t *testing.T, paddr unsafe.Pointer) { + addr := (*Uint64)(paddr) + v := addr.Load() + vlo := v & ((1 << 32) - 1) + vhi := v >> 32 + if vlo != vhi { + t.Fatalf("Uint64: %#x != %#x", vlo, vhi) + } + new := v + 1 + 1<<32 + addr.Store(new) +} + func hammerStoreLoadUintptr(t *testing.T, paddr unsafe.Pointer) { addr := (*uintptr)(paddr) v := LoadUintptr(addr) @@ -1155,6 +2161,33 @@ func hammerStoreLoadUintptr(t *testing.T, paddr unsafe.Pointer) { StoreUintptr(addr, new) } +//go:nocheckptr +func hammerStoreLoadUintptrMethod(t *testing.T, paddr unsafe.Pointer) { + addr := (*Uintptr)(paddr) + v := addr.Load() + new := v + if arch32 { + vlo := v & ((1 << 16) - 1) + vhi := v >> 16 + if vlo != vhi { + t.Fatalf("Uintptr: %#x != %#x", vlo, vhi) + } + new = v + 1 + 1<<16 + if vlo == 1e4 { + new = 0 + } + } else { + vlo := v & ((1 << 32) - 1) + vhi := v >> 32 + if vlo != vhi { + t.Fatalf("Uintptr: %#x != %#x", vlo, vhi) + } + inc := uint64(1 + 1<<32) + new = v + uintptr(inc) + } + addr.Store(new) +} + // This code is just testing that LoadPointer/StorePointer operate // atomically; it's not actually calculating pointers. // @@ -1185,12 +2218,47 @@ func hammerStoreLoadPointer(t *testing.T, paddr unsafe.Pointer) { StorePointer(addr, unsafe.Pointer(new)) } +// This code is just testing that LoadPointer/StorePointer operate +// atomically; it's not actually calculating pointers. +// +//go:nocheckptr +func hammerStoreLoadPointerMethod(t *testing.T, paddr unsafe.Pointer) { + addr := (*Pointer[byte])(paddr) + v := uintptr(unsafe.Pointer(addr.Load())) + new := v + if arch32 { + vlo := v & ((1 << 16) - 1) + vhi := v >> 16 + if vlo != vhi { + t.Fatalf("Pointer: %#x != %#x", vlo, vhi) + } + new = v + 1 + 1<<16 + if vlo == 1e4 { + new = 0 + } + } else { + vlo := v & ((1 << 32) - 1) + vhi := v >> 32 + if vlo != vhi { + t.Fatalf("Pointer: %#x != %#x", vlo, vhi) + } + inc := uint64(1 + 1<<32) + new = v + uintptr(inc) + } + addr.Store((*byte)(unsafe.Pointer(new))) +} + func TestHammerStoreLoad(t *testing.T) { var tests []func(*testing.T, unsafe.Pointer) tests = append(tests, hammerStoreLoadInt32, hammerStoreLoadUint32, - hammerStoreLoadUintptr, hammerStoreLoadPointer) + hammerStoreLoadUintptr, hammerStoreLoadPointer, + hammerStoreLoadInt32Method, hammerStoreLoadUint32Method, + hammerStoreLoadUintptrMethod, hammerStoreLoadPointerMethod, + ) if test64err == nil { - tests = append(tests, hammerStoreLoadInt64, hammerStoreLoadUint64) + tests = append(tests, hammerStoreLoadInt64, hammerStoreLoadUint64, + hammerStoreLoadInt64Method, hammerStoreLoadUint64Method, + ) } n := int(1e6) if testing.Short() { @@ -1430,42 +2498,99 @@ func TestUnaligned64(t *testing.T) { p := (*uint64)(unsafe.Pointer(&x[1])) // misaligned shouldPanic(t, "LoadUint64", func() { LoadUint64(p) }) + shouldPanic(t, "LoadUint64Method", func() { (*Uint64)(unsafe.Pointer(p)).Load() }) shouldPanic(t, "StoreUint64", func() { StoreUint64(p, 1) }) + shouldPanic(t, "StoreUint64Method", func() { (*Uint64)(unsafe.Pointer(p)).Store(1) }) shouldPanic(t, "CompareAndSwapUint64", func() { CompareAndSwapUint64(p, 1, 2) }) + shouldPanic(t, "CompareAndSwapUint64Method", func() { (*Uint64)(unsafe.Pointer(p)).CompareAndSwap(1, 2) }) shouldPanic(t, "AddUint64", func() { AddUint64(p, 3) }) + shouldPanic(t, "AddUint64Method", func() { (*Uint64)(unsafe.Pointer(p)).Add(3) }) +} + +func TestAutoAligned64(t *testing.T) { + var signed struct { + _ uint32 + i Int64 + } + if o := reflect.TypeOf(&signed).Elem().Field(1).Offset; o != 8 { + t.Fatalf("Int64 offset = %d, want 8", o) + } + if p := reflect.ValueOf(&signed).Elem().Field(1).Addr().Pointer(); p&7 != 0 { + t.Fatalf("Int64 pointer = %#x, want 8-aligned", p) + } + + var unsigned struct { + _ uint32 + i Uint64 + } + if o := reflect.TypeOf(&unsigned).Elem().Field(1).Offset; o != 8 { + t.Fatalf("Uint64 offset = %d, want 8", o) + } + if p := reflect.ValueOf(&unsigned).Elem().Field(1).Addr().Pointer(); p&7 != 0 { + t.Fatalf("Int64 pointer = %#x, want 8-aligned", p) + } } func TestNilDeref(t *testing.T) { funcs := [...]func(){ func() { CompareAndSwapInt32(nil, 0, 0) }, + func() { (*Int32)(nil).CompareAndSwap(0, 0) }, func() { CompareAndSwapInt64(nil, 0, 0) }, + func() { (*Int64)(nil).CompareAndSwap(0, 0) }, func() { CompareAndSwapUint32(nil, 0, 0) }, + func() { (*Uint32)(nil).CompareAndSwap(0, 0) }, func() { CompareAndSwapUint64(nil, 0, 0) }, + func() { (*Uint64)(nil).CompareAndSwap(0, 0) }, func() { CompareAndSwapUintptr(nil, 0, 0) }, + func() { (*Uintptr)(nil).CompareAndSwap(0, 0) }, func() { CompareAndSwapPointer(nil, nil, nil) }, + func() { (*Pointer[byte])(nil).CompareAndSwap(nil, nil) }, func() { SwapInt32(nil, 0) }, + func() { (*Int32)(nil).Swap(0) }, func() { SwapUint32(nil, 0) }, + func() { (*Uint32)(nil).Swap(0) }, func() { SwapInt64(nil, 0) }, + func() { (*Int64)(nil).Swap(0) }, func() { SwapUint64(nil, 0) }, + func() { (*Uint64)(nil).Swap(0) }, func() { SwapUintptr(nil, 0) }, + func() { (*Uintptr)(nil).Swap(0) }, func() { SwapPointer(nil, nil) }, + func() { (*Pointer[byte])(nil).Swap(nil) }, func() { AddInt32(nil, 0) }, + func() { (*Int32)(nil).Add(0) }, func() { AddUint32(nil, 0) }, + func() { (*Uint32)(nil).Add(0) }, func() { AddInt64(nil, 0) }, + func() { (*Int64)(nil).Add(0) }, func() { AddUint64(nil, 0) }, + func() { (*Uint64)(nil).Add(0) }, func() { AddUintptr(nil, 0) }, + func() { (*Uintptr)(nil).Add(0) }, func() { LoadInt32(nil) }, + func() { (*Int32)(nil).Load() }, func() { LoadInt64(nil) }, + func() { (*Int64)(nil).Load() }, func() { LoadUint32(nil) }, + func() { (*Uint32)(nil).Load() }, func() { LoadUint64(nil) }, + func() { (*Uint64)(nil).Load() }, func() { LoadUintptr(nil) }, + func() { (*Uintptr)(nil).Load() }, func() { LoadPointer(nil) }, + func() { (*Pointer[byte])(nil).Load() }, func() { StoreInt32(nil, 0) }, + func() { (*Int32)(nil).Store(0) }, func() { StoreInt64(nil, 0) }, + func() { (*Int64)(nil).Store(0) }, func() { StoreUint32(nil, 0) }, + func() { (*Uint32)(nil).Store(0) }, func() { StoreUint64(nil, 0) }, + func() { (*Uint64)(nil).Store(0) }, func() { StoreUintptr(nil, 0) }, + func() { (*Uintptr)(nil).Store(0) }, func() { StorePointer(nil, nil) }, + func() { (*Pointer[byte])(nil).Store(nil) }, } for _, f := range funcs { func() { diff --git a/src/sync/atomic/type.go b/src/sync/atomic/type.go new file mode 100644 index 0000000000..f7b8f5a3b7 --- /dev/null +++ b/src/sync/atomic/type.go @@ -0,0 +1,191 @@ +// Copyright 2022 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package atomic + +import "unsafe" + +// A Bool is an atomic boolean value. +// The zero value is false. +type Bool struct { + _ noCopy + v uint32 +} + +// Load atomically loads and returns the value stored in x. +func (x *Bool) Load() bool { return LoadUint32(&x.v) != 0 } + +// Store atomically stores val into x. +func (x *Bool) Store(val bool) { StoreUint32(&x.v, b32(val)) } + +// Swap atomically stores new into x and returns the previous value. +func (x *Bool) Swap(new bool) (old bool) { return SwapUint32(&x.v, b32(new)) != 0 } + +// CompareAndSwap executes the compare-and-swap operation for the boolean value x. +func (x *Bool) CompareAndSwap(old, new bool) (swapped bool) { + return CompareAndSwapUint32(&x.v, b32(old), b32(new)) +} + +// b32 returns a uint32 0 or 1 representing b. +func b32(b bool) uint32 { + if b { + return 1 + } + return 0 +} + +// A Pointer is an atomic pointer of type *T. The zero value is a nil *T. +type Pointer[T any] struct { + _ noCopy + v unsafe.Pointer +} + +// Load atomically loads and returns the value stored in x. +func (x *Pointer[T]) Load() *T { return (*T)(LoadPointer(&x.v)) } + +// Store atomically stores val into x. +func (x *Pointer[T]) Store(val *T) { StorePointer(&x.v, unsafe.Pointer(val)) } + +// Swap atomically stores new into x and returns the previous value. +func (x *Pointer[T]) Swap(new *T) (old *T) { return (*T)(SwapPointer(&x.v, unsafe.Pointer(new))) } + +// CompareAndSwap executes the compare-and-swap operation for x. +func (x *Pointer[T]) CompareAndSwap(old, new *T) (swapped bool) { + return CompareAndSwapPointer(&x.v, unsafe.Pointer(old), unsafe.Pointer(new)) +} + +// An Int32 is an atomic int32. The zero value is zero. +type Int32 struct { + _ noCopy + v int32 +} + +// Load atomically loads and returns the value stored in x. +func (x *Int32) Load() int32 { return LoadInt32(&x.v) } + +// Store atomically stores val into x. +func (x *Int32) Store(val int32) { StoreInt32(&x.v, val) } + +// Swap atomically stores new into x and returns the previous value. +func (x *Int32) Swap(new int32) (old int32) { return SwapInt32(&x.v, new) } + +// CompareAndSwap executes the compare-and-swap operation for x. +func (x *Int32) CompareAndSwap(old, new int32) (swapped bool) { + return CompareAndSwapInt32(&x.v, old, new) +} + +// Add atomically adds delta to x and returns the new value. +func (x *Int32) Add(delta int32) (new int32) { return AddInt32(&x.v, delta) } + +// An Int64 is an atomic int64. The zero value is zero. +type Int64 struct { + _ noCopy + _ align64 + v int64 +} + +// Load atomically loads and returns the value stored in x. +func (x *Int64) Load() int64 { return LoadInt64(&x.v) } + +// Store atomically stores val into x. +func (x *Int64) Store(val int64) { StoreInt64(&x.v, val) } + +// Swap atomically stores new into x and returns the previous value. +func (x *Int64) Swap(new int64) (old int64) { return SwapInt64(&x.v, new) } + +// CompareAndSwap executes the compare-and-swap operation for x. +func (x *Int64) CompareAndSwap(old, new int64) (swapped bool) { + return CompareAndSwapInt64(&x.v, old, new) +} + +// Add atomically adds delta to x and returns the new value. +func (x *Int64) Add(delta int64) (new int64) { return AddInt64(&x.v, delta) } + +// An Uint32 is an atomic uint32. The zero value is zero. +type Uint32 struct { + _ noCopy + v uint32 +} + +// Load atomically loads and returns the value stored in x. +func (x *Uint32) Load() uint32 { return LoadUint32(&x.v) } + +// Store atomically stores val into x. +func (x *Uint32) Store(val uint32) { StoreUint32(&x.v, val) } + +// Swap atomically stores new into x and returns the previous value. +func (x *Uint32) Swap(new uint32) (old uint32) { return SwapUint32(&x.v, new) } + +// CompareAndSwap executes the compare-and-swap operation for x. +func (x *Uint32) CompareAndSwap(old, new uint32) (swapped bool) { + return CompareAndSwapUint32(&x.v, old, new) +} + +// Add atomically adds delta to x and returns the new value. +func (x *Uint32) Add(delta uint32) (new uint32) { return AddUint32(&x.v, delta) } + +// An Uint64 is an atomic uint64. The zero value is zero. +type Uint64 struct { + _ noCopy + _ align64 + v uint64 +} + +// Load atomically loads and returns the value stored in x. +func (x *Uint64) Load() uint64 { return LoadUint64(&x.v) } + +// Store atomically stores val into x. +func (x *Uint64) Store(val uint64) { StoreUint64(&x.v, val) } + +// Swap atomically stores new into x and returns the previous value. +func (x *Uint64) Swap(new uint64) (old uint64) { return SwapUint64(&x.v, new) } + +// CompareAndSwap executes the compare-and-swap operation for x. +func (x *Uint64) CompareAndSwap(old, new uint64) (swapped bool) { + return CompareAndSwapUint64(&x.v, old, new) +} + +// Add atomically adds delta to x and returns the new value. +func (x *Uint64) Add(delta uint64) (new uint64) { return AddUint64(&x.v, delta) } + +// An Uintptr is an atomic uintptr. The zero value is zero. +type Uintptr struct { + _ noCopy + v uintptr +} + +// Load atomically loads and returns the value stored in x. +func (x *Uintptr) Load() uintptr { return LoadUintptr(&x.v) } + +// Store atomically stores val into x. +func (x *Uintptr) Store(val uintptr) { StoreUintptr(&x.v, val) } + +// Swap atomically stores new into x and returns the previous value. +func (x *Uintptr) Swap(new uintptr) (old uintptr) { return SwapUintptr(&x.v, new) } + +// CompareAndSwap executes the compare-and-swap operation for x. +func (x *Uintptr) CompareAndSwap(old, new uintptr) (swapped bool) { + return CompareAndSwapUintptr(&x.v, old, new) +} + +// Add atomically adds delta to x and returns the new value. +func (x *Uintptr) Add(delta uintptr) (new uintptr) { return AddUintptr(&x.v, delta) } + +// noCopy may be added to structs which must not be copied +// after the first use. +// +// See https://golang.org/issues/8005#issuecomment-190753527 +// for details. +// +// Note that it must not be embedded, due to the Lock and Unlock methods. +type noCopy struct{} + +// Lock is a no-op used by -copylocks checker from `go vet`. +func (*noCopy) Lock() {} +func (*noCopy) Unlock() {} + +// align64 may be added to structs that must be 64-bit aligned. +// This struct is recognized by a special case in the compiler +// and will not work if copied to any other package. +type align64 struct{}