mirror
/
go
mirror of https://github.com/golang/go.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

runtime/internal/atomic: add 32-bit And/Or 

These will be used in a following CL to perform larger bit clear and bit
set than And8/Or8.

Change-Id: I60f7b1099e29b69eb64add77564faee862880a8d
Reviewed-on: https://go-review.googlesource.com/c/go/+/260977
Run-TryBot: Michael Pratt <mpratt@google.com>
TryBot-Result: Go Bot <gobot@golang.org>
Reviewed-by: Cherry Zhang <cherryyz@google.com>
Trust: Michael Pratt <mpratt@google.com>
This commit is contained in:
Michael Pratt 2020-10-08 14:38:39 -04:00
parent 7e01b3b387
commit ad61343f88
20 changed files with 400 additions and 8 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

16
src/runtime/internal/atomic/asm_386.s
View File

@ -243,3 +243,19 @@ TEXT ·Store8(SB), NOSPLIT, $0-5
MOVB val+4(FP), AX
XCHGB AX, 0(BX)
RET
// func Or(addr *uint32, v uint32)
TEXT ·Or(SB), NOSPLIT, $0-8
MOVL ptr+0(FP), AX
MOVL val+4(FP), BX
LOCK
ORL BX, (AX)
RET
// func And(addr *uint32, v uint32)
TEXT ·And(SB), NOSPLIT, $0-8
MOVL ptr+0(FP), AX
MOVL val+4(FP), BX
LOCK
ANDL BX, (AX)
RET

16
src/runtime/internal/atomic/asm_amd64.s
View File

@ -169,3 +169,19 @@ TEXT runtime∕internal∕atomic·And8(SB), NOSPLIT, $0-9
LOCK
ANDB BX, (AX)
RET
// func Or(addr *uint32, v uint32)
TEXT runtime∕internal∕atomic·Or(SB), NOSPLIT, $0-12
MOVQ ptr+0(FP), AX
MOVL val+8(FP), BX
LOCK
ORL BX, (AX)
RET
// func And(addr *uint32, v uint32)
TEXT runtime∕internal∕atomic·And(SB), NOSPLIT, $0-12
MOVQ ptr+0(FP), AX
MOVL val+8(FP), BX
LOCK
ANDL BX, (AX)
RET

26
src/runtime/internal/atomic/asm_mips64x.s
View File

@ -243,3 +243,29 @@ TEXT ·And8(SB), NOSPLIT, $0-9
BEQ R4, -4(PC)
SYNC
RET
// func Or(addr *uint32, v uint32)
TEXT ·Or(SB), NOSPLIT, $0-12
MOVV ptr+0(FP), R1
MOVW val+8(FP), R2
SYNC
LL (R1), R3
OR R2, R3
SC R3, (R1)
BEQ R3, -4(PC)
SYNC
RET
// func And(addr *uint32, v uint32)
TEXT ·And(SB), NOSPLIT, $0-12
MOVV ptr+0(FP), R1
MOVW val+8(FP), R2
SYNC
LL (R1), R3
AND R2, R3
SC R3, (R1)
BEQ R3, -4(PC)
SYNC
RET

26
src/runtime/internal/atomic/asm_mipsx.s
View File

@ -172,3 +172,29 @@ try_and8:
BEQ R4, try_and8
SYNC
RET
// func Or(addr *uint32, v uint32)
TEXT ·Or(SB), NOSPLIT, $0-8
MOVW ptr+0(FP), R1
MOVW val+4(FP), R2
SYNC
LL (R1), R3
OR R2, R3
SC R3, (R1)
BEQ R3, -4(PC)
SYNC
RET
// func And(addr *uint32, v uint32)
TEXT ·And(SB), NOSPLIT, $0-8
MOVW ptr+0(FP), R1
MOVW val+4(FP), R2
SYNC
LL (R1), R3
AND R2, R3
SC R3, (R1)
BEQ R3, -4(PC)
SYNC
RET

30
src/runtime/internal/atomic/asm_ppc64x.s
View File

@ -222,8 +222,32 @@ TEXT runtime∕internal∕atomic·And8(SB), NOSPLIT, $0-9
MOVBZ val+8(FP), R4
LWSYNC
again:
LBAR (R3),R6
AND R4,R6
STBCCC R6,(R3)
LBAR (R3), R6
AND R4, R6
STBCCC R6, (R3)
BNE again
RET
// func Or(addr *uint32, v uint32)
TEXT runtime∕internal∕atomic·Or(SB), NOSPLIT, $0-12
MOVD ptr+0(FP), R3
MOVW val+8(FP), R4
LWSYNC
again:
LWAR (R3), R6
OR R4, R6
STWCCC R6, (R3)
BNE again
RET
// func And(addr *uint32, v uint32)
TEXT runtime∕internal∕atomic·And(SB), NOSPLIT, $0-12
MOVD ptr+0(FP), R3
MOVW val+8(FP), R4
LWSYNC
again:
LWAR (R3),R6
AND R4, R6
STWCCC R6, (R3)
BNE again
RET

22
src/runtime/internal/atomic/asm_s390x.s
View File

@ -174,8 +174,8 @@ TEXT ·Xchguintptr(SB), NOSPLIT, $0-24
// func Or8(addr *uint8, v uint8)
TEXT ·Or8(SB), NOSPLIT, $0-9
MOVD ptr+0(FP), R3
MOVBZ val+8(FP), R4
MOVD ptr+0(FP), R3
MOVBZ val+8(FP), R4
// We don't have atomic operations that work on individual bytes so we
// need to align addr down to a word boundary and create a mask
// containing v to OR with the entire word atomically.
@ -188,8 +188,8 @@ TEXT ·Or8(SB), NOSPLIT, $0-9
// func And8(addr *uint8, v uint8)
TEXT ·And8(SB), NOSPLIT, $0-9
MOVD ptr+0(FP), R3
MOVBZ val+8(FP), R4
MOVD ptr+0(FP), R3
MOVBZ val+8(FP), R4
// We don't have atomic operations that work on individual bytes so we
// need to align addr down to a word boundary and create a mask
// containing v to AND with the entire word atomically.
@ -200,3 +200,17 @@ TEXT ·And8(SB), NOSPLIT, $0-9
RLL R5, R4, R4 // R4 = rotl(R4, R5)
LAN R4, R6, 0(R3) // R6 = *R3; *R3 &= R4; (atomic)
RET
// func Or(addr *uint32, v uint32)
TEXT ·Or(SB), NOSPLIT, $0-12
MOVD ptr+0(FP), R3
MOVW val+8(FP), R4
LAO R4, R6, 0(R3) // R6 = *R3; *R3 |= R4; (atomic)
RET
// func And(addr *uint32, v uint32)
TEXT ·And(SB), NOSPLIT, $0-12
MOVD ptr+0(FP), R3
MOVW val+8(FP), R4
LAN R4, R6, 0(R3) // R6 = *R3; *R3 &= R4; (atomic)
RET

6
src/runtime/internal/atomic/atomic_386.go
View File

@ -69,6 +69,12 @@ func And8(ptr *uint8, val uint8)
//go:noescape
func Or8(ptr *uint8, val uint8)
//go:noescape
func And(ptr *uint32, val uint32)
//go:noescape
func Or(ptr *uint32, val uint32)
// NOTE: Do not add atomicxor8 (XOR is not idempotent).
//go:noescape

6
src/runtime/internal/atomic/atomic_amd64.go
View File

@ -77,6 +77,12 @@ func And8(ptr *uint8, val uint8)
//go:noescape
func Or8(ptr *uint8, val uint8)
//go:noescape
func And(ptr *uint32, val uint32)
//go:noescape
func Or(ptr *uint32, val uint32)
// NOTE: Do not add atomicxor8 (XOR is not idempotent).
//go:noescape

20
src/runtime/internal/atomic/atomic_arm.go
View File

@ -182,6 +182,26 @@ func And8(addr *uint8, v uint8) {
}
}
//go:nosplit
func Or(addr *uint32, v uint32) {
for {
old := *addr
if Cas(addr, old, old|v) {
return
}
}
}
//go:nosplit
func And(addr *uint32, v uint32) {
for {
old := *addr
if Cas(addr, old, old&v) {
return
}
}
}
//go:nosplit
func armcas(ptr *uint32, old, new uint32) bool

6
src/runtime/internal/atomic/atomic_arm64.go
View File

@ -53,6 +53,12 @@ func Or8(ptr *uint8, val uint8)
//go:noescape
func And8(ptr *uint8, val uint8)
//go:noescape
func And(ptr *uint32, val uint32)
//go:noescape
func Or(ptr *uint32, val uint32)
//go:noescape
func Cas64(ptr *uint64, old, new uint64) bool

19
src/runtime/internal/atomic/atomic_arm64.s
View File

@ -164,3 +164,22 @@ TEXT ·Or8(SB), NOSPLIT, $0-9
CBNZ R3, -3(PC)
RET
// func And(addr *uint32, v uint32)
TEXT ·And(SB), NOSPLIT, $0-12
MOVD ptr+0(FP), R0
MOVW val+8(FP), R1
LDAXRW (R0), R2
AND R1, R2
STLXRW R2, (R0), R3
CBNZ R3, -3(PC)
RET
// func Or(addr *uint32, v uint32)
TEXT ·Or(SB), NOSPLIT, $0-12
MOVD ptr+0(FP), R0
MOVW val+8(FP), R1
LDAXRW (R0), R2
ORR R1, R2
STLXRW R2, (R0), R3
CBNZ R3, -3(PC)
RET

6
src/runtime/internal/atomic/atomic_mips64x.go
View File

@ -55,6 +55,12 @@ func Or8(ptr *uint8, val uint8)
// NOTE: Do not add atomicxor8 (XOR is not idempotent).
//go:noescape
func And(ptr *uint32, val uint32)
//go:noescape
func Or(ptr *uint32, val uint32)
//go:noescape
func Cas64(ptr *uint64, old, new uint64) bool

6
src/runtime/internal/atomic/atomic_mipsx.go
View File

@ -141,6 +141,12 @@ func And8(ptr *uint8, val uint8)
//go:noescape
func Or8(ptr *uint8, val uint8)
//go:noescape
func And(ptr *uint32, val uint32)
//go:noescape
func Or(ptr *uint32, val uint32)
//go:noescape
func Store(ptr *uint32, val uint32)

6
src/runtime/internal/atomic/atomic_ppc64x.go
View File

@ -55,6 +55,12 @@ func Or8(ptr *uint8, val uint8)
// NOTE: Do not add atomicxor8 (XOR is not idempotent).
//go:noescape
func And(ptr *uint32, val uint32)
//go:noescape
func Or(ptr *uint32, val uint32)
//go:noescape
func Cas64(ptr *uint64, old, new uint64) bool

6
src/runtime/internal/atomic/atomic_riscv64.go
View File

@ -51,6 +51,12 @@ func Or8(ptr *uint8, val uint8)
//go:noescape
func And8(ptr *uint8, val uint8)
//go:noescape
func And(ptr *uint32, val uint32)
//go:noescape
func Or(ptr *uint32, val uint32)
//go:noescape
func Cas64(ptr *uint64, old, new uint64) bool

14
src/runtime/internal/atomic/atomic_riscv64.s
View File

@ -242,3 +242,17 @@ TEXT ·Or8(SB), NOSPLIT, $0-9
SLL A2, A1
AMOORW A1, (A0), ZERO
RET
// func And(ptr *uint32, val uint32)
TEXT ·And(SB), NOSPLIT, $0-12
MOV ptr+0(FP), A0
MOVW val+8(FP), A1
AMOANDW A1, (A0), ZERO
RET
// func Or(ptr *uint32, val uint32)
TEXT ·Or(SB), NOSPLIT, $0-12
MOV ptr+0(FP), A0
MOVW val+8(FP), A1
AMOORW A1, (A0), ZERO
RET

6
src/runtime/internal/atomic/atomic_s390x.go
View File

@ -91,6 +91,12 @@ func Or8(ptr *uint8, val uint8)
// NOTE: Do not add atomicxor8 (XOR is not idempotent).
//go:noescape
func And(ptr *uint32, val uint32)
//go:noescape
func Or(ptr *uint32, val uint32)
//go:noescape
func Xadd(ptr *uint32, delta int32) uint32

119
src/runtime/internal/atomic/atomic_test.go
View File

@ -150,6 +150,45 @@ func TestAnd8(t *testing.T) {
}
}
func TestAnd(t *testing.T) {
// Basic sanity check.
x := uint32(0xffffffff)
for i := uint32(0); i < 32; i++ {
atomic.And(&x, ^(1 << i))
if r := uint32(0xffffffff) << (i + 1); x != r {
t.Fatalf("clearing bit %#x: want %#x, got %#x", uint32(1<<i), r, x)
}
}
// Set every bit in array to 1.
a := make([]uint32, 1<<12)
for i := range a {
a[i] = 0xffffffff
}
// Clear array bit-by-bit in different goroutines.
done := make(chan bool)
for i := 0; i < 32; i++ {
m := ^uint32(1 << i)
go func() {
for i := range a {
atomic.And(&a[i], m)
}
done <- true
}()
}
for i := 0; i < 32; i++ {
<-done
}
// Check that the array has been totally cleared.
for i, v := range a {
if v != 0 {
t.Fatalf("a[%v] not cleared: want %#x, got %#x", i, uint32(0), v)
}
}
}
func TestOr8(t *testing.T) {
// Basic sanity check.
x := uint8(0)
@ -186,7 +225,43 @@ func TestOr8(t *testing.T) {
}
}
func TestBitwiseContended(t *testing.T) {
func TestOr(t *testing.T) {
// Basic sanity check.
x := uint32(0)
for i := uint32(0); i < 32; i++ {
atomic.Or(&x, 1<<i)
if r := (uint32(1) << (i + 1)) - 1; x != r {
t.Fatalf("setting bit %#x: want %#x, got %#x", uint32(1)<<i, r, x)
}
}
// Start with every bit in array set to 0.
a := make([]uint32, 1<<12)
// Set every bit in array bit-by-bit in different goroutines.
done := make(chan bool)
for i := 0; i < 32; i++ {
m := uint32(1 << i)
go func() {
for i := range a {
atomic.Or(&a[i], m)
}
done <- true
}()
}
for i := 0; i < 32; i++ {
<-done
}
// Check that the array has been totally set.
for i, v := range a {
if v != 0xffffffff {
t.Fatalf("a[%v] not fully set: want %#x, got %#x", i, uint32(0xffffffff), v)
}
}
}
func TestBitwiseContended8(t *testing.T) {
// Start with every bit in array set to 0.
a := make([]uint8, 16)
@ -228,6 +303,48 @@ func TestBitwiseContended(t *testing.T) {
}
}
func TestBitwiseContended(t *testing.T) {
// Start with every bit in array set to 0.
a := make([]uint32, 16)
// Iterations to try.
N := 1 << 16
if testing.Short() {
N = 1 << 10
}
// Set and then clear every bit in the array bit-by-bit in different goroutines.
done := make(chan bool)
for i := 0; i < 32; i++ {
m := uint32(1 << i)
go func() {
for n := 0; n < N; n++ {
for i := range a {
atomic.Or(&a[i], m)
if atomic.Load(&a[i])&m != m {
t.Errorf("a[%v] bit %#x not set", i, m)
}
atomic.And(&a[i], ^m)
if atomic.Load(&a[i])&m != 0 {
t.Errorf("a[%v] bit %#x not clear", i, m)
}
}
}
done <- true
}()
}
for i := 0; i < 32; i++ {
<-done
}
// Check that the array has been totally cleared.
for i, v := range a {
if v != 0 {
t.Fatalf("a[%v] not cleared: want %#x, got %#x", i, uint32(0), v)
}
}
}
func TestStorepNoWB(t *testing.T) {
var p [2]*int
for i := range p {

12
src/runtime/internal/atomic/atomic_wasm.go
View File

@ -131,6 +131,18 @@ func Or8(ptr *uint8, val uint8) {
// NOTE: Do not add atomicxor8 (XOR is not idempotent).
//go:nosplit
//go:noinline
func And(ptr *uint32, val uint32) {
*ptr = *ptr & val
}
//go:nosplit
//go:noinline
func Or(ptr *uint32, val uint32) {
*ptr = *ptr | val
}
//go:nosplit
//go:noinline
func Cas64(ptr *uint64, old, new uint64) bool {

40
src/runtime/internal/atomic/bench_test.go
View File

@ -51,6 +51,14 @@ func BenchmarkAnd8(b *testing.B) {
}
}
func BenchmarkAnd(b *testing.B) {
var x [128]uint32 // give x its own cache line
sink = &x
for i := 0; i < b.N; i++ {
atomic.And(&x[63], uint32(i))
}
}
func BenchmarkAnd8Parallel(b *testing.B) {
var x [512]uint8 // give byte its own cache line
sink = &x
@ -63,6 +71,18 @@ func BenchmarkAnd8Parallel(b *testing.B) {
})
}
func BenchmarkAndParallel(b *testing.B) {
var x [128]uint32 // give x its own cache line
sink = &x
b.RunParallel(func(pb *testing.PB) {
i := uint32(0)
for pb.Next() {
atomic.And(&x[63], i)
i++
}
})
}
func BenchmarkOr8(b *testing.B) {
var x [512]uint8 // give byte its own cache line
sink = &x
@ -71,6 +91,14 @@ func BenchmarkOr8(b *testing.B) {
}
}
func BenchmarkOr(b *testing.B) {
var x [128]uint32 // give x its own cache line
sink = &x
for i := 0; i < b.N; i++ {
atomic.Or(&x[63], uint32(i))
}
}
func BenchmarkOr8Parallel(b *testing.B) {
var x [512]uint8 // give byte its own cache line
sink = &x
@ -83,6 +111,18 @@ func BenchmarkOr8Parallel(b *testing.B) {
})
}
func BenchmarkOrParallel(b *testing.B) {
var x [128]uint32 // give x its own cache line
sink = &x
b.RunParallel(func(pb *testing.PB) {
i := uint32(0)
for pb.Next() {
atomic.Or(&x[63], i)
i++
}
})
}
func BenchmarkXadd(b *testing.B) {
var x uint32
ptr := &x