mirror of https://github.com/golang/go.git
runtime: use vDSO for clock_gettime on linux/arm64
Use the __vdso_clock_gettime fast path via the vDSO on linux/arm64 to speed up nanotime and walltime. This results in the following performance improvement for time.Now on Cavium ThunderX: name old time/op new time/op delta TimeNow 442ns ± 0% 163ns ± 0% -63.16% (p=0.000 n=10+10) And benchmarks on VDSO BenchmarkClockVDSOAndFallbackPaths/vDSO 10000000 166 ns/op BenchmarkClockVDSOAndFallbackPaths/Fallback 3000000 456 ns/op Change-Id: I326118c6dff865eaa0569fc45d1fc1ff95cb74f6 Reviewed-on: https://go-review.googlesource.com/99855 Run-TryBot: Tobias Klauser <tobias.klauser@gmail.com> Run-TryBot: Ian Lance Taylor <iant@golang.org> TryBot-Result: Gobot Gobot <gobot@golang.org> Reviewed-by: Ian Lance Taylor <iant@golang.org>
This commit is contained in:
Meng Zhuo
Ian Lance Taylor
parent
b63b0f2b75
commit
ea59ebd338
|
|
@ -3,7 +3,7 @@
|
|||
// license that can be found in the LICENSE file.
|
||||
|
||||
// +build linux
|
||||
// +build !386,!amd64,!arm
|
||||
// +build !386,!amd64,!arm,!arm64
|
||||
|
||||
package runtime
|
||||
|
||||
|
|
|
|||
|
|
@ -12,6 +12,9 @@
|
|||
|
||||
#define AT_FDCWD -100
|
||||
|
||||
#define CLOCK_REALTIME 0
|
||||
#define CLOCK_MONOTONIC 1
|
||||
|
||||
#define SYS_exit 93
|
||||
#define SYS_read 63
|
||||
#define SYS_write 64
|
||||
|
|
@ -180,23 +183,87 @@ TEXT runtime·mincore(SB),NOSPLIT|NOFRAME,$0-28
|
|||
|
||||
// func walltime() (sec int64, nsec int32)
|
||||
TEXT runtime·walltime(SB),NOSPLIT,$24-12
|
||||
MOVW $0, R0 // CLOCK_REALTIME
|
||||
MOVD RSP, R20 // R20 is unchanged by C code
|
||||
MOVD RSP, R1
|
||||
|
||||
MOVD g_m(g), R21 // R21 = m
|
||||
|
||||
// Set vdsoPC and vdsoSP for SIGPROF traceback.
|
||||
MOVD LR, m_vdsoPC(R21)
|
||||
MOVD R20, m_vdsoSP(R21)
|
||||
|
||||
MOVD m_curg(R21), R0
|
||||
CMP g, R0
|
||||
BNE noswitch
|
||||
|
||||
MOVD m_g0(R21), R3
|
||||
MOVD (g_sched+gobuf_sp)(R3), R1 // Set RSP to g0 stack
|
||||
|
||||
noswitch:
|
||||
SUB $16, R1
|
||||
BIC $15, R1 // Align for C code
|
||||
MOVD R1, RSP
|
||||
|
||||
MOVW $CLOCK_REALTIME, R0
|
||||
MOVD runtime·vdsoClockgettimeSym(SB), R2
|
||||
CBZ R2, fallback
|
||||
BL (R2)
|
||||
B finish
|
||||
|
||||
fallback:
|
||||
MOVD $SYS_clock_gettime, R8
|
||||
SVC
|
||||
|
||||
finish:
|
||||
MOVD 0(RSP), R3 // sec
|
||||
MOVD 8(RSP), R5 // nsec
|
||||
|
||||
MOVD R20, RSP // restore SP
|
||||
MOVD $0, m_vdsoSP(R21) // clear vdsoSP
|
||||
|
||||
MOVD R3, sec+0(FP)
|
||||
MOVW R5, nsec+8(FP)
|
||||
RET
|
||||
|
||||
TEXT runtime·nanotime(SB),NOSPLIT,$24-8
|
||||
MOVW $1, R0 // CLOCK_MONOTONIC
|
||||
MOVD RSP, R20 // R20 is unchanged by C code
|
||||
MOVD RSP, R1
|
||||
|
||||
MOVD g_m(g), R21 // R21 = m
|
||||
|
||||
// Set vdsoPC and vdsoSP for SIGPROF traceback.
|
||||
MOVD LR, m_vdsoPC(R21)
|
||||
MOVD R20, m_vdsoSP(R21)
|
||||
|
||||
MOVD m_curg(R21), R0
|
||||
CMP g, R0
|
||||
BNE noswitch
|
||||
|
||||
MOVD m_g0(R21), R3
|
||||
MOVD (g_sched+gobuf_sp)(R3), R1 // Set RSP to g0 stack
|
||||
|
||||
noswitch:
|
||||
SUB $16, R1
|
||||
BIC $15, R1
|
||||
MOVD R1, RSP
|
||||
|
||||
MOVW $CLOCK_MONOTONIC, R0
|
||||
MOVD runtime·vdsoClockgettimeSym(SB), R2
|
||||
CBZ R2, fallback
|
||||
BL (R2)
|
||||
B finish
|
||||
|
||||
fallback:
|
||||
MOVD $SYS_clock_gettime, R8
|
||||
SVC
|
||||
|
||||
finish:
|
||||
MOVD 0(RSP), R3 // sec
|
||||
MOVD 8(RSP), R5 // nsec
|
||||
|
||||
MOVD R20, RSP // restore SP
|
||||
MOVD $0, m_vdsoSP(R21) // clear vdsoSP
|
||||
|
||||
// sec is in R3, nsec in R5
|
||||
// return nsec in R3
|
||||
MOVD $1000000000, R4
|
||||
|
|
|
|||
|
|
@ -3,7 +3,7 @@
|
|||
// license that can be found in the LICENSE file.
|
||||
|
||||
// +build linux
|
||||
// +build amd64
|
||||
// +build amd64 arm64
|
||||
|
||||
package runtime
|
||||
|
||||
|
|
|
|||
|
|
@ -2,7 +2,7 @@
|
|||
// Use of this source code is governed by a BSD-style
|
||||
// license that can be found in the LICENSE file.
|
||||
|
||||
// +build linux,!386,!amd64,!arm !linux
|
||||
// +build linux,!386,!amd64,!arm,!arm64 !linux
|
||||
|
||||
package runtime
|
||||
|
||||
|
|
|
|||
|
|
@ -3,7 +3,7 @@
|
|||
// license that can be found in the LICENSE file.
|
||||
|
||||
// +build linux
|
||||
// +build 386 amd64 arm
|
||||
// +build 386 amd64 arm arm64
|
||||
|
||||
package runtime
|
||||
|
||||
|
|
@ -96,8 +96,6 @@ type vdsoInfo struct {
|
|||
verdef *elfVerdef
|
||||
}
|
||||
|
||||
var linux26 = vdsoVersionKey{"LINUX_2.6", 0x3ae75f6}
|
||||
|
||||
// see vdso_linux_*.go for vdsoSymbolKeys[] and vdso*Sym vars
|
||||
|
||||
func vdsoInitFromSysinfoEhdr(info *vdsoInfo, hdr *elfEhdr) {
|
||||
|
|
@ -220,7 +218,6 @@ func vdsoParseSymbols(info *vdsoInfo, version int32) {
|
|||
if k.name != gostringnocopy(&info.symstrings[sym.st_name]) {
|
||||
return false
|
||||
}
|
||||
|
||||
// Check symbol version.
|
||||
if info.versym != nil && version != 0 && int32(info.versym[symIndex]&0x7fff) != version {
|
||||
return false
|
||||
|
|
@ -276,7 +273,7 @@ func vdsoauxv(tag, val uintptr) {
|
|||
// when passed to the three functions below.
|
||||
info1 := (*vdsoInfo)(noescape(unsafe.Pointer(&info)))
|
||||
vdsoInitFromSysinfoEhdr(info1, (*elfEhdr)(unsafe.Pointer(val)))
|
||||
vdsoParseSymbols(info1, vdsoFindVersion(info1, &linux26))
|
||||
vdsoParseSymbols(info1, vdsoFindVersion(info1, &vdsoLinuxVersion))
|
||||
}
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -11,11 +11,11 @@ const (
|
|||
vdsoArrayMax = 1<<31 - 1
|
||||
)
|
||||
|
||||
var vdsoLinuxVersion = vdsoVersionKey{"LINUX_2.6", 0x3ae75f6}
|
||||
|
||||
var vdsoSymbolKeys = []vdsoSymbolKey{
|
||||
{"__vdso_clock_gettime", 0xd35ec75, 0x6e43a318, &vdsoClockgettimeSym},
|
||||
}
|
||||
|
||||
// initialize to fall back to syscall
|
||||
var (
|
||||
vdsoClockgettimeSym uintptr = 0
|
||||
)
|
||||
var vdsoClockgettimeSym uintptr = 0
|
||||
|
|
|
|||
|
|
@ -10,6 +10,8 @@ const (
|
|||
vdsoArrayMax = 1<<50 - 1
|
||||
)
|
||||
|
||||
var vdsoLinuxVersion = vdsoVersionKey{"LINUX_2.6", 0x3ae75f6}
|
||||
|
||||
var vdsoSymbolKeys = []vdsoSymbolKey{
|
||||
{"__vdso_gettimeofday", 0x315ca59, 0xb01bca00, &vdsoGettimeofdaySym},
|
||||
{"__vdso_clock_gettime", 0xd35ec75, 0x6e43a318, &vdsoClockgettimeSym},
|
||||
|
|
|
|||
|
|
@ -11,6 +11,8 @@ const (
|
|||
vdsoArrayMax = 1<<31 - 1
|
||||
)
|
||||
|
||||
var vdsoLinuxVersion = vdsoVersionKey{"LINUX_2.6", 0x3ae75f6}
|
||||
|
||||
var vdsoSymbolKeys = []vdsoSymbolKey{
|
||||
{"__vdso_clock_gettime", 0xd35ec75, 0x6e43a318, &vdsoClockgettimeSym},
|
||||
}
|
||||
|
|
|
|||
|
|
@ -0,0 +1,21 @@
|
|||
// Copyright 2018 The Go Authors. All rights reserved.
|
||||
// Use of this source code is governed by a BSD-style
|
||||
// license that can be found in the LICENSE file.
|
||||
|
||||
package runtime
|
||||
|
||||
const (
|
||||
// vdsoArrayMax is the byte-size of a maximally sized array on this architecture.
|
||||
// See cmd/compile/internal/arm64/galign.go arch.MAXWIDTH initialization.
|
||||
vdsoArrayMax = 1<<50 - 1
|
||||
)
|
||||
|
||||
// key and version at man 7 vdso : aarch64
|
||||
var vdsoLinuxVersion = vdsoVersionKey{"LINUX_2.6.39", 0x75fcb89}
|
||||
|
||||
var vdsoSymbolKeys = []vdsoSymbolKey{
|
||||
{"__kernel_clock_gettime", 0xd35ec75, 0x6e43a318, &vdsoClockgettimeSym},
|
||||
}
|
||||
|
||||
// initialize to fall back to syscall
|
||||
var vdsoClockgettimeSym uintptr = 0
|
||||
|
|
@ -3,7 +3,7 @@
|
|||
// license that can be found in the LICENSE file.
|
||||
|
||||
// +build linux
|
||||
// +build 386 amd64 arm
|
||||
// +build 386 amd64 arm arm64
|
||||
|
||||
package runtime_test
|
||||
|
||||
|
|
|
|||
Loading…
Reference in New Issue