diff --git a/src/runtime/mbitmap.go b/src/runtime/mbitmap.go index db43e482d2..2d2abca643 100644 --- a/src/runtime/mbitmap.go +++ b/src/runtime/mbitmap.go @@ -387,6 +387,18 @@ func (h heapBits) initSpan(size, n, total uintptr) { throw("initSpan: unaligned length") } nbyte := total / heapBitmapScale + if ptrSize == 8 && size == ptrSize { + end := h.bitp + bitp := subtractb(end, nbyte-1) + for { + *bitp = bitPointerAll + if bitp == end { + break + } + bitp = add1(bitp) + } + return + } memclr(unsafe.Pointer(subtractb(h.bitp, nbyte-1)), nbyte) } @@ -443,33 +455,34 @@ func heapBitsSweepSpan(base, size, n uintptr, f func(uintptr)) { switch { default: throw("heapBitsSweepSpan") - case size == ptrSize: + case ptrSize == 8 && size == ptrSize: // Consider mark bits in all four 2-bit entries of each bitmap byte. bitp := h.bitp for i := uintptr(0); i < n; i += 4 { x := uint32(*bitp) + // Note that unlike the other size cases, we leave the pointer bits set here. + // These are initialized during initSpan when the span is created and left + // in place the whole time the span is used for pointer-sized objects. + // That lets heapBitsSetType avoid an atomic update to set the pointer bit + // during allocation. if x&bitMarked != 0 { x &^= bitMarked } else { - x &^= bitPointer f(base + i*ptrSize) } if x&(bitMarked<