diff --git a/src/runtime/mbitmap.go b/src/runtime/mbitmap.go index 61e1254bed..234aa9509a 100644 --- a/src/runtime/mbitmap.go +++ b/src/runtime/mbitmap.go @@ -8,7 +8,8 @@ // // Stack frames and global variables in the data and bss sections are described // by 1-bit bitmaps in which 0 means uninteresting and 1 means live pointer -// to be visited during GC. +// to be visited during GC. The bits in each byte are consumed starting with +// the low bit: 1<<0, 1<<1, and so on. // // Heap bitmap // @@ -19,8 +20,6 @@ // That is, the byte at address start-1 holds the 2-bit entries for the four words // start through start+3*ptrSize, the byte at start-2 holds the entries for // start+4*ptrSize through start+7*ptrSize, and so on. -// In each byte, the low 2 bits describe the first word, the next 2 bits describe -// the next word, and so on. // // In each 2-bit entry, the lower bit holds the same information as in the 1-bit // bitmaps: 0 means uninteresting and 1 means live pointer to be visited during GC. @@ -33,6 +32,11 @@ // This 00 is called the ``dead'' encoding: it signals that the rest of the words // in the object are uninteresting to the garbage collector. // +// The 2-bit entries are split when written into the byte, so that the top half +// of the byte contains 4 mark bits and the bottom half contains 4 pointer bits. +// This form allows a copy from the 1-bit to the 4-bit form to keep the +// pointer bits contiguous, instead of having to space them out. +// // The code makes use of the fact that the zero value for a heap bitmap // has no live pointer bit set and is (depending on position), not marked, // not checkmarked, and is the dead encoding. @@ -65,11 +69,15 @@ package runtime import "unsafe" const ( - bitPointer = 1 - bitMarked = 2 + bitPointer = 1 << 0 + bitMarked = 1 << 4 - heapBitsWidth = 2 // heap bitmap bits to describe one pointer - heapBitmapScale = ptrSize * (8 / heapBitsWidth) // number of data bytes described by one heap bitmap byte + heapBitsShift = 1 // shift offset between successive bitPointer or bitMarked entries + heapBitmapScale = ptrSize * (8 / 2) // number of data bytes described by one heap bitmap byte + + // all mark/pointer bits in a byte + bitMarkedAll = bitMarked | bitMarked<> h.shift) - if b&(bitPointer|bitPointer<>(heapBitsWidth+h.shift))&bitMarked != 0 + return (*h.bitp>>(heapBitsShift+h.shift))&bitMarked != 0 } // setCheckmarked sets the checkmarked bit. @@ -307,7 +315,7 @@ func (h heapBits) setCheckmarked(size uintptr) { atomicor8(h.bitp, bitPointer< 2*ptrSize { x = 0 @@ -454,10 +462,10 @@ func heapBitsSweepSpan(base, size, n uintptr, f func(uintptr)) { } bitp = subtractb(bitp, step) x = uint32(*bitp) - if x&(bitMarked<<4) != 0 { - x &^= bitMarked << 4 + if x&(bitMarked<<(2*heapBitsShift)) != 0 { + x &^= bitMarked << (2 * heapBitsShift) } else { - x &^= 0xf0 + x &^= (bitMarked|bitPointer)<<(2*heapBitsShift) | (bitMarked|bitPointer)<<(3*heapBitsShift) f(base + (i+1)*size) if size > 2*ptrSize { *subtractb(bitp, 1) = 0 @@ -554,12 +562,12 @@ func heapBitsSetType(x, size, dataSize uintptr, typ *_type) { if size == 2*ptrSize { if typ.size == ptrSize { // 2-element slice of pointer. - atomicor8(h.bitp, (bitPointer|bitPointer<= nw { goto Phase3 } @@ -724,7 +732,7 @@ func heapBitsSetType(x, size, dataSize uintptr, typ *_type) { b >>= 4 nb -= 4 - case ptrSize == 8 && h.shift == 4: + case ptrSize == 8 && h.shift == 2: // Ptrmask and heap bitmap are misaligned. // The bits for the first two words are in a byte shared with another object // and must be updated atomically. @@ -735,7 +743,7 @@ func heapBitsSetType(x, size, dataSize uintptr, typ *_type) { // not with its mark bit. Since there is only one allocation // from a given span at a time, we should be able to set // these bits non-atomically. Not worth the risk right now. - hb = (b&1)<<4 | (b&2)<<(4+heapBitsWidth-1) // bits being prepared for *h.bitp + hb = (b & 3) << (2 * heapBitsShift) b >>= 2 nb -= 2 // Note: no bitMarker in hb because the first two words don't get markers from us. @@ -763,8 +771,8 @@ func heapBitsSetType(x, size, dataSize uintptr, typ *_type) { // if w+4 >= nw, then b has only nw-w bits, // but we'll stop at the break and then truncate // appropriately in Phase 3. - hb = b&1 | (b&2)<<(heapBitsWidth-1) | (b&4)<<(2*heapBitsWidth-2) | (b&8)<<(3*heapBitsWidth-3) - hb |= bitMarked | bitMarked<= nw { break } @@ -803,8 +811,8 @@ func heapBitsSetType(x, size, dataSize uintptr, typ *_type) { } // Emit bitmap byte. - hb = b&1 | (b&2)<<(heapBitsWidth-1) | (b&4)<<(2*heapBitsWidth-2) | (b&8)<<(3*heapBitsWidth-3) - hb |= bitMarked | bitMarked<= nw { break } @@ -822,15 +830,16 @@ Phase3: // then we must write a ``dead'' entry to the next bitmap byte. if frag := (nw - w) % 4; frag != 0 { // Data ends at least one word early. - hb &= 1<<(heapBitsWidth*frag) - 1 + mask := uintptr(1)<= size { break } - have = (*h.bitp >> h.shift) & 3 + have = (*h.bitp >> h.shift) & (bitPointer | bitMarked) if i == dataSize/ptrSize || i/ndata == count-1 && j >= nptr { want = 0 // dead marker } else { @@ -883,8 +892,9 @@ Phase4: println("mismatch writing bits for", *typ._string, "x", dataSize/typ.size) print("typ.size=", typ.size, " typ.ptrdata=", typ.ptrdata, " dataSize=", dataSize, " size=", size, "\n") print("w=", w, " nw=", nw, " b=", hex(b), " nb=", nb, " hb=", hex(hb), "\n") - h = heapBitsForAddr(x) - print("initial bits h.bitp=", h.bitp, " h.shift=", h.shift, "\n") + h0 := heapBitsForAddr(x) + print("initial bits h0.bitp=", h0.bitp, " h0.shift=", h0.shift, "\n") + print("current bits h.bitp=", h.bitp, " h.shift=", h.shift, " *h.bitp=", hex(*h.bitp), "\n") print("ptrmask=", ptrmask, " p=", p, " endp=", endp, " endnb=", endnb, " pbits=", hex(pbits), " b=", hex(b), " nb=", nb, "\n") println("at word", i, "offset", i*ptrSize, "have", have, "want", want) throw("bad heapBitsSetType")