diff --git a/src/reflect/all_test.go b/src/reflect/all_test.go index f8ffaae8e1..d4c3e4e588 100644 --- a/src/reflect/all_test.go +++ b/src/reflect/all_test.go @@ -5399,6 +5399,9 @@ func verifyGCBitsSlice(t *testing.T, typ Type, cap int, bits []byte) { for len(bits) > 2 && bits[len(bits)-1] == 0 { bits = bits[:len(bits)-1] } + if len(bits) == 2 && bits[0] == 0 && bits[1] == 0 { + bits = bits[:0] + } if !bytes.Equal(heapBits, bits) { t.Errorf("heapBits incorrect for make(%v, 0, %v)\nhave %v\nwant %v", typ, cap, heapBits, bits) } diff --git a/src/runtime/cgocall.go b/src/runtime/cgocall.go index 887343edd1..6dceff09ef 100644 --- a/src/runtime/cgocall.go +++ b/src/runtime/cgocall.go @@ -559,7 +559,7 @@ func cgoCheckUnknownPointer(p unsafe.Pointer, msg string) (base, i uintptr) { } n := span.elemsize for i = uintptr(0); i < n; i += sys.PtrSize { - if i >= 2*sys.PtrSize && !hbits.morePointers() { + if i != 1*sys.PtrSize && !hbits.morePointers() { // No more possible pointers. break } diff --git a/src/runtime/gcinfo_test.go b/src/runtime/gcinfo_test.go index 9a61b4f2b2..011f005403 100644 --- a/src/runtime/gcinfo_test.go +++ b/src/runtime/gcinfo_test.go @@ -66,7 +66,7 @@ func verifyGCInfo(t *testing.T, name string, p interface{}, mask0 []byte) { } func padDead(mask []byte) []byte { - // Because the dead bit isn't encoded until the third word, + // Because the dead bit isn't encoded in the second word, // and because on 32-bit systems a one-word allocation // uses a two-word block, the pointer info for a one-word // object needs to be expanded to include an extra scalar @@ -81,6 +81,9 @@ func trimDead(mask []byte) []byte { for len(mask) > 2 && mask[len(mask)-1] == typeScalar { mask = mask[:len(mask)-1] } + if len(mask) == 2 && mask[0] == typeScalar && mask[1] == typeScalar { + mask = mask[:0] + } return mask } diff --git a/src/runtime/heapdump.go b/src/runtime/heapdump.go index 4afe663418..c317b5f969 100644 --- a/src/runtime/heapdump.go +++ b/src/runtime/heapdump.go @@ -714,7 +714,7 @@ func makeheapobjbv(p uintptr, size uintptr) bitvector { i := uintptr(0) hbits := heapBitsForAddr(p) for ; i < nptr; i++ { - if i >= 2 && !hbits.morePointers() { + if i != 1 && !hbits.morePointers() { break // end of object } if hbits.isPointer() { diff --git a/src/runtime/malloc.go b/src/runtime/malloc.go index c9cc82192d..bb17919fd0 100644 --- a/src/runtime/malloc.go +++ b/src/runtime/malloc.go @@ -700,7 +700,7 @@ func mallocgc(size uintptr, typ *_type, needzero bool) unsafe.Pointer { var scanSize uintptr if noscan { - heapBitsSetTypeNoScan(uintptr(x), size) + heapBitsSetTypeNoScan(uintptr(x)) } else { // If allocating a defer+arg block, now that we've picked a malloc size // large enough to hold everything, cut the "asked for" size down to diff --git a/src/runtime/mbitmap.go b/src/runtime/mbitmap.go index 0bfb184945..8061e1d138 100644 --- a/src/runtime/mbitmap.go +++ b/src/runtime/mbitmap.go @@ -24,13 +24,15 @@ // In each 2-bit entry, the lower bit holds the same information as in the 1-bit // bitmaps: 0 means uninteresting and 1 means live pointer to be visited during GC. // The meaning of the high bit depends on the position of the word being described -// in its allocated object. In the first word, the high bit is unused. +// in its allocated object. In all words *except* the second word, the +// high bit indicates that the object is still being described. In +// these words, if a bit pair with a high bit 0 is encountered, the +// low bit can also be assumed to be 0, and the object description is +// over. This 00 is called the ``dead'' encoding: it signals that the +// rest of the words in the object are uninteresting to the garbage +// collector. +// // In the second word, the high bit is the GC ``checkmarked'' bit (see below). -// In the third and later words, the high bit indicates that the object is still -// being described. In these words, if a bit pair with a high bit 0 is encountered, -// the low bit can also be assumed to be 0, and the object description is over. -// This 00 is called the ``dead'' encoding: it signals that the rest of the words -// in the object are uninteresting to the garbage collector. // // The 2-bit entries are split when written into the byte, so that the top half // of the byte contains 4 high bits and the bottom half contains 4 low (pointer) @@ -76,7 +78,7 @@ import ( const ( bitPointer = 1 << 0 - bitMarked = 1 << 4 + bitMarked = 1 << 4 // TODO: Rename bitScan. heapBitsShift = 1 // shift offset between successive bitPointer or bitMarked entries heapBitmapScale = sys.PtrSize * (8 / 2) // number of data bytes described by one heap bitmap byte @@ -490,7 +492,7 @@ func (h heapBits) bits() uint32 { // morePointers returns true if this word and all remaining words in this object // are scalars. -// h must not describe the first or second word of the object. +// h must not describe the second word of the object. func (h heapBits) morePointers() bool { return h.bits()&bitMarked != 0 } @@ -512,22 +514,7 @@ func (h heapBits) hasPointers(size uintptr) bool { if size == sys.PtrSize { // 1-word objects are always pointers return true } - // Otherwise, at least a 2-word object, and at least 2-word aligned, - // so h.shift is either 0 or 2, so we know we can get the bits for the - // first two words out of *h.bitp. - // If either of the first two words is a pointer, not pointer free. - b := uint32(*h.bitp >> h.shift) - if b&(bitPointer|bitPointer<>h.shift)&bitMarked != 0 } // isCheckmarked reports whether the heap bits have the checkmarked bit set. @@ -720,9 +707,9 @@ func typeBitsBulkBarrier(typ *_type, p, size uintptr) { // TODO(rsc): Perhaps introduce a different heapBitsSpan type. // initSpan initializes the heap bitmap for a span. -// It clears all mark and checkmark bits. +// It clears all checkmark bits. // If this is a span of pointer-sized objects, it initializes all -// words to pointer (and there are no dead bits). +// words to pointer/scan. // Otherwise, it initializes all words to scalar/dead. func (h heapBits) initSpan(s *mspan) { size, n, total := s.layout() @@ -745,7 +732,7 @@ func (h heapBits) initSpan(s *mspan) { end := h.bitp bitp := subtractb(end, nbyte-1) for { - *bitp = bitPointerAll + *bitp = bitPointerAll | bitMarkedAll if bitp == end { break } @@ -897,6 +884,9 @@ func heapBitsSetType(x, size, dataSize uintptr, typ *_type) { if !h.isPointer() { throw("heapBitsSetType: pointer bit missing") } + if !h.morePointers() { + throw("heapBitsSetType: scan bit missing") + } } return } @@ -924,17 +914,17 @@ func heapBitsSetType(x, size, dataSize uintptr, typ *_type) { // unused second word. if gcphase == _GCoff { *h.bitp &^= (bitPointer | bitMarked | ((bitPointer | bitMarked) << heapBitsShift)) << h.shift - *h.bitp |= bitPointer << h.shift + *h.bitp |= (bitPointer | bitMarked) << h.shift } else { atomic.And8(h.bitp, ^uint8((bitPointer|bitMarked|((bitPointer|bitMarked)<= nw { goto Phase3 } @@ -1169,14 +1167,18 @@ func heapBitsSetType(x, size, dataSize uintptr, typ *_type) { // from a given span at a time, we should be able to set // these bits non-atomically. Not worth the risk right now. hb = (b & (bitPointer | bitPointer<>= 2 nb -= 2 - // Note: no bitMarker in hb because the first two words don't get markers from us. + // Note: no bitMarker for second word because that's + // the checkmark. if gcphase == _GCoff { - *hbitp &^= uint8((bitPointer | (bitPointer << heapBitsShift)) << (2 * heapBitsShift)) + *hbitp &^= uint8((bitPointer | bitMarked | (bitPointer << heapBitsShift)) << (2 * heapBitsShift)) *hbitp |= uint8(hb) } else { - atomic.And8(hbitp, ^(uint8(bitPointer|bitPointer<>(j%8))&1 != 0 { want |= bitPointer } - if i >= 2 { + if i != 1 { want |= bitMarked } else { have &^= bitMarked @@ -1366,39 +1368,11 @@ Phase4: } } -// heapBitsSetTypeNoScan marks x as noscan. For objects with 1 or 2 -// words set their bitPointers to off (0). -// All other objects have the first 3 bitPointers set to -// off (0) and the scan word in the third word -// also set to off (0). -func heapBitsSetTypeNoScan(x, size uintptr) { +// heapBitsSetTypeNoScan marks x as noscan by setting the first word +// of x in the heap bitmap to scalar/dead. +func heapBitsSetTypeNoScan(x uintptr) { h := heapBitsForAddr(uintptr(x)) - bitp := h.bitp - - if sys.PtrSize == 8 && size == sys.PtrSize { - // If this is truely noScan the tinyAlloc logic should have noticed - // and combined such objects. - throw("noscan object is too small") - } else if size%(4*sys.PtrSize) == 0 { - *bitp &^= bitPointer | bitPointer< 2*sys.PtrSize { - *bitp &^= (bitPointer | bitMarked) << (2 * heapBitsShift) - } - } else if h.shift == 2 { - *bitp &^= bitPointer<<(2*heapBitsShift) | bitPointer<<(3*heapBitsShift) - if size > 2*sys.PtrSize { - bitp = subtract1(bitp) - *bitp &^= bitPointer | bitMarked - } - } else { - throw("Type has unrecognized size") - } - } else { - throw("Type has unrecognized size") - } + *h.bitp &^= (bitPointer | bitMarked) << h.shift } var debugPtrmask struct { @@ -1804,12 +1778,6 @@ Run: dst = subtract1(dst) bits >>= 4 } - // Clear the mark bits in the first two entries. - // They are the actual mark and checkmark bits, - // not non-dead markers. It simplified the code - // above to set the marker in every bit written and - // then clear these two as a special case at the end. - *dstStart &^= bitMarked | bitMarked<= 2*sys.PtrSize && !hbits.morePointers() { + if i != 1*sys.PtrSize && !hbits.morePointers() { mask = mask[:i/sys.PtrSize] break } diff --git a/src/runtime/mgcmark.go b/src/runtime/mgcmark.go index 8c8ce67fbf..af3205ab23 100644 --- a/src/runtime/mgcmark.go +++ b/src/runtime/mgcmark.go @@ -1132,7 +1132,7 @@ func scanobject(b uintptr, gcw *gcWork) { // in the type bit for the one word. The only one-word objects // are pointers, or else they'd be merged with other non-pointer // data into larger allocations. - if i >= 2*sys.PtrSize && !hbits.morePointers() { + if i != 1*sys.PtrSize && !hbits.morePointers() { break // no more pointers in this object } if !hbits.isPointer() {