diff --git a/src/pkg/runtime/malloc.c b/src/pkg/runtime/malloc.c index f4143669e7..ca8ebf5dd6 100644 --- a/src/pkg/runtime/malloc.c +++ b/src/pkg/runtime/malloc.c @@ -21,7 +21,10 @@ MHeap runtime·mheap; #pragma dataflag NOPTR MStats runtime·memstats; +static Type* notype; + void runtime·cmallocgc(uintptr size, Type *typ, uint32 flag, void **ret); +void runtime·gc_notype_ptr(Eface*); void* runtime·mallocgc(uintptr size, Type *typ, uint32 flag) @@ -31,6 +34,8 @@ runtime·mallocgc(uintptr size, Type *typ, uint32 flag) // Call into the Go version of mallocgc. // TODO: maybe someday we can get rid of this. It is // probably the only location where we run Go code on the M stack. + if((flag&FlagNoScan) == 0 && typ == nil) + typ = notype; runtime·cmallocgc(size, typ, flag, &ret); return ret; } @@ -124,6 +129,7 @@ runtime·mallocinit(void) uintptr limit; uint64 i; bool reserved; + Eface notype_eface; p = nil; p_size = 0; @@ -251,6 +257,9 @@ runtime·mallocinit(void) // Initialize the rest of the allocator. runtime·MHeap_Init(&runtime·mheap); g->m->mcache = runtime·allocmcache(); + + runtime·gc_notype_ptr(¬ype_eface); + notype = notype_eface.type; } void* diff --git a/src/pkg/runtime/malloc.go b/src/pkg/runtime/malloc.go index 84c69abde7..152b3b6b68 100644 --- a/src/pkg/runtime/malloc.go +++ b/src/pkg/runtime/malloc.go @@ -225,66 +225,56 @@ func gomallocgc(size uintptr, typ *_type, flags int) unsafe.Pointer { *xbits |= (bitsPointer << 2) << shift goto marked } - if typ != nil && (uintptr(typ.gc[0])|uintptr(typ.gc[1])) != 0 && uintptr(typ.size) > ptrSize { - if typ.kind&kindGCProg != 0 { - nptr := (uintptr(typ.size) + ptrSize - 1) / ptrSize - masksize := nptr - if masksize%2 != 0 { - masksize *= 2 // repeated - } - masksize = masksize * pointersPerByte / 8 // 4 bits per word - masksize++ // unroll flag in the beginning - if masksize > maxGCMask && typ.gc[1] != 0 { - // If the mask is too large, unroll the program directly - // into the GC bitmap. It's 7 times slower than copying - // from the pre-unrolled mask, but saves 1/16 of type size - // memory for the mask. - mp := acquirem() - mp.ptrarg[0] = x - mp.ptrarg[1] = unsafe.Pointer(typ) - mp.scalararg[0] = uint(size) - mp.scalararg[1] = uint(size0) - onM(&unrollgcproginplace_m) - releasem(mp) - goto marked - } - ptrmask = (*uint8)(unsafe.Pointer(uintptr(typ.gc[0]))) - // Check whether the program is already unrolled. - if uintptr(goatomicloadp(unsafe.Pointer(ptrmask)))&0xff == 0 { - mp := acquirem() - mp.ptrarg[0] = unsafe.Pointer(typ) - onM(&unrollgcprog_m) - releasem(mp) - } - ptrmask = (*uint8)(add(unsafe.Pointer(ptrmask), 1)) // skip the unroll flag byte - } else { - ptrmask = (*uint8)(unsafe.Pointer(&typ.gc[0])) // embed mask + if typ.kind&kindGCProg != 0 { + nptr := (uintptr(typ.size) + ptrSize - 1) / ptrSize + masksize := nptr + if masksize%2 != 0 { + masksize *= 2 // repeated } - if size == 2*ptrSize { - xbitsb := (*uint8)(add(unsafe.Pointer(xbits), shift/8)) - *xbitsb = *ptrmask | bitBoundary + masksize = masksize * pointersPerByte / 8 // 4 bits per word + masksize++ // unroll flag in the beginning + if masksize > maxGCMask && typ.gc[1] != 0 { + // If the mask is too large, unroll the program directly + // into the GC bitmap. It's 7 times slower than copying + // from the pre-unrolled mask, but saves 1/16 of type size + // memory for the mask. + mp := acquirem() + mp.ptrarg[0] = x + mp.ptrarg[1] = unsafe.Pointer(typ) + mp.scalararg[0] = uint(size) + mp.scalararg[1] = uint(size0) + onM(&unrollgcproginplace_m) + releasem(mp) goto marked } - te = uintptr(typ.size) / ptrSize - // If the type occupies odd number of words, its mask is repeated. - if te%2 == 0 { - te /= 2 + ptrmask = (*uint8)(unsafe.Pointer(uintptr(typ.gc[0]))) + // Check whether the program is already unrolled. + if uintptr(goatomicloadp(unsafe.Pointer(ptrmask)))&0xff == 0 { + mp := acquirem() + mp.ptrarg[0] = unsafe.Pointer(typ) + onM(&unrollgcprog_m) + releasem(mp) } + ptrmask = (*uint8)(add(unsafe.Pointer(ptrmask), 1)) // skip the unroll flag byte + } else { + ptrmask = (*uint8)(unsafe.Pointer(&typ.gc[0])) // embed mask } if size == 2*ptrSize { xbitsb := (*uint8)(add(unsafe.Pointer(xbits), shift/8)) - *xbitsb = (bitsPointer << 2) | (bitsPointer << 6) | bitBoundary + *xbitsb = *ptrmask | bitBoundary goto marked } + te = uintptr(typ.size) / ptrSize + // If the type occupies odd number of words, its mask is repeated. + if te%2 == 0 { + te /= 2 + } // Copy pointer bitmask into the bitmap. for i := uintptr(0); i < size0; i += 2 * ptrSize { - v := uint8((bitsPointer << 2) | (bitsPointer << 6)) - if ptrmask != nil { - v = *(*uint8)(add(unsafe.Pointer(ptrmask), ti)) - ti++ - if ti == te { - ti = 0 - } + v := *(*uint8)(add(unsafe.Pointer(ptrmask), ti)) + ti++ + if ti == te { + ti = 0 } if i == 0 { v |= bitBoundary diff --git a/src/pkg/runtime/mgc0.go b/src/pkg/runtime/mgc0.go index 624485d18b..496725f6a7 100644 --- a/src/pkg/runtime/mgc0.go +++ b/src/pkg/runtime/mgc0.go @@ -19,6 +19,15 @@ func gc_itab_ptr(ret *interface{}) { *ret = (*itab)(nil) } +// Type used for "conservative" allocations in C code. +type notype [8]*byte + +// Called from C. Returns the Go type used for C allocations w/o type. +func gc_notype_ptr(ret *interface{}) { + var x notype + *ret = x +} + func timenow() (sec int64, nsec int32) func gc_unixnanotime(now *int64) {