diff --git a/src/runtime/mgc.go b/src/runtime/mgc.go index 2fe013294d..122b160c23 100644 --- a/src/runtime/mgc.go +++ b/src/runtime/mgc.go @@ -196,6 +196,13 @@ type gcControllerState struct { // end of the cycle. scanWork int64 + // bgScanCredit is the scan work credit accumulated by the + // concurrent background scan. This credit is accumulated by + // the background scan and stolen by mutator assists. This is + // updated atomically. Updates occur in bounded batches, since + // it is both written and read throughout the cycle. + bgScanCredit int64 + // workRatioAvg is a moving average of the scan work ratio // (scan work per byte marked). workRatioAvg float64 @@ -205,6 +212,7 @@ type gcControllerState struct { // for a new GC cycle. func (c *gcControllerState) startCycle() { c.scanWork = 0 + c.bgScanCredit = 0 // If this is the first GC cycle or we're operating on a very // small heap, fake heap_marked so it looks like next_gc is @@ -235,6 +243,13 @@ func (c *gcControllerState) endCycle() { c.workRatioAvg = workRatioWeight*workRatio + (1-workRatioWeight)*c.workRatioAvg } +// gcBgCreditSlack is the amount of scan work credit background +// scanning can accumulate locally before updating +// gcController.bgScanCredit. Lower values give mutator assists more +// accurate accounting of background scanning. Higher values reduce +// memory contention. +const gcBgCreditSlack = 2000 + // Determine whether to initiate a GC. // If the GC is already working no need to trigger another one. // This should establish a feedback loop where if the GC does not @@ -440,7 +455,7 @@ func gc(mode int) { tMark = nanotime() } var gcw gcWork - gcDrain(&gcw) + gcDrain(&gcw, gcBgCreditSlack) gcw.dispose() // Despite the barrier in gcDrain, gcDrainNs may still // be doing work at this point. This is okay because @@ -649,7 +664,7 @@ func gcMark(start_time int64) { gchelperstart() parfordo(work.markfor) var gcw gcWork - gcDrain(&gcw) + gcDrain(&gcw, -1) gcw.dispose() if work.full != 0 { @@ -831,7 +846,7 @@ func gchelper() { parfordo(work.markfor) if gcphase != _GCscan { var gcw gcWork - gcDrain(&gcw) // blocks in getfull + gcDrain(&gcw, -1) // blocks in getfull gcw.dispose() } diff --git a/src/runtime/mgcmark.go b/src/runtime/mgcmark.go index 660a7d4033..38a24ff0e8 100644 --- a/src/runtime/mgcmark.go +++ b/src/runtime/mgcmark.go @@ -354,12 +354,23 @@ func scanframeworker(frame *stkframe, unused unsafe.Pointer, gcw *gcWork) { // gcDrain scans objects in work buffers, blackening grey // objects until all work buffers have been drained. +// If flushScanCredit != -1, gcDrain flushes accumulated scan work +// credit to gcController.bgScanCredit whenever gcw's local scan work +// credit exceeds flushScanCredit. //go:nowritebarrier -func gcDrain(gcw *gcWork) { +func gcDrain(gcw *gcWork, flushScanCredit int64) { if gcphase != _GCmark && gcphase != _GCmarktermination { throw("scanblock phase incorrect") } + var lastScanFlush, nextScanFlush int64 + if flushScanCredit != -1 { + lastScanFlush = gcw.scanWork + nextScanFlush = lastScanFlush + flushScanCredit + } else { + nextScanFlush = int64(^uint64(0) >> 1) + } + for { // If another proc wants a pointer, give it some. if work.nwait > 0 && work.full == 0 { @@ -378,6 +389,20 @@ func gcDrain(gcw *gcWork) { // into an empty wbuf in scanobject so there could be // a performance hit as we keep fetching fresh wbufs. scanobject(b, 0, nil, gcw) + + // Flush background scan work credit to the global + // account if we've accumulated enough locally so + // mutator assists can draw on it. + if gcw.scanWork >= nextScanFlush { + credit := gcw.scanWork - lastScanFlush + xaddint64(&gcController.bgScanCredit, credit) + lastScanFlush = gcw.scanWork + nextScanFlush = lastScanFlush + flushScanCredit + } + } + if flushScanCredit != -1 { + credit := gcw.scanWork - lastScanFlush + xaddint64(&gcController.bgScanCredit, credit) } checknocurrentwbuf() }