mirror
/
go
mirror of https://github.com/golang/go.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

runtime: do not collect GC roots explicitly 

Currently we collect (add) all roots into a global array in a single-threaded GC phase.
This hinders parallelism.
With this change we just kick off parallel for for number_of_goroutines+5 iterations.
Then parallel for callback decides whether it needs to scan stack of a goroutine
scan data segment, scan finalizers, etc. This eliminates the single-threaded phase entirely.
This requires to store all goroutines in an array instead of a linked list
(to allow direct indexing).
This CL also removes DebugScan functionality. It is broken because it uses
unbounded stack, so it can not run on g0. When it was working, I've found
it helpless for debugging issues because the two algorithms are too different now.
This change would require updating the DebugScan, so it's simpler to just delete it.

With 8 threads this change reduces GC pause by ~6%, while keeping cputime roughly the same.

garbage-8
allocated                 2987886      2989221      +0.04%
allocs                      62885        62887      +0.00%
cputime                  21286000     21272000      -0.07%
gc-pause-one             26633247     24885421      -6.56%
gc-pause-total             873570       811264      -7.13%
rss                     242089984    242515968      +0.18%
sys-gc                   13934336     13869056      -0.47%
sys-heap                205062144    205062144      +0.00%
sys-other                12628288     12628288      +0.00%
sys-stack                11534336     11927552      +3.41%
sys-total               243159104    243487040      +0.13%
time                      2809477      2740795      -2.44%

R=golang-codereviews, rsc
CC=cshapiro, golang-codereviews, khr
https://golang.org/cl/46860043
This commit is contained in:
Dmitriy Vyukov 2014-01-21 13:06:57 +04:00
parent 0e027fca42
commit cb133c6607
4 changed files with 217 additions and 334 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

465
src/pkg/runtime/mgc0.c
View File

@ -17,7 +17,6 @@
enum {
Debug = 0,
DebugMark = 0, // run second pass to check mark
CollectStats = 0,
ScanStackByFrames = 1,
IgnorePreciseGC = 0,
@ -40,6 +39,13 @@ enum {
BitsPointer = 1,
BitsIface = 2,
BitsEface = 3,
RootData = 0,
RootBss = 1,
RootFinalizers = 2,
RootSpanTypes = 3,
RootFlushCaches = 4,
RootCount = 5,
};
static struct
@ -190,7 +196,10 @@ static Workbuf* getfull(Workbuf*);
static void putempty(Workbuf*);
static Workbuf* handoff(Workbuf*);
static void gchelperstart(void);
static void scanstack(G* gp, void *scanbuf);
static void addfinroots(void *wbufp, void *v);
static void flushallmcaches(void);
static void scanframe(Stkframe *frame, void *wbufp);
static void addstackroots(G *gp, Workbuf **wbufp);
static struct {
uint64 full; // lock-free list of full blocks
@ -200,7 +209,6 @@ static struct {
int64 tstart;
volatile uint32 nwait;
volatile uint32 ndone;
volatile uint32 debugmarkdone;
Note alldone;
ParFor *markfor;
ParFor *sweepfor;
@ -208,16 +216,11 @@ static struct {
Lock;
byte *chunk;
uintptr nchunk;
Obj *roots;
uint32 nroot;
uint32 rootcap;
} work;
enum {
GC_DEFAULT_PTR = GC_NUM_INSTR,
GC_CHAN,
GC_G_PTR,
GC_NUM_INSTR2
};
@ -636,9 +639,6 @@ static uintptr defaultProg[2] = {PtrSize, GC_DEFAULT_PTR};
// Hchan program
static uintptr chanProg[2] = {0, GC_CHAN};
// G* program
static uintptr gptrProg[2] = {0, GC_G_PTR};
// Local variables of a program fragment or loop
typedef struct Frame Frame;
struct Frame {
@ -707,15 +707,11 @@ checkptr(void *obj, uintptr objti)
// a work list in the Workbuf* structures and loops in the main function
// body. Keeping an explicit work list is easier on the stack allocator and
// more efficient.
//
// wbuf: current work buffer
// wp: storage for next queued pointer (write pointer)
// nobj: number of queued objects
static void
scanblock(Workbuf *wbuf, Obj *wp, uintptr nobj, bool keepworking)
scanblock(Workbuf *wbuf, bool keepworking)
{
byte *b, *arena_start, *arena_used;
uintptr n, i, end_b, elemsize, size, ti, objti, count, type;
uintptr n, i, end_b, elemsize, size, ti, objti, count, type, nobj;
uintptr *pc, precise_type, nominal_size;
uintptr *chan_ret, chancap;
void *obj;
@ -728,6 +724,7 @@ scanblock(Workbuf *wbuf, Obj *wp, uintptr nobj, bool keepworking)
Iface *iface;
Hchan *chan;
ChanType *chantype;
Obj *wp;
if(sizeof(Workbuf) % PageSize != 0)
runtime·throw("scanblock: size of Workbuf is suboptimal");
@ -741,6 +738,14 @@ scanblock(Workbuf *wbuf, Obj *wp, uintptr nobj, bool keepworking)
precise_type = false;
nominal_size = 0;
if(wbuf) {
nobj = wbuf->nobj;
wp = &wbuf->obj[nobj];
} else {
nobj = 0;
wp = nil;
}
// Initialize sbuf
scanbuffers = &bufferList[m->helpgc];
@ -1075,11 +1080,6 @@ scanblock(Workbuf *wbuf, Obj *wp, uintptr nobj, bool keepworking)
pc = chan_ret;
continue;
case GC_G_PTR:
obj = (void*)stack_top.b;
scanstack(obj, &sbuf);
goto next_block;
default:
runtime·throw("scanblock: invalid GC instruction");
return;
@ -1124,82 +1124,6 @@ scanblock(Workbuf *wbuf, Obj *wp, uintptr nobj, bool keepworking)
}
}
// debug_scanblock is the debug copy of scanblock.
// it is simpler, slower, single-threaded, recursive,
// and uses bitSpecial as the mark bit.
static void
debug_scanblock(byte *b, uintptr n)
{
byte *obj, *p;
void **vp;
uintptr size, *bitp, bits, shift, i, xbits, off;
MSpan *s;
if(!DebugMark)
runtime·throw("debug_scanblock without DebugMark");
if((intptr)n < 0) {
runtime·printf("debug_scanblock %p %D\n", b, (int64)n);
runtime·throw("debug_scanblock");
}
// Align b to a word boundary.
off = (uintptr)b & (PtrSize-1);
if(off != 0) {
b += PtrSize - off;
n -= PtrSize - off;
}
vp = (void**)b;
n /= PtrSize;
for(i=0; i<n; i++) {
obj = (byte*)vp[i];
// Words outside the arena cannot be pointers.
if((byte*)obj < runtime·mheap.arena_start || (byte*)obj >= runtime·mheap.arena_used)
continue;
// Round down to word boundary.
obj = (void*)((uintptr)obj & ~((uintptr)PtrSize-1));
// Consult span table to find beginning.
s = runtime·MHeap_LookupMaybe(&runtime·mheap, obj);
if(s == nil)
continue;
p = (byte*)((uintptr)s->start<<PageShift);
size = s->elemsize;
if(s->sizeclass == 0) {
obj = p;
} else {
int32 i = ((byte*)obj - p)/size;
obj = p+i*size;
}
// Now that we know the object header, reload bits.
off = (uintptr*)obj - (uintptr*)runtime·mheap.arena_start;
bitp = (uintptr*)runtime·mheap.arena_start - off/wordsPerBitmapWord - 1;
shift = off % wordsPerBitmapWord;
xbits = *bitp;
bits = xbits >> shift;
// Now we have bits, bitp, and shift correct for
// obj pointing at the base of the object.
// If not allocated or already marked, done.
if((bits & bitAllocated) == 0 || (bits & bitSpecial) != 0) // NOTE: bitSpecial not bitMarked
continue;
*bitp |= bitSpecial<<shift;
if(!(bits & bitMarked))
runtime·printf("found unmarked block %p in %p\n", obj, vp+i);
// If object has no pointers, don't need to scan further.
if((bits & bitScan) == 0)
continue;
debug_scanblock(obj, size);
}
}
// Append obj to the work buffer.
// _wbuf, _wp, _nobj are input/output parameters and are specifying the work buffer.
static void
@ -1255,19 +1179,92 @@ enqueue(Obj obj, Workbuf **_wbuf, Obj **_wp, uintptr *_nobj)
*_nobj = nobj;
}
static void
enqueue1(Workbuf **wbufp, Obj obj)
{
Workbuf *wbuf;
wbuf = *wbufp;
if(wbuf->nobj >= nelem(wbuf->obj))
*wbufp = wbuf = getempty(wbuf);
wbuf->obj[wbuf->nobj++] = obj;
}
static void
markroot(ParFor *desc, uint32 i)
{
Obj *wp;
Workbuf *wbuf;
uintptr nobj;
FinBlock *fb;
MSpan **allspans, *s;
uint32 spanidx;
G *gp;
USED(&desc);
wp = nil;
wbuf = nil;
nobj = 0;
enqueue(work.roots[i], &wbuf, &wp, &nobj);
scanblock(wbuf, wp, nobj, false);
wbuf = getempty(nil);
switch(i) {
case RootData:
enqueue1(&wbuf, (Obj){data, edata - data, (uintptr)gcdata});
break;
case RootBss:
enqueue1(&wbuf, (Obj){bss, ebss - bss, (uintptr)gcbss});
break;
case RootFinalizers:
for(fb=allfin; fb; fb=fb->alllink)
enqueue1(&wbuf, (Obj){(byte*)fb->fin, fb->cnt*sizeof(fb->fin[0]), 0});
break;
case RootSpanTypes:
// mark span types and MSpan.specials (to walk spans only once)
allspans = runtime·mheap.allspans;
for(spanidx=0; spanidx<runtime·mheap.nspan; spanidx++) {
Special *sp;
SpecialFinalizer *spf;
s = allspans[spanidx];
if(s->state != MSpanInUse)
continue;
// The garbage collector ignores type pointers stored in MSpan.types:
// - Compiler-generated types are stored outside of heap.
// - The reflect package has runtime-generated types cached in its data structures.
// The garbage collector relies on finding the references via that cache.
if(s->types.compression == MTypes_Words || s->types.compression == MTypes_Bytes)
markonly((byte*)s->types.data);
for(sp = s->specials; sp != nil; sp = sp->next) {
if(sp->kind != KindSpecialFinalizer)
continue;
// don't mark finalized object, but scan it so we
// retain everything it points to.
spf = (SpecialFinalizer*)sp;
enqueue1(&wbuf, (Obj){(void*)((s->start << PageShift) + spf->offset), s->elemsize, 0});
enqueue1(&wbuf, (Obj){(void*)&spf->fn, PtrSize, 0});
enqueue1(&wbuf, (Obj){(void*)&spf->fint, PtrSize, 0});
enqueue1(&wbuf, (Obj){(void*)&spf->ot, PtrSize, 0});
}
}
break;
case RootFlushCaches:
flushallmcaches();
break;
default:
// the rest is scanning goroutine stacks
if(i - RootCount >= runtime·allglen)
runtime·throw("markroot: bad index");
gp = runtime·allg[i - RootCount];
// remember when we've first observed the G blocked
// needed only to output in traceback
if((gp->status == Gwaiting || gp->status == Gsyscall) && gp->waitsince == 0)
gp->waitsince = work.tstart;
addstackroots(gp, &wbuf);
break;
}
if(wbuf)
scanblock(wbuf, false);
}
// Get an empty work buffer off the work.empty list,
@ -1364,30 +1361,6 @@ handoff(Workbuf *b)
return b1;
}
static void
addroot(Obj obj)
{
uint32 cap;
Obj *new;
if(work.nroot >= work.rootcap) {
cap = PageSize/sizeof(Obj);
if(cap < 2*work.rootcap)
cap = 2*work.rootcap;
new = (Obj*)runtime·SysAlloc(cap*sizeof(Obj), &mstats.gc_sys);
if(new == nil)
runtime·throw("runtime: cannot allocate memory");
if(work.roots != nil) {
runtime·memmove(new, work.roots, work.rootcap*sizeof(Obj));
runtime·SysFree(work.roots, work.rootcap*sizeof(Obj), &mstats.gc_sys);
}
work.roots = new;
work.rootcap = cap;
}
work.roots[work.nroot] = obj;
work.nroot++;
}
extern byte pclntab[]; // base for f->ptrsoff
typedef struct BitVector BitVector;
@ -1427,7 +1400,7 @@ stackmapdata(StackMap *stackmap, int32 n)
// Scans an interface data value when the interface type indicates
// that it is a pointer.
static void
scaninterfacedata(uintptr bits, byte *scanp, bool afterprologue, Scanbuf *sbuf)
scaninterfacedata(uintptr bits, byte *scanp, bool afterprologue, void *wbufp)
{
Itab *tab;
Type *type;
@ -1443,14 +1416,12 @@ scaninterfacedata(uintptr bits, byte *scanp, bool afterprologue, Scanbuf *sbuf)
return;
}
}
*sbuf->obj.pos++ = (Obj){scanp+PtrSize, PtrSize, 0};
if(sbuf->obj.pos == sbuf->obj.end)
flushobjbuf(sbuf);
enqueue1(wbufp, (Obj){scanp+PtrSize, PtrSize, 0});
}
// Starting from scanp, scans words corresponding to set bits.
static void
scanbitvector(byte *scanp, BitVector *bv, bool afterprologue, Scanbuf *sbuf)
scanbitvector(byte *scanp, BitVector *bv, bool afterprologue, void *wbufp)
{
uintptr word, bits;
uint32 *wordp;
@ -1467,12 +1438,10 @@ scanbitvector(byte *scanp, BitVector *bv, bool afterprologue, Scanbuf *sbuf)
for(; i > 0; i--) {
bits = word & 3;
if(bits != BitsNoPointer && *(void**)scanp != nil)
if(bits == BitsPointer) {
*sbuf->obj.pos++ = (Obj){scanp, PtrSize, 0};
if(sbuf->obj.pos == sbuf->obj.end)
flushobjbuf(sbuf);
} else
scaninterfacedata(bits, scanp, afterprologue, sbuf);
if(bits == BitsPointer)
enqueue1(wbufp, (Obj){scanp, PtrSize, 0});
else
scaninterfacedata(bits, scanp, afterprologue, wbufp);
word >>= BitsPerPointer;
scanp += PtrSize;
}
@ -1481,10 +1450,9 @@ scanbitvector(byte *scanp, BitVector *bv, bool afterprologue, Scanbuf *sbuf)
// Scan a stack frame: local variables and function arguments/results.
static void
scanframe(Stkframe *frame, void *arg)
scanframe(Stkframe *frame, void *wbufp)
{
Func *f;
Scanbuf *sbuf;
StackMap *stackmap;
BitVector *bv;
uintptr size;
@ -1504,7 +1472,6 @@ scanframe(Stkframe *frame, void *arg)
pcdata = 0;
}
sbuf = arg;
// Scan local variables if stack frame has been allocated.
// Use pointer information if known.
afterprologue = (frame->varp > (byte*)frame->sp);
@ -1513,25 +1480,23 @@ scanframe(Stkframe *frame, void *arg)
if(stackmap == nil) {
// No locals information, scan everything.
size = frame->varp - (byte*)frame->sp;
*sbuf->obj.pos++ = (Obj){frame->varp - size, size, 0};
if(sbuf->obj.pos == sbuf->obj.end)
flushobjbuf(sbuf);
enqueue1(wbufp, (Obj){frame->varp - size, size, 0});
} else if(stackmap->n < 0) {
// Locals size information, scan just the locals.
size = -stackmap->n;
*sbuf->obj.pos++ = (Obj){frame->varp - size, size, 0};
if(sbuf->obj.pos == sbuf->obj.end)
flushobjbuf(sbuf); } else if(stackmap->n > 0) {
enqueue1(wbufp, (Obj){frame->varp - size, size, 0});
} else if(stackmap->n > 0) {
// Locals bitmap information, scan just the pointers in
// locals.
if(pcdata < 0 || pcdata >= stackmap->n) {
// don't know where we are
runtime·printf("pcdata is %d and %d stack map entries\n", pcdata, stackmap->n);
runtime·throw("addframeroots: bad symbol table");
runtime·printf("pcdata is %d and %d stack map entries for %s (targetpc=%p)\n",
pcdata, stackmap->n, runtime·funcname(f), targetpc);
runtime·throw("scanframe: bad symbol table");
}
bv = stackmapdata(stackmap, pcdata);
size = (bv->n * PtrSize) / BitsPerPointer;
scanbitvector(frame->varp - size, bv, afterprologue, sbuf);
scanbitvector(frame->varp - size, bv, afterprologue, wbufp);
}
}
@ -1540,22 +1505,13 @@ scanframe(Stkframe *frame, void *arg)
stackmap = runtime·funcdata(f, FUNCDATA_ArgsPointerMaps);
if(stackmap != nil) {
bv = stackmapdata(stackmap, pcdata);
scanbitvector(frame->argp, bv, false, sbuf);
} else {
*sbuf->obj.pos++ = (Obj){frame->argp, frame->arglen, 0};
if(sbuf->obj.pos == sbuf->obj.end)
flushobjbuf(sbuf);
}
scanbitvector(frame->argp, bv, false, wbufp);
} else
enqueue1(wbufp, (Obj){frame->argp, frame->arglen, 0});
}
static void
scanstack(G* gp, void *scanbuf)
{
runtime·gentraceback(~(uintptr)0, ~(uintptr)0, 0, gp, 0, nil, 0x7fffffff, scanframe, scanbuf, false);
}
static void
addstackroots(G *gp)
addstackroots(G *gp, Workbuf **wbufp)
{
M *mp;
int32 n;
@ -1564,6 +1520,20 @@ addstackroots(G *gp)
void *base;
uintptr size;
switch(gp->status){
default:
runtime·printf("unexpected G.status %d (goroutine %p %D)\n", gp->status, gp, gp->goid);
runtime·throw("mark - bad status");
case Gdead:
return;
case Grunning:
runtime·throw("mark - world not stopped");
case Grunnable:
case Gsyscall:
case Gwaiting:
break;
}
if(gp == g)
runtime·throw("can't scan our own stack");
if((mp = gp->m) != nil && mp->helpgc)
@ -1585,13 +1555,13 @@ addstackroots(G *gp)
guard = gp->stackguard;
// For function about to start, context argument is a root too.
if(gp->sched.ctxt != 0 && runtime·mlookup(gp->sched.ctxt, &base, &size, nil))
addroot((Obj){base, size, 0});
enqueue1(wbufp, (Obj){base, size, 0});
}
if(ScanStackByFrames) {
USED(sp);
USED(stk);
USED(guard);
addroot((Obj){(byte*)gp, PtrSize, (uintptr)gptrProg});
runtime·gentraceback(~(uintptr)0, ~(uintptr)0, 0, gp, 0, nil, 0x7fffffff, scanframe, wbufp, false);
} else {
n = 0;
while(stk) {
@ -1599,7 +1569,7 @@ addstackroots(G *gp)
runtime·printf("scanstack inconsistent: g%D#%d sp=%p not in [%p,%p]\n", gp->goid, n, sp, guard-StackGuard, stk);
runtime·throw("scanstack");
}
addroot((Obj){(byte*)sp, (uintptr)stk - sp, (uintptr)defaultProg | PRECISE | LOOP});
enqueue1(wbufp, (Obj){(byte*)sp, (uintptr)stk - sp, (uintptr)defaultProg | PRECISE | LOOP});
sp = stk->gobuf.sp;
guard = stk->stackguard;
stk = (Stktop*)stk->stackbase;
@ -1608,116 +1578,6 @@ addstackroots(G *gp)
}
}
static void
addroots(void)
{
G *gp;
FinBlock *fb;
MSpan *s, **allspans;
uint32 spanidx;
Special *sp;
SpecialFinalizer *spf;
work.nroot = 0;
// data & bss
// TODO(atom): load balancing
addroot((Obj){data, edata - data, (uintptr)gcdata});
addroot((Obj){bss, ebss - bss, (uintptr)gcbss});
// MSpan.types
allspans = runtime·mheap.allspans;
for(spanidx=0; spanidx<runtime·mheap.nspan; spanidx++) {
s = allspans[spanidx];
if(s->state == MSpanInUse) {
// The garbage collector ignores type pointers stored in MSpan.types:
// - Compiler-generated types are stored outside of heap.
// - The reflect package has runtime-generated types cached in its data structures.
// The garbage collector relies on finding the references via that cache.
switch(s->types.compression) {
case MTypes_Empty:
case MTypes_Single:
break;
case MTypes_Words:
case MTypes_Bytes:
markonly((byte*)s->types.data);
break;
}
}
}
// MSpan.specials
allspans = runtime·mheap.allspans;
for(spanidx=0; spanidx<runtime·mheap.nspan; spanidx++) {
s = allspans[spanidx];
if(s->state != MSpanInUse)
continue;
for(sp = s->specials; sp != nil; sp = sp->next) {
switch(sp->kind) {
case KindSpecialFinalizer:
spf = (SpecialFinalizer*)sp;
// don't mark finalized object, but scan it so we
// retain everything it points to.
addroot((Obj){(void*)((s->start << PageShift) + spf->offset), s->elemsize, 0});
addroot((Obj){(void*)&spf->fn, PtrSize, 0});
addroot((Obj){(void*)&spf->fint, PtrSize, 0});
addroot((Obj){(void*)&spf->ot, PtrSize, 0});
break;
case KindSpecialProfile:
break;
}
}
}
// stacks
for(gp=runtime·allg; gp!=nil; gp=gp->alllink) {
switch(gp->status){
default:
runtime·printf("unexpected G.status %d\n", gp->status);
runtime·throw("mark - bad status");
case Gdead:
break;
case Grunning:
runtime·throw("mark - world not stopped");
case Grunnable:
case Gsyscall:
case Gwaiting:
addstackroots(gp);
break;
}
// remember when we've first observed the G blocked
// needed only to output in traceback
if((gp->status == Gwaiting || gp->status == Gsyscall) && gp->waitsince == 0)
gp->waitsince = work.tstart;
}
for(fb=allfin; fb; fb=fb->alllink)
addroot((Obj){(byte*)fb->fin, fb->cnt*sizeof(fb->fin[0]), 0});
}
static void
addfreelists(void)
{
int32 i;
P *p, **pp;
MCache *c;
MLink *m;
// Mark objects in the MCache of each P so we don't collect them.
for(pp=runtime·allp; p=*pp; pp++) {
c = p->mcache;
if(c==nil)
continue;
for(i = 0; i < NumSizeClasses; i++) {
for(m = c->list[i].list; m != nil; m = m->next) {
markonly(m);
}
}
}
// Note: the sweeper will mark objects in each span's freelist.
}
void
runtime·queuefinalizer(byte *p, FuncVal *fn, uintptr nret, Type *fint, PtrType *ot)
{
@ -1844,11 +1704,6 @@ sweepspan(ParFor *desc, uint32 idx)
continue;
if((bits & bitMarked) != 0) {
if(DebugMark) {
if(!(bits & bitSpecial))
runtime·printf("found spurious mark on %p\n", p);
*bitp &= ~(bitSpecial<<shift);
}
*bitp &= ~(bitMarked<<shift);
continue;
}
@ -1981,13 +1836,7 @@ runtime·gchelper(void)
runtime·parfordo(work.markfor);
// help other threads scan secondary blocks
scanblock(nil, nil, 0, true);
if(DebugMark) {
// wait while the main thread executes mark(debug_scanblock)
while(runtime·atomicload(&work.debugmarkdone) == 0)
runtime·usleep(10);
}
scanblock(nil, true);
runtime·parfordo(work.sweepfor);
bufferList[m->helpgc].busy = 0;
@ -2023,13 +1872,26 @@ cachestats(void)
}
}
static void
flushallmcaches(void)
{
P *p, **pp;
MCache *c;
// Flush MCache's to MCentral.
for(pp=runtime·allp; p=*pp; pp++) {
c = p->mcache;
if(c==nil)
continue;
runtime·MCache_ReleaseAll(c);
}
}
static void
updatememstats(GCStats *stats)
{
M *mp;
MSpan *s;
MCache *c;
P *p, **pp;
int32 i;
uint64 stacks_inuse, smallfree;
uint64 *src, *dst;
@ -2070,12 +1932,7 @@ updatememstats(GCStats *stats)
}
// Flush MCache's to MCentral.
for(pp=runtime·allp; p=*pp; pp++) {
c = p->mcache;
if(c==nil)
continue;
runtime·MCache_ReleaseAll(c);
}
flushallmcaches();
// Aggregate local stats.
cachestats();
@ -2278,11 +2135,8 @@ gc(struct gc_args *args)
work.nwait = 0;
work.ndone = 0;
work.debugmarkdone = 0;
work.nproc = runtime·gcprocs();
addroots();
addfreelists();
runtime·parforsetup(work.markfor, work.nproc, work.nroot, nil, false, markroot);
runtime·parforsetup(work.markfor, work.nproc, RootCount + runtime·allglen, nil, false, markroot);
runtime·parforsetup(work.sweepfor, work.nproc, runtime·mheap.nspan, nil, true, sweepspan);
if(work.nproc > 1) {
runtime·noteclear(&work.alldone);
@ -2293,13 +2147,8 @@ gc(struct gc_args *args)
gchelperstart();
runtime·parfordo(work.markfor);
scanblock(nil, nil, 0, true);
scanblock(nil, true);
if(DebugMark) {
for(i=0; i<work.nroot; i++)
debug_scanblock(work.roots[i].p, work.roots[i].n);
runtime·atomicstore(&work.debugmarkdone, 1);
}
t2 = runtime·nanotime();
runtime·parfordo(work.sweepfor);

5
src/pkg/runtime/mprof.goc
View File

@ -436,7 +436,7 @@ saveg(uintptr pc, uintptr sp, G *gp, TRecord *r)
}
func GoroutineProfile(b Slice) (n int, ok bool) {
uintptr pc, sp;
uintptr pc, sp, i;
TRecord *r;
G *gp;
@ -455,7 +455,8 @@ func GoroutineProfile(b Slice) (n int, ok bool) {
ok = true;
r = (TRecord*)b.array;
saveg(pc, sp, g, r++);
for(gp = runtime·allg; gp != nil; gp = gp->alllink) {
for(i = 0; i < runtime·allglen; i++) {
gp = runtime·allg[i];
if(gp == g || gp->status == Gdead)
continue;
saveg(~(uintptr)0, ~(uintptr)0, gp, r++);

77
src/pkg/runtime/proc.c
View File

@ -67,8 +67,7 @@ int32 runtime·gomaxprocs;
uint32 runtime·needextram;
bool runtime·iscgo;
M runtime·m0;
G runtime·g0; // idle goroutine for m0
G* runtime·allg;
G runtime·g0; // idle goroutine for m0
G* runtime·lastg;
M* runtime·allm;
M* runtime·extram;
@ -76,6 +75,11 @@ int8* runtime·goos;
int32 runtime·ncpu;
static int32 newprocs;
static Lock allglock; // the following vars are protected by this lock or by stoptheworld
G** runtime·allg;
uintptr runtime·allglen;
static uintptr allgcap;
void runtime·mstart(void);
static void runqput(P*, G*);
static G* runqget(P*);
@ -115,6 +119,7 @@ static bool preemptall(void);
static bool preemptone(P*);
static bool exitsyscallfast(void);
static bool haveexperiment(int8*);
static void allgadd(G*);
// The bootstrap sequence is:
//
@ -278,6 +283,7 @@ runtime·tracebackothers(G *me)
{
G *gp;
int32 traceback;
uintptr i;
traceback = runtime·gotraceback(nil);
@ -288,7 +294,9 @@ runtime·tracebackothers(G *me)
runtime·traceback(~(uintptr)0, ~(uintptr)0, 0, gp);
}
for(gp = runtime·allg; gp != nil; gp = gp->alllink) {
runtime·lock(&allglock);
for(i = 0; i < runtime·allglen; i++) {
gp = runtime·allg[i];
if(gp == me || gp == m->curg || gp->status == Gdead)
continue;
if(gp->issystem && traceback < 2)
@ -301,6 +309,7 @@ runtime·tracebackothers(G *me)
} else
runtime·traceback(~(uintptr)0, ~(uintptr)0, 0, gp);
}
runtime·unlock(&allglock);
}
static void
@ -792,13 +801,7 @@ runtime·newextram(void)
if(raceenabled)
gp->racectx = runtime·racegostart(runtime·newextram);
// put on allg for garbage collector
runtime·lock(&runtime·sched);
if(runtime·lastg == nil)
runtime·allg = gp;
else
runtime·lastg->alllink = gp;
runtime·lastg = gp;
runtime·unlock(&runtime·sched);
allgadd(gp);
// Add m to the extra list.
mnext = lockextra(true);
@ -1766,13 +1769,7 @@ runtime·newproc1(FuncVal *fn, byte *argp, int32 narg, int32 nret, void *callerp
runtime·throw("invalid stack in newg");
} else {
newg = runtime·malg(StackMin);
runtime·lock(&runtime·sched);
if(runtime·lastg == nil)
runtime·allg = newg;
else
runtime·lastg->alllink = newg;
runtime·lastg = newg;
runtime·unlock(&runtime·sched);
allgadd(newg);
}
sp = (byte*)newg->stackbase;
@ -1805,6 +1802,31 @@ runtime·newproc1(FuncVal *fn, byte *argp, int32 narg, int32 nret, void *callerp
return newg;
}
static void
allgadd(G *gp)
{
G **new;
uintptr cap;
runtime·lock(&allglock);
if(runtime·allglen >= allgcap) {
cap = 4096/sizeof(new[0]);
if(cap < 2*allgcap)
cap = 2*allgcap;
new = runtime·malloc(cap*sizeof(new[0]));
if(new == nil)
runtime·throw("runtime: cannot allocate memory");
if(runtime·allg != nil) {
runtime·memmove(new, runtime·allg, runtime·allglen*sizeof(new[0]));
runtime·free(runtime·allg);
}
runtime·allg = new;
allgcap = cap;
}
runtime·allg[runtime·allglen++] = gp;
runtime·unlock(&allglock);
}
// Put on gfree list.
// If local list is too long, transfer a batch to the global list.
static void
@ -1994,19 +2016,21 @@ runtime·gcount(void)
{
G *gp;
int32 n, s;
uintptr i;
n = 0;
runtime·lock(&runtime·sched);
runtime·lock(&allglock);
// TODO(dvyukov): runtime.NumGoroutine() is O(N).
// We do not want to increment/decrement centralized counter in newproc/goexit,
// just to make runtime.NumGoroutine() faster.
// Compromise solution is to introduce per-P counters of active goroutines.
for(gp = runtime·allg; gp; gp = gp->alllink) {
for(i = 0; i < runtime·allglen; i++) {
gp = runtime·allg[i];
s = gp->status;
if(s == Grunnable || s == Grunning || s == Gsyscall || s == Gwaiting)
n++;
}
runtime·unlock(&runtime·sched);
runtime·unlock(&allglock);
return n;
}
@ -2345,6 +2369,7 @@ checkdead(void)
{
G *gp;
int32 run, grunning, s;
uintptr i;
// -1 for sysmon
run = runtime·sched.mcount - runtime·sched.nmidle - runtime·sched.nmidlelocked - 1;
@ -2356,17 +2381,21 @@ checkdead(void)
runtime·throw("checkdead: inconsistent counts");
}
grunning = 0;
for(gp = runtime·allg; gp; gp = gp->alllink) {
runtime·lock(&allglock);
for(i = 0; i < runtime·allglen; i++) {
gp = runtime·allg[i];
if(gp->isbackground)
continue;
s = gp->status;
if(s == Gwaiting)
grunning++;
else if(s == Grunnable || s == Grunning || s == Gsyscall) {
runtime·unlock(&allglock);
runtime·printf("checkdead: find g %D in status %d\n", gp->goid, s);
runtime·throw("checkdead: runnable g");
}
}
runtime·unlock(&allglock);
if(grunning == 0) // possible if main goroutine calls runtime·Goexit()
runtime·exit(0);
m->throwing = -1; // do not dump full stacks
@ -2553,6 +2582,7 @@ runtime·schedtrace(bool detailed)
int64 now;
int64 id1, id2, id3;
int32 i, t, h;
uintptr gi;
int8 *fmt;
M *mp, *lockedm;
G *gp, *lockedg;
@ -2620,13 +2650,16 @@ runtime·schedtrace(bool detailed)
mp->mallocing, mp->throwing, mp->gcing, mp->locks, mp->dying, mp->helpgc,
mp->spinning, id3);
}
for(gp = runtime·allg; gp; gp = gp->alllink) {
runtime·lock(&allglock);
for(gi = 0; gi < runtime·allglen; gi++) {
gp = runtime·allg[gi];
mp = gp->m;
lockedm = gp->lockedm;
runtime·printf(" G%D: status=%d(%s) m=%d lockedm=%d\n",
gp->goid, gp->status, gp->waitreason, mp ? mp->id : -1,
lockedm ? lockedm->id : -1);
}
runtime·unlock(&allglock);
runtime·unlock(&runtime·sched);
}

4
src/pkg/runtime/runtime.h
View File

@ -263,7 +263,6 @@ struct G
uintptr stackguard; // same as stackguard0, but not set to StackPreempt
uintptr stack0;
uintptr stacksize;
G* alllink; // on allg
void* param; // passed parameter on wakeup
int16 status;
int64 goid;
@ -719,7 +718,8 @@ bool runtime·topofstack(Func*);
*/
extern String runtime·emptystring;
extern uintptr runtime·zerobase;
extern G* runtime·allg;
extern G** runtime·allg;
extern uintptr runtime·allglen;
extern G* runtime·lastg;
extern M* runtime·allm;
extern P** runtime·allp;