diff --git a/src/runtime/mcache.go b/src/runtime/mcache.go index ec9ccb4abb..9ff4259ce9 100644 --- a/src/runtime/mcache.go +++ b/src/runtime/mcache.go @@ -24,8 +24,6 @@ type mcache struct { stackcache [_NumStackOrders]stackfreelist - sudogcache *sudog - // Local allocator stats, flushed during GC. local_nlookup uintptr // number of pointer lookups local_largefree uintptr // bytes freed for large objects (>maxsmallsize) diff --git a/src/runtime/mgc.go b/src/runtime/mgc.go index 830bf879d4..5417d3a291 100644 --- a/src/runtime/mgc.go +++ b/src/runtime/mgc.go @@ -628,6 +628,19 @@ func clearpools() { poolcleanup() } + // Clear central sudog cache. + // Leave per-P caches alone, they have strictly bounded size. + // Disconnect cached list before dropping it on the floor, + // so that a dangling ref to one entry does not pin all of them. + lock(&sched.sudoglock) + var sg, sgnext *sudog + for sg = sched.sudogcache; sg != nil; sg = sgnext { + sgnext = sg.next + sg.next = nil + } + sched.sudogcache = nil + unlock(&sched.sudoglock) + for _, p := range &allp { if p == nil { break @@ -636,15 +649,6 @@ func clearpools() { if c := p.mcache; c != nil { c.tiny = nil c.tinyoffset = 0 - - // disconnect cached list before dropping it on the floor, - // so that a dangling ref to one entry does not pin all of them. - var sg, sgnext *sudog - for sg = c.sudogcache; sg != nil; sg = sgnext { - sgnext = sg.next - sg.next = nil - } - c.sudogcache = nil } // clear defer pools diff --git a/src/runtime/proc.go b/src/runtime/proc.go index d251c314d4..d83b1bebf4 100644 --- a/src/runtime/proc.go +++ b/src/runtime/proc.go @@ -167,17 +167,6 @@ func goready(gp *g) { //go:nosplit func acquireSudog() *sudog { - c := gomcache() - s := c.sudogcache - if s != nil { - if s.elem != nil { - throw("acquireSudog: found s.elem != nil in cache") - } - c.sudogcache = s.next - s.next = nil - return s - } - // Delicate dance: the semaphore implementation calls // acquireSudog, acquireSudog calls new(sudog), // new calls malloc, malloc can call the garbage collector, @@ -187,12 +176,31 @@ func acquireSudog() *sudog { // The acquirem/releasem increments m.locks during new(sudog), // which keeps the garbage collector from being invoked. mp := acquirem() - p := new(sudog) - if p.elem != nil { - throw("acquireSudog: found p.elem != nil after new") + pp := mp.p + if len(pp.sudogcache) == 0 { + lock(&sched.sudoglock) + // First, try to grab a batch from central cache. + for len(pp.sudogcache) < cap(pp.sudogcache)/2 && sched.sudogcache != nil { + s := sched.sudogcache + sched.sudogcache = s.next + s.next = nil + pp.sudogcache = append(pp.sudogcache, s) + } + unlock(&sched.sudoglock) + // If the central cache is empty, allocate a new one. + if len(pp.sudogcache) == 0 { + pp.sudogcache = append(pp.sudogcache, new(sudog)) + } + } + ln := len(pp.sudogcache) + s := pp.sudogcache[ln-1] + pp.sudogcache[ln-1] = nil + pp.sudogcache = pp.sudogcache[:ln-1] + if s.elem != nil { + throw("acquireSudog: found s.elem != nil in cache") } releasem(mp) - return p + return s } //go:nosplit @@ -216,9 +224,30 @@ func releaseSudog(s *sudog) { if gp.param != nil { throw("runtime: releaseSudog with non-nil gp.param") } - c := gomcache() - s.next = c.sudogcache - c.sudogcache = s + mp := acquirem() // avoid rescheduling to another P + pp := mp.p + if len(pp.sudogcache) == cap(pp.sudogcache) { + // Transfer half of local cache to the central cache. + var first, last *sudog + for len(pp.sudogcache) > cap(pp.sudogcache)/2 { + ln := len(pp.sudogcache) + p := pp.sudogcache[ln-1] + pp.sudogcache[ln-1] = nil + pp.sudogcache = pp.sudogcache[:ln-1] + if first == nil { + first = p + } else { + last.next = p + } + last = p + } + lock(&sched.sudoglock) + last.next = sched.sudogcache + sched.sudogcache = first + unlock(&sched.sudoglock) + } + pp.sudogcache = append(pp.sudogcache, s) + releasem(mp) } // funcPC returns the entry PC of the function f. diff --git a/src/runtime/proc1.go b/src/runtime/proc1.go index f3248a5351..906528c0ab 100644 --- a/src/runtime/proc1.go +++ b/src/runtime/proc1.go @@ -2483,6 +2483,7 @@ func procresize(nprocs int32) *p { pp = new(p) pp.id = i pp.status = _Pgcstop + pp.sudogcache = pp.sudogbuf[:0] atomicstorep(unsafe.Pointer(&allp[i]), unsafe.Pointer(pp)) } if pp.mcache == nil { @@ -2521,6 +2522,10 @@ func procresize(nprocs int32) *p { } sched.runqsize++ } + for i := range &p.sudogbuf { + p.sudogbuf[i] = nil + } + p.sudogcache = p.sudogbuf[:0] freemcache(p.mcache) p.mcache = nil gfpurge(p) diff --git a/src/runtime/runtime2.go b/src/runtime/runtime2.go index ea2d55dbb6..81d39fb48e 100644 --- a/src/runtime/runtime2.go +++ b/src/runtime/runtime2.go @@ -329,6 +329,9 @@ type p struct { gfree *g gfreecnt int32 + sudogcache []*sudog + sudogbuf [128]*sudog + tracebuf *traceBuf pad [64]byte @@ -365,6 +368,10 @@ type schedt struct { gfree *g ngfree int32 + // Central cache of sudog structs. + sudoglock mutex + sudogcache *sudog + gcwaiting uint32 // gc is waiting to run stopwait int32 stopnote note