cybercyst/go
1
0
Fork 0
mirror of https://github.com/golang/go.git synced 2025-05-28 19:02:22 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

runtime: use typed memmove (write barriers) for chan, map, interface content

Browse Source 
Found with GODEBUG=wbshadow=2 mode.
Eventually that will run automatically, but right now
it still detects other missing write barriers.

Change-Id: Iea83d693480c2f3008b4e80d55821acff65970a6
Reviewed-on: https://go-review.googlesource.com/2277
Reviewed-by: Keith Randall <khr@golang.org>
This commit is contained in:
Russ Cox 2014-12-29 10:07:47 -05:00
parent bcadab9349
commit 54bb4dc390
5 changed files with 53 additions and 37 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
src/runtime/chan.go
View File

@ -146,7 +146,7 @@ func chansend(t *chantype, c *hchan, ep unsafe.Pointer, block bool, callerpc uin
recvg := sg.g
if sg.elem != nil {
memmove(unsafe.Pointer(sg.elem), ep, uintptr(c.elemsize))
typedmemmove(c.elemtype, unsafe.Pointer(sg.elem), ep)
sg.elem = nil
}
recvg.param = unsafe.Pointer(sg)
@ -234,7 +234,7 @@ func chansend(t *chantype, c *hchan, ep unsafe.Pointer, block bool, callerpc uin
raceacquire(chanbuf(c, c.sendx))
racerelease(chanbuf(c, c.sendx))
}
memmove(chanbuf(c, c.sendx), ep, uintptr(c.elemsize))
typedmemmove(c.elemtype, chanbuf(c, c.sendx), ep)
c.sendx++
if c.sendx == c.dataqsiz {
c.sendx = 0
@ -379,7 +379,7 @@ func chanrecv(t *chantype, c *hchan, ep unsafe.Pointer, block bool) (selected, r
unlock(&c.lock)
if ep != nil {
memmove(ep, sg.elem, uintptr(c.elemsize))
typedmemmove(c.elemtype, ep, sg.elem)
}
sg.elem = nil
gp := sg.g
@ -484,7 +484,7 @@ func chanrecv(t *chantype, c *hchan, ep unsafe.Pointer, block bool) (selected, r
racerelease(chanbuf(c, c.recvx))
}
if ep != nil {
memmove(ep, chanbuf(c, c.recvx), uintptr(c.elemsize))
typedmemmove(c.elemtype, ep, chanbuf(c, c.recvx))
}
memclr(chanbuf(c, c.recvx), uintptr(c.elemsize))

16
src/runtime/hashmap.go
View File

@ -435,13 +435,13 @@ again:
continue
}
// already have a mapping for key. Update it.
memmove(k2, key, uintptr(t.key.size))
typedmemmove(t.key, k2, key)
v := add(unsafe.Pointer(b), dataOffset+bucketCnt*uintptr(t.keysize)+i*uintptr(t.valuesize))
v2 := v
if t.indirectvalue {
v2 = *((*unsafe.Pointer)(v2))
}
memmove(v2, val, uintptr(t.elem.size))
typedmemmove(t.elem, v2, val)
return
}
ovf := b.overflow(t)
@ -486,8 +486,8 @@ again:
*(*unsafe.Pointer)(insertv) = vmem
insertv = vmem
}
memmove(insertk, key, uintptr(t.key.size))
memmove(insertv, val, uintptr(t.elem.size))
typedmemmove(t.key, insertk, key)
typedmemmove(t.elem, insertv, val)
*inserti = top
h.count++
}
@ -846,12 +846,12 @@ func evacuate(t *maptype, h *hmap, oldbucket uintptr) {
if t.indirectkey {
*(*unsafe.Pointer)(xk) = k2 // copy pointer
} else {
memmove(xk, k, uintptr(t.key.size)) // copy value
typedmemmove(t.key, xk, k) // copy value
}
if t.indirectvalue {
*(*unsafe.Pointer)(xv) = *(*unsafe.Pointer)(v)
} else {
memmove(xv, v, uintptr(t.elem.size))
typedmemmove(t.elem, xv, v)
}
xi++
xk = add(xk, uintptr(t.keysize))
@ -873,12 +873,12 @@ func evacuate(t *maptype, h *hmap, oldbucket uintptr) {
if t.indirectkey {
*(*unsafe.Pointer)(yk) = k2
} else {
memmove(yk, k, uintptr(t.key.size))
typedmemmove(t.key, yk, k)
}
if t.indirectvalue {
*(*unsafe.Pointer)(yv) = *(*unsafe.Pointer)(v)
} else {
memmove(yv, v, uintptr(t.elem.size))
typedmemmove(t.elem, yv, v)
}
yi++
yk = add(yk, uintptr(t.keysize))

49
src/runtime/iface.go
View File

@ -132,16 +132,15 @@ func typ2Itab(t *_type, inter *interfacetype, cache **itab) *itab {
}
func convT2E(t *_type, elem unsafe.Pointer) (e interface{}) {
size := uintptr(t.size)
ep := (*eface)(unsafe.Pointer(&e))
if isDirectIface(t) {
ep._type = t
memmove(unsafe.Pointer(&ep.data), elem, size)
typedmemmove(t, unsafe.Pointer(&ep.data), elem)
} else {
x := newobject(t)
// TODO: We allocate a zeroed object only to overwrite it with
// actual data. Figure out how to avoid zeroing. Also below in convT2I.
memmove(x, elem, size)
typedmemmove(t, x, elem)
ep._type = t
ep.data = x
}
@ -154,14 +153,13 @@ func convT2I(t *_type, inter *interfacetype, cache **itab, elem unsafe.Pointer)
tab = getitab(inter, t, false)
atomicstorep(unsafe.Pointer(cache), unsafe.Pointer(tab))
}
size := uintptr(t.size)
pi := (*iface)(unsafe.Pointer(&i))
if isDirectIface(t) {
pi.tab = tab
memmove(unsafe.Pointer(&pi.data), elem, size)
typedmemmove(t, unsafe.Pointer(&pi.data), elem)
} else {
x := newobject(t)
memmove(x, elem, size)
typedmemmove(t, x, elem)
pi.tab = tab
pi.data = x
}
@ -180,11 +178,15 @@ func assertI2T(t *_type, i fInterface) (r struct{}) {
if tab._type != t {
panic(&TypeAssertionError{*tab.inter.typ._string, *tab._type._string, *t._string, ""})
}
size := uintptr(t.size)
// NOTE(rsc): If this changes to take a pointer argument
// instead of using &r, these calls need to change to be
// typedmemmove (the first can be just writebarrierptr).
// Until then, it is very important that no blocking operation
// happens between the memmove and the return.
if isDirectIface(t) {
memmove(unsafe.Pointer(&r), unsafe.Pointer(&ip.data), size)
memmove(unsafe.Pointer(&r), unsafe.Pointer(&ip.data), uintptr(t.size))
} else {
memmove(unsafe.Pointer(&r), ip.data, size)
memmove(unsafe.Pointer(&r), ip.data, uintptr(t.size))
}
return
}
@ -192,19 +194,23 @@ func assertI2T(t *_type, i fInterface) (r struct{}) {
//go:nosplit
func assertI2T2(t *_type, i fInterface) (r byte) {
ip := (*iface)(unsafe.Pointer(&i))
size := uintptr(t.size)
ok := (*bool)(add(unsafe.Pointer(&r), size))
ok := (*bool)(add(unsafe.Pointer(&r), uintptr(t.size)))
tab := ip.tab
if tab == nil || tab._type != t {
*ok = false
memclr(unsafe.Pointer(&r), size)
memclr(unsafe.Pointer(&r), uintptr(t.size))
return
}
*ok = true
// NOTE(rsc): If this changes to take a pointer argument
// instead of using &r, these calls need to change to be
// typedmemmove (the first can be just writebarrierptr).
// Until then, it is very important that no blocking operation
// happens between the memmove and the return.
if isDirectIface(t) {
memmove(unsafe.Pointer(&r), unsafe.Pointer(&ip.data), size)
memmove(unsafe.Pointer(&r), unsafe.Pointer(&ip.data), uintptr(t.size))
} else {
memmove(unsafe.Pointer(&r), ip.data, size)
memmove(unsafe.Pointer(&r), ip.data, uintptr(t.size))
}
return
}
@ -224,11 +230,15 @@ func assertE2T(t *_type, e interface{}) (r struct{}) {
if ep._type != t {
panic(&TypeAssertionError{"", *ep._type._string, *t._string, ""})
}
size := uintptr(t.size)
// NOTE(rsc): If this changes to take a pointer argument
// instead of using &r, these calls need to change to be
// typedmemmove (the first can be just writebarrierptr).
// Until then, it is very important that no blocking operation
// happens between the memmove and the return.
if isDirectIface(t) {
memmove(unsafe.Pointer(&r), unsafe.Pointer(&ep.data), size)
memmove(unsafe.Pointer(&r), unsafe.Pointer(&ep.data), uintptr(t.size))
} else {
memmove(unsafe.Pointer(&r), ep.data, size)
memmove(unsafe.Pointer(&r), ep.data, uintptr(t.size))
}
return
}
@ -244,6 +254,11 @@ func assertE2T2(t *_type, e interface{}) (r byte) {
return
}
*ok = true
// NOTE(rsc): If this changes to take a pointer argument
// instead of using &r, these calls need to change to be
// typedmemmove (the first can be just writebarrierptr).
// Until then, it is very important that no blocking operation
// happens between the memmove and the return.
if isDirectIface(t) {
memmove(unsafe.Pointer(&r), unsafe.Pointer(&ep.data), size)
} else {

9
src/runtime/mgc0.go
View File

@ -304,8 +304,9 @@ func typedmemmove(typ *_type, dst, src unsafe.Pointer) {
} else {
*(*uintptr)(dst) = *(*uintptr)(src)
}
dst = add(dst, ptrSize)
src = add(src, ptrSize)
// TODO(rsc): The noescape calls should be unnecessary.
dst = add(noescape(dst), ptrSize)
src = add(noescape(src), ptrSize)
if i+1 == nptr {
break
}
@ -315,8 +316,8 @@ func typedmemmove(typ *_type, dst, src unsafe.Pointer) {
} else {
*(*uintptr)(dst) = *(*uintptr)(src)
}
dst = add(dst, ptrSize)
src = add(src, ptrSize)
dst = add(noescape(dst), ptrSize)
src = add(noescape(src), ptrSize)
}
})
}

8
src/runtime/select.go
View File

@ -449,7 +449,7 @@ asyncrecv:
*cas.receivedp = true
}
if cas.elem != nil {
memmove(cas.elem, chanbuf(c, c.recvx), uintptr(c.elemsize))
typedmemmove(c.elemtype, cas.elem, chanbuf(c, c.recvx))
}
memclr(chanbuf(c, c.recvx), uintptr(c.elemsize))
c.recvx++
@ -477,7 +477,7 @@ asyncsend:
racerelease(chanbuf(c, c.sendx))
raceReadObjectPC(c.elemtype, cas.elem, cas.pc, chansendpc)
}
memmove(chanbuf(c, c.sendx), cas.elem, uintptr(c.elemsize))
typedmemmove(c.elemtype, chanbuf(c, c.sendx), cas.elem)
c.sendx++
if c.sendx == c.dataqsiz {
c.sendx = 0
@ -512,7 +512,7 @@ syncrecv:
*cas.receivedp = true
}
if cas.elem != nil {
memmove(cas.elem, sg.elem, uintptr(c.elemsize))
typedmemmove(c.elemtype, cas.elem, sg.elem)
}
sg.elem = nil
gp = sg.g
@ -548,7 +548,7 @@ syncsend:
print("syncsend: sel=", sel, " c=", c, "\n")
}
if sg.elem != nil {
memmove(sg.elem, cas.elem, uintptr(c.elemsize))
typedmemmove(c.elemtype, sg.elem, cas.elem)
}
sg.elem = nil
gp = sg.g