cmd/compile: expand calls cleanup

Convert expand calls into a smaller number of focused
recursive rewrites, and rely on an enhanced version of
"decompose" to clean up afterwards.

Debugging information seems to emerge intact.

Change-Id: Ic46da4207e3a4da5c8e2c47b637b0e35abbe56bb
Reviewed-on: https://go-review.googlesource.com/c/go/+/507295
Run-TryBot: David Chase <drchase@google.com>
Reviewed-by: Cherry Mui <cherryyz@google.com>
Reviewed-by: Keith Randall <khr@golang.org>
TryBot-Result: Gopher Robot <gobot@golang.org>

src/cmd/compile/internal/ssa/_gen/dec.rules

@@ -91,3 +91,109 @@
     (OffPtr <typ.BytePtrPtr> [config.PtrSize] dst)
     data
     (Store {typ.Uintptr} dst itab mem))
+
+// Helpers for expand calls
+// Some of these are copied from generic.rules
+
+(IMake _typ (StructMake1 val)) => (IMake _typ val)
+(StructSelect [0] (IData x)) => (IData x)
+
+(StructSelect (StructMake1 x)) => x
+(StructSelect [0] (StructMake2 x _)) => x
+(StructSelect [1] (StructMake2 _ x)) => x
+(StructSelect [0] (StructMake3 x _ _)) => x
+(StructSelect [1] (StructMake3 _ x _)) => x
+(StructSelect [2] (StructMake3 _ _ x)) => x
+(StructSelect [0] (StructMake4 x _ _ _)) => x
+(StructSelect [1] (StructMake4 _ x _ _)) => x
+(StructSelect [2] (StructMake4 _ _ x _)) => x
+(StructSelect [3] (StructMake4 _ _ _ x)) => x
+
+// Special case coming from immediate interface rewriting
+// Typical case: (StructSelect [0] (IData (IMake typ dat)) rewrites to (StructSelect [0] dat)
+// but because the interface is immediate, the type of "IData" is a one-element struct containing
+// a pointer that is not the pointer type of dat (can be a *uint8).
+// More annoying case: (ArraySelect[0] (StructSelect[0] isAPtr))
+// There, result of the StructSelect is an Array (not a pointer) and
+// the pre-rewrite input to the ArraySelect is a struct, not a pointer.
+(StructSelect [0] x) && x.Type.IsPtr()  => x
+(ArraySelect [0] x) && x.Type.IsPtr()  => x
+
+// These, too.  Bits is bits.
+(ArrayMake1 x) && x.Type.IsPtr() => x
+(StructMake1 x) && x.Type.IsPtr() => x
+
+(Store dst (StructMake1 <t> f0) mem) =>
+  (Store {t.FieldType(0)} (OffPtr <t.FieldType(0).PtrTo()> [0] dst) f0 mem)
+(Store dst (StructMake2 <t> f0 f1) mem) =>
+  (Store {t.FieldType(1)}
+    (OffPtr <t.FieldType(1).PtrTo()> [t.FieldOff(1)] dst)
+    f1
+    (Store {t.FieldType(0)}
+      (OffPtr <t.FieldType(0).PtrTo()> [0] dst)
+        f0 mem))
+(Store dst (StructMake3 <t> f0 f1 f2) mem) =>
+  (Store {t.FieldType(2)}
+    (OffPtr <t.FieldType(2).PtrTo()> [t.FieldOff(2)] dst)
+    f2
+    (Store {t.FieldType(1)}
+      (OffPtr <t.FieldType(1).PtrTo()> [t.FieldOff(1)] dst)
+      f1
+      (Store {t.FieldType(0)}
+        (OffPtr <t.FieldType(0).PtrTo()> [0] dst)
+          f0 mem)))
+(Store dst (StructMake4 <t> f0 f1 f2 f3) mem) =>
+  (Store {t.FieldType(3)}
+    (OffPtr <t.FieldType(3).PtrTo()> [t.FieldOff(3)] dst)
+    f3
+    (Store {t.FieldType(2)}
+      (OffPtr <t.FieldType(2).PtrTo()> [t.FieldOff(2)] dst)
+      f2
+      (Store {t.FieldType(1)}
+        (OffPtr <t.FieldType(1).PtrTo()> [t.FieldOff(1)] dst)
+        f1
+        (Store {t.FieldType(0)}
+          (OffPtr <t.FieldType(0).PtrTo()> [0] dst)
+            f0 mem))))
+
+(ArraySelect (ArrayMake1 x)) => x
+(ArraySelect [0] (IData x)) => (IData x)
+
+(Store dst (ArrayMake1 e) mem) => (Store {e.Type} dst e mem)
+
+// NOTE removed must-not-be-SSA condition.
+(ArraySelect [i] x:(Load <t> ptr mem)) =>
+  @x.Block (Load <v.Type> (OffPtr <v.Type.PtrTo()> [t.Elem().Size()*i] ptr) mem)
+
+(StringPtr x:(Load <t> ptr mem)) && t.IsString() => @x.Block (Load <typ.BytePtr> ptr mem)
+(StringLen x:(Load <t> ptr mem)) && t.IsString() => @x.Block (Load <typ.Int>
+      (OffPtr <typ.IntPtr> [config.PtrSize] ptr)
+      mem)
+
+// NOTE removed must-not-be-SSA condition.
+(StructSelect [i] x:(Load <t> ptr mem)) =>
+  @x.Block (Load <v.Type> (OffPtr <v.Type.PtrTo()> [t.FieldOff(int(i))] ptr) mem)
+
+(ITab x:(Load <t> ptr mem)) && t.IsInterface() => @x.Block (Load <typ.Uintptr> ptr mem)
+
+(IData x:(Load <t> ptr mem)) && t.IsInterface() => @x.Block (Load <typ.BytePtr>
+      (OffPtr <typ.BytePtrPtr> [config.PtrSize] ptr)
+      mem)
+
+(SlicePtr x:(Load <t> ptr mem)) && t.IsSlice() => @x.Block (Load <t.Elem().PtrTo()> ptr mem)
+(SliceLen x:(Load <t> ptr mem)) && t.IsSlice() => @x.Block (Load <typ.Int>
+      (OffPtr <typ.IntPtr> [config.PtrSize] ptr)
+      mem)
+(SliceCap x:(Load <t> ptr mem)) && t.IsSlice() => @x.Block (Load <typ.Int>
+      (OffPtr <typ.IntPtr> [2*config.PtrSize] ptr)
+      mem)
+
+(ComplexReal x:(Load <t> ptr mem)) && t.IsComplex() && t.Size() == 8 => @x.Block (Load <typ.Float32> ptr mem)
+(ComplexImag x:(Load <t> ptr mem)) && t.IsComplex() && t.Size() == 8 => @x.Block (Load <typ.Float32>
+      (OffPtr <typ.Float32Ptr> [4] ptr)
+      mem)
+
+(ComplexReal x:(Load <t> ptr mem)) && t.IsComplex() && t.Size() == 16 => @x.Block (Load <typ.Float64> ptr mem)
+(ComplexImag x:(Load <t> ptr mem)) && t.IsComplex() && t.Size() == 16 => @x.Block (Load <typ.Float64>
+      (OffPtr <typ.Float64Ptr> [8] ptr)
+      mem)

src/cmd/compile/internal/ssa/compile.go

@@ -472,8 +472,8 @@ var passes = [...]pass{
 	{name: "nilcheckelim", fn: nilcheckelim},
 	{name: "prove", fn: prove},
 	{name: "early fuse", fn: fuseEarly},
-	{name: "decompose builtin", fn: decomposeBuiltIn, required: true},
 	{name: "expand calls", fn: expandCalls, required: true},
+	{name: "decompose builtin", fn: postExpandCallsDecompose, required: true},
 	{name: "softfloat", fn: softfloat, required: true},
 	{name: "late opt", fn: opt, required: true}, // TODO: split required rules and optimizing rules
 	{name: "dead auto elim", fn: elimDeadAutosGeneric},
@@ -547,6 +547,8 @@ var passOrder = [...]constraint{
 	{"generic cse", "tighten"},
 	// checkbce needs the values removed
 	{"generic deadcode", "check bce"},
+	// decompose builtin now also cleans up after expand calls
+	{"expand calls", "decompose builtin"},
 	// don't run optimization pass until we've decomposed builtin objects
 	{"decompose builtin", "late opt"},
 	// decompose builtin is the last pass that may introduce new float ops, so run softfloat after it

src/cmd/compile/internal/ssa/op.go

@@ -238,7 +238,7 @@ func (a *AuxCall) RegsOfArg(which int64) []abi.RegIndex {
 	return a.abiInfo.InParam(int(which)).Registers
 }
 
-// NameOfResult returns the type of result which (indexed 0, 1, etc).
+// NameOfResult returns the ir.Name of result which (indexed 0, 1, etc).
 func (a *AuxCall) NameOfResult(which int64) *ir.Name {
 	return a.abiInfo.OutParam(int(which)).Name
 }

src/cmd/compile/internal/ssa/rewritedec.go

@@ -6,12 +6,18 @@ import "cmd/compile/internal/types"
 
 func rewriteValuedec(v *Value) bool {
 	switch v.Op {
+	case OpArrayMake1:
+		return rewriteValuedec_OpArrayMake1(v)
+	case OpArraySelect:
+		return rewriteValuedec_OpArraySelect(v)
 	case OpComplexImag:
 		return rewriteValuedec_OpComplexImag(v)
 	case OpComplexReal:
 		return rewriteValuedec_OpComplexReal(v)
 	case OpIData:
 		return rewriteValuedec_OpIData(v)
+	case OpIMake:
+		return rewriteValuedec_OpIMake(v)
 	case OpITab:
 		return rewriteValuedec_OpITab(v)
 	case OpLoad:
@@ -30,11 +36,92 @@ func rewriteValuedec(v *Value) bool {
 		return rewriteValuedec_OpStringLen(v)
 	case OpStringPtr:
 		return rewriteValuedec_OpStringPtr(v)
+	case OpStructMake1:
+		return rewriteValuedec_OpStructMake1(v)
+	case OpStructSelect:
+		return rewriteValuedec_OpStructSelect(v)
+	}
+	return false
+}
+func rewriteValuedec_OpArrayMake1(v *Value) bool {
+	v_0 := v.Args[0]
+	// match: (ArrayMake1 x)
+	// cond: x.Type.IsPtr()
+	// result: x
+	for {
+		x := v_0
+		if !(x.Type.IsPtr()) {
+			break
+		}
+		v.copyOf(x)
+		return true
+	}
+	return false
+}
+func rewriteValuedec_OpArraySelect(v *Value) bool {
+	v_0 := v.Args[0]
+	b := v.Block
+	// match: (ArraySelect [0] x)
+	// cond: x.Type.IsPtr()
+	// result: x
+	for {
+		if auxIntToInt64(v.AuxInt) != 0 {
+			break
+		}
+		x := v_0
+		if !(x.Type.IsPtr()) {
+			break
+		}
+		v.copyOf(x)
+		return true
+	}
+	// match: (ArraySelect (ArrayMake1 x))
+	// result: x
+	for {
+		if v_0.Op != OpArrayMake1 {
+			break
+		}
+		x := v_0.Args[0]
+		v.copyOf(x)
+		return true
+	}
+	// match: (ArraySelect [0] (IData x))
+	// result: (IData x)
+	for {
+		if auxIntToInt64(v.AuxInt) != 0 || v_0.Op != OpIData {
+			break
+		}
+		x := v_0.Args[0]
+		v.reset(OpIData)
+		v.AddArg(x)
+		return true
+	}
+	// match: (ArraySelect [i] x:(Load <t> ptr mem))
+	// result: @x.Block (Load <v.Type> (OffPtr <v.Type.PtrTo()> [t.Elem().Size()*i] ptr) mem)
+	for {
+		i := auxIntToInt64(v.AuxInt)
+		x := v_0
+		if x.Op != OpLoad {
+			break
+		}
+		t := x.Type
+		mem := x.Args[1]
+		ptr := x.Args[0]
+		b = x.Block
+		v0 := b.NewValue0(v.Pos, OpLoad, v.Type)
+		v.copyOf(v0)
+		v1 := b.NewValue0(v.Pos, OpOffPtr, v.Type.PtrTo())
+		v1.AuxInt = int64ToAuxInt(t.Elem().Size() * i)
+		v1.AddArg(ptr)
+		v0.AddArg2(v1, mem)
+		return true
 	}
 	return false
 }
 func rewriteValuedec_OpComplexImag(v *Value) bool {
 	v_0 := v.Args[0]
+	b := v.Block
+	typ := &b.Func.Config.Types
 	// match: (ComplexImag (ComplexMake _ imag ))
 	// result: imag
 	for {
@@ -45,10 +132,58 @@ func rewriteValuedec_OpComplexImag(v *Value) bool {
 		v.copyOf(imag)
 		return true
 	}
+	// match: (ComplexImag x:(Load <t> ptr mem))
+	// cond: t.IsComplex() && t.Size() == 8
+	// result: @x.Block (Load <typ.Float32> (OffPtr <typ.Float32Ptr> [4] ptr) mem)
+	for {
+		x := v_0
+		if x.Op != OpLoad {
+			break
+		}
+		t := x.Type
+		mem := x.Args[1]
+		ptr := x.Args[0]
+		if !(t.IsComplex() && t.Size() == 8) {
+			break
+		}
+		b = x.Block
+		v0 := b.NewValue0(v.Pos, OpLoad, typ.Float32)
+		v.copyOf(v0)
+		v1 := b.NewValue0(v.Pos, OpOffPtr, typ.Float32Ptr)
+		v1.AuxInt = int64ToAuxInt(4)
+		v1.AddArg(ptr)
+		v0.AddArg2(v1, mem)
+		return true
+	}
+	// match: (ComplexImag x:(Load <t> ptr mem))
+	// cond: t.IsComplex() && t.Size() == 16
+	// result: @x.Block (Load <typ.Float64> (OffPtr <typ.Float64Ptr> [8] ptr) mem)
+	for {
+		x := v_0
+		if x.Op != OpLoad {
+			break
+		}
+		t := x.Type
+		mem := x.Args[1]
+		ptr := x.Args[0]
+		if !(t.IsComplex() && t.Size() == 16) {
+			break
+		}
+		b = x.Block
+		v0 := b.NewValue0(v.Pos, OpLoad, typ.Float64)
+		v.copyOf(v0)
+		v1 := b.NewValue0(v.Pos, OpOffPtr, typ.Float64Ptr)
+		v1.AuxInt = int64ToAuxInt(8)
+		v1.AddArg(ptr)
+		v0.AddArg2(v1, mem)
+		return true
+	}
 	return false
 }
 func rewriteValuedec_OpComplexReal(v *Value) bool {
 	v_0 := v.Args[0]
+	b := v.Block
+	typ := &b.Func.Config.Types
 	// match: (ComplexReal (ComplexMake real _ ))
 	// result: real
 	for {
@@ -59,10 +194,53 @@ func rewriteValuedec_OpComplexReal(v *Value) bool {
 		v.copyOf(real)
 		return true
 	}
+	// match: (ComplexReal x:(Load <t> ptr mem))
+	// cond: t.IsComplex() && t.Size() == 8
+	// result: @x.Block (Load <typ.Float32> ptr mem)
+	for {
+		x := v_0
+		if x.Op != OpLoad {
+			break
+		}
+		t := x.Type
+		mem := x.Args[1]
+		ptr := x.Args[0]
+		if !(t.IsComplex() && t.Size() == 8) {
+			break
+		}
+		b = x.Block
+		v0 := b.NewValue0(v.Pos, OpLoad, typ.Float32)
+		v.copyOf(v0)
+		v0.AddArg2(ptr, mem)
+		return true
+	}
+	// match: (ComplexReal x:(Load <t> ptr mem))
+	// cond: t.IsComplex() && t.Size() == 16
+	// result: @x.Block (Load <typ.Float64> ptr mem)
+	for {
+		x := v_0
+		if x.Op != OpLoad {
+			break
+		}
+		t := x.Type
+		mem := x.Args[1]
+		ptr := x.Args[0]
+		if !(t.IsComplex() && t.Size() == 16) {
+			break
+		}
+		b = x.Block
+		v0 := b.NewValue0(v.Pos, OpLoad, typ.Float64)
+		v.copyOf(v0)
+		v0.AddArg2(ptr, mem)
+		return true
+	}
 	return false
 }
 func rewriteValuedec_OpIData(v *Value) bool {
 	v_0 := v.Args[0]
+	b := v.Block
+	config := b.Func.Config
+	typ := &b.Func.Config.Types
 	// match: (IData (IMake _ data))
 	// result: data
 	for {
@@ -73,10 +251,52 @@ func rewriteValuedec_OpIData(v *Value) bool {
 		v.copyOf(data)
 		return true
 	}
+	// match: (IData x:(Load <t> ptr mem))
+	// cond: t.IsInterface()
+	// result: @x.Block (Load <typ.BytePtr> (OffPtr <typ.BytePtrPtr> [config.PtrSize] ptr) mem)
+	for {
+		x := v_0
+		if x.Op != OpLoad {
+			break
+		}
+		t := x.Type
+		mem := x.Args[1]
+		ptr := x.Args[0]
+		if !(t.IsInterface()) {
+			break
+		}
+		b = x.Block
+		v0 := b.NewValue0(v.Pos, OpLoad, typ.BytePtr)
+		v.copyOf(v0)
+		v1 := b.NewValue0(v.Pos, OpOffPtr, typ.BytePtrPtr)
+		v1.AuxInt = int64ToAuxInt(config.PtrSize)
+		v1.AddArg(ptr)
+		v0.AddArg2(v1, mem)
+		return true
+	}
+	return false
+}
+func rewriteValuedec_OpIMake(v *Value) bool {
+	v_1 := v.Args[1]
+	v_0 := v.Args[0]
+	// match: (IMake _typ (StructMake1 val))
+	// result: (IMake _typ val)
+	for {
+		_typ := v_0
+		if v_1.Op != OpStructMake1 {
+			break
+		}
+		val := v_1.Args[0]
+		v.reset(OpIMake)
+		v.AddArg2(_typ, val)
+		return true
+	}
 	return false
 }
 func rewriteValuedec_OpITab(v *Value) bool {
 	v_0 := v.Args[0]
+	b := v.Block
+	typ := &b.Func.Config.Types
 	// match: (ITab (IMake itab _))
 	// result: itab
 	for {
@@ -87,6 +307,26 @@ func rewriteValuedec_OpITab(v *Value) bool {
 		v.copyOf(itab)
 		return true
 	}
+	// match: (ITab x:(Load <t> ptr mem))
+	// cond: t.IsInterface()
+	// result: @x.Block (Load <typ.Uintptr> ptr mem)
+	for {
+		x := v_0
+		if x.Op != OpLoad {
+			break
+		}
+		t := x.Type
+		mem := x.Args[1]
+		ptr := x.Args[0]
+		if !(t.IsInterface()) {
+			break
+		}
+		b = x.Block
+		v0 := b.NewValue0(v.Pos, OpLoad, typ.Uintptr)
+		v.copyOf(v0)
+		v0.AddArg2(ptr, mem)
+		return true
+	}
 	return false
 }
 func rewriteValuedec_OpLoad(v *Value) bool {
@@ -209,6 +449,9 @@ func rewriteValuedec_OpLoad(v *Value) bool {
 }
 func rewriteValuedec_OpSliceCap(v *Value) bool {
 	v_0 := v.Args[0]
+	b := v.Block
+	config := b.Func.Config
+	typ := &b.Func.Config.Types
 	// match: (SliceCap (SliceMake _ _ cap))
 	// result: cap
 	for {
@@ -219,10 +462,36 @@ func rewriteValuedec_OpSliceCap(v *Value) bool {
 		v.copyOf(cap)
 		return true
 	}
+	// match: (SliceCap x:(Load <t> ptr mem))
+	// cond: t.IsSlice()
+	// result: @x.Block (Load <typ.Int> (OffPtr <typ.IntPtr> [2*config.PtrSize] ptr) mem)
+	for {
+		x := v_0
+		if x.Op != OpLoad {
+			break
+		}
+		t := x.Type
+		mem := x.Args[1]
+		ptr := x.Args[0]
+		if !(t.IsSlice()) {
+			break
+		}
+		b = x.Block
+		v0 := b.NewValue0(v.Pos, OpLoad, typ.Int)
+		v.copyOf(v0)
+		v1 := b.NewValue0(v.Pos, OpOffPtr, typ.IntPtr)
+		v1.AuxInt = int64ToAuxInt(2 * config.PtrSize)
+		v1.AddArg(ptr)
+		v0.AddArg2(v1, mem)
+		return true
+	}
 	return false
 }
 func rewriteValuedec_OpSliceLen(v *Value) bool {
 	v_0 := v.Args[0]
+	b := v.Block
+	config := b.Func.Config
+	typ := &b.Func.Config.Types
 	// match: (SliceLen (SliceMake _ len _))
 	// result: len
 	for {
@@ -233,10 +502,34 @@ func rewriteValuedec_OpSliceLen(v *Value) bool {
 		v.copyOf(len)
 		return true
 	}
+	// match: (SliceLen x:(Load <t> ptr mem))
+	// cond: t.IsSlice()
+	// result: @x.Block (Load <typ.Int> (OffPtr <typ.IntPtr> [config.PtrSize] ptr) mem)
+	for {
+		x := v_0
+		if x.Op != OpLoad {
+			break
+		}
+		t := x.Type
+		mem := x.Args[1]
+		ptr := x.Args[0]
+		if !(t.IsSlice()) {
+			break
+		}
+		b = x.Block
+		v0 := b.NewValue0(v.Pos, OpLoad, typ.Int)
+		v.copyOf(v0)
+		v1 := b.NewValue0(v.Pos, OpOffPtr, typ.IntPtr)
+		v1.AuxInt = int64ToAuxInt(config.PtrSize)
+		v1.AddArg(ptr)
+		v0.AddArg2(v1, mem)
+		return true
+	}
 	return false
 }
 func rewriteValuedec_OpSlicePtr(v *Value) bool {
 	v_0 := v.Args[0]
+	b := v.Block
 	// match: (SlicePtr (SliceMake ptr _ _ ))
 	// result: ptr
 	for {
@@ -247,6 +540,26 @@ func rewriteValuedec_OpSlicePtr(v *Value) bool {
 		v.copyOf(ptr)
 		return true
 	}
+	// match: (SlicePtr x:(Load <t> ptr mem))
+	// cond: t.IsSlice()
+	// result: @x.Block (Load <t.Elem().PtrTo()> ptr mem)
+	for {
+		x := v_0
+		if x.Op != OpLoad {
+			break
+		}
+		t := x.Type
+		mem := x.Args[1]
+		ptr := x.Args[0]
+		if !(t.IsSlice()) {
+			break
+		}
+		b = x.Block
+		v0 := b.NewValue0(v.Pos, OpLoad, t.Elem().PtrTo())
+		v.copyOf(v0)
+		v0.AddArg2(ptr, mem)
+		return true
+	}
 	return false
 }
 func rewriteValuedec_OpSlicePtrUnchecked(v *Value) bool {
@@ -393,10 +706,141 @@ func rewriteValuedec_OpStore(v *Value) bool {
 		v.AddArg3(v0, data, v1)
 		return true
 	}
+	// match: (Store dst (StructMake1 <t> f0) mem)
+	// result: (Store {t.FieldType(0)} (OffPtr <t.FieldType(0).PtrTo()> [0] dst) f0 mem)
+	for {
+		dst := v_0
+		if v_1.Op != OpStructMake1 {
+			break
+		}
+		t := v_1.Type
+		f0 := v_1.Args[0]
+		mem := v_2
+		v.reset(OpStore)
+		v.Aux = typeToAux(t.FieldType(0))
+		v0 := b.NewValue0(v.Pos, OpOffPtr, t.FieldType(0).PtrTo())
+		v0.AuxInt = int64ToAuxInt(0)
+		v0.AddArg(dst)
+		v.AddArg3(v0, f0, mem)
+		return true
+	}
+	// match: (Store dst (StructMake2 <t> f0 f1) mem)
+	// result: (Store {t.FieldType(1)} (OffPtr <t.FieldType(1).PtrTo()> [t.FieldOff(1)] dst) f1 (Store {t.FieldType(0)} (OffPtr <t.FieldType(0).PtrTo()> [0] dst) f0 mem))
+	for {
+		dst := v_0
+		if v_1.Op != OpStructMake2 {
+			break
+		}
+		t := v_1.Type
+		f1 := v_1.Args[1]
+		f0 := v_1.Args[0]
+		mem := v_2
+		v.reset(OpStore)
+		v.Aux = typeToAux(t.FieldType(1))
+		v0 := b.NewValue0(v.Pos, OpOffPtr, t.FieldType(1).PtrTo())
+		v0.AuxInt = int64ToAuxInt(t.FieldOff(1))
+		v0.AddArg(dst)
+		v1 := b.NewValue0(v.Pos, OpStore, types.TypeMem)
+		v1.Aux = typeToAux(t.FieldType(0))
+		v2 := b.NewValue0(v.Pos, OpOffPtr, t.FieldType(0).PtrTo())
+		v2.AuxInt = int64ToAuxInt(0)
+		v2.AddArg(dst)
+		v1.AddArg3(v2, f0, mem)
+		v.AddArg3(v0, f1, v1)
+		return true
+	}
+	// match: (Store dst (StructMake3 <t> f0 f1 f2) mem)
+	// result: (Store {t.FieldType(2)} (OffPtr <t.FieldType(2).PtrTo()> [t.FieldOff(2)] dst) f2 (Store {t.FieldType(1)} (OffPtr <t.FieldType(1).PtrTo()> [t.FieldOff(1)] dst) f1 (Store {t.FieldType(0)} (OffPtr <t.FieldType(0).PtrTo()> [0] dst) f0 mem)))
+	for {
+		dst := v_0
+		if v_1.Op != OpStructMake3 {
+			break
+		}
+		t := v_1.Type
+		f2 := v_1.Args[2]
+		f0 := v_1.Args[0]
+		f1 := v_1.Args[1]
+		mem := v_2
+		v.reset(OpStore)
+		v.Aux = typeToAux(t.FieldType(2))
+		v0 := b.NewValue0(v.Pos, OpOffPtr, t.FieldType(2).PtrTo())
+		v0.AuxInt = int64ToAuxInt(t.FieldOff(2))
+		v0.AddArg(dst)
+		v1 := b.NewValue0(v.Pos, OpStore, types.TypeMem)
+		v1.Aux = typeToAux(t.FieldType(1))
+		v2 := b.NewValue0(v.Pos, OpOffPtr, t.FieldType(1).PtrTo())
+		v2.AuxInt = int64ToAuxInt(t.FieldOff(1))
+		v2.AddArg(dst)
+		v3 := b.NewValue0(v.Pos, OpStore, types.TypeMem)
+		v3.Aux = typeToAux(t.FieldType(0))
+		v4 := b.NewValue0(v.Pos, OpOffPtr, t.FieldType(0).PtrTo())
+		v4.AuxInt = int64ToAuxInt(0)
+		v4.AddArg(dst)
+		v3.AddArg3(v4, f0, mem)
+		v1.AddArg3(v2, f1, v3)
+		v.AddArg3(v0, f2, v1)
+		return true
+	}
+	// match: (Store dst (StructMake4 <t> f0 f1 f2 f3) mem)
+	// result: (Store {t.FieldType(3)} (OffPtr <t.FieldType(3).PtrTo()> [t.FieldOff(3)] dst) f3 (Store {t.FieldType(2)} (OffPtr <t.FieldType(2).PtrTo()> [t.FieldOff(2)] dst) f2 (Store {t.FieldType(1)} (OffPtr <t.FieldType(1).PtrTo()> [t.FieldOff(1)] dst) f1 (Store {t.FieldType(0)} (OffPtr <t.FieldType(0).PtrTo()> [0] dst) f0 mem))))
+	for {
+		dst := v_0
+		if v_1.Op != OpStructMake4 {
+			break
+		}
+		t := v_1.Type
+		f3 := v_1.Args[3]
+		f0 := v_1.Args[0]
+		f1 := v_1.Args[1]
+		f2 := v_1.Args[2]
+		mem := v_2
+		v.reset(OpStore)
+		v.Aux = typeToAux(t.FieldType(3))
+		v0 := b.NewValue0(v.Pos, OpOffPtr, t.FieldType(3).PtrTo())
+		v0.AuxInt = int64ToAuxInt(t.FieldOff(3))
+		v0.AddArg(dst)
+		v1 := b.NewValue0(v.Pos, OpStore, types.TypeMem)
+		v1.Aux = typeToAux(t.FieldType(2))
+		v2 := b.NewValue0(v.Pos, OpOffPtr, t.FieldType(2).PtrTo())
+		v2.AuxInt = int64ToAuxInt(t.FieldOff(2))
+		v2.AddArg(dst)
+		v3 := b.NewValue0(v.Pos, OpStore, types.TypeMem)
+		v3.Aux = typeToAux(t.FieldType(1))
+		v4 := b.NewValue0(v.Pos, OpOffPtr, t.FieldType(1).PtrTo())
+		v4.AuxInt = int64ToAuxInt(t.FieldOff(1))
+		v4.AddArg(dst)
+		v5 := b.NewValue0(v.Pos, OpStore, types.TypeMem)
+		v5.Aux = typeToAux(t.FieldType(0))
+		v6 := b.NewValue0(v.Pos, OpOffPtr, t.FieldType(0).PtrTo())
+		v6.AuxInt = int64ToAuxInt(0)
+		v6.AddArg(dst)
+		v5.AddArg3(v6, f0, mem)
+		v3.AddArg3(v4, f1, v5)
+		v1.AddArg3(v2, f2, v3)
+		v.AddArg3(v0, f3, v1)
+		return true
+	}
+	// match: (Store dst (ArrayMake1 e) mem)
+	// result: (Store {e.Type} dst e mem)
+	for {
+		dst := v_0
+		if v_1.Op != OpArrayMake1 {
+			break
+		}
+		e := v_1.Args[0]
+		mem := v_2
+		v.reset(OpStore)
+		v.Aux = typeToAux(e.Type)
+		v.AddArg3(dst, e, mem)
+		return true
+	}
 	return false
 }
 func rewriteValuedec_OpStringLen(v *Value) bool {
 	v_0 := v.Args[0]
+	b := v.Block
+	config := b.Func.Config
+	typ := &b.Func.Config.Types
 	// match: (StringLen (StringMake _ len))
 	// result: len
 	for {
@@ -407,10 +851,35 @@ func rewriteValuedec_OpStringLen(v *Value) bool {
 		v.copyOf(len)
 		return true
 	}
+	// match: (StringLen x:(Load <t> ptr mem))
+	// cond: t.IsString()
+	// result: @x.Block (Load <typ.Int> (OffPtr <typ.IntPtr> [config.PtrSize] ptr) mem)
+	for {
+		x := v_0
+		if x.Op != OpLoad {
+			break
+		}
+		t := x.Type
+		mem := x.Args[1]
+		ptr := x.Args[0]
+		if !(t.IsString()) {
+			break
+		}
+		b = x.Block
+		v0 := b.NewValue0(v.Pos, OpLoad, typ.Int)
+		v.copyOf(v0)
+		v1 := b.NewValue0(v.Pos, OpOffPtr, typ.IntPtr)
+		v1.AuxInt = int64ToAuxInt(config.PtrSize)
+		v1.AddArg(ptr)
+		v0.AddArg2(v1, mem)
+		return true
+	}
 	return false
 }
 func rewriteValuedec_OpStringPtr(v *Value) bool {
 	v_0 := v.Args[0]
+	b := v.Block
+	typ := &b.Func.Config.Types
 	// match: (StringPtr (StringMake ptr _))
 	// result: ptr
 	for {
@@ -421,6 +890,191 @@ func rewriteValuedec_OpStringPtr(v *Value) bool {
 		v.copyOf(ptr)
 		return true
 	}
+	// match: (StringPtr x:(Load <t> ptr mem))
+	// cond: t.IsString()
+	// result: @x.Block (Load <typ.BytePtr> ptr mem)
+	for {
+		x := v_0
+		if x.Op != OpLoad {
+			break
+		}
+		t := x.Type
+		mem := x.Args[1]
+		ptr := x.Args[0]
+		if !(t.IsString()) {
+			break
+		}
+		b = x.Block
+		v0 := b.NewValue0(v.Pos, OpLoad, typ.BytePtr)
+		v.copyOf(v0)
+		v0.AddArg2(ptr, mem)
+		return true
+	}
+	return false
+}
+func rewriteValuedec_OpStructMake1(v *Value) bool {
+	v_0 := v.Args[0]
+	// match: (StructMake1 x)
+	// cond: x.Type.IsPtr()
+	// result: x
+	for {
+		x := v_0
+		if !(x.Type.IsPtr()) {
+			break
+		}
+		v.copyOf(x)
+		return true
+	}
+	return false
+}
+func rewriteValuedec_OpStructSelect(v *Value) bool {
+	v_0 := v.Args[0]
+	b := v.Block
+	// match: (StructSelect [0] (IData x))
+	// result: (IData x)
+	for {
+		if auxIntToInt64(v.AuxInt) != 0 || v_0.Op != OpIData {
+			break
+		}
+		x := v_0.Args[0]
+		v.reset(OpIData)
+		v.AddArg(x)
+		return true
+	}
+	// match: (StructSelect (StructMake1 x))
+	// result: x
+	for {
+		if v_0.Op != OpStructMake1 {
+			break
+		}
+		x := v_0.Args[0]
+		v.copyOf(x)
+		return true
+	}
+	// match: (StructSelect [0] (StructMake2 x _))
+	// result: x
+	for {
+		if auxIntToInt64(v.AuxInt) != 0 || v_0.Op != OpStructMake2 {
+			break
+		}
+		x := v_0.Args[0]
+		v.copyOf(x)
+		return true
+	}
+	// match: (StructSelect [1] (StructMake2 _ x))
+	// result: x
+	for {
+		if auxIntToInt64(v.AuxInt) != 1 || v_0.Op != OpStructMake2 {
+			break
+		}
+		x := v_0.Args[1]
+		v.copyOf(x)
+		return true
+	}
+	// match: (StructSelect [0] (StructMake3 x _ _))
+	// result: x
+	for {
+		if auxIntToInt64(v.AuxInt) != 0 || v_0.Op != OpStructMake3 {
+			break
+		}
+		x := v_0.Args[0]
+		v.copyOf(x)
+		return true
+	}
+	// match: (StructSelect [1] (StructMake3 _ x _))
+	// result: x
+	for {
+		if auxIntToInt64(v.AuxInt) != 1 || v_0.Op != OpStructMake3 {
+			break
+		}
+		x := v_0.Args[1]
+		v.copyOf(x)
+		return true
+	}
+	// match: (StructSelect [2] (StructMake3 _ _ x))
+	// result: x
+	for {
+		if auxIntToInt64(v.AuxInt) != 2 || v_0.Op != OpStructMake3 {
+			break
+		}
+		x := v_0.Args[2]
+		v.copyOf(x)
+		return true
+	}
+	// match: (StructSelect [0] (StructMake4 x _ _ _))
+	// result: x
+	for {
+		if auxIntToInt64(v.AuxInt) != 0 || v_0.Op != OpStructMake4 {
+			break
+		}
+		x := v_0.Args[0]
+		v.copyOf(x)
+		return true
+	}
+	// match: (StructSelect [1] (StructMake4 _ x _ _))
+	// result: x
+	for {
+		if auxIntToInt64(v.AuxInt) != 1 || v_0.Op != OpStructMake4 {
+			break
+		}
+		x := v_0.Args[1]
+		v.copyOf(x)
+		return true
+	}
+	// match: (StructSelect [2] (StructMake4 _ _ x _))
+	// result: x
+	for {
+		if auxIntToInt64(v.AuxInt) != 2 || v_0.Op != OpStructMake4 {
+			break
+		}
+		x := v_0.Args[2]
+		v.copyOf(x)
+		return true
+	}
+	// match: (StructSelect [3] (StructMake4 _ _ _ x))
+	// result: x
+	for {
+		if auxIntToInt64(v.AuxInt) != 3 || v_0.Op != OpStructMake4 {
+			break
+		}
+		x := v_0.Args[3]
+		v.copyOf(x)
+		return true
+	}
+	// match: (StructSelect [0] x)
+	// cond: x.Type.IsPtr()
+	// result: x
+	for {
+		if auxIntToInt64(v.AuxInt) != 0 {
+			break
+		}
+		x := v_0
+		if !(x.Type.IsPtr()) {
+			break
+		}
+		v.copyOf(x)
+		return true
+	}
+	// match: (StructSelect [i] x:(Load <t> ptr mem))
+	// result: @x.Block (Load <v.Type> (OffPtr <v.Type.PtrTo()> [t.FieldOff(int(i))] ptr) mem)
+	for {
+		i := auxIntToInt64(v.AuxInt)
+		x := v_0
+		if x.Op != OpLoad {
+			break
+		}
+		t := x.Type
+		mem := x.Args[1]
+		ptr := x.Args[0]
+		b = x.Block
+		v0 := b.NewValue0(v.Pos, OpLoad, v.Type)
+		v.copyOf(v0)
+		v1 := b.NewValue0(v.Pos, OpOffPtr, v.Type.PtrTo())
+		v1.AuxInt = int64ToAuxInt(t.FieldOff(int(i)))
+		v1.AddArg(ptr)
+		v0.AddArg2(v1, mem)
+		return true
+	}
 	return false
 }
 func rewriteBlockdec(b *Block) bool {

src/cmd/compile/internal/ssagen/ssa.go

@@ -526,7 +526,7 @@ func buildssa(fn *ir.Func, worker int) *ssa.Func {
 						s.store(n.Type(), s.decladdrs[n], v)
 					} else { // Too big for SSA.
 						// Brute force, and early, do a bunch of stores from registers
-						// TODO fix the nasty storeArgOrLoad recursion in ssa/expand_calls.go so this Just Works with store of a big Arg.
+						// Note that expand calls knows about this and doesn't trouble itself with larger-than-SSA-able Args in registers.
 						s.storeParameterRegsToStack(s.f.ABISelf, paramAssignment, n, s.decladdrs[n], false)
 					}
 				}

test/abi/more_intstar_input.go

@@ -12,10 +12,6 @@
 
 package main
 
-import (
-	"fmt"
-)
-
 var sink int
 
 //go:registerparams
@@ -33,12 +29,12 @@ func G(a, b, c, d, e, f, g, h, i, j, k, l, m *int) {
 	var scratch [1000 * 100]int
 	I := *c - *e - *l // zero.
 	scratch[I] = *d
-	fmt.Println("Got this far!")
+	println("Got this far!")
 	sink += scratch[0]
 }
 
 func main() {
 	a, b, c, d, e, f, g, h, i, j, k, l, m := 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13
 	F(&a, &b, &c, &d, &e, &f, &g, &h, &i, &j, &k, &l, &m)
-	fmt.Printf("Sink = %d\n", sink-7)
+	println("Sink =", sink-7)
 }

test/abi/reg_not_ssa.go

@@ -0,0 +1,40 @@
+// run
+
+//go:build !wasm
+// +build !wasm
+
+// Copyright 2023 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package main
+
+// small enough for registers, too large for SSA
+type T struct {
+	a, b, c, d, e int
+}
+
+//go:noinline
+func F() {
+	a, b := g(), g()
+	h(b, b)
+	h(a, g())
+	if a.a == 1 {
+		a = g()
+	}
+	h(a, a)
+}
+
+//go:noinline
+func g() T {
+	return T{1, 2, 3, 4, 5}
+}
+
+//go:noinline
+func h(s, t T) {
+	if s != t {
+		println("NEQ")
+	}
+}
+
+func main() { F() }

test/codegen/condmove.go

@@ -57,7 +57,7 @@ func cmov16bit(x, y uint16) uint16 {
 	}
 	// amd64:"CMOVW(HI|CS)"
 	// arm64:"CSNEG\t(LS|HS)"
-	// ppc64x:"ISEL\t[$]0"
+	// ppc64x:"ISEL\t[$][01]"
 	// wasm:"Select"
 	return x
 }

test/codegen/spectre.go

@@ -13,12 +13,12 @@ func IndexArray(x *[10]int, i int) int {
 }
 
 func IndexString(x string, i int) byte {
-	// amd64:`CMOVQLS`
+	// amd64:`CMOVQ(LS|CC)`
 	return x[i]
 }
 
 func IndexSlice(x []float64, i int) float64 {
-	// amd64:`CMOVQLS`
+	// amd64:`CMOVQ(LS|CC)`
 	return x[i]
 }