cmd/compile/internal/types2: disallow type cycles through type parameter lists

If we reach a generic type that is part of a cycle
and we are in a type parameter list, we have a cycle
through a type parameter list, which is invalid.

Fixes #49439.

Change-Id: Ia6cf97e1748ca0c0e61c02841202050091365b0b
Reviewed-on: https://go-review.googlesource.com/c/go/+/361922
Trust: Robert Griesemer <gri@golang.org>
Reviewed-by: Robert Findley <rfindley@google.com>

src/cmd/compile/internal/types2/check.go

@@ -46,6 +46,7 @@ type context struct {
 	pos           syntax.Pos                // if valid, identifiers are looked up as if at position pos (used by Eval)
 	iota          constant.Value            // value of iota in a constant declaration; nil otherwise
 	errpos        syntax.Pos                // if valid, identifier position of a constant with inherited initializer
+	inTParamList  bool                      // set if inside a type parameter list
 	sig           *Signature                // function signature if inside a function; nil otherwise
 	isPanic       map[*syntax.CallExpr]bool // set of panic call expressions (used for termination check)
 	hasLabel      bool                      // set if a function makes use of labels (only ~1% of functions); unused outside functions

src/cmd/compile/internal/types2/decl.go

@@ -228,13 +228,23 @@ func (check *Checker) validCycle(obj Object) (valid bool) {
 	assert(obj.color() >= grey)
 	start := obj.color() - grey // index of obj in objPath
 	cycle := check.objPath[start:]
-	nval := 0 // number of (constant or variable) values in the cycle
-	ndef := 0 // number of type definitions in the cycle
+	tparCycle := false // if set, the cycle is through a type parameter list
+	nval := 0          // number of (constant or variable) values in the cycle; valid if !generic
+	ndef := 0          // number of type definitions in the cycle; valid if !generic
+loop:
 	for _, obj := range cycle {
 		switch obj := obj.(type) {
 		case *Const, *Var:
 			nval++
 		case *TypeName:
+			// If we reach a generic type that is part of a cycle
+			// and we are in a type parameter list, we have a cycle
+			// through a type parameter list, which is invalid.
+			if check.inTParamList && isGeneric(obj.typ) {
+				tparCycle = true
+				break loop
+			}
+
 			// Determine if the type name is an alias or not. For
 			// package-level objects, use the object map which
 			// provides syntactic information (which doesn't rely
@@ -262,7 +272,11 @@ func (check *Checker) validCycle(obj Object) (valid bool) {
 
 	if check.conf.Trace {
 		check.trace(obj.Pos(), "## cycle detected: objPath = %s->%s (len = %d)", pathString(cycle), obj.Name(), len(cycle))
+		if tparCycle {
+			check.trace(obj.Pos(), "## cycle contains: generic type in a type parameter list")
+		} else {
 			check.trace(obj.Pos(), "## cycle contains: %d values, %d type definitions", nval, ndef)
+		}
 		defer func() {
 			if !valid {
 				check.trace(obj.Pos(), "=> error: cycle is invalid")
@@ -270,6 +284,7 @@ func (check *Checker) validCycle(obj Object) (valid bool) {
 		}()
 	}
 
+	if !tparCycle {
 		// A cycle involving only constants and variables is invalid but we
 		// ignore them here because they are reported via the initialization
 		// cycle check.
@@ -283,6 +298,7 @@ func (check *Checker) validCycle(obj Object) (valid bool) {
 		if nval == 0 && ndef > 0 {
 			return true
 		}
+	}
 
 	check.cycleError(cycle)
 	return false
@@ -624,6 +640,19 @@ func (check *Checker) collectTypeParams(dst **TypeParamList, list []*syntax.Fiel
 	// Example: type T[P T[P]] interface{}
 	*dst = bindTParams(tparams)
 
+	// Signal to cycle detection that we are in a type parameter list.
+	// We can only be inside one type parameter list at any given time:
+	// function closures may appear inside a type parameter list but they
+	// cannot be generic, and their bodies are processed in delayed and
+	// sequential fashion. Note that with each new declaration, we save
+	// the existing context and restore it when done; thus inTParamList
+	// is true exactly only when we are in a specific type parameter list.
+	assert(!check.inTParamList)
+	check.inTParamList = true
+	defer func() {
+		check.inTParamList = false
+	}()
+
 	// Keep track of bounds for later validation.
 	var bound Type
 	var bounds []Type

src/cmd/compile/internal/types2/testdata/fixedbugs/issue45550.go2

@@ -4,7 +4,7 @@
 
 package p
 
-type Builder[T interface{ struct{ Builder[T] } }] struct{}
+type Builder /* ERROR illegal cycle */ [T interface{ struct{ Builder[T] } }] struct{}
 type myBuilder struct {
-	Builder[myBuilder /* ERROR myBuilder does not satisfy */]
+	Builder[myBuilder]
 }

src/cmd/compile/internal/types2/testdata/fixedbugs/issue46461.go2

@@ -5,16 +5,16 @@
 package p
 
 // test case 1
-type T[U interface{ M() T[U] }] int
+type T /* ERROR illegal cycle */ [U interface{ M() T[U] }] int
 
 type X int
 
 func (X) M() T[X] { return 0 }
 
 // test case 2
-type A[T interface{ A[T] }] interface{}
+type A /* ERROR illegal cycle */ [T interface{ A[T] }] interface{}
 
 // test case 3
-type A2[U interface{ A2[U] }] interface{ M() A2[U] }
+type A2 /* ERROR illegal cycle */ [U interface{ A2[U] }] interface{ M() A2[U] }
 
 type I interface{ A2[I]; M() A2[I] }

src/cmd/compile/internal/types2/testdata/fixedbugs/issue47796.go2

@@ -6,16 +6,16 @@ package p
 
 // parameterized types with self-recursive constraints
 type (
-	T1[P T1[P]]                            interface{}
-	T2[P, Q T2[P, Q]]                      interface{}
+	T1 /* ERROR illegal cycle */ [P T1[P]]                            interface{}
+	T2 /* ERROR illegal cycle */ [P, Q T2[P, Q]]                      interface{}
 	T3[P T2[P, Q], Q interface{ ~string }] interface{}
 
-	T4a[P T4a[P]]                                                        interface{ ~int }
-	T4b[P T4b[int]]                                                      interface{ ~int }
-	T4c[P T4c[string /* ERROR string does not satisfy T4c\[string\] */]] interface{ ~int }
+	T4a /* ERROR illegal cycle */ [P T4a[P]]                                                        interface{ ~int }
+	T4b /* ERROR illegal cycle */ [P T4b[int]]                                                      interface{ ~int }
+	T4c /* ERROR illegal cycle */ [P T4c[string]] interface{ ~int }
 
 	// mutually recursive constraints
-	T5[P T6[P]] interface{ int }
+	T5 /* ERROR illegal cycle */ [P T6[P]] interface{ int }
 	T6[P T5[P]] interface{ int }
 )
 
@@ -28,6 +28,6 @@ var (
 
 // test case from issue
 
-type Eq[a Eq[a]] interface {
+type Eq /* ERROR illegal cycle */ [a Eq[a]] interface {
 	Equal(that a) bool
 }

src/cmd/compile/internal/types2/testdata/fixedbugs/issue48529.go2

@@ -4,7 +4,7 @@
 
 package p
 
-type T[U interface{ M() T /* ERROR "got 2 arguments but 1 type parameters" */ [U, int] }] int
+type T /* ERROR illegal cycle */ [U interface{ M() T[U, int] }] int
 
 type X int
 

src/cmd/compile/internal/types2/testdata/fixedbugs/issue49439.go2

@@ -0,0 +1,26 @@
+// Copyright 2021 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package p
+
+import "unsafe"
+
+type T0 /* ERROR illegal cycle */ [P T0[P]] struct{}
+
+type T1 /* ERROR illegal cycle */ [P T2[P]] struct{}
+type T2[P T1[P]] struct{}
+
+type T3 /* ERROR illegal cycle */ [P interface{ ~struct{ f T3[int] } }] struct{}
+
+// valid cycle in M
+type N[P M[P]] struct{}
+type M[Q any] struct { F *M[Q] }
+
+// "crazy" case
+type TC[P [unsafe.Sizeof(func() {
+        type T [P [unsafe.Sizeof(func(){})]byte] struct{}
+})]byte] struct{}
+
+// test case from issue
+type X /* ERROR illegal cycle */ [T any, PT X[T]] interface{}

test/typeparam/issue46461.go

@@ -1,4 +1,4 @@
-// compile -G=3
+// errorcheck -G=3
 
 // Copyright 2021 The Go Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style
@@ -6,7 +6,7 @@
 
 package p
 
-type T[U interface{ M() T[U] }] int
+type T[U interface{ M() T[U] }] int // ERROR "invalid recursive type T"
 
 type X int
 

test/typeparam/issue46461b.dir/a.go

@@ -4,4 +4,4 @@
 
 package a
 
-type T[U interface{ M() T[U] }] int
+type T[U interface{ M() int }] int

test/typeparam/issue46461b.dir/b.go

@@ -8,4 +8,6 @@ import "./a"
 
 type X int
 
-func (X) M() a.T[X] { return 0 }
+func (X) M() int { return 0 }
+
+type _ a.T[X]

test/typeparam/issue48280.dir/a.go

@@ -4,7 +4,7 @@
 
 package a
 
-type I[T I[T]] interface {
+type I[T any] interface {
 	F() T
 }
 

test/typeparam/issue48306.dir/a.go

@@ -4,6 +4,6 @@
 
 package a
 
-type I[T I[T]] interface {
+type I[T any] interface {
 	F() T
 }