cmd/compile/internal/types2: avoid infinite expansion for invalid recursive generic types

The algorithm for detecting invalid recursive types that
expand indefinitely suffered from the exact problem is was
intended to detect: if the indefinite expansion is happening
through type parameters, the algorithm ended up in an infinite
sequence of instantiations. (This is only a problem for generic
types).

Changed the algorithm to always only consider the "original"
uninstantiated types. This avoids the problem but it will also
not detect some invalid recursive generic types anymore. That
requires a more sophisticated type flow analysis.
Opened #48962 to track.

Addressed with help from @findleyr.

For #48951.

Change-Id: Ie29cea8f810dae55153dbb1b17c9390cd823c2d9
Reviewed-on: https://go-review.googlesource.com/c/go/+/355732
Trust: Robert Griesemer <gri@golang.org>
Reviewed-by: Robert Findley <rfindley@google.com>

src/cmd/compile/internal/types2/decl.go

@@ -330,7 +330,16 @@ func (check *Checker) validType(typ Type, path []Object) typeInfo {
 		}
 
 	case *Named:
-		t.resolve(check.conf.Context)
+		// If t is parameterized, we should be considering the instantiated (expanded)
+		// form of t, but in general we can't with this algorithm: if t is an invalid
+		// type it may be so because it infinitely expands through a type parameter.
+		// Instantiating such a type would lead to an infinite sequence of instantiations.
+		// In general, we need "type flow analysis" to recognize those cases.
+		// Example: type A[T any] struct{ x A[*T] } (issue #48951)
+		// In this algorithm we always only consider the orginal, uninstantiated type.
+		// This won't recognize some invalid cases with parameterized types, but it
+		// will terminate.
+		t = t.orig
 
 		// don't touch the type if it is from a different package or the Universe scope
 		// (doing so would lead to a race condition - was issue #35049)
@@ -359,7 +368,7 @@ func (check *Checker) validType(typ Type, path []Object) typeInfo {
 					check.cycleError(path[i:])
 					t.info = invalid
 					t.underlying = Typ[Invalid]
-					return t.info
+					return invalid
 				}
 			}
 			panic("cycle start not found")

src/cmd/compile/internal/types2/testdata/check/issues.go2

@@ -145,8 +145,8 @@ type List3[TElem any] struct {
 }
 
 // Infinite generic type declarations must lead to an error.
-type inf1[T any] struct{ _ inf1 /* ERROR illegal cycle */ [T] }
-type inf2[T any] struct{ inf2 /* ERROR illegal cycle */ [T] }
+type inf1 /* ERROR illegal cycle */ [T any] struct{ _ inf1[T] }
+type inf2 /* ERROR illegal cycle */ [T any] struct{ inf2[T] }
 
 // The implementation of conversions T(x) between integers and floating-point
 // numbers checks that both T and x have either integer or floating-point

src/cmd/compile/internal/types2/testdata/fixedbugs/issue39634.go2

@@ -37,8 +37,8 @@ func main7() { var _ foo7 = x7[int]{} }
 // func main8() {}
 
 // crash 9
-type foo9[A any] interface { foo9 /* ERROR illegal cycle */ [A] }
-func _() { var _ = new(foo9 /* ERROR illegal cycle */ [int]) }
+type foo9 /* ERROR illegal cycle */ [A any] interface { foo9[A] }
+func _() { var _ = new(foo9[int]) }
 
 // crash 12
 var u /* ERROR cycle */ , i [func /* ERROR used as value */ /* ERROR used as value */ (u, c /* ERROR undeclared */ /* ERROR undeclared */ ) {}(0, len /* ERROR must be called */ /* ERROR must be called */ )]c /* ERROR undeclared */ /* ERROR undeclared */

src/cmd/compile/internal/types2/testdata/fixedbugs/issue39938.go2

@@ -3,6 +3,8 @@
 // license that can be found in the LICENSE file.
 
 // Check "infinite expansion" cycle errors across instantiated types.
+// We can't detect these errors anymore at the moment. See #48962 for
+// details.
 
 package p
 
@@ -11,11 +13,11 @@ type E1[P any] *P
 type E2[P any] struct{ _ P }
 type E3[P any] struct{ _ *P }
 
-type T0 /* ERROR illegal cycle */ struct {
+type T0 /* illegal cycle */ struct {
         _ E0[T0]
 }
 
-type T0_ /* ERROR illegal cycle */ struct {
+type T0_ /* illegal cycle */ struct {
         E0[T0_]
 }
 
@@ -23,7 +25,7 @@ type T1 struct {
         _ E1[T1]
 }
 
-type T2 /* ERROR illegal cycle */ struct {
+type T2 /* illegal cycle */ struct {
         _ E2[T2]
 }
 
@@ -33,7 +35,7 @@ type T3 struct {
 
 // some more complex cases
 
-type T4 /* ERROR illegal cycle */ struct {
+type T4 /* illegal cycle */ struct {
 	_ E0[E2[T4]]
 }
 
@@ -41,7 +43,7 @@ type T5 struct {
 	_ E0[E2[E0[E1[E2[[10]T5]]]]]
 }
 
-type T6 /* ERROR illegal cycle */ struct {
+type T6 /* illegal cycle */ struct {
 	_ E0[[10]E2[E0[E2[E2[T6]]]]]
 }
 

src/cmd/compile/internal/types2/testdata/fixedbugs/issue48951.go2

@@ -0,0 +1,21 @@
+// Copyright 2020 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package p
+
+type (
+        A1 /* ERROR illegal cycle */ [P any] [10]A1[P]
+        A2 /* ERROR illegal cycle */ [P any] [10]A2[*P]
+        A3[P any] [10]*A3[P]
+
+        L1[P any] []L1[P]
+
+        S1 /* ERROR illegal cycle */ [P any] struct{ f S1[P] }
+        S2 /* ERROR illegal cycle */ [P any] struct{ f S2[*P] } // like example in issue
+        S3[P any] struct{ f *S3[P] }
+
+        I1 /* ERROR illegal cycle */ [P any] interface{ I1[P] }
+        I2 /* ERROR illegal cycle */ [P any] interface{ I2[*P] }
+        I3[P any] interface{ *I3 /* ERROR interface contains type constraints */ [P] }
+)