go/types: fix type substitution

When instantiating a type, cache a new version with a valid (non-nil)
underlying type so that we don't run into a missing underlying type
if instantiation is self-recursive.

Added more tests, also for issues found while debugging.

Change-Id: I520a4f71e60bd9020977276db078582d41a677f1
Reviewed-on: https://team-review.git.corp.google.com/c/golang/go2-dev/+/723252
Reviewed-by: Robert Griesemer <gri@google.com>

src/go/types/NOTES

@@ -4,6 +4,9 @@ so we have a better track record. I only switched to this file in Nov 2019, henc
 ----------------------------------------------------------------------------------------------------
 TODO
 
+- fix endless instantiation when printing: type T(type P) T(P)
+- report error for infinite type: type A(type P) struct { _ A(P) }
+- review all direct accesses to Named.underlying and verify that they are still correct
 - track contract origin in interface bounds for better error messages
 - better error message when we need parentheses around a parameterized function parameter type
 - revisit uses of (raw)lookupFieldOrMethod with respect to the addressable flag (can we get rid of it?)

src/go/types/subst.go

@@ -306,6 +306,8 @@ func (subst *subster) typ(typ Type) Type {
 			}
 		}
 
+		assert(t.underlying != nil)
+
 		if t.tparams == nil {
 			dump(">>> %s is not parameterized", t)
 			return t // type is not parameterized
@@ -358,14 +360,15 @@ func (subst *subster) typ(typ Type) Type {
 
 		// create a new named type and populate caches to avoid endless recursion
 		tname := NewTypeName(subst.pos, t.obj.pkg, t.obj.name, nil)
-		named := subst.check.NewNamed(tname, nil, nil)
-		named.tparams = t.tparams // new type is still parameterized
+		named := subst.check.NewNamed(tname, t.underlying, t.methods) // method signatures are updated lazily
+		named.tparams = t.tparams                                     // new type is still parameterized
 		named.targs = new_targs
 		subst.check.typMap[h] = named
 		subst.cache[t] = named
+
+		// do the substitution
 		dump(">>> subst %s with %s (new: %s)", t.underlying, subst.smap, new_targs)
 		named.underlying = subst.typ(t.underlying)
-		named.methods = t.methods // method signatures are updated lazily
 
 		return named
 
@@ -373,6 +376,7 @@ func (subst *subster) typ(typ Type) Type {
 		return subst.smap.lookup(t)
 
 	case *instance:
+		// TODO(gri) can we avoid the expansion here and just substitute the type parameters?
 		return subst.typ(t.expand())
 
 	default:

src/go/types/testdata/issues.go2

@@ -174,14 +174,14 @@ func (l *List(T)) Init() {
 }
 
 // This is the original program causing the same issue.
-type Element(type TElem) struct {
-	next, prev *Element(TElem)
+type Element2(type TElem) struct {
+	next, prev *Element2(TElem)
 	list *List2(TElem)
 	Value TElem
 }
 
 type List2(type TElem) struct {
-	root Element(TElem)
+	root Element2(TElem)
 	len  int
 }
 
@@ -191,3 +191,29 @@ func (l *List2(TElem)) Init() *List2(TElem) {
 	l.len = 0
 	return l
 }
+
+// Self-recursive instantiations must work correctly.
+type A(type P) struct { _ *A(P) }
+
+type AB(type P) struct { _ *BA(P) }
+type BA(type P) struct { _ *AB(P) }
+
+// And a variation that also caused a problem with an
+// unresolved underlying type.
+type Element3(type TElem) struct {
+	next, prev *Element3(TElem)
+	list *List3(TElem)
+	Value TElem
+}
+
+func (e *Element3(TElem)) Next() *Element3(TElem) {
+	if p := e.next; e.list != nil && p != &e.list.root {
+		return p
+	}
+	return nil
+}
+
+type List3(type TElem) struct {
+	root Element3(TElem)
+	len  int
+}

src/go/types/testdata/todos.go2

@@ -13,30 +13,36 @@ contract _C(T) {
 	* /* ERROR not yet supported */ T m()
 }
 
-// indexing on generic types containing type parameters in their type list
-// is not yet supported
+// Indexing on generic types containing type parameters in their type list
+// is not yet supported.
 func _(type T interface { type T })(x T) {
         _ = x /* ERROR type parameter */ /* ERROR cannot index */ [0]
 }
 
-// channel sends and receives on generic types is not yet supported
+// Indexing a generic type with an array type bound should check length.
+// (Example by mdempsky@.)
+func _(type T interface { type [10]int })(x T) {
+	_ = x[20] // this should report a compile-time error
+}
+
+// Channel sends and receives on generic types is not yet supported.
 func _(type T interface{ type chan int })(ch T) {
 	ch <- /* ERROR cannot send */ 0
 	_ = <- ch /* ERROR cannot receive */
 }
 
-// pointer indirection of generic types is not yet supported
+// Pointer indirection of generic types is not yet supported.
 func _(type T interface{ type *int })(p T) {
 	_ = *p /* ERROR cannot indirect */
 }
 
-// calling of a generic variable is not yet supported
+// Calling of a generic variable is not yet supported.
 func _(type T interface{ type func() })(f T) {
 	f /* ERROR cannot call */ ()
 	go f /* ERROR cannot call */ ()
 }
 
-// need to investigate the exact nature of a generic type (is it a named type)?
+// Need to investigate the exact nature of a generic type (is it a named type)?
 func _(type T interface{ type int})(x T) {
 	type myint int
 	var _ int = x /* ERROR cannot use */
@@ -56,3 +62,6 @@ func _() {
    _ = [](T1(int)){}                          // this works
    _ = [](T2(int, string)){}                  // T2(int, float) cannot be a conversion - should not need ()'s
 }
+
+// Infinite generic type declarations should lead to an error.
+type inf(type T) struct{ _ inf(T) } // should report an error

src/go/types/testdata/typeinst.go2

@@ -44,3 +44,8 @@ var _ List(List(List(int)))
 type T3(type P) List(P)
 
 var _ T3(int) = T3(int)(List(int){1, 2, 3})
+
+// Self-recursive generic types are not permitted
+
+type self1(type P) self1 /* ERROR illegal cycle */ (P)
+type self2(type P) *self2(P) // this is ok