[dev.go2go] go/types: fix computation of type hash

Fixes #39982.

Change-Id: I4e7b52c34bf8df63f2063dc2504a8125ca7585a0
Reviewed-on: https://go-review.googlesource.com/c/go/+/240737
Reviewed-by: Robert Griesemer <gri@golang.org>

src/go/types/check_test.go

@@ -138,6 +138,7 @@ var tests = [][]string{
 	{"fixedbugs/issue39938.go2"},
 	{"fixedbugs/issue39948.go2"},
 	{"fixedbugs/issue39976.go2"},
+	{"fixedbugs/issue39982.go2"},
 }
 
 var fset = token.NewFileSet()

src/go/types/errors.go

@@ -127,7 +127,7 @@ func stripAnnotations(s string) string {
 	var b strings.Builder
 	for _, r := range s {
 		// strip #'s and subscript digits
-		if r != '#' && !('₀' <= r && r < '₀'+10) { // '₀' == U+2080
+		if r != instanceMarker && !('₀' <= r && r < '₀'+10) { // '₀' == U+2080
 			b.WriteRune(r)
 		}
 	}

src/go/types/fixedbugs/issue39982.go2

@@ -0,0 +1,36 @@
+// Copyright 2020 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package p
+
+type (
+	T(type _) struct{}
+	S(type _) struct {
+		data T(*T(int))
+	}
+)
+
+func _() {
+	_ = S(int){
+		data: T(*T(int)){},
+	}
+}
+
+// full test case from issue
+
+type (
+	Element(type TElem) struct{}
+
+	entry(type K comparable) struct{}
+
+	Cache(type K comparable) struct {
+		data map[K]*Element(*entry(K))
+	}
+)
+
+func _() {
+	_ = Cache(int){
+		data: make(map[int](*Element(*entry(int)))),
+	}
+}

src/go/types/subst.go

@@ -425,7 +425,21 @@ func instantiatedHash(typ *Named, targs []Type) string {
 	buf.WriteByte('(')
 	writeTypeList(&buf, targs, nil, nil)
 	buf.WriteByte(')')
-	return buf.String()
+
+	// With respect to the represented type, whether a
+	// type is fully expanded or stored as instance
+	// does not matter - they are the same types.
+	// Remove the instanceMarkers printed for instances.
+	res := buf.Bytes()
+	i := 0
+	for _, b := range res {
+		if b != instanceMarker {
+			res[i] = b
+			i++
+		}
+	}
+
+	return string(res[:i])
 }
 
 func typeListString(list []Type) string {

src/go/types/typestring.go

@@ -76,6 +76,10 @@ func WriteType(buf *bytes.Buffer, typ Type, qf Qualifier) {
 	writeType(buf, typ, qf, make([]Type, 0, 8))
 }
 
+// instanceMarker is the prefix for an instantiated type
+// in "non-evaluated" instance form.
+const instanceMarker = '#'
+
 func writeType(buf *bytes.Buffer, typ Type, qf Qualifier, visited []Type) {
 	// Theoretically, this is a quadratic lookup algorithm, but in
 	// practice deeply nested composite types with unnamed component
@@ -281,7 +285,7 @@ func writeType(buf *bytes.Buffer, typ Type, qf Qualifier, visited []Type) {
 		buf.WriteString(s + subscript(t.id))
 
 	case *instance:
-		buf.WriteByte('#') // indicate "non-evaluated" syntactic instance
+		buf.WriteByte(instanceMarker) // indicate "non-evaluated" syntactic instance
 		writeTypeName(buf, t.base.obj, qf)
 		buf.WriteByte('(')
 		writeTypeList(buf, t.targs, qf, visited)