go/go2go, cmd/go2go: build a GOPATH of translated packages

Also start a testsuite for cmd/go2go.

Change-Id: Ib693c79b3a7427ac2f6be3264469274157a39851

src/cmd/go2go/doc.go

@@ -16,4 +16,15 @@
 //      test       translate and then run "go test packages"
 //      translate  translate .go2 files into .go files for listed packages
 //
+// A package is expected to contain .go2 files but no .go files.
+//
+// Non-local imported packages will be first looked up using the GO2PATH
+// environment variable, which should point to a GOPATH-like directory.
+// For example, import "x" will first look for $GO2PATH/src/x.
+// If not found in GO2PATH, imports will be looked up in the usual way.
+// If an import includes .go2 files, it will be translated into .go files.
+//
+// Translation into standard Go requires generating Go code with mangled names.
+// The mangled names will always include Odia (Oriya) digits, such as ୦ and ୮.
+// Do not use Oriya digits in identifiers in your own code.
 package main

src/cmd/go2go/go2go_test.go

@@ -0,0 +1,122 @@
+// Copyright 2020 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package main_test
+
+import (
+	"fmt"
+	"internal/testenv"
+	"io/ioutil"
+	"os"
+	"os/exec"
+	"path/filepath"
+	"sort"
+	"sync"
+	"testing"
+)
+
+func TestMain(m *testing.M) {
+	os.Exit(testMain(m))
+}
+
+func testMain(m *testing.M) int {
+	dir, err := ioutil.TempDir("", "go2gotest")
+	if err != nil {
+		fmt.Fprintln(os.Stderr, err)
+		os.Exit(1)
+	}
+	defer os.RemoveAll(dir)
+	testTempDir = dir
+
+	return m.Run()
+}
+
+// testTempDir is a temporary directory the tests can use.
+var testTempDir string
+
+// testGo2go is the version of cmd/go2go run by the tests.
+var testGo2go string
+
+// testGo2goOnce ensures that testGo2go is built only once.
+var testGo2goOnce sync.Once
+
+// testGo2goErr is an error that occurred when building testGo2go.
+// In the normal case this is nil.
+var testGo2goErr error
+
+// buildGo2go builds an up-to-date version of cmd/go2go.
+// This is not run from TestMain because it's simpler if it has a *testing.T.
+func buildGo2go(t *testing.T) {
+	t.Helper()
+	testenv.MustHaveGoBuild(t)
+	testGo2goOnce.Do(func() {
+		testGo2go = filepath.Join(testTempDir, "go2go.exe")
+		t.Logf("running [go build -o %s]", testGo2go)
+		out, err := exec.Command(testenv.GoToolPath(t), "build", "-o", testGo2go).CombinedOutput()
+		if len(out) > 0 {
+			t.Logf("%s", out)
+		}
+		testGo2goErr = err
+	})
+	if testGo2goErr != nil {
+		t.Fatal("failed to build testgo2go program:", testGo2goErr)
+	}
+}
+
+func TestGO2PATH(t *testing.T) {
+	buildGo2go(t)
+
+	copyFile := func(path string, info os.FileInfo, err error) error {
+		if err != nil {
+			t.Fatal(err)
+		}
+		newPath := filepath.Join(testTempDir, path)
+		if info.IsDir() {
+			if err := os.MkdirAll(newPath, 0755); err != nil {
+				t.Fatal(err)
+			}
+			return nil
+		}
+		data, err := ioutil.ReadFile(path)
+		if err != nil {
+			t.Fatal(err)
+		}
+		if err := ioutil.WriteFile(newPath, data, 0444); err != nil {
+			t.Fatal(err)
+		}
+		return nil
+	}
+
+	if err := filepath.Walk("testdata/go2path/src", copyFile); err != nil {
+		t.Fatal(err)
+	}
+
+	d, err := os.Open(filepath.Join(testTempDir, "testdata/go2path/src"))
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer d.Close()
+	dirs, err := d.Readdirnames(-1)
+	if err != nil {
+		t.Fatal(err)
+	}
+	sort.Strings(dirs)
+
+	for _, dir := range dirs {
+		t.Run(dir, func(t *testing.T) {
+			cmd := exec.Command(testGo2go, "test")
+			cmd.Dir = filepath.Join(testTempDir, "testdata", "go2path", "src", dir)
+			cmd.Env = append(os.Environ(), "GO2PATH=" + filepath.Join(testTempDir, "testdata", "go2path"))
+			t.Logf("running [%s test] in %s", testGo2go, cmd.Dir)
+			out, err := cmd.CombinedOutput()
+			if len(out) > 0 {
+				t.Log(dir)
+				t.Logf("%s", out)
+			}
+			if err != nil {
+				t.Errorf("error testing %s: %v", dir, err)
+			}
+		})
+	}
+}

src/cmd/go2go/main.go

@@ -75,6 +75,14 @@ func main() {
 		cmd.Stdout = os.Stdout
 		cmd.Stderr = os.Stderr
 		cmd.Dir = rundir
+		gopath := importerTmpdir
+		if oldGopath := os.Getenv("GOPATH"); oldGopath != "" {
+			gopath += ":" + oldGopath
+		}
+		cmd.Env = append(os.Environ(),
+			"GOPATH=" + gopath,
+			"GO111MODULE=off",
+		)
 		if err := cmd.Run(); err != nil {
 			die(fmt.Sprintf("%s %v failed: %v", gotool, args, err))
 		}

src/cmd/go2go/testdata/go2path/src/contracts/contracts.go2

@@ -0,0 +1,15 @@
+// Copyright 2020 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// Package contracts defines some useful contracts.
+package contracts
+
+// The Ordered contract permits any ordered type: any type that supports
+// the operations <, <=, >=, >, as well as == and !=.
+contract Ordered(T) {
+	T int, int8, int16, int32, int64,
+		uint, uint8, uint16, uint32, uint64, uintptr,
+		float32, float64,
+		string
+}

src/cmd/go2go/testdata/go2path/src/gsort/gsort.go2

@@ -0,0 +1,51 @@
+// Copyright 2020 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// Package gsort provides primitives for sorting slices of any type.
+package gsort
+
+import (
+	"sort"
+
+	"contracts"
+)
+
+// orderedSlice is a slice of values of some ordered type.
+type orderedSlice(type Elem contracts.Ordered) []Elem
+
+// orderedSlice implements sort.Interface.
+
+func (s orderedSlice(Elem)) Len() int           { return len(s) }
+func (s orderedSlice(Elem)) Less(i, j int) bool {
+	if s[i] < s[j] {
+		return true
+	}
+	isNaN := func(f Elem) bool { return f != f }
+	if isNaN(s[i]) && !isNaN(s[j]) {
+		return true
+	}
+	return false
+}
+func (s orderedSlice(Elem)) Swap(i, j int)      { s[i], s[j] = s[j], s[i] }
+
+// OrderedSlice sorts a slice of any ordered type in ascending order.
+func OrderedSlice(type Elem contracts.Ordered)(s []Elem) {
+	sort.Sort(orderedSlice(Elem)(s))
+}
+
+// sliceFn implements sort.Interface for a slice of any type with an
+// explicit less-than function.
+type sliceFn(type Elem) struct {
+	s    []Elem
+	less func(Elem, Elem) bool
+}
+
+func (s sliceFn(Elem)) Len() int           { return len(s.s) }
+func (s sliceFn(Elem)) Less(i, j int) bool { return s.less(s.s[i], s.s[j]) }
+func (s sliceFn(Elem)) Swap(i, j int)      { s.s[i], s.s[j] = s.s[j], s.s[i] }
+
+// SliceFn sorts a slice of any type according to a less-than function.
+func SliceFn(type Elem)(s []Elem, less func(Elem, Elem) bool) {
+	sort.Sort(sliceFn(Elem){s, less})
+}

src/cmd/go2go/testdata/go2path/src/gsort/gsort_test.go2

@@ -0,0 +1,92 @@
+// Copyright 2020 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package gsort
+
+import (
+	"math"
+	"sort"
+	"testing"
+
+	"contracts"
+	"slices"
+)
+
+// Test data copied from the standard library sort package.
+
+var ints = []int{74, 59, 238, -784, 9845, 959, 905, 0, 0, 42, 7586, -5467984, 7586}
+var float64s = []float64{74.3, 59.0, math.Inf(1), 238.2, -784.0, 2.3, math.NaN(), math.NaN(), math.Inf(-1), 9845.768, -959.7485, 905, 7.8, 7.8}
+var strings = []string{"", "Hello", "foo", "bar", "foo", "f00", "%*&^*&^&", "***"}
+
+func TestSortOrderedInts(t *testing.T) {
+	testOrdered(int)(t, ints, sort.Ints)
+}
+
+func TestSortOrderedFloat64s(t *testing.T) {
+	testOrdered(float64)(t, float64s, sort.Float64s)
+}
+
+func TestSortOrderedStrings(t *testing.T) {
+	testOrdered(string)(t, strings, sort.Strings)
+}
+
+func testOrdered(type Elem contracts.Ordered)(t *testing.T, s []Elem, sorter func([]Elem)) {
+	s1 := make([]Elem, len(s))
+	copy(s1, s)
+	s2 := make([]Elem, len(s))
+	copy(s2, s)
+	OrderedSlice(Elem)(s1)
+	sorter(s2)
+	if !slices.Equal(Elem)(s1, s2) {
+		t.Fatalf("got %v, want %v", s1, s2)
+	}
+	for i := len(s1) - 1; i > 0; i-- {
+		if s1[i] < s1[i-1] {
+			t.Fatalf("element %d (%v) < element %d (%v)", i, s1[i], i - 1, s1[i - 1])
+		}
+	}
+}
+
+var slicesToSort = [][]int {
+	[]int{1, 2},
+	[]int{3, 2, 1},
+	[]int{1},
+	[]int{1, 3},
+	[]int{1, 2, 3},
+}
+
+var sortedSlices = [][]int {
+	[]int{1},
+	[]int{1, 2},
+	[]int{1, 3},
+	[]int{1, 2, 3},
+	[]int{3, 2, 1},
+}
+
+func sorter(s1, s2 []int) bool {
+	switch {
+	case len(s1) < len(s2):
+		return true
+	case len(s1) > len(s2):
+		return false
+	}
+	for i := range s1 {
+		switch {
+		case s1[i] < s2[i]:
+			return true
+		case s1[i] > s2[i]:
+			return false
+		}
+	}
+	return false
+}
+
+func TestSortSliceFn(t *testing.T) {
+	c := make([][]int, len(slicesToSort))
+	copy(c, slicesToSort)
+	SliceFn([]int)(c, sorter)
+	if !slices.EqualFn([]int)(c, sortedSlices, func(a, b []int) bool { return slices.Equal(int)(a, b) }) {
+		t.Errorf("got %v, want %v", c, sortedSlices)
+	}
+}

src/cmd/go2go/testdata/go2path/src/slices/slices.go2

@@ -0,0 +1,40 @@
+// Copyright 2020 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package slices
+
+import "contracts"
+
+// Equal reports whether two slices are equal: the same length and all
+// elements equal. All floating point NaNs are considered equal.
+func Equal(type Elem contracts.Ordered)(s1, s2 []Elem) bool {
+	if len(s1) != len(s2) {
+		return false
+	}
+	for i, v1 := range s1 {
+		v2 := s2[i]
+		if v1 != v2 {
+			isNaN := func(f Elem) bool { return f != f }
+			if !isNaN(v1) || !isNaN(v2) {
+				return false
+			}
+		}
+	}
+	return true
+}
+
+// EqualFn reports whether two slices are equal using a comparision
+// function on each element.
+func EqualFn(type Elem)(s1, s2 []Elem, eq func(Elem, Elem) bool) bool {
+	if len(s1) != len(s2) {
+		return false
+	}
+	for i, v1 := range s1 {
+		v2 := s2[i]
+		if !eq(v1, v2) {
+			return false
+		}
+	}
+	return true
+}

src/go/go2go/importer.go

@@ -14,7 +14,6 @@ import (
 	"io/ioutil"
 	"log"
 	"os"
-	"path"
 	"path/filepath"
 	"strings"
 )
@@ -47,6 +46,7 @@ type Importer struct {
 var _ types.ImporterFrom = &Importer{}
 
 // NewImporter returns a new Importer.
+// The tmpdir will become a GOPATH with translated files.
 func NewImporter(tmpdir string) *Importer {
 	return &Importer{
 		tmpdir:       tmpdir,
@@ -76,6 +76,14 @@ func (imp *Importer) ImportFrom(importPath, dir string, mode types.ImportMode) (
 		return imp.localImport(importPath, dir)
 	}
 
+	if imp.translated[importPath] != "" {
+		tpkg, ok := imp.packages[importPath]
+		if !ok {
+			return nil, fmt.Errorf("circular import when processing %q", importPath)
+		}
+		return tpkg, nil
+	}
+
 	var pdir string
 	if go2path := os.Getenv("GO2PATH"); go2path != "" {
 		pdir = imp.findFromPath(go2path, importPath)
@@ -114,11 +122,15 @@ func (imp *Importer) ImportFrom(importPath, dir string, mode types.ImportMode) (
 	}
 
 	if len(gofiles) > 0 {
-		return nil, fmt.Errorf("import path %q (directory %q) has both .go and .go2 files", importPath, pdir)
+		for _, gofile := range gofiles {
+			if err := checkGoFile(pdir, gofile); err != nil {
+				return nil, err
+			}
+		}
 	}
 
-	tdir, err := ioutil.TempDir(imp.tmpdir, path.Base(importPath))
-	if err != nil {
+	tdir := filepath.Join(imp.tmpdir, "src", importPath)
+	if err := os.MkdirAll(tdir, 0755); err != nil {
 		return nil, err
 	}
 	for _, name := range names {
@@ -131,13 +143,13 @@ func (imp *Importer) ImportFrom(importPath, dir string, mode types.ImportMode) (
 		}
 	}
 
-	tpkgs, err := rewriteToPkgs(imp, dir)
+	imp.translated[importPath] = tdir
+
+	tpkgs, err := rewriteToPkgs(imp, tdir)
 	if err != nil {
 		return nil, err
 	}
 
-	imp.translated[importPath] = tdir
-
 	switch len(tpkgs) {
 	case 1:
 		return tpkgs[0], nil

src/go/go2go/instantiate.go

@@ -500,6 +500,19 @@ func (t *translator) instantiateExpr(ta *typeArgs, e ast.Expr) ast.Expr {
 			Type: typ,
 			Body: body,
 		}
+	case *ast.CompositeLit:
+		typ := t.instantiateExpr(ta, e.Type)
+		elts, changed := t.instantiateExprList(ta, e.Elts)
+		if typ == e.Type && !changed {
+			return e
+		}
+		return &ast.CompositeLit{
+			Type:       typ,
+			Lbrace:     e.Lbrace,
+			Elts:       elts,
+			Rbrace:     e.Rbrace,
+			Incomplete: e.Incomplete,
+		}
 	case *ast.ParenExpr:
 		x := t.instantiateExpr(ta, e.X)
 		if x == e.X {

src/go/go2go/names.go

@@ -30,6 +30,7 @@ var nameCodes = map[rune]int{
 	']': 7,
 	'(': 8,
 	')': 9,
+	'.': 10,
 }
 
 // instantiatedName returns the name of a newly instantiated function.
@@ -46,16 +47,19 @@ func (t *translator) instantiatedName(qid qualifiedIdent, types []types.Type) (s
 
 		// We have to uniquely translate s into a valid Go identifier.
 		// This is not possible in general but we assume that
-		// identifiers will not contain
+		// identifiers will not contain nameSep or nameIntro.
 		for _, r := range s {
+			if r == nameSep || r == nameIntro {
+				panic(fmt.Sprintf("identifier %q contains mangling rune %c", s, r))
+			}
 			if unicode.IsLetter(r) || unicode.IsDigit(r) || r == '_' {
 				sb.WriteRune(r)
 			} else {
 				code, ok := nameCodes[r]
 				if !ok {
-					panic(fmt.Sprintf("unexpected type string character %q in %q", r, s))
+					panic(fmt.Sprintf("%s: unexpected type string character %q in %q", t.fset.Position(qid.ident.Pos()), r, s))
 				}
-				fmt.Fprintf(&sb, "%c%d", nameIntro, code)
+				fmt.Fprintf(&sb, "%c%x", nameIntro, code)
 			}
 		}
 	}

src/go/go2go/rewrite.go

@@ -57,7 +57,7 @@ type translator struct {
 	importer           *Importer
 	info               *types.Info
 	types              map[ast.Expr]types.Type
-	instantiations     map[qualifiedIdent][]*instantiation
+	instantiations     map[string][]*instantiation
 	newDecls           []ast.Decl
 	typeInstantiations map[types.Type][]*typeInstantiation
 
@@ -115,7 +115,7 @@ func rewriteAST(fset *token.FileSet, importer *Importer, info *types.Info, file
 		importer:           importer,
 		info:               info,
 		types:              make(map[ast.Expr]types.Type),
-		instantiations:     make(map[qualifiedIdent][]*instantiation),
+		instantiations:     make(map[string][]*instantiation),
 		typeInstantiations: make(map[types.Type][]*typeInstantiation),
 	}
 	t.translate(file)
@@ -144,10 +144,49 @@ func rewriteAST(fset *token.FileSet, importer *Importer, info *types.Info, file
 		}
 		for _, spec := range gen.Specs {
 			imp := spec.(*ast.ImportSpec)
-			path := strings.TrimPrefix(strings.TrimSuffix(imp.Path.Value, `"`), `"`)
-			if _, ok := importer.lookupPackage(path); !ok {
+			if imp.Name != nil && imp.Name.Name == "_" {
 				continue
 			}
+			path := strings.TrimPrefix(strings.TrimSuffix(imp.Path.Value, `"`), `"`)
+
+			var tok token.Token
+			var importableName string
+			if _, ok := importer.lookupPackage(path); ok {
+				tok = token.TYPE
+				importableName = t.importableName()
+			} else {
+				fileDir := filepath.Dir(fset.Position(file.Name.Pos()).Filename)
+				pkg, err := importer.ImportFrom(path, fileDir, 0)
+				if err != nil {
+					return err
+				}
+				scope := pkg.Scope()
+				names := scope.Names()
+			nameLoop:
+				for _, name := range names {
+					if !token.IsExported(name) {
+						continue
+					}
+					obj := scope.Lookup(name)
+					switch obj.(type) {
+					case *types.TypeName:
+						tok = token.TYPE
+						importableName = name
+						break nameLoop
+					case *types.Var:
+						tok = token.VAR
+						importableName = name
+						break nameLoop
+					case *types.Const:
+						tok = token.CONST
+						importableName = name
+						break nameLoop
+					}
+				}
+				if importableName == "" {
+					return fmt.Errorf("can't find any importable name in package %q", path)
+				}
+			}
 
 			var name string
 			if imp.Name != nil {
@@ -155,20 +194,35 @@ func rewriteAST(fset *token.FileSet, importer *Importer, info *types.Info, file
 			} else {
 				name = filepath.Base(path)
 			}
-			file.Decls = append(file.Decls,
-				&ast.GenDecl{
-					Tok:   token.VAR,
-					Specs: []ast.Spec{
-						&ast.ValueSpec{
-							Names: []*ast.Ident{
-								ast.NewIdent("_"),
-							},
-							Type:  &ast.SelectorExpr{
-								X:   ast.NewIdent(name),
-								Sel: ast.NewIdent(t.importableName()),
-							},
+			var spec ast.Spec
+			switch tok {
+			case token.CONST, token.VAR:
+				spec = &ast.ValueSpec{
+					Names:  []*ast.Ident{
+						ast.NewIdent("_"),
+					},
+					Values: []ast.Expr{
+						&ast.SelectorExpr{
+							X:   ast.NewIdent(name),
+							Sel: ast.NewIdent(importableName),
 						},
 					},
+				}
+			case token.TYPE:
+				spec = &ast.TypeSpec{
+					Name: ast.NewIdent("_"),
+					Type: &ast.SelectorExpr{
+						X:   ast.NewIdent(name),
+						Sel: ast.NewIdent(importableName),
+					},
+				}
+			default:
+				panic("can't happen")
+			}
+			file.Decls = append(file.Decls,
+				&ast.GenDecl{
+					Tok: tok,
+					Specs: []ast.Spec{spec},
 				})
 		}
 	}
@@ -292,6 +346,20 @@ func (t *translator) translateStmt(ps *ast.Stmt) {
 		t.translateExpr(&s.Cond)
 		t.translateBlockStmt(s.Body)
 		t.translateStmt(&s.Else)
+	case *ast.CaseClause:
+		t.translateExprList(s.List)
+		t.translateStmtList(s.Body)
+	case *ast.SwitchStmt:
+		t.translateStmt(&s.Init)
+		t.translateExpr(&s.Tag)
+		t.translateBlockStmt(s.Body)
+	case *ast.TypeSwitchStmt:
+		t.translateStmt(&s.Init)
+		t.translateStmt(&s.Assign)
+		t.translateBlockStmt(s.Body)
+	case *ast.CommClause:
+		t.translateStmt(&s.Comm)
+		t.translateStmtList(s.Body)
 	case *ast.ForStmt:
 		t.translateStmt(&s.Init)
 		t.translateExpr(&s.Cond)
@@ -323,6 +391,14 @@ func (t *translator) translateStmt(ps *ast.Stmt) {
 	}
 }
 
+// translateStmtList translates a list of statements from Go with
+// contracts to Go 1.
+func (t *translator) translateStmtList(sl []ast.Stmt) {
+	for i := range sl {
+		t.translateStmt(&sl[i])
+	}
+}
+
 // translateExpr translates an expression from Go with contracts to Go 1.
 func (t *translator) translateExpr(pe *ast.Expr) {
 	if t.err != nil {
@@ -414,7 +490,8 @@ func (t *translator) translateFunctionInstantiation(pe *ast.Expr) {
 	qid := t.instantiatedIdent(call)
 	argList, typeList, typeArgs := t.instantiationTypes(call)
 
-	instantiations := t.instantiations[qid]
+	key := qid.String()
+	instantiations := t.instantiations[key]
 	for _, inst := range instantiations {
 		if t.sameTypes(typeList, inst.types) {
 			*pe = inst.decl
@@ -432,7 +509,7 @@ func (t *translator) translateFunctionInstantiation(pe *ast.Expr) {
 		types: typeList,
 		decl:  instIdent,
 	}
-	t.instantiations[qid] = append(instantiations, n)
+	t.instantiations[key] = append(instantiations, n)
 
 	if typeArgs {
 		*pe = instIdent

test/gen/g006.go2

@@ -76,7 +76,7 @@ func testOrdered(type Elem Ordered)(name string, s []Elem, sorter func([]Elem))
 	return ok
 }
 
-func sliceEq(type Elem Ordered)(s1, s2[]Elem) bool {
+func sliceEq(type Elem Ordered)(s1, s2 []Elem) bool {
 	for i, v1 := range s1 {
 		v2 := s2[i]
 		if v1 != v2 {