cmd/go2go, go/go2go: new framework for translating generics to Go

Adds a new command go2go, invocable after installation as "go tool go2go".
The go2go command supports build, test, run, and translate subcommands.
These will translate .go2 files to .go files.
No actual translation is done yet; the files are simply rewritten.
So this is a framework that works if the .go2 files are pure Go 1 code.

Change-Id: I8183fd276c4c87577c428794fb87e8dde1fa398d

src/cmd/go2go/doc.go

@@ -0,0 +1,19 @@
+// Copyright 2020 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// go2go is a command for trying out generic Go code.
+// It supports a small number of commands similar to cmd/go.
+//
+// Usage:
+//
+//	go2go <command> [arguments]
+//
+// The commands are:
+//
+//	build      translate and then run "go build packages"
+//	run        translate and then run a list of files
+//      test       translate and then run "go test packages"
+//      translate  translate .go2 files into .go files for listed packages
+//
+package main

src/cmd/go2go/main.go

@@ -0,0 +1,127 @@
+// Copyright 2020 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package main
+
+import (
+	"flag"
+	"fmt"
+	"io/ioutil"
+	"os"
+	"os/exec"
+	"path/filepath"
+	"runtime"
+	"strings"
+)
+
+var gotool = filepath.Join(runtime.GOROOT(), "bin", "go")
+
+var cmds = map[string]bool{
+	"build":     true,
+	"run":       true,
+	"test":      true,
+	"translate": true,
+}
+
+func main() {
+	flag.Parse()
+
+	args := flag.Args()
+	if len(args) < 1 {
+		usage()
+	}
+
+	if !cmds[args[0]] {
+		usage()
+	}
+
+	var rundir string
+	if args[0] == "run" {
+		tmpdir := copyToTmpdir(args[1:])
+		defer os.RemoveAll(tmpdir)
+		translate(tmpdir)
+		nargs := []string{"run"}
+		for _, arg := range args[1:] {
+			base := filepath.Base(arg)
+			f := strings.TrimSuffix(base, ".go2") + ".go"
+			nargs = append(nargs, f)
+		}
+		args = nargs
+		rundir = tmpdir
+	} else {
+		for _, dir := range expandPackages(args[1:]) {
+			translate(dir)
+		}
+	}
+
+	if args[0] != "translate" {
+		cmd := exec.Command(gotool, args...)
+		cmd.Stdin = os.Stdin
+		cmd.Stdout = os.Stdout
+		cmd.Stderr = os.Stderr
+		cmd.Dir = rundir
+		if err := cmd.Run(); err != nil {
+			die(fmt.Sprintf("%s %v failed: %v", gotool, args, err))
+		}
+	}
+}
+
+// expandPackages returns a list of directories expanded from packages.
+func expandPackages(pkgs []string) []string {
+	if len(pkgs) == 0 {
+		return []string{"."}
+	}
+	var dirs []string
+	for _, pkg := range pkgs {
+		cmd := exec.Command(gotool, "list", "-f", "{{.Dir}}", pkg)
+		cmd.Stderr = os.Stderr
+		out, err := cmd.Output()
+		if err != nil {
+			die(fmt.Sprintf("%s list %q failed: %v", gotool, pkg, err))
+		}
+		dirs = append(dirs, strings.Split(string(out), "\n")...)
+	}
+	return dirs
+}
+
+// copyToTmpdir copies files into a temporary directory.
+func copyToTmpdir(files []string) string {
+	if len(files) == 0 {
+		die("no files to run")
+	}
+	tmpdir, err := ioutil.TempDir("", "go2go-run")
+	if err != nil {
+		die(err.Error())
+	}
+	for _, file := range files {
+		data, err := ioutil.ReadFile(file)
+		if err != nil {
+			die(err.Error())
+		}
+		if err := ioutil.WriteFile(filepath.Join(tmpdir, filepath.Base(file)), data, 0444); err != nil {
+			die(err.Error())
+		}
+	}
+	return tmpdir
+}
+
+// usage reports a usage message and exits with failure.
+func usage() {
+	fmt.Fprint(os.Stderr, `Usage: go2go <command> [arguments]
+
+The commands are:
+
+	build      translate and build packages
+	run        translate and run list of files
+	test       translate and test packages
+	translate  translate .go2 files into .go files
+`)
+	os.Exit(1)
+}
+
+// die reports an error and exits.
+func die(msg string) {
+	fmt.Fprintln(os.Stderr, msg)
+	os.Exit(1)
+}

src/cmd/go2go/translate.go

@@ -0,0 +1,16 @@
+// Copyright 2020 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package main
+
+import (
+	"go/go2go"
+)
+
+// translate writes .go files for all .go2 files in dir.
+func translate(dir string) {
+	if err := go2go.Rewrite(dir); err != nil {
+		die(err.Error())
+	}
+}

src/go/go2go/go2go.go

@@ -0,0 +1,178 @@
+// Copyright 2020 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// Package go2go rewrites polymorphic code into non-polymorphic code.
+package go2go
+
+import (
+	"fmt"
+	"go/ast"
+	"go/importer"
+	"go/parser"
+	"go/token"
+	"go/types"
+	"io"
+	"path/filepath"
+	"os"
+	"sort"
+	"strings"
+)
+
+// rewritePrefix is what we put at the start of each newly generated .go file.
+const rewritePrefix = "// Code generated by go2go; DO NOT EDIT.\n\n"
+
+// Rewrite rewrites the contents of a single directory.
+// It looks for all files with the extension .go2, and parses
+// them as a single package. It writes out a .go file with any
+// polymorphic code rewritten into normal code.
+func Rewrite(dir string) error {
+	go2files, gofiles, err := go2Files(dir)
+	if err != nil {
+		return err
+	}
+
+	fset := token.NewFileSet()
+	pkgs, err := parseFiles(dir, go2files, fset)
+	if err != nil {
+		return err
+	}
+
+	type fileAST struct {
+		name string
+		ast  *ast.File
+	}
+	type gpkg struct {
+		tpkg *types.Package
+		pkgfiles []fileAST
+	}
+
+	var tpkgs []*gpkg
+	for name, pkg := range pkgs {
+		pkgfiles := make([]fileAST, 0, len(pkg.Files))
+		for n, f := range pkg.Files {
+			pkgfiles = append(pkgfiles, fileAST{n, f})
+		}
+		sort.Slice(pkgfiles, func(i, j int) bool {
+			return pkgfiles[i].name < pkgfiles[j].name
+		})
+
+		asts := make([]*ast.File, 0, len(pkgfiles))
+		for _, a := range pkgfiles {
+			asts = append(asts, a.ast)
+		}
+
+		conf := types.Config{Importer: importer.Default()}
+		var info types.Info
+		tpkg, err := conf.Check(name, fset, asts, &info)
+		if err != nil {
+			return err
+		}
+
+		tpkgs = append(tpkgs, &gpkg{tpkg: tpkg, pkgfiles: pkgfiles})
+	}
+
+	if err := checkAndRemoveGofiles(dir, gofiles); err != nil {
+		return err
+	}
+
+	for _, tpkg := range tpkgs {
+		for _, pkgfile := range tpkg.pkgfiles {
+			if err := rewrite(dir, fset, pkgfile.name, pkgfile.ast); err != nil {
+				return err
+			}
+		}
+	}
+
+	return nil
+}
+
+// go2Files returns the list of files in dir with a .go2 extension
+// and a list of files with a .go extension.
+// This returns an error if it finds any .go files that do not start
+// with rewritePrefix.
+func go2Files(dir string) (go2files []string, gofiles []string, err error) {
+	f, err := os.Open(dir)
+	if err != nil {
+		return nil, nil, err
+	}
+	defer f.Close()
+
+	files, err := f.Readdirnames(0)
+	if err != nil {
+		return nil, nil, fmt.Errorf("reading directory %s: %w", dir, err)
+
+	}
+
+	go2files = make([]string, 0, len(files))
+	gofiles = make([]string, 0, len(files))
+	for _, f := range files {
+		switch filepath.Ext(f) {
+		case ".go2":
+			go2files = append(go2files, f)
+		case ".go":
+			gofiles = append(gofiles, f)
+		}
+	}
+
+	return go2files, gofiles, nil
+}
+
+// checkAndRemoveGofiles looks through all the .go files.
+// Any .go file that starts with rewritePrefix is removed.
+// Any other .go file is reported as an error.
+// This is intended to make it harder for go2go to break a
+// traditional Go package.
+func checkAndRemoveGofiles(dir string, gofiles []string) error {
+	for _, f := range gofiles {
+		if err := checkGoFile(dir, f); err != nil {
+			return err
+		}
+		if err := os.Remove(filepath.Join(dir, f)); err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+// checkGofile reports an error if the file does not start with rewritePrefix.
+func checkGoFile(dir, f string) error {
+	o, err := os.Open(filepath.Join(dir, f))
+	if err != nil {
+		return err
+	}
+	defer o.Close()
+	var buf [100]byte
+	n, err := o.Read(buf[:])
+	if n > 0 && !strings.HasPrefix(string(buf[:n]), rewritePrefix) {
+		return fmt.Errorf("Go file %s was not created by go2go", f)
+	}
+	if err != nil && err != io.EOF {
+		return err
+	}
+	return nil
+}
+
+// parseFiles parses a list of .go2 files.
+func parseFiles(dir string, go2files []string, fset *token.FileSet) (map[string]*ast.Package, error) {
+	pkgs := make(map[string]*ast.Package)
+	for _, go2f := range go2files {
+		filename := filepath.Join(dir, go2f)
+		pf, err := parser.ParseFile(fset, filename, nil, 0)
+		if err != nil {
+			return nil, err
+		}
+
+		name := pf.Name.Name
+		pkg, ok := pkgs[name]
+		if !ok {
+			pkg = &ast.Package{
+				Name:  name,
+				Files: make(map[string]*ast.File),
+			}
+			pkgs[name] = pkg
+		}
+		pkg.Files[filename] = pf
+	}
+	return pkgs, nil
+}

src/go/go2go/rewrite.go

@@ -0,0 +1,41 @@
+// Copyright 2020 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package go2go
+
+import (
+	"bufio"
+	"fmt"
+	"go/ast"
+	"go/format"
+	"go/token"
+	"path/filepath"
+	"os"
+	"strings"
+)
+
+// rewrite rewrites the contents of one file.
+func rewrite(dir string, fset *token.FileSet, filename string, ast *ast.File) (err error) {
+	filename = filepath.Base(filename)
+	goFile := strings.TrimSuffix(filename, filepath.Ext(filename)) + ".go"
+	o, err := os.Create(filepath.Join(dir, goFile))
+	if err != nil {
+		return err
+	}
+	defer func() {
+		if closeErr := o.Close(); err == nil {
+			err = closeErr
+		}
+	}()
+
+	w := bufio.NewWriter(o)
+	defer func() {
+		if flushErr := w.Flush(); err == nil {
+			err = flushErr
+		}
+	}()
+	fmt.Fprintln(w, rewritePrefix)
+
+	return format.Node(w, fset, ast)
+}