internal/proxydir: add an internal package for file-based proxies

Both packagestest and the gopls regtests need to write module data to
the filesystem in proxy structure. Since this seems like a common and
self-contained concerned, factor this out into a shared package.

Change-Id: I5275dbc0cd7b13290061e8bb559d6dd287fbb275
Reviewed-on: https://go-review.googlesource.com/c/tools/+/227841
Run-TryBot: Robert Findley <rfindley@google.com>
TryBot-Result: Gobot Gobot <gobot@golang.org>
Reviewed-by: Rebecca Stambler <rstambler@golang.org>

go/packages/packagestest/modules.go

@@ -5,7 +5,6 @@
 package packagestest
 
 import (
-	"archive/zip"
 	"context"
 	"fmt"
 	"io/ioutil"
@@ -17,6 +16,7 @@ import (
 
 	"golang.org/x/tools/internal/gocommand"
 	"golang.org/x/tools/internal/packagesinternal"
+	"golang.org/x/tools/internal/proxydir"
 )
 
 // Modules is the exporter that produces module layouts.
@@ -138,7 +138,7 @@ func (modules) Finalize(exported *Exported) error {
 	}
 
 	// Zip up all the secondary modules into the proxy dir.
-	proxyDir := filepath.Join(exported.temp, "modproxy")
+	modProxyDir := filepath.Join(exported.temp, "modproxy")
 	for module, files := range exported.written {
 		if module == exported.primary {
 			continue
@@ -150,8 +150,7 @@ func (modules) Finalize(exported *Exported) error {
 			module = v.module
 			version = v.version
 		}
-		dir := filepath.Join(proxyDir, module, "@v")
-		if err := writeModuleProxy(dir, module, version, files); err != nil {
+		if err := writeModuleFiles(modProxyDir, module, version, files); err != nil {
 			return fmt.Errorf("creating module proxy dir for %v: %v", module, err)
 		}
 	}
@@ -164,7 +163,7 @@ func (modules) Finalize(exported *Exported) error {
 	exported.Config.Env = append(exported.Config.Env,
 		"GO111MODULE=on",
 		"GOPATH="+filepath.Join(exported.temp, "modcache"),
-		"GOPROXY="+proxyDirToURL(proxyDir),
+		"GOPROXY="+proxydir.ToURL(modProxyDir),
 		"GOSUMDB=off",
 	)
 	gocmdRunner := &gocommand.Runner{}
@@ -185,65 +184,16 @@ func (modules) Finalize(exported *Exported) error {
 	return nil
 }
 
-// writeModuleProxy creates a directory in the proxy dir for a module.
-func writeModuleProxy(dir, module, ver string, files map[string]string) error {
-	if err := os.MkdirAll(dir, 0755); err != nil {
-		return err
-	}
-
-	// the modproxy checks for versions by looking at the "list" file,
-	// since we are supporting multiple versions, create the file if it does not exist or
-	// append the version number to the preexisting file.
-	f, err := os.OpenFile(filepath.Join(dir, "list"), os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0644)
-	if err != nil {
-		return err
-	}
-	if _, err := f.WriteString(ver + "\n"); err != nil {
-		return err
-	}
-
-	// go.mod, copied from the file written in Finalize.
-	modContents, err := ioutil.ReadFile(files["go.mod"])
-	if err != nil {
-		return err
-	}
-	if err := ioutil.WriteFile(filepath.Join(dir, ver+".mod"), modContents, 0644); err != nil {
-		return err
-	}
-
-	// info file, just the bare bones.
-	infoContents := []byte(fmt.Sprintf(`{"Version": "%v", "Time":"2017-12-14T13:08:43Z"}`, ver))
-	if err := ioutil.WriteFile(filepath.Join(dir, ver+".info"), infoContents, 0644); err != nil {
-		return err
-	}
-
-	// zip of all the source files.
-	f, err = os.OpenFile(filepath.Join(dir, ver+".zip"), os.O_CREATE|os.O_WRONLY, 0644)
-	if err != nil {
-		return err
-	}
-	z := zip.NewWriter(f)
-	for name, path := range files {
-		zf, err := z.Create(module + "@" + ver + "/" + name)
-		if err != nil {
-			return err
-		}
+func writeModuleFiles(rootDir, module, ver string, filePaths map[string]string) error {
+	fileData := make(map[string][]byte)
+	for name, path := range filePaths {
 		contents, err := ioutil.ReadFile(path)
 		if err != nil {
 			return err
 		}
-		if _, err := zf.Write(contents); err != nil {
-			return err
-		}
+		fileData[name] = contents
 	}
-	if err := z.Close(); err != nil {
-		return err
-	}
-	if err := f.Close(); err != nil {
-		return err
-	}
-
-	return nil
+	return proxydir.WriteModuleVersion(rootDir, module, ver, fileData)
 }
 
 func modCache(exported *Exported) string {

internal/proxydir/proxydir.go

@@ -0,0 +1,79 @@
+// Copyright 2020 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// Package proxydir provides functions for writing module data to a directory
+// in proxy format, so that it can be used as a module proxy by setting
+// GOPROXY="file://<dir>".
+package proxydir
+
+import (
+	"archive/zip"
+	"fmt"
+	"io"
+	"io/ioutil"
+	"os"
+	"path/filepath"
+)
+
+// WriteModuleVersion creates a directory in the proxy dir for a module.
+func WriteModuleVersion(rootDir, module, ver string, files map[string][]byte) (rerr error) {
+	dir := filepath.Join(rootDir, module, "@v")
+	if err := os.MkdirAll(dir, 0755); err != nil {
+		return err
+	}
+
+	// The go command checks for versions by looking at the "list" file.  Since
+	// we are supporting multiple versions, create this file if it does not exist
+	// or append the version number to the preexisting file.
+	f, err := os.OpenFile(filepath.Join(dir, "list"), os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0644)
+	if err != nil {
+		return err
+	}
+	defer checkClose("list file", f, &rerr)
+	if _, err := f.WriteString(ver + "\n"); err != nil {
+		return err
+	}
+
+	// Serve the go.mod file on the <version>.mod url, if it exists. Otherwise,
+	// serve a stub.
+	modContents, ok := files["go.mod"]
+	if !ok {
+		modContents = []byte("module " + module)
+	}
+	if err := ioutil.WriteFile(filepath.Join(dir, ver+".mod"), modContents, 0644); err != nil {
+		return err
+	}
+
+	// info file, just the bare bones.
+	infoContents := []byte(fmt.Sprintf(`{"Version": "%v", "Time":"2017-12-14T13:08:43Z"}`, ver))
+	if err := ioutil.WriteFile(filepath.Join(dir, ver+".info"), infoContents, 0644); err != nil {
+		return err
+	}
+
+	// zip of all the source files.
+	f, err = os.OpenFile(filepath.Join(dir, ver+".zip"), os.O_CREATE|os.O_WRONLY, 0644)
+	if err != nil {
+		return err
+	}
+	defer checkClose("zip file", f, &rerr)
+	z := zip.NewWriter(f)
+	defer checkClose("zip writer", z, &rerr)
+	for name, contents := range files {
+		zf, err := z.Create(module + "@" + ver + "/" + name)
+		if err != nil {
+			return err
+		}
+		if _, err := zf.Write(contents); err != nil {
+			return err
+		}
+	}
+
+	return nil
+}
+
+func checkClose(name string, closer io.Closer, err *error) {
+	if cerr := closer.Close(); cerr != nil && *err == nil {
+		*err = fmt.Errorf("closing %s: %v", name, cerr)
+	}
+}

internal/proxydir/proxydir_112.go

@@ -1,14 +1,15 @@
 // +build !go1.13
 
-// Copyright 2019 The Go Authors. All rights reserved.
+// Copyright 2020 The Go Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style
 // license that can be found in the LICENSE file.
 
-package packagestest
+package proxydir
 
 import "path/filepath"
 
-func proxyDirToURL(dir string) string {
+// ToURL returns the file uri for a proxy directory.
+func ToURL(dir string) string {
 	// Prior to go1.13, the Go command on Windows only accepted GOPROXY file URLs
 	// of the form file://C:/path/to/proxy. This was incorrect: when parsed, "C:"
 	// is interpreted as the host. See golang.org/issue/6027. This has been

internal/proxydir/proxydir_113.go

@@ -1,17 +1,18 @@
 // +build go1.13
 
-// Copyright 2018 The Go Authors. All rights reserved.
+// Copyright 2020 The Go Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style
 // license that can be found in the LICENSE file.
 
-package packagestest
+package proxydir
 
 import (
 	"path/filepath"
 	"strings"
 )
 
-func proxyDirToURL(dir string) string {
+// ToURL returns the file uri for a proxy directory.
+func ToURL(dir string) string {
 	// file URLs on Windows must start with file:///. See golang.org/issue/6027.
 	path := filepath.ToSlash(dir)
 	if !strings.HasPrefix(path, "/") {

internal/proxydir/proxydir_test.go

@@ -0,0 +1,112 @@
+// Copyright 2020 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package proxydir
+
+import (
+	"archive/zip"
+	"fmt"
+	"io/ioutil"
+	"os"
+	"path/filepath"
+	"strings"
+	"testing"
+)
+
+func TestWriteModuleVersion(t *testing.T) {
+	tests := []struct {
+		modulePath, version string
+		files               map[string][]byte
+	}{
+		{
+			modulePath: "mod.test/module",
+			version:    "v1.2.3",
+			files: map[string][]byte{
+				"go.mod":   []byte("module mod.com\n\ngo 1.12"),
+				"const.go": []byte("package module\n\nconst Answer = 42"),
+			},
+		},
+		{
+			modulePath: "mod.test/module",
+			version:    "v1.2.4",
+			files: map[string][]byte{
+				"go.mod":   []byte("module mod.com\n\ngo 1.12"),
+				"const.go": []byte("package module\n\nconst Answer = 43"),
+			},
+		},
+		{
+			modulePath: "mod.test/nogomod",
+			version:    "v0.9.0",
+			files: map[string][]byte{
+				"const.go": []byte("package module\n\nconst Other = \"Other\""),
+			},
+		},
+	}
+	dir, err := ioutil.TempDir("", "proxydirtest-")
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer os.RemoveAll(dir)
+	for _, test := range tests {
+		// Since we later assert on the contents of /list, don't use subtests.
+		if err := WriteModuleVersion(dir, test.modulePath, test.version, test.files); err != nil {
+			t.Fatal(err)
+		}
+		rootDir := filepath.Join(dir, filepath.FromSlash(test.modulePath), "@v")
+		gomod, err := ioutil.ReadFile(filepath.Join(rootDir, test.version+".mod"))
+		if err != nil {
+			t.Fatal(err)
+		}
+		wantMod, ok := test.files["go.mod"]
+		if !ok {
+			wantMod = []byte("module " + test.modulePath)
+		}
+		if got, want := string(gomod), string(wantMod); got != want {
+			t.Errorf("reading %s/@v/%s.mod: got %q, want %q", test.modulePath, test.version, got, want)
+		}
+		zr, err := zip.OpenReader(filepath.Join(rootDir, test.version+".zip"))
+		if err != nil {
+			t.Fatal(err)
+		}
+		defer zr.Close()
+
+		for _, zf := range zr.File {
+			r, err := zf.Open()
+			if err != nil {
+				t.Fatal(err)
+			}
+			defer r.Close()
+			content, err := ioutil.ReadAll(r)
+			if err != nil {
+				t.Fatal(err)
+			}
+			name := strings.TrimPrefix(zf.Name, fmt.Sprintf("%s@%s/", test.modulePath, test.version))
+			if got, want := string(content), string(test.files[name]); got != want {
+				t.Errorf("unzipping %q: got %q, want %q", zf.Name, got, want)
+			}
+			delete(test.files, name)
+		}
+		for name := range test.files {
+			t.Errorf("file %q not present in the module zip", name)
+		}
+	}
+
+	lists := []struct {
+		modulePath, want string
+	}{
+		{"mod.test/module", "v1.2.3\nv1.2.4\n"},
+		{"mod.test/nogomod", "v0.9.0\n"},
+	}
+
+	for _, test := range lists {
+		fp := filepath.Join(dir, filepath.FromSlash(test.modulePath), "@v", "list")
+		list, err := ioutil.ReadFile(fp)
+		if err != nil {
+			t.Fatal(err)
+		}
+		if got := string(list); got != test.want {
+			t.Errorf("%q/@v/list: got %q, want %q", test.modulePath, got, test.want)
+		}
+	}
+}