cmd/go: don't fetch files missing sums in readonly mode

If the go command needs a .mod or .zip file in -mod=readonly mode
(now the default), and that file doesn't have a hash in the main
module's go.sum file, the go command will now report an error before
fetching the file, rather than at the end when failing to update
go.sum. The error says specifically which entry is missing.

If this error is encountered when loading the build list, it will
suggest 'go mod tidy'.

If this error is encountered when loading a specific package (an
import or command line argument), the error will mention that package
and will suggest 'go mod tidy' or 'go get -d'.

Fixes #41934
Fixes #41935

Change-Id: I96ec2ef9258bd4bade9915c43d47e6243c376a81
Reviewed-on: https://go-review.googlesource.com/c/go/+/262341
Reviewed-by: Bryan C. Mills <bcmills@google.com>
Trust: Jay Conrod <jayconrod@google.com>

src/cmd/go/internal/load/pkg.go

@@ -254,8 +254,8 @@ func (p *Package) setLoadPackageDataError(err error, path string, stk *ImportSta
 	// package's source files themselves (scanner errors).
 	//
 	// TODO(matloob): Perhaps make each of those the errors in the first group
-	// (including modload.ImportMissingError, and the corresponding
-	// "cannot find package %q in any of" GOPATH-mode error
+	// (including modload.ImportMissingError, ImportMissingSumError, and the
+	// corresponding "cannot find package %q in any of" GOPATH-mode error
 	// produced in build.(*Context).Import; modload.AmbiguousImportError,
 	// and modload.PackageNotInModuleError; and the malformed module path errors
 	// produced in golang.org/x/mod/module.CheckMod) implement an interface
@@ -430,6 +430,7 @@ type ImportPathError interface {
 var (
 	_ ImportPathError = (*importError)(nil)
 	_ ImportPathError = (*modload.ImportMissingError)(nil)
+	_ ImportPathError = (*modload.ImportMissingSumError)(nil)
 )
 
 type importError struct {

src/cmd/go/internal/modfetch/fetch.go

@@ -428,6 +428,28 @@ func readGoSum(dst map[module.Version][]string, file string, data []byte) error
 	return nil
 }
 
+// HaveSum returns true if the go.sum file contains an entry for mod.
+// The entry's hash must be generated with a known hash algorithm.
+// mod.Version may have a "/go.mod" suffix to distinguish sums for
+// .mod and .zip files.
+func HaveSum(mod module.Version) bool {
+	goSum.mu.Lock()
+	defer goSum.mu.Unlock()
+	inited, err := initGoSum()
+	if err != nil || !inited {
+		return false
+	}
+	for _, h := range goSum.m[mod] {
+		if !strings.HasPrefix(h, "h1:") {
+			continue
+		}
+		if !goSum.status[modSum{mod, h}].dirty {
+			return true
+		}
+	}
+	return false
+}
+
 // checkMod checks the given module's checksum.
 func checkMod(mod module.Version) {
 	if cfg.GOMODCACHE == "" {

src/cmd/go/internal/modload/import.go

@@ -124,6 +124,31 @@ func (e *AmbiguousImportError) Error() string {
 	return buf.String()
 }
 
+// ImportMissingSumError is reported in readonly mode when we need to check
+// if a module in the build list contains a package, but we don't have a sum
+// for its .zip file.
+type ImportMissingSumError struct {
+	importPath   string
+	found, inAll bool
+}
+
+func (e *ImportMissingSumError) Error() string {
+	var message string
+	if e.found {
+		message = fmt.Sprintf("missing go.sum entry needed to verify package %s is provided by exactly one module", e.importPath)
+	} else {
+		message = fmt.Sprintf("missing go.sum entry for module providing package %s", e.importPath)
+	}
+	if e.inAll {
+		return message + "; try 'go mod tidy' to add it"
+	}
+	return message
+}
+
+func (e *ImportMissingSumError) ImportPath() string {
+	return e.importPath
+}
+
 type invalidImportError struct {
 	importPath string
 	err        error
@@ -208,13 +233,23 @@ func importFromBuildList(ctx context.Context, path string) (m module.Version, di
 	// Check each module on the build list.
 	var dirs []string
 	var mods []module.Version
+	haveSumErr := false
 	for _, m := range buildList {
 		if !maybeInModule(path, m.Path) {
 			// Avoid possibly downloading irrelevant modules.
 			continue
 		}
-		root, isLocal, err := fetch(ctx, m)
+		needSum := true
+		root, isLocal, err := fetch(ctx, m, needSum)
 		if err != nil {
+			if sumErr := (*sumMissingError)(nil); errors.As(err, &sumErr) {
+				// We are missing a sum needed to fetch a module in the build list.
+				// We can't verify that the package is unique, and we may not find
+				// the package at all. Keep checking other modules to decide which
+				// error to report.
+				haveSumErr = true
+				continue
+			}
 			// Report fetch error.
 			// Note that we don't know for sure this module is necessary,
 			// but it certainly _could_ provide the package, and even if we
@@ -230,12 +265,15 @@ func importFromBuildList(ctx context.Context, path string) (m module.Version, di
 			dirs = append(dirs, dir)
 		}
 	}
+	if len(mods) > 1 {
+		return module.Version{}, "", &AmbiguousImportError{importPath: path, Dirs: dirs, Modules: mods}
+	}
+	if haveSumErr {
+		return module.Version{}, "", &ImportMissingSumError{importPath: path, found: len(mods) > 0}
+	}
 	if len(mods) == 1 {
 		return mods[0], dirs[0], nil
 	}
-	if len(mods) > 0 {
-		return module.Version{}, "", &AmbiguousImportError{importPath: path, Dirs: dirs, Modules: mods}
-	}
 
 	return module.Version{}, "", &ImportMissingError{Path: path, isStd: pathIsStd}
 }
@@ -306,7 +344,8 @@ func queryImport(ctx context.Context, path string) (module.Version, error) {
 			return len(mods[i].Path) > len(mods[j].Path)
 		})
 		for _, m := range mods {
-			root, isLocal, err := fetch(ctx, m)
+			needSum := true
+			root, isLocal, err := fetch(ctx, m, needSum)
 			if err != nil {
 				// Report fetch error as above.
 				return module.Version{}, err
@@ -473,9 +512,14 @@ func dirInModule(path, mpath, mdir string, isLocal bool) (dir string, haveGoFile
 // fetch downloads the given module (or its replacement)
 // and returns its location.
 //
+// needSum indicates whether the module may be downloaded in readonly mode
+// without a go.sum entry. It should only be false for modules fetched
+// speculatively (for example, for incompatible version filtering). The sum
+// will still be verified normally.
+//
 // The isLocal return value reports whether the replacement,
 // if any, is local to the filesystem.
-func fetch(ctx context.Context, mod module.Version) (dir string, isLocal bool, err error) {
+func fetch(ctx context.Context, mod module.Version, needSum bool) (dir string, isLocal bool, err error) {
 	if mod == Target {
 		return ModRoot(), true, nil
 	}
@@ -505,6 +549,18 @@ func fetch(ctx context.Context, mod module.Version) (dir string, isLocal bool, e
 		mod = r
 	}
 
+	if cfg.BuildMod == "readonly" && needSum && !modfetch.HaveSum(mod) {
+		return "", false, module.VersionError(mod, &sumMissingError{})
+	}
+
 	dir, err = modfetch.Download(ctx, mod)
 	return dir, false, err
 }
+
+type sumMissingError struct {
+	suggestion string
+}
+
+func (e *sumMissingError) Error() string {
+	return "missing go.sum entry" + e.suggestion
+}

src/cmd/go/internal/modload/load.go

@@ -276,6 +276,8 @@ func LoadPackages(ctx context.Context, opts PackageOpts, patterns ...string) (ma
 			if pkg.flags.has(pkgInAll) {
 				if imErr := (*ImportMissingError)(nil); errors.As(pkg.err, &imErr) {
 					imErr.inAll = true
+				} else if sumErr := (*ImportMissingSumError)(nil); errors.As(pkg.err, &sumErr) {
+					sumErr.inAll = true
 				}
 			}
 

src/cmd/go/internal/modload/modfile.go

@@ -406,8 +406,11 @@ type retraction struct {
 // taking into account any replacements for m, exclusions of its dependencies,
 // and/or vendoring.
 //
-// goModSummary cannot be used on the Target module, as its requirements
-// may change.
+// m must be a version in the module graph, reachable from the Target module.
+// In readonly mode, the go.sum file must contain an entry for m's go.mod file
+// (or its replacement). goModSummary must not be called for the Target module
+// itself, as its requirements may change. Use rawGoModSummary for other
+// module versions.
 //
 // The caller must not modify the returned summary.
 func goModSummary(m module.Version) (*modFileSummary, error) {
@@ -442,6 +445,13 @@ func goModSummary(m module.Version) (*modFileSummary, error) {
 	if actual.Path == "" {
 		actual = m
 	}
+	if cfg.BuildMod == "readonly" && actual.Version != "" {
+		key := module.Version{Path: actual.Path, Version: actual.Version + "/go.mod"}
+		if !modfetch.HaveSum(key) {
+			suggestion := fmt.Sprintf("; try 'go mod download %s' to add it", m.Path)
+			return nil, module.VersionError(actual, &sumMissingError{suggestion: suggestion})
+		}
+	}
 	summary, err := rawGoModSummary(actual)
 	if err != nil {
 		return nil, err

src/cmd/go/internal/modload/query.go

@@ -599,7 +599,8 @@ func QueryPattern(ctx context.Context, pattern, query string, current func(strin
 				return r, err
 			}
 			r.Mod.Version = r.Rev.Version
-			root, isLocal, err := fetch(ctx, r.Mod)
+			needSum := true
+			root, isLocal, err := fetch(ctx, r.Mod, needSum)
 			if err != nil {
 				return r, err
 			}
@@ -816,7 +817,8 @@ func (e *PackageNotInModuleError) ImportPath() string {
 
 // ModuleHasRootPackage returns whether module m contains a package m.Path.
 func ModuleHasRootPackage(ctx context.Context, m module.Version) (bool, error) {
-	root, isLocal, err := fetch(ctx, m)
+	needSum := false
+	root, isLocal, err := fetch(ctx, m, needSum)
 	if err != nil {
 		return false, err
 	}
@@ -825,7 +827,8 @@ func ModuleHasRootPackage(ctx context.Context, m module.Version) (bool, error) {
 }
 
 func versionHasGoMod(ctx context.Context, m module.Version) (bool, error) {
-	root, _, err := fetch(ctx, m)
+	needSum := false
+	root, _, err := fetch(ctx, m, needSum)
 	if err != nil {
 		return false, err
 	}

src/cmd/go/internal/modload/search.go

@@ -156,7 +156,8 @@ func matchPackages(ctx context.Context, m *search.Match, tags map[string]bool, f
 			isLocal = true
 		} else {
 			var err error
-			root, isLocal, err = fetch(ctx, mod)
+			needSum := true
+			root, isLocal, err = fetch(ctx, mod, needSum)
 			if err != nil {
 				m.AddError(err)
 				continue

src/cmd/go/testdata/script/mod_install_pkg_version.txt

@@ -16,7 +16,7 @@ env GO111MODULE=auto
 cd m
 cp go.mod go.mod.orig
 ! go list -m all
-stderr 'example.com/cmd@v1.1.0-doesnotexist:.*404 Not Found'
+stderr '^go: example.com/cmd@v1.1.0-doesnotexist: missing go.sum entry; try ''go mod download example.com/cmd'' to add it$'
 go install example.com/cmd/a@latest
 cmp go.mod go.mod.orig
 exists $GOPATH/bin/a$GOEXE
@@ -67,9 +67,9 @@ cd tmp
 go mod init tmp
 go mod edit -require=rsc.io/fortune@v1.0.0
 ! go install -mod=readonly $GOPATH/pkg/mod/rsc.io/fortune@v1.0.0
-stderr '^go: updates to go.sum needed, disabled by -mod=readonly$'
+stderr '^go: rsc.io/fortune@v1.0.0: missing go.sum entry; try ''go mod download rsc.io/fortune'' to add it$'
 ! go install -mod=readonly ../../pkg/mod/rsc.io/fortune@v1.0.0
-stderr '^go: updates to go.sum needed, disabled by -mod=readonly$'
+stderr '^go: rsc.io/fortune@v1.0.0: missing go.sum entry; try ''go mod download rsc.io/fortune'' to add it$'
 go get -d rsc.io/fortune@v1.0.0
 go install -mod=readonly $GOPATH/pkg/mod/rsc.io/fortune@v1.0.0
 exists $GOPATH/bin/fortune$GOEXE

src/cmd/go/testdata/script/mod_load_badchain.txt

@@ -40,6 +40,19 @@ module m
 go 1.13
 
 require example.com/badchain/a v1.0.0
+-- go.sum --
+example.com/badchain/a v1.0.0 h1:iJDLiHLmpQgr9Zrv+44UqywAE2IG6WkHnH4uG08vf+s=
+example.com/badchain/a v1.0.0/go.mod h1:6/gnCYHdVrs6mUgatUYUSbuHxEY+/yWedmTggLz23EI=
+example.com/badchain/a v1.1.0 h1:cPxQpsOjaIrn05yDfl4dFFgGSbjYmytLqtIIBfTsEqA=
+example.com/badchain/a v1.1.0/go.mod h1:T15b2BEK+RY7h7Lr2dgS38p1pgH5/t7Kf5nQXBlcW/A=
+example.com/badchain/b v1.0.0 h1:kjDVlBxpjQavYxHE7ECCyyXhfwsfhWIqvghfRgPktSA=
+example.com/badchain/b v1.0.0/go.mod h1:sYsH934pMc3/A2vQZh019qrWmp4+k87l3O0VFUYqL+I=
+example.com/badchain/b v1.1.0 h1:iEALV+DRN62FArnYylBR4YwCALn/hCdITvhdagHa0L4=
+example.com/badchain/b v1.1.0/go.mod h1:mlCgKO7lRZ+ijwMFIBFRPCGt5r5oqCcHdhSSE0VL4uY=
+example.com/badchain/c v1.0.0 h1:lOeUHQKR7SboSH7Bj6eIDWoNHaDQXI0T2GfaH2x9fNA=
+example.com/badchain/c v1.0.0/go.mod h1:4U3gzno17SaQ2koSVNxITu9r60CeLSgye9y4/5LnfOE=
+example.com/badchain/c v1.1.0 h1:VtTg1g7fOutWKHQf+ag04KLRpdMGSfQ9s9tagVtGW14=
+example.com/badchain/c v1.1.0/go.mod h1:tyoJj5qh+qtb48sflwdVvk4R+OjPQEY2UJOoibsVLPk=
 -- use/use.go --
 package use
 

src/cmd/go/testdata/script/mod_load_replace_mismatch.txt

@@ -2,7 +2,7 @@
 # the original module and its location, report an error with all three paths.
 # In particular, the "required as" path should be the original.
 # Verifies golang.org/issue/38220.
-! go list .
+! go mod download
 cmp stderr want
 
 -- go.mod --

src/cmd/go/testdata/script/mod_readonly.txt

@@ -41,7 +41,8 @@ go list -m all
 
 # -mod=readonly should reject inconsistent go.mod files
 # (ones that would be rewritten).
-go mod edit -require rsc.io/sampler@v1.2.0
+go get -d rsc.io/sampler@v1.2.0
+go mod edit -require rsc.io/quote@v1.5.2
 cp go.mod go.mod.inconsistent
 ! go list
 stderr 'go: updates to go.mod needed, disabled by -mod=readonly'

src/cmd/go/testdata/script/mod_sum_readonly.txt

@@ -0,0 +1,87 @@
+# Test that go.sum does not get updated when -mod=readonly flag is set
+env GO111MODULE=on
+
+# When a sum is needed to load the build list, we get an error for the
+# specific module. The .mod file is not downloaded, and go.sum is not written.
+! go list -m all
+stderr '^go: rsc.io/quote@v1.5.2: missing go.sum entry; try ''go mod download rsc.io/quote'' to add it$'
+! exists $GOPATH/pkg/mod/cache/download/rsc.io/quote/@v/v1.5.2.mod
+! exists go.sum
+
+# If go.sum exists but contains hashes from an algorithm we don't know about,
+# we should see the same error.
+cp go.sum.h2only go.sum
+! go list -m all
+stderr '^go: rsc.io/quote@v1.5.2: missing go.sum entry; try ''go mod download rsc.io/quote'' to add it$'
+! exists $GOPATH/pkg/mod/cache/download/rsc.io/quote/@v/v1.5.2.mod
+cmp go.sum go.sum.h2only
+rm go.sum
+
+# If we replace a module, we should see a missing sum error for the replacement.
+cp go.mod go.mod.orig
+go mod edit -replace rsc.io/quote@v1.5.2=rsc.io/quote@v1.5.1
+! go list -m all
+stderr '^go: rsc.io/quote@v1.5.2 \(replaced by rsc.io/quote@v1.5.1\): missing go.sum entry; try ''go mod download rsc.io/quote'' to add it$'
+! exists $GOPATH/pkg/mod/cache/download/rsc.io/quote/@v/v1.5.1.mod
+! exists go.sum
+cp go.mod.orig go.mod
+
+# Control: when sums are present, loading the build list downloads .mod files.
+cp go.sum.buildlistonly go.sum
+go list -m all
+exists $GOPATH/pkg/mod/cache/download/rsc.io/quote/@v/v1.5.2.mod
+
+
+# When a sum is needed to load a .mod file for a package outside the build list,
+# we get a generic missing import error.
+! go list example.com/doesnotexist
+stderr '^no required module provides package example.com/doesnotexist; try ''go get -d example.com/doesnotexist'' to add it$'
+
+# When a sum is needed to load a .zip file, we get a more specific error.
+# The .zip file is not downloaded.
+! go list rsc.io/quote
+stderr '^missing go.sum entry for module providing package rsc.io/quote$'
+! exists $GOPATH/pkg/mod/cache/download/rsc.io/quote/@v/v1.5.2.zip
+
+# The error is attached to the package from the missing module. We can load
+# a package that imports it without that error.
+go list -e -deps -f '{{.ImportPath}}{{with .Error}} {{.Err}}{{end}}' .
+stdout '^m$'
+stdout '^rsc.io/quote missing go.sum entry for module providing package rsc.io/quote; try ''go mod tidy'' to add it$'
+! exists $GOPATH/pkg/mod/cache/download/rsc.io/quote/@v/v1.5.2.zip
+
+# go.sum should not have been written.
+cmp go.sum go.sum.buildlistonly
+
+# Control: when sums are present, 'go list' downloads .zip files.
+cp go.sum.tidy go.sum
+go list .
+exists $GOPATH/pkg/mod/cache/download/rsc.io/quote/@v/v1.5.2.zip
+
+-- go.mod --
+module m
+
+go 1.15
+
+require rsc.io/quote v1.5.2
+-- use.go --
+package use
+
+import _ "rsc.io/quote"
+-- go.sum.h2only --
+golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h2:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
+rsc.io/quote v1.5.2/go.mod h2:LzX7hefJvL54yjefDEDHNONDjII0t9xZLPXsUe+TKr0=
+rsc.io/sampler v1.3.0/go.mod h2:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA=
+-- go.sum.buildlistonly --
+golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
+rsc.io/quote v1.5.2/go.mod h1:LzX7hefJvL54yjefDEDHNONDjII0t9xZLPXsUe+TKr0=
+rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA=
+-- go.sum.tidy --
+golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c h1:pvCbr/wm8HzDD3fVywevekufpn6tCGPY3spdHeZJEsw=
+golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
+rsc.io/quote v1.5.2 h1:3fEykkD9k7lYzXqCYrwGAf7iNhbk4yCjHmKBN9td4L0=
+rsc.io/quote v1.5.2/go.mod h1:LzX7hefJvL54yjefDEDHNONDjII0t9xZLPXsUe+TKr0=
+rsc.io/sampler v1.3.0 h1:HLGR/BgEtI3r0uymSP/nl2uPLsUnNJX8toRyhfpBTII=
+rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA=
+rsc.io/testonly v1.0.0 h1:K/VWHdO+Jv7woUXG0GzVNx1czBXUt3Ib1deaMn+xk64=
+rsc.io/testonly v1.0.0/go.mod h1:OqmGbIFOcF+XrFReLOGZ6BhMM7uMBiQwZsyNmh74SzY=

src/cmd/go/testdata/script/mod_sumdb.txt

@@ -17,7 +17,7 @@ stderr 'SECURITY ERROR\nThis download does NOT match the one reported by the che
 ! go get -d golang.org/x/text
 
 go mod edit -require rsc.io/quote@v1.5.2
-! go list all
+! go mod tidy
 stderr 'go: rsc.io/quote@v1.5.2: verifying go.mod: checksum mismatch'
 stderr 'SECURITY ERROR\n'
 

src/cmd/go/testdata/script/sum_readonly.txt

@@ -1,29 +0,0 @@
-# Test that go.sum does not get updated when -mod=readonly flag is set
-env GO111MODULE=on
-
-go get -d rsc.io/quote
-go mod tidy
-
-# go.sum != dirty; -mod=readonly
-go list -mod=readonly
-
-# dirty up go.sum by removing it.
-rm go.sum
-
-# go.sum == dirty; -mod=readonly
-! go list -mod=readonly
-
-stderr 'go: updates to go.sum needed, disabled by -mod=readonly'
-
--- go.mod --
-module m
-
--- main.go --
-
-package main
-
-import "rsc.io/quote"
-
-func main() {
-    println(quote.Hello())
-}