internal/govulncheck: copy from x/vuln repo

Currently, gopls maintains an edited copy of the govulncheck command
logic in the internal/vulncheck directory, along with the necessary
glue to make it work with gopls.

This CL is the first in a sequence that will make it easier for gopls
to use that logic.

It creates a new package, internal/govulncheck, adds a script to copy
the corresponding package from the x/vuln repo, and removes the cache
in internal/vulncheck in favor of the copied one.

Although it might appear simpler to copy the .go files directly into
internal/vulncheck, that would require editing the package directives
in the files, and has the risk of overwriting files.

Change-Id: I00f726f7b142048da2407f212873420df54844b3
Reviewed-on: https://go-review.googlesource.com/c/tools/+/405997
TryBot-Result: Gopher Robot <gobot@golang.org>
Reviewed-by: Hyang-Ah Hana Kim <hyangah@gmail.com>
Run-TryBot: Jonathan Amsterdam <jba@google.com>
gopls-CI: kokoro <noreply+kokoro@google.com>

gopls/internal/govulncheck/README.md

@@ -0,0 +1,17 @@
+# internal/govulncheck package
+
+This package is a literal copy of the cmd/govulncheck/internal/govulncheck
+package in the vuln repo (https://go.googlesource.com/vuln).
+
+The `copy.sh` does the copying, after removing all .go files here. To use it:
+
+1. Clone the vuln repo to a directory next to the directory holding this repo
+   (tools). After doing that your directory structure should look something like
+   ```
+   ~/repos/x/tools/gopls/...
+   ~/repos/x/vuln/...
+   ```
+
+2. cd to this directory.
+
+3. Run `copy.sh`.

gopls/internal/govulncheck/cache.go

@@ -2,7 +2,11 @@
 // Use of this source code is governed by a BSD-style
 // license that can be found in the LICENSE file.
 
-package vulncheck
+//go:build go1.18
+// +build go1.18
+
+// Package govulncheck supports the govulncheck command.
+package govulncheck
 
 import (
 	"encoding/json"
@@ -17,8 +21,6 @@ import (
 	"golang.org/x/vuln/osv"
 )
 
-// copy from x/vuln/cmd/govulncheck/cache.go
-
 // The cache uses a single JSON index file for each vulnerability database
 // which contains the map from packages to the time the last
 // vulnerability for that package was added/modified and the time that
@@ -37,19 +39,22 @@ import (
 // $GOPATH/pkg/mod/cache/download/vulndb/{db hostname}/{import path}/vulns.json
 //   []*osv.Entry
 
-// fsCache is a thread-safe file-system cache implementing osv.Cache
+// FSCache is a thread-safe file-system cache implementing osv.Cache
 //
 // TODO: use something like cmd/go/internal/lockedfile for thread safety?
-type fsCache struct {
+type FSCache struct {
 	mu      sync.Mutex
 	rootDir string
 }
 
+// Assert that *FSCache implements client.Cache.
+var _ client.Cache = (*FSCache)(nil)
+
 // use cfg.GOMODCACHE available in cmd/go/internal?
 var defaultCacheRoot = filepath.Join(build.Default.GOPATH, "/pkg/mod/cache/download/vulndb")
 
-func defaultCache() *fsCache {
-	return &fsCache{rootDir: defaultCacheRoot}
+func DefaultCache() *FSCache {
+	return &FSCache{rootDir: defaultCacheRoot}
 }
 
 type cachedIndex struct {
@@ -57,7 +62,7 @@ type cachedIndex struct {
 	Index     client.DBIndex
 }
 
-func (c *fsCache) ReadIndex(dbName string) (client.DBIndex, time.Time, error) {
+func (c *FSCache) ReadIndex(dbName string) (client.DBIndex, time.Time, error) {
 	c.mu.Lock()
 	defer c.mu.Unlock()
 
@@ -75,7 +80,7 @@ func (c *fsCache) ReadIndex(dbName string) (client.DBIndex, time.Time, error) {
 	return index.Index, index.Retrieved, nil
 }
 
-func (c *fsCache) WriteIndex(dbName string, index client.DBIndex, retrieved time.Time) error {
+func (c *FSCache) WriteIndex(dbName string, index client.DBIndex, retrieved time.Time) error {
 	c.mu.Lock()
 	defer c.mu.Unlock()
 
@@ -96,7 +101,7 @@ func (c *fsCache) WriteIndex(dbName string, index client.DBIndex, retrieved time
 	return nil
 }
 
-func (c *fsCache) ReadEntries(dbName string, p string) ([]*osv.Entry, error) {
+func (c *FSCache) ReadEntries(dbName string, p string) ([]*osv.Entry, error) {
 	c.mu.Lock()
 	defer c.mu.Unlock()
 
@@ -114,7 +119,7 @@ func (c *fsCache) ReadEntries(dbName string, p string) ([]*osv.Entry, error) {
 	return entries, nil
 }
 
-func (c *fsCache) WriteEntries(dbName string, p string, entries []*osv.Entry) error {
+func (c *FSCache) WriteEntries(dbName string, p string, entries []*osv.Entry) error {
 	c.mu.Lock()
 	defer c.mu.Unlock()
 

gopls/internal/govulncheck/cache_test.go

@@ -0,0 +1,165 @@
+// Copyright 2021 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+//go:build go1.18
+// +build go1.18
+
+package govulncheck
+
+import (
+	"fmt"
+	"os"
+	"path/filepath"
+	"reflect"
+	"testing"
+	"time"
+
+	"golang.org/x/sync/errgroup"
+	"golang.org/x/vuln/client"
+	"golang.org/x/vuln/osv"
+)
+
+func TestCache(t *testing.T) {
+	tmpDir := t.TempDir()
+
+	cache := &FSCache{rootDir: tmpDir}
+	dbName := "vulndb.golang.org"
+
+	_, _, err := cache.ReadIndex(dbName)
+	if err != nil {
+		t.Fatalf("ReadIndex failed for non-existent database: %v", err)
+	}
+
+	if err = os.Mkdir(filepath.Join(tmpDir, dbName), 0777); err != nil {
+		t.Fatalf("os.Mkdir failed: %v", err)
+	}
+	_, _, err = cache.ReadIndex(dbName)
+	if err != nil {
+		t.Fatalf("ReadIndex failed for database without cached index: %v", err)
+	}
+
+	now := time.Now()
+	expectedIdx := client.DBIndex{
+		"a.vuln.example.com": time.Time{}.Add(time.Hour),
+		"b.vuln.example.com": time.Time{}.Add(time.Hour * 2),
+		"c.vuln.example.com": time.Time{}.Add(time.Hour * 3),
+	}
+	if err = cache.WriteIndex(dbName, expectedIdx, now); err != nil {
+		t.Fatalf("WriteIndex failed to write index: %v", err)
+	}
+
+	idx, retrieved, err := cache.ReadIndex(dbName)
+	if err != nil {
+		t.Fatalf("ReadIndex failed for database with cached index: %v", err)
+	}
+	if !reflect.DeepEqual(idx, expectedIdx) {
+		t.Errorf("ReadIndex returned unexpected index, got:\n%s\nwant:\n%s", idx, expectedIdx)
+	}
+	if !retrieved.Equal(now) {
+		t.Errorf("ReadIndex returned unexpected retrieved: got %s, want %s", retrieved, now)
+	}
+
+	if _, err = cache.ReadEntries(dbName, "vuln.example.com"); err != nil {
+		t.Fatalf("ReadEntires failed for non-existent package: %v", err)
+	}
+
+	expectedEntries := []*osv.Entry{
+		{ID: "001"},
+		{ID: "002"},
+		{ID: "003"},
+	}
+	if err := cache.WriteEntries(dbName, "vuln.example.com", expectedEntries); err != nil {
+		t.Fatalf("WriteEntries failed: %v", err)
+	}
+
+	entries, err := cache.ReadEntries(dbName, "vuln.example.com")
+	if err != nil {
+		t.Fatalf("ReadEntries failed for cached package: %v", err)
+	}
+	if !reflect.DeepEqual(entries, expectedEntries) {
+		t.Errorf("ReadEntries returned unexpected entries, got:\n%v\nwant:\n%v", entries, expectedEntries)
+	}
+}
+
+func TestConcurrency(t *testing.T) {
+	tmpDir := t.TempDir()
+
+	cache := &FSCache{rootDir: tmpDir}
+	dbName := "vulndb.golang.org"
+
+	g := new(errgroup.Group)
+	for i := 0; i < 1000; i++ {
+		i := i
+		g.Go(func() error {
+			id := i % 5
+			p := fmt.Sprintf("package%d", id)
+
+			entries, err := cache.ReadEntries(dbName, p)
+			if err != nil {
+				return err
+			}
+
+			err = cache.WriteEntries(dbName, p, append(entries, &osv.Entry{ID: fmt.Sprint(id)}))
+			if err != nil {
+				return err
+			}
+			return nil
+		})
+	}
+
+	if err := g.Wait(); err != nil {
+		t.Errorf("error in parallel cache entries read/write: %v", err)
+	}
+
+	// sanity checking
+	for i := 0; i < 5; i++ {
+		id := fmt.Sprint(i)
+		p := fmt.Sprintf("package%s", id)
+
+		es, err := cache.ReadEntries(dbName, p)
+		if err != nil {
+			t.Fatalf("failed to read entries: %v", err)
+		}
+		for _, e := range es {
+			if e.ID != id {
+				t.Errorf("want %s ID for vuln entry; got %s", id, e.ID)
+			}
+		}
+	}
+
+	// do similar for cache index
+	start := time.Now()
+	for i := 0; i < 1000; i++ {
+		i := i
+		g.Go(func() error {
+			id := i % 5
+			p := fmt.Sprintf("package%v", id)
+
+			idx, _, err := cache.ReadIndex(dbName)
+			if err != nil {
+				return err
+			}
+
+			if idx == nil {
+				idx = client.DBIndex{}
+			}
+
+			// sanity checking
+			if rt, ok := idx[p]; ok && rt.Before(start) {
+				return fmt.Errorf("unexpected past time in index: %v before start %v", rt, start)
+			}
+
+			now := time.Now()
+			idx[p] = now
+			if err := cache.WriteIndex(dbName, idx, now); err != nil {
+				return err
+			}
+			return nil
+		})
+	}
+
+	if err := g.Wait(); err != nil {
+		t.Errorf("error in parallel cache index read/write: %v", err)
+	}
+}

gopls/internal/govulncheck/copy.sh

@@ -0,0 +1,13 @@
+#!/bin/bash -eu
+
+# Copyright 2020 The Go Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style
+# license that can be found in the LICENSE file.
+
+set -o pipefail
+
+# Copy golang.org/x/vuln/cmd/govulncheck/internal/govulncheck into this directory.
+# Assume the x/vuln repo is a sibling of the tools repo.
+
+rm -f *.go
+cp ../../../../vuln/cmd/govulncheck/internal/govulncheck/*.go .

gopls/internal/vulncheck/command.go

@@ -14,6 +14,7 @@ import (
 	"strings"
 
 	"golang.org/x/tools/go/packages"
+	gvc "golang.org/x/tools/gopls/internal/govulncheck"
 	"golang.org/x/tools/internal/lsp/command"
 	"golang.org/x/vuln/client"
 	"golang.org/x/vuln/vulncheck"
@@ -28,7 +29,7 @@ func govulncheck(ctx context.Context, cfg *packages.Config, args command.Vulnche
 		args.Pattern = "."
 	}
 
-	dbClient, err := client.NewClient(findGOVULNDB(cfg), client.Options{HTTPCache: defaultCache()})
+	dbClient, err := client.NewClient(findGOVULNDB(cfg), client.Options{HTTPCache: gvc.DefaultCache()})
 	if err != nil {
 		return res, err
 	}