diff --git a/gopls/internal/govulncheck/README.md b/gopls/internal/govulncheck/README.md new file mode 100644 index 0000000000..d8339c506f --- /dev/null +++ b/gopls/internal/govulncheck/README.md @@ -0,0 +1,17 @@ +# internal/govulncheck package + +This package is a literal copy of the cmd/govulncheck/internal/govulncheck +package in the vuln repo (https://go.googlesource.com/vuln). + +The `copy.sh` does the copying, after removing all .go files here. To use it: + +1. Clone the vuln repo to a directory next to the directory holding this repo + (tools). After doing that your directory structure should look something like + ``` + ~/repos/x/tools/gopls/... + ~/repos/x/vuln/... + ``` + +2. cd to this directory. + +3. Run `copy.sh`. diff --git a/gopls/internal/vulncheck/cache.go b/gopls/internal/govulncheck/cache.go similarity index 82% rename from gopls/internal/vulncheck/cache.go rename to gopls/internal/govulncheck/cache.go index 39a38fb068..404c356732 100644 --- a/gopls/internal/vulncheck/cache.go +++ b/gopls/internal/govulncheck/cache.go @@ -2,7 +2,11 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. -package vulncheck +//go:build go1.18 +// +build go1.18 + +// Package govulncheck supports the govulncheck command. +package govulncheck import ( "encoding/json" @@ -17,8 +21,6 @@ import ( "golang.org/x/vuln/osv" ) -// copy from x/vuln/cmd/govulncheck/cache.go - // The cache uses a single JSON index file for each vulnerability database // which contains the map from packages to the time the last // vulnerability for that package was added/modified and the time that @@ -37,19 +39,22 @@ import ( // $GOPATH/pkg/mod/cache/download/vulndb/{db hostname}/{import path}/vulns.json // []*osv.Entry -// fsCache is a thread-safe file-system cache implementing osv.Cache +// FSCache is a thread-safe file-system cache implementing osv.Cache // // TODO: use something like cmd/go/internal/lockedfile for thread safety? -type fsCache struct { +type FSCache struct { mu sync.Mutex rootDir string } +// Assert that *FSCache implements client.Cache. +var _ client.Cache = (*FSCache)(nil) + // use cfg.GOMODCACHE available in cmd/go/internal? var defaultCacheRoot = filepath.Join(build.Default.GOPATH, "/pkg/mod/cache/download/vulndb") -func defaultCache() *fsCache { - return &fsCache{rootDir: defaultCacheRoot} +func DefaultCache() *FSCache { + return &FSCache{rootDir: defaultCacheRoot} } type cachedIndex struct { @@ -57,7 +62,7 @@ type cachedIndex struct { Index client.DBIndex } -func (c *fsCache) ReadIndex(dbName string) (client.DBIndex, time.Time, error) { +func (c *FSCache) ReadIndex(dbName string) (client.DBIndex, time.Time, error) { c.mu.Lock() defer c.mu.Unlock() @@ -75,7 +80,7 @@ func (c *fsCache) ReadIndex(dbName string) (client.DBIndex, time.Time, error) { return index.Index, index.Retrieved, nil } -func (c *fsCache) WriteIndex(dbName string, index client.DBIndex, retrieved time.Time) error { +func (c *FSCache) WriteIndex(dbName string, index client.DBIndex, retrieved time.Time) error { c.mu.Lock() defer c.mu.Unlock() @@ -96,7 +101,7 @@ func (c *fsCache) WriteIndex(dbName string, index client.DBIndex, retrieved time return nil } -func (c *fsCache) ReadEntries(dbName string, p string) ([]*osv.Entry, error) { +func (c *FSCache) ReadEntries(dbName string, p string) ([]*osv.Entry, error) { c.mu.Lock() defer c.mu.Unlock() @@ -114,7 +119,7 @@ func (c *fsCache) ReadEntries(dbName string, p string) ([]*osv.Entry, error) { return entries, nil } -func (c *fsCache) WriteEntries(dbName string, p string, entries []*osv.Entry) error { +func (c *FSCache) WriteEntries(dbName string, p string, entries []*osv.Entry) error { c.mu.Lock() defer c.mu.Unlock() diff --git a/gopls/internal/govulncheck/cache_test.go b/gopls/internal/govulncheck/cache_test.go new file mode 100644 index 0000000000..5a25c78102 --- /dev/null +++ b/gopls/internal/govulncheck/cache_test.go @@ -0,0 +1,165 @@ +// Copyright 2021 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +//go:build go1.18 +// +build go1.18 + +package govulncheck + +import ( + "fmt" + "os" + "path/filepath" + "reflect" + "testing" + "time" + + "golang.org/x/sync/errgroup" + "golang.org/x/vuln/client" + "golang.org/x/vuln/osv" +) + +func TestCache(t *testing.T) { + tmpDir := t.TempDir() + + cache := &FSCache{rootDir: tmpDir} + dbName := "vulndb.golang.org" + + _, _, err := cache.ReadIndex(dbName) + if err != nil { + t.Fatalf("ReadIndex failed for non-existent database: %v", err) + } + + if err = os.Mkdir(filepath.Join(tmpDir, dbName), 0777); err != nil { + t.Fatalf("os.Mkdir failed: %v", err) + } + _, _, err = cache.ReadIndex(dbName) + if err != nil { + t.Fatalf("ReadIndex failed for database without cached index: %v", err) + } + + now := time.Now() + expectedIdx := client.DBIndex{ + "a.vuln.example.com": time.Time{}.Add(time.Hour), + "b.vuln.example.com": time.Time{}.Add(time.Hour * 2), + "c.vuln.example.com": time.Time{}.Add(time.Hour * 3), + } + if err = cache.WriteIndex(dbName, expectedIdx, now); err != nil { + t.Fatalf("WriteIndex failed to write index: %v", err) + } + + idx, retrieved, err := cache.ReadIndex(dbName) + if err != nil { + t.Fatalf("ReadIndex failed for database with cached index: %v", err) + } + if !reflect.DeepEqual(idx, expectedIdx) { + t.Errorf("ReadIndex returned unexpected index, got:\n%s\nwant:\n%s", idx, expectedIdx) + } + if !retrieved.Equal(now) { + t.Errorf("ReadIndex returned unexpected retrieved: got %s, want %s", retrieved, now) + } + + if _, err = cache.ReadEntries(dbName, "vuln.example.com"); err != nil { + t.Fatalf("ReadEntires failed for non-existent package: %v", err) + } + + expectedEntries := []*osv.Entry{ + {ID: "001"}, + {ID: "002"}, + {ID: "003"}, + } + if err := cache.WriteEntries(dbName, "vuln.example.com", expectedEntries); err != nil { + t.Fatalf("WriteEntries failed: %v", err) + } + + entries, err := cache.ReadEntries(dbName, "vuln.example.com") + if err != nil { + t.Fatalf("ReadEntries failed for cached package: %v", err) + } + if !reflect.DeepEqual(entries, expectedEntries) { + t.Errorf("ReadEntries returned unexpected entries, got:\n%v\nwant:\n%v", entries, expectedEntries) + } +} + +func TestConcurrency(t *testing.T) { + tmpDir := t.TempDir() + + cache := &FSCache{rootDir: tmpDir} + dbName := "vulndb.golang.org" + + g := new(errgroup.Group) + for i := 0; i < 1000; i++ { + i := i + g.Go(func() error { + id := i % 5 + p := fmt.Sprintf("package%d", id) + + entries, err := cache.ReadEntries(dbName, p) + if err != nil { + return err + } + + err = cache.WriteEntries(dbName, p, append(entries, &osv.Entry{ID: fmt.Sprint(id)})) + if err != nil { + return err + } + return nil + }) + } + + if err := g.Wait(); err != nil { + t.Errorf("error in parallel cache entries read/write: %v", err) + } + + // sanity checking + for i := 0; i < 5; i++ { + id := fmt.Sprint(i) + p := fmt.Sprintf("package%s", id) + + es, err := cache.ReadEntries(dbName, p) + if err != nil { + t.Fatalf("failed to read entries: %v", err) + } + for _, e := range es { + if e.ID != id { + t.Errorf("want %s ID for vuln entry; got %s", id, e.ID) + } + } + } + + // do similar for cache index + start := time.Now() + for i := 0; i < 1000; i++ { + i := i + g.Go(func() error { + id := i % 5 + p := fmt.Sprintf("package%v", id) + + idx, _, err := cache.ReadIndex(dbName) + if err != nil { + return err + } + + if idx == nil { + idx = client.DBIndex{} + } + + // sanity checking + if rt, ok := idx[p]; ok && rt.Before(start) { + return fmt.Errorf("unexpected past time in index: %v before start %v", rt, start) + } + + now := time.Now() + idx[p] = now + if err := cache.WriteIndex(dbName, idx, now); err != nil { + return err + } + return nil + }) + } + + if err := g.Wait(); err != nil { + t.Errorf("error in parallel cache index read/write: %v", err) + } +} diff --git a/gopls/internal/govulncheck/copy.sh b/gopls/internal/govulncheck/copy.sh new file mode 100755 index 0000000000..24ed45bfe5 --- /dev/null +++ b/gopls/internal/govulncheck/copy.sh @@ -0,0 +1,13 @@ +#!/bin/bash -eu + +# Copyright 2020 The Go Authors. All rights reserved. +# Use of this source code is governed by a BSD-style +# license that can be found in the LICENSE file. + +set -o pipefail + +# Copy golang.org/x/vuln/cmd/govulncheck/internal/govulncheck into this directory. +# Assume the x/vuln repo is a sibling of the tools repo. + +rm -f *.go +cp ../../../../vuln/cmd/govulncheck/internal/govulncheck/*.go . diff --git a/gopls/internal/vulncheck/command.go b/gopls/internal/vulncheck/command.go index 3fd9d03682..459ecca44c 100644 --- a/gopls/internal/vulncheck/command.go +++ b/gopls/internal/vulncheck/command.go @@ -14,6 +14,7 @@ import ( "strings" "golang.org/x/tools/go/packages" + gvc "golang.org/x/tools/gopls/internal/govulncheck" "golang.org/x/tools/internal/lsp/command" "golang.org/x/vuln/client" "golang.org/x/vuln/vulncheck" @@ -28,7 +29,7 @@ func govulncheck(ctx context.Context, cfg *packages.Config, args command.Vulnche args.Pattern = "." } - dbClient, err := client.NewClient(findGOVULNDB(cfg), client.Options{HTTPCache: defaultCache()}) + dbClient, err := client.NewClient(findGOVULNDB(cfg), client.Options{HTTPCache: gvc.DefaultCache()}) if err != nil { return res, err }