diff --git a/gopls/internal/regtest/workspace_test.go b/gopls/internal/regtest/workspace_test.go index 4e9559dba6..9eaff5596e 100644 --- a/gopls/internal/regtest/workspace_test.go +++ b/gopls/internal/regtest/workspace_test.go @@ -158,6 +158,16 @@ replace random.org => %s } const workspaceModuleProxy = ` +-- example.com@v1.2.3/go.mod -- +module example.com + +go 1.12 +-- example.com@v1.2.3/blah/blah.go -- +package blah + +func SaySomething() { + fmt.Println("something") +} -- b.com@v1.2.3/go.mod -- module b.com @@ -364,6 +374,9 @@ func Hello() int { } func TestUseGoplsMod(t *testing.T) { + // This test validates certain functionality related to using a gopls.mod + // file to specify workspace modules. + testenv.NeedsGo1Point(t, 14) const multiModule = ` -- moda/a/go.mod -- module a.com @@ -384,6 +397,7 @@ func main() { -- modb/go.mod -- module b.com +require example.com v1.2.3 -- modb/b/b.go -- package b @@ -404,12 +418,18 @@ replace a.com => $SANDBOX_WORKDIR/moda/a WithProxyFiles(workspaceModuleProxy), WithModes(Experimental), ).run(t, multiModule, func(t *testing.T, env *Env) { + // Initially, the gopls.mod should cause only the a.com module to be + // loaded. Validate this by jumping to a definition in b.com and ensuring + // that we go to the module cache. env.OpenFile("moda/a/a.go") - original, _ := env.GoToDefinition("moda/a/a.go", env.RegexpSearch("moda/a/a.go", "Hello")) - if want := "b.com@v1.2.3/b/b.go"; !strings.HasSuffix(original, want) { - t.Errorf("expected %s, got %v", want, original) + location, _ := env.GoToDefinition("moda/a/a.go", env.RegexpSearch("moda/a/a.go", "Hello")) + if want := "b.com@v1.2.3/b/b.go"; !strings.HasSuffix(location, want) { + t.Errorf("expected %s, got %v", want, location) } workdir := env.Sandbox.Workdir.RootURI().SpanURI().Filename() + + // Now, modify the gopls.mod file on disk to activate the b.com module in + // the workspace. env.WriteWorkspaceFile("gopls.mod", fmt.Sprintf(`module gopls-workspace require ( @@ -426,9 +446,41 @@ replace b.com => %s/modb env.DiagnosticAtRegexp("modb/b/b.go", "x"), ), ) - newLocation, _ := env.GoToDefinition("moda/a/a.go", env.RegexpSearch("moda/a/a.go", "Hello")) - if want := "modb/b/b.go"; !strings.HasSuffix(newLocation, want) { - t.Errorf("expected %s, got %v", want, newLocation) + env.OpenFile("modb/go.mod") + // Check that go.mod diagnostics picked up the newly active mod file. + env.Await(env.DiagnosticAtRegexp("modb/go.mod", `require example.com v1.2.3`)) + // ...and that jumping to definition now goes to b.com in the workspace. + location, _ = env.GoToDefinition("moda/a/a.go", env.RegexpSearch("moda/a/a.go", "Hello")) + if want := "modb/b/b.go"; !strings.HasSuffix(location, want) { + t.Errorf("expected %s, got %v", want, location) + } + + // Now, let's modify the gopls.mod *overlay* (not on disk), and verify that + // this change is also picked up. + env.OpenFile("gopls.mod") + env.SetBufferContent("gopls.mod", fmt.Sprintf(`module gopls-workspace + +require ( + a.com v0.0.0-goplsworkspace +) + +replace a.com => %s/moda/a +`, workdir)) + env.Await(CompletedWork(lsp.DiagnosticWorkTitle(lsp.FromDidChange), 1)) + // TODO: diagnostics are not being cleared from the old go.mod location, + // because it's not treated as a 'deleted' file. Uncomment this after + // fixing. + /* + env.Await(OnceMet( + CompletedWork(lsp.DiagnosticWorkTitle(lsp.FromDidChange), 1), + EmptyDiagnostics("modb/go.mod"), + )) + */ + + // Just as before, check that we now jump to the module cache. + location, _ = env.GoToDefinition("moda/a/a.go", env.RegexpSearch("moda/a/a.go", "Hello")) + if want := "b.com@v1.2.3/b/b.go"; !strings.HasSuffix(location, want) { + t.Errorf("expected %s, got %v", want, location) } }) } diff --git a/gopls/internal/regtest/wrappers.go b/gopls/internal/regtest/wrappers.go index 53f79527d8..d7ca87fbe8 100644 --- a/gopls/internal/regtest/wrappers.go +++ b/gopls/internal/regtest/wrappers.go @@ -92,6 +92,13 @@ func (e *Env) EditBuffer(name string, edits ...fake.Edit) { } } +func (e *Env) SetBufferContent(name string, content string) { + e.T.Helper() + if err := e.Editor.SetBufferContent(e.Ctx, name, content); err != nil { + e.T.Fatal(err) + } +} + // RegexpRange returns the range of the first match for re in the buffer // specified by name, calling t.Fatal on any error. It first searches for the // position in open buffers, then in workspace files. diff --git a/internal/lsp/cache/cache.go b/internal/lsp/cache/cache.go index c2b4fe52f2..5e6180d73f 100644 --- a/internal/lsp/cache/cache.go +++ b/internal/lsp/cache/cache.go @@ -79,14 +79,10 @@ func (c *Cache) getFile(ctx context.Context, uri span.URI) (*fileHandle, error) return fh, nil } - select { - case ioLimit <- struct{}{}: - case <-ctx.Done(): - return nil, ctx.Err() + fh, err := readFile(ctx, uri, fi.ModTime()) + if err != nil { + return nil, err } - defer func() { <-ioLimit }() - - fh = readFile(ctx, uri, fi.ModTime()) c.fileMu.Lock() c.fileContent[uri] = fh c.fileMu.Unlock() @@ -96,7 +92,14 @@ func (c *Cache) getFile(ctx context.Context, uri span.URI) (*fileHandle, error) // ioLimit limits the number of parallel file reads per process. var ioLimit = make(chan struct{}, 128) -func readFile(ctx context.Context, uri span.URI, modTime time.Time) *fileHandle { +func readFile(ctx context.Context, uri span.URI, modTime time.Time) (*fileHandle, error) { + select { + case ioLimit <- struct{}{}: + case <-ctx.Done(): + return nil, ctx.Err() + } + defer func() { <-ioLimit }() + ctx, done := event.Start(ctx, "cache.readFile", tag.File.Of(uri.Filename())) _ = ctx defer done() @@ -106,14 +109,14 @@ func readFile(ctx context.Context, uri span.URI, modTime time.Time) *fileHandle return &fileHandle{ modTime: modTime, err: err, - } + }, nil } return &fileHandle{ modTime: modTime, uri: uri, bytes: data, hash: hashContents(data), - } + }, nil } func (c *Cache) NewSession(ctx context.Context) *Session { diff --git a/internal/lsp/cache/imports.go b/internal/lsp/cache/imports.go index 5039eac520..48216ab1d3 100644 --- a/internal/lsp/cache/imports.go +++ b/internal/lsp/cache/imports.go @@ -33,34 +33,31 @@ func (s *importsState) runProcessEnvFunc(ctx context.Context, snapshot *snapshot // Use temporary go.mod files, but always go to disk for the contents. // Rebuilding the cache is expensive, and we don't want to do it for // transient changes. - var modFH, sumFH source.FileHandle + var modFH source.FileHandle + var gosum []byte var modFileIdentifier string var err error - // TODO(heschik): Change the goimports logic to use a persistent workspace + // TODO(rfindley): Change the goimports logic to use a persistent workspace // module for workspace module mode. // // Get the go.mod file that corresponds to this view's root URI. This is // broken because it assumes that the view's root is a module, but this is // not more broken than the previous state--it is a temporary hack that // should be removed ASAP. - var match *moduleRoot - for _, m := range snapshot.modules { - if m.rootURI == snapshot.view.rootURI { - match = m + var matchURI span.URI + for modURI := range snapshot.workspace.activeModFiles() { + if dirURI(modURI) == snapshot.view.rootURI { + matchURI = modURI } } - if match != nil { - modFH, err = snapshot.GetFile(ctx, match.modURI) + // TODO(rFindley): should it be an error if matchURI is empty? + if matchURI != "" { + modFH, err = snapshot.GetFile(ctx, matchURI) if err != nil { return err } modFileIdentifier = modFH.FileIdentity().Hash - if match.sumURI != "" { - sumFH, err = snapshot.GetFile(ctx, match.sumURI) - if err != nil { - return err - } - } + gosum = snapshot.goSum(ctx, matchURI) } // v.goEnv is immutable -- changes make a new view. Options can change. // We can't compare build flags directly because we may add -modfile. @@ -87,7 +84,7 @@ func (s *importsState) runProcessEnvFunc(ctx context.Context, snapshot *snapshot } s.cachedModFileIdentifier = modFileIdentifier s.cachedBuildFlags = currentBuildFlags - s.cleanupProcessEnv, err = s.populateProcessEnv(ctx, snapshot, modFH, sumFH) + s.cleanupProcessEnv, err = s.populateProcessEnv(ctx, snapshot, modFH, gosum) if err != nil { return err } @@ -125,7 +122,7 @@ func (s *importsState) runProcessEnvFunc(ctx context.Context, snapshot *snapshot // populateProcessEnv sets the dynamically configurable fields for the view's // process environment. Assumes that the caller is holding the s.view.importsMu. -func (s *importsState) populateProcessEnv(ctx context.Context, snapshot *snapshot, modFH, sumFH source.FileHandle) (cleanup func(), err error) { +func (s *importsState) populateProcessEnv(ctx context.Context, snapshot *snapshot, modFH source.FileHandle, gosum []byte) (cleanup func(), err error) { cleanup = func() {} pe := s.processEnv @@ -166,7 +163,7 @@ func (s *importsState) populateProcessEnv(ctx context.Context, snapshot *snapsho // Add -modfile to the build flags, if we are using it. if snapshot.workspaceMode()&tempModfile != 0 && modFH != nil { var tmpURI span.URI - tmpURI, cleanup, err = tempModFile(modFH, sumFH) + tmpURI, cleanup, err = tempModFile(modFH, gosum) if err != nil { return nil, err } diff --git a/internal/lsp/cache/load.go b/internal/lsp/cache/load.go index 0d07070df0..f5bb720197 100644 --- a/internal/lsp/cache/load.go +++ b/internal/lsp/cache/load.go @@ -191,14 +191,11 @@ func (s *snapshot) tempWorkspaceModule(ctx context.Context) (_ span.URI, cleanup if s.workspaceMode()&usesWorkspaceModule == 0 { return "", cleanup, nil } - wsModuleHandle, err := s.getWorkspaceModuleHandle(ctx) - if err != nil { - return "", nil, err - } - file, err := wsModuleHandle.build(ctx, s) + file, err := s.workspace.modFile(ctx, s) if err != nil { return "", nil, err } + content, err := file.Format() if err != nil { return "", cleanup, err diff --git a/internal/lsp/cache/mod.go b/internal/lsp/cache/mod.go index 6d84c501c2..0e4ed4e3dd 100644 --- a/internal/lsp/cache/mod.go +++ b/internal/lsp/cache/mod.go @@ -98,24 +98,26 @@ func (s *snapshot) ParseMod(ctx context.Context, modFH source.FileHandle) (*sour return pmh.parse(ctx, s) } -func (s *snapshot) sumFH(ctx context.Context, modFH source.FileHandle) (source.FileHandle, error) { +// goSum reads the go.sum file for the go.mod file at modURI, if it exists. If +// it doesn't exist, it returns nil. +func (s *snapshot) goSum(ctx context.Context, modURI span.URI) []byte { // Get the go.sum file, either from the snapshot or directly from the // cache. Avoid (*snapshot).GetFile here, as we don't want to add // nonexistent file handles to the snapshot if the file does not exist. - sumURI := span.URIFromPath(sumFilename(modFH.URI())) + sumURI := span.URIFromPath(sumFilename(modURI)) var sumFH source.FileHandle = s.FindFile(sumURI) if sumFH == nil { var err error sumFH, err = s.view.session.cache.getFile(ctx, sumURI) if err != nil { - return nil, err + return nil } } - _, err := sumFH.Read() + content, err := sumFH.Read() if err != nil { - return nil, err + return nil } - return sumFH, nil + return content } func sumFilename(modURI span.URI) string { diff --git a/internal/lsp/cache/session.go b/internal/lsp/cache/session.go index 733c937215..678e9c42de 100644 --- a/internal/lsp/cache/session.go +++ b/internal/lsp/cache/session.go @@ -7,8 +7,6 @@ package cache import ( "context" "fmt" - "os" - "path/filepath" "strconv" "strings" "sync" @@ -171,10 +169,8 @@ func (s *Session) createView(ctx context.Context, name string, folder span.URI, return nil, nil, func() {}, err } - // If workspace module mode is enabled, find all of the modules in the - // workspace. By default, we just find the root module. - var modules map[span.URI]*moduleRoot - modules, err = findWorkspaceModules(ctx, ws.rootURI, options) + // Build the gopls workspace, collecting active modules in the view. + workspace, err := newWorkspace(ctx, ws.rootURI, s, options.ExperimentalWorkspaceModule) if err != nil { return nil, nil, func() {}, err } @@ -225,11 +221,9 @@ func (s *Session) createView(ctx context.Context, name string, folder span.URI, modTidyHandles: make(map[span.URI]*modTidyHandle), modUpgradeHandles: make(map[span.URI]*modUpgradeHandle), modWhyHandles: make(map[span.URI]*modWhyHandle), - modules: modules, + workspace: workspace, } - v.snapshot.workspaceDirectories = v.snapshot.findWorkspaceDirectories(ctx) - // Initialize the view without blocking. initCtx, initCancel := context.WithCancel(xcontext.Detach(ctx)) v.initCancelFirstAttempt = initCancel @@ -242,52 +236,6 @@ func (s *Session) createView(ctx context.Context, name string, folder span.URI, return v, snapshot, snapshot.generation.Acquire(ctx), nil } -// findWorkspaceModules walks the view's root folder, looking for go.mod files. -// Any that are found are added to the view's set of modules, which are then -// used to construct the workspace module. -// -// It assumes that the caller has not yet created the view, and therefore does -// not lock any of the internal data structures before accessing them. -// -// TODO(rstambler): Check overlays for go.mod files. -func findWorkspaceModules(ctx context.Context, root span.URI, options *source.Options) (map[span.URI]*moduleRoot, error) { - // Walk the view's folder to find all modules in the view. - modules := make(map[span.URI]*moduleRoot) - if !options.ExperimentalWorkspaceModule { - path := filepath.Join(root.Filename(), "go.mod") - if info, _ := os.Stat(path); info != nil { - if m := getViewModule(ctx, root, span.URIFromPath(path), options); m != nil { - modules[m.rootURI] = m - } - } - return modules, nil - } - return modules, filepath.Walk(root.Filename(), func(path string, info os.FileInfo, err error) error { - if err != nil { - // Probably a permission error. Keep looking. - return filepath.SkipDir - } - // For any path that is not the workspace folder, check if the path - // would be ignored by the go command. Vendor directories also do not - // contain workspace modules. - if info.IsDir() && path != root.Filename() { - suffix := strings.TrimPrefix(path, root.Filename()) - switch { - case checkIgnored(suffix), - strings.Contains(filepath.ToSlash(suffix), "/vendor/"): - return filepath.SkipDir - } - } - // We're only interested in go.mod files. - if filepath.Base(path) == "go.mod" { - if m := getViewModule(ctx, root, span.URIFromPath(path), options); m != nil { - modules[m.rootURI] = m - } - } - return nil - }) -} - // View returns the view by name. func (s *Session) View(name string) source.View { s.viewMu.Lock() @@ -439,8 +387,14 @@ func (s *Session) ModifyFiles(ctx context.Context, changes []source.FileModifica return err } +type fileChange struct { + content []byte + exists bool + fileHandle source.VersionedFileHandle +} + func (s *Session) DidModifyFiles(ctx context.Context, changes []source.FileModification) (map[span.URI]source.View, map[source.View]source.Snapshot, []func(), []span.URI, error) { - views := make(map[*View]map[span.URI]source.VersionedFileHandle) + views := make(map[*View]map[span.URI]*fileChange) bestViews := map[span.URI]source.View{} // Keep track of deleted files so that we can clear their diagnostics. // A file might be re-created after deletion, so only mark files that @@ -485,23 +439,28 @@ func (s *Session) DidModifyFiles(ctx context.Context, changes []source.FileModif return nil, nil, nil, nil, err } if _, ok := views[view]; !ok { - views[view] = make(map[span.URI]source.VersionedFileHandle) + views[view] = make(map[span.URI]*fileChange) } - var ( - fh source.VersionedFileHandle - ok bool - ) - if fh, ok = overlays[c.URI]; ok { - views[view][c.URI] = fh + if fh, ok := overlays[c.URI]; ok { + views[view][c.URI] = &fileChange{ + content: fh.text, + exists: true, + fileHandle: fh, + } delete(deletions, c.URI) } else { fsFile, err := s.cache.getFile(ctx, c.URI) if err != nil { return nil, nil, nil, nil, err } - fh = &closedFile{fsFile} - views[view][c.URI] = fh - if _, err := fh.Read(); err != nil { + content, err := fsFile.Read() + fh := &closedFile{fsFile} + views[view][c.URI] = &fileChange{ + content: content, + exists: err == nil, + fileHandle: fh, + } + if err != nil { deletions[c.URI] = struct{}{} } } @@ -510,8 +469,8 @@ func (s *Session) DidModifyFiles(ctx context.Context, changes []source.FileModif snapshots := map[source.View]source.Snapshot{} var releases []func() - for view, uris := range views { - snapshot, release := view.invalidateContent(ctx, uris, forceReloadMetadata) + for view, changed := range views { + snapshot, release := view.invalidateContent(ctx, changed, forceReloadMetadata) snapshots[view] = snapshot releases = append(releases, release) } diff --git a/internal/lsp/cache/snapshot.go b/internal/lsp/cache/snapshot.go index f26772ec2c..ff06817e21 100644 --- a/internal/lsp/cache/snapshot.go +++ b/internal/lsp/cache/snapshot.go @@ -78,10 +78,6 @@ type snapshot struct { // when the view is created. workspacePackages map[packageID]packagePath - // workspaceDirectories are the directories containing workspace packages. - // They are the view's root, as well as any replace targets. - workspaceDirectories map[span.URI]struct{} - // unloadableFiles keeps track of files that we've failed to load. unloadableFiles map[span.URI]struct{} @@ -96,12 +92,7 @@ type snapshot struct { modUpgradeHandles map[span.URI]*modUpgradeHandle modWhyHandles map[span.URI]*modWhyHandle - // modules is the set of modules currently in this workspace. - modules map[span.URI]*moduleRoot - - // workspaceModuleHandle keeps track of the in-memory representation of the - // go.mod file for the workspace module. - workspaceModuleHandle *workspaceModuleHandle + workspace *workspace } type packageKey struct { @@ -128,14 +119,14 @@ func (s *snapshot) FileSet() *token.FileSet { func (s *snapshot) ModFiles() []span.URI { var uris []span.URI - for _, m := range s.modules { - uris = append(uris, m.modURI) + for modURI := range s.workspace.activeModFiles() { + uris = append(uris, modURI) } return uris } func (s *snapshot) ValidBuildConfiguration() bool { - return validBuildConfiguration(s.view.rootURI, &s.view.workspaceInformation, s.modules) + return validBuildConfiguration(s.view.rootURI, &s.view.workspaceInformation, s.workspace.activeModFiles()) } // workspaceMode describes the way in which the snapshot's workspace should @@ -152,7 +143,7 @@ func (s *snapshot) workspaceMode() workspaceMode { // If the view is not in a module and contains no modules, but still has a // valid workspace configuration, do not create the workspace module. // It could be using GOPATH or a different build system entirely. - if len(s.modules) == 0 && validBuildConfiguration { + if len(s.workspace.activeModFiles()) == 0 && validBuildConfiguration { return mode } mode |= moduleMode @@ -255,11 +246,9 @@ func (s *snapshot) goCommandInvocation(ctx context.Context, mode source.Invocati // the passed-in working dir. if mode == source.ForTypeChecking { if s.workspaceMode()&usesWorkspaceModule == 0 { - var mod *moduleRoot - for _, m := range s.modules { // range to access the only element - mod = m + for m := range s.workspace.activeModFiles() { // range to access the only element + modURI = m } - modURI = mod.modURI } else { var tmpDir span.URI var err error @@ -291,9 +280,8 @@ func (s *snapshot) goCommandInvocation(ctx context.Context, mode source.Invocati return "", nil, cleanup, err } // Use the go.sum if it happens to be available. - sumFH, _ := s.sumFH(ctx, modFH) - - tmpURI, cleanup, err = tempModFile(modFH, sumFH) + gosum := s.goSum(ctx, modURI) + tmpURI, cleanup, err = tempModFile(modFH, gosum) if err != nil { return "", nil, cleanup, err } @@ -601,14 +589,7 @@ func (s *snapshot) workspacePackageIDs() (ids []packageID) { } func (s *snapshot) WorkspaceDirectories(ctx context.Context) []span.URI { - s.mu.Lock() - defer s.mu.Unlock() - - var dirs []span.URI - for d := range s.workspaceDirectories { - dirs = append(dirs, d) - } - return dirs + return s.workspace.dirs(ctx, s) } func (s *snapshot) WorkspacePackages(ctx context.Context) ([]source.Package, error) { @@ -684,12 +665,12 @@ func (s *snapshot) CachedImportPaths(ctx context.Context) (map[string]source.Pac func (s *snapshot) GoModForFile(ctx context.Context, uri span.URI) span.URI { var match span.URI - for _, m := range s.modules { - if !isSubdirectory(m.rootURI.Filename(), uri.Filename()) { + for modURI := range s.workspace.activeModFiles() { + if !isSubdirectory(dirURI(modURI).Filename(), uri.Filename()) { continue } - if len(m.modURI) > len(match) { - match = m.modURI + if len(modURI) > len(match) { + match = modURI } } return match @@ -810,7 +791,7 @@ func (s *snapshot) FindFile(uri span.URI) source.VersionedFileHandle { // GetVersionedFile returns a File for the given URI. If the file is unknown it // is added to the managed set. // -// GetFile succeeds even if the file does not exist. A non-nil error return +// GetVersionedFile succeeds even if the file does not exist. A non-nil error return // indicates some type of internal error, for example if ctx is cancelled. func (s *snapshot) GetVersionedFile(ctx context.Context, uri span.URI) (source.VersionedFileHandle, error) { f, err := s.view.getFile(uri) @@ -992,32 +973,32 @@ func generationName(v *View, snapshotID uint64) string { return fmt.Sprintf("v%v/%v", v.id, snapshotID) } -func (s *snapshot) clone(ctx context.Context, withoutURIs map[span.URI]source.VersionedFileHandle, forceReloadMetadata bool) (*snapshot, reinitializeView) { +func (s *snapshot) clone(ctx context.Context, changes map[span.URI]*fileChange, forceReloadMetadata bool) (*snapshot, reinitializeView) { + newWorkspace, workspaceChanged := s.workspace.invalidate(ctx, changes) + s.mu.Lock() defer s.mu.Unlock() newGen := s.view.session.cache.store.Generation(generationName(s.view, s.id+1)) result := &snapshot{ - id: s.id + 1, - generation: newGen, - view: s.view, - builtin: s.builtin, - ids: make(map[span.URI][]packageID), - importedBy: make(map[packageID][]packageID), - metadata: make(map[packageID]*metadata), - packages: make(map[packageKey]*packageHandle), - actions: make(map[actionKey]*actionHandle), - files: make(map[span.URI]source.VersionedFileHandle), - goFiles: make(map[parseKey]*parseGoHandle), - workspaceDirectories: make(map[span.URI]struct{}), - workspacePackages: make(map[packageID]packagePath), - unloadableFiles: make(map[span.URI]struct{}), - parseModHandles: make(map[span.URI]*parseModHandle), - modTidyHandles: make(map[span.URI]*modTidyHandle), - modUpgradeHandles: make(map[span.URI]*modUpgradeHandle), - modWhyHandles: make(map[span.URI]*modWhyHandle), - modules: make(map[span.URI]*moduleRoot), - workspaceModuleHandle: s.workspaceModuleHandle, + id: s.id + 1, + generation: newGen, + view: s.view, + builtin: s.builtin, + ids: make(map[span.URI][]packageID), + importedBy: make(map[packageID][]packageID), + metadata: make(map[packageID]*metadata), + packages: make(map[packageKey]*packageHandle), + actions: make(map[actionKey]*actionHandle), + files: make(map[span.URI]source.VersionedFileHandle), + goFiles: make(map[parseKey]*parseGoHandle), + workspacePackages: make(map[packageID]packagePath), + unloadableFiles: make(map[span.URI]struct{}), + parseModHandles: make(map[span.URI]*parseModHandle), + modTidyHandles: make(map[span.URI]*modTidyHandle), + modUpgradeHandles: make(map[span.URI]*modUpgradeHandle), + modWhyHandles: make(map[span.URI]*modWhyHandle), + workspace: newWorkspace, } if s.builtin != nil { @@ -1028,6 +1009,7 @@ func (s *snapshot) clone(ctx context.Context, withoutURIs map[span.URI]source.Ve for k, v := range s.files { result.files[k] = v } + // Copy the set of unloadable files. for k, v := range s.unloadableFiles { result.unloadableFiles[k] = v @@ -1036,13 +1018,9 @@ func (s *snapshot) clone(ctx context.Context, withoutURIs map[span.URI]source.Ve for k, v := range s.parseModHandles { result.parseModHandles[k] = v } - // Copy all of the workspace directories. They may be reset later. - for k, v := range s.workspaceDirectories { - result.workspaceDirectories[k] = v - } for k, v := range s.goFiles { - if _, ok := withoutURIs[k.file.URI]; ok { + if _, ok := changes[k.file.URI]; ok { continue } newGen.Inherit(v.handle) @@ -1053,53 +1031,55 @@ func (s *snapshot) clone(ctx context.Context, withoutURIs map[span.URI]source.Ve // Copy all of the go.mod-related handles. They may be invalidated later, // so we inherit them at the end of the function. for k, v := range s.modTidyHandles { - if _, ok := withoutURIs[k]; ok { + if _, ok := changes[k]; ok { continue } result.modTidyHandles[k] = v } for k, v := range s.modUpgradeHandles { - if _, ok := withoutURIs[k]; ok { + if _, ok := changes[k]; ok { continue } result.modUpgradeHandles[k] = v } for k, v := range s.modWhyHandles { - if _, ok := withoutURIs[k]; ok { + if _, ok := changes[k]; ok { continue } result.modWhyHandles[k] = v } - // Add all of the modules now. They may be deleted or added to later. - for k, v := range s.modules { - result.modules[k] = v - } - - var modulesChanged, shouldReinitializeView bool + var reinitialize reinitializeView // directIDs keeps track of package IDs that have directly changed. // It maps id->invalidateMetadata. directIDs := map[packageID]bool{} - for withoutURI, currentFH := range withoutURIs { + // Invalidate all package metadata if the workspace module has changed. + if workspaceChanged { + reinitialize = definitelyReinit + for k := range s.metadata { + directIDs[k] = true + } + } + for uri, change := range changes { // The original FileHandle for this URI is cached on the snapshot. - originalFH := s.files[withoutURI] + originalFH := s.files[uri] // Check if the file's package name or imports have changed, // and if so, invalidate this file's packages' metadata. - invalidateMetadata := forceReloadMetadata || s.shouldInvalidateMetadata(ctx, result, originalFH, currentFH) + invalidateMetadata := forceReloadMetadata || s.shouldInvalidateMetadata(ctx, result, originalFH, change.fileHandle) // Mark all of the package IDs containing the given file. // TODO: if the file has moved into a new package, we should invalidate that too. - filePackages := guessPackagesForURI(withoutURI, s.ids) + filePackages := guessPackagesForURI(uri, s.ids) for _, id := range filePackages { directIDs[id] = directIDs[id] || invalidateMetadata } // Invalidate the previous modTidyHandle if any of the files have been // saved or if any of the metadata has been invalidated. - if invalidateMetadata || fileWasSaved(originalFH, currentFH) { + if invalidateMetadata || fileWasSaved(originalFH, change.fileHandle) { // TODO(rstambler): Only delete mod handles for which the // withoutURI is relevant. for k := range s.modTidyHandles { @@ -1112,75 +1092,22 @@ func (s *snapshot) clone(ctx context.Context, withoutURIs map[span.URI]source.Ve delete(result.modWhyHandles, k) } } - currentExists := true - if _, err := currentFH.Read(); os.IsNotExist(err) { - currentExists = false - } - // If the file invalidation is for a go.mod. originalFH is nil if the - // file is newly created. - currentMod := currentExists && currentFH.Kind() == source.Mod - originalMod := originalFH != nil && originalFH.Kind() == source.Mod - if currentMod || originalMod { - modulesChanged = true - + if isGoMod(uri) { // If the view's go.mod file's contents have changed, invalidate // the metadata for every known package in the snapshot. - if invalidateMetadata { - for k := range s.metadata { - directIDs[k] = true - } - // If a go.mod file in the workspace has changed, we need to - // rebuild the workspace module. - result.workspaceModuleHandle = nil - } - delete(result.parseModHandles, withoutURI) - - // Check if this is a newly created go.mod file. When a new module - // is created, we have to retry the initial workspace load. - rootURI := span.URIFromPath(filepath.Dir(withoutURI.Filename())) - if currentMod { - if _, ok := result.modules[rootURI]; !ok { - if m := getViewModule(ctx, s.view.rootURI, currentFH.URI(), s.view.Options()); m != nil { - result.modules[m.rootURI] = m - shouldReinitializeView = true - } - - } - } else if originalMod { - // Similarly, we need to retry the IWL if a go.mod in the workspace - // was deleted. - if _, ok := result.modules[rootURI]; ok { - delete(result.modules, rootURI) - shouldReinitializeView = true - } + delete(result.parseModHandles, uri) + if _, ok := result.workspace.activeModFiles()[uri]; ok && reinitialize < maybeReinit { + reinitialize = maybeReinit } } - // Keep track of the creations and deletions of go.sum files. - // Creating a go.sum without an associated go.mod has no effect on the - // set of modules. - currentSum := currentExists && currentFH.Kind() == source.Sum - originalSum := originalFH != nil && originalFH.Kind() == source.Sum - if currentSum || originalSum { - rootURI := span.URIFromPath(filepath.Dir(withoutURI.Filename())) - if currentSum { - if mod, ok := result.modules[rootURI]; ok { - mod.sumURI = currentFH.URI() - } - } else if originalSum { - if mod, ok := result.modules[rootURI]; ok { - mod.sumURI = "" - } - } - } - // Handle the invalidated file; it may have new contents or not exist. - if !currentExists { - delete(result.files, withoutURI) + if !change.exists { + delete(result.files, uri) } else { - result.files[withoutURI] = currentFH + result.files[uri] = change.fileHandle } // Make sure to remove the changed file from the unloadable set. - delete(result.unloadableFiles, withoutURI) + delete(result.unloadableFiles, uri) } // Invalidate reverse dependencies too. @@ -1208,12 +1135,6 @@ func (s *snapshot) clone(ctx context.Context, withoutURIs map[span.URI]source.Ve addRevDeps(id, invalidateMetadata) } - // When modules change, we need to recompute their workspace directories, - // as replace directives may have changed. - if modulesChanged { - result.workspaceDirectories = result.findWorkspaceDirectories(ctx) - } - // Copy the package type information. for k, v := range s.packages { if _, ok := transitiveIDs[k.id]; ok { @@ -1292,20 +1213,9 @@ copyIDs: for _, v := range result.parseModHandles { newGen.Inherit(v.handle) } - if result.workspaceModuleHandle != nil { - newGen.Inherit(result.workspaceModuleHandle.handle) - } // Don't bother copying the importedBy graph, // as it changes each time we update metadata. - var reinitialize reinitializeView - if modulesChanged { - reinitialize = maybeReinit - } - if shouldReinitializeView { - reinitialize = definitelyReinit - } - // If the snapshot's workspace mode has changed, the packages loaded using // the previous mode are no longer relevant, so clear them out. if s.workspaceMode() != result.workspaceMode() { @@ -1443,45 +1353,6 @@ func (s *snapshot) shouldInvalidateMetadata(ctx context.Context, newSnapshot *sn return false } -// findWorkspaceDirectoriesLocked returns all of the directories that are -// considered to be part of the view's workspace. For GOPATH workspaces, this -// is just the view's root. For modules-based workspaces, this is the module -// root and any replace targets. It also returns the parseModHandle for the -// view's go.mod file if it has one. -// -// It assumes that the file handle is the view's go.mod file, if it has one. -// The caller need not be holding the snapshot's mutex, but it might be. -func (s *snapshot) findWorkspaceDirectories(ctx context.Context) map[span.URI]struct{} { - // If the view does not have a go.mod file, only the root directory - // is known. In GOPATH mode, we should really watch the entire GOPATH, - // but that's too expensive. - dirs := map[span.URI]struct{}{ - s.view.rootURI: {}, - } - for _, m := range s.modules { - fh, err := s.GetFile(ctx, m.modURI) - if err != nil { - continue - } - // Ignore parse errors. An invalid go.mod is not fatal. - // TODO(rstambler): Try to preserve existing watched directories as - // much as possible, otherwise we will thrash when a go.mod is edited. - mod, err := s.ParseMod(ctx, fh) - if err != nil { - continue - } - for _, r := range mod.File.Replace { - // We may be replacing a module with a different version, not a path - // on disk. - if r.New.Version != "" { - continue - } - dirs[span.URIFromPath(r.New.Path)] = struct{}{} - } - } - return dirs -} - func (s *snapshot) BuiltinPackage(ctx context.Context) (*source.BuiltinPackage, error) { s.AwaitInitialized(ctx) @@ -1533,111 +1404,40 @@ func (s *snapshot) buildBuiltinPackage(ctx context.Context, goFiles []string) er return nil } -type workspaceModuleHandle struct { - handle *memoize.Handle -} - -type workspaceModuleData struct { - file *modfile.File - err error -} - -type workspaceModuleKey string - -func (wmh *workspaceModuleHandle) build(ctx context.Context, snapshot *snapshot) (*modfile.File, error) { - v, err := wmh.handle.Get(ctx, snapshot.generation, snapshot) +// BuildGoplsMod generates a go.mod file for all modules in the workspace. It +// bypasses any existing gopls.mod. +func BuildGoplsMod(ctx context.Context, root span.URI, fs source.FileSource) (*modfile.File, error) { + allModules, err := findAllModules(ctx, root) if err != nil { return nil, err } - data := v.(*workspaceModuleData) - return data.file, data.err + return buildWorkspaceModFile(ctx, allModules, fs) } -func (s *snapshot) getWorkspaceModuleHandle(ctx context.Context) (*workspaceModuleHandle, error) { - s.mu.Lock() - wsModule := s.workspaceModuleHandle - s.mu.Unlock() - if wsModule != nil { - return wsModule, nil - } - var fhs []source.FileHandle - for _, mod := range s.modules { - fh, err := s.GetFile(ctx, mod.modURI) - if err != nil { - return nil, err - } - fhs = append(fhs, fh) - } - goplsModURI := span.URIFromPath(filepath.Join(s.view.Folder().Filename(), "gopls.mod")) - goplsModFH, err := s.GetFile(ctx, goplsModURI) - if err != nil { - return nil, err - } - _, err = goplsModFH.Read() - switch { - case err == nil: - // We have a gopls.mod. Our handle only depends on it. - fhs = []source.FileHandle{goplsModFH} - case os.IsNotExist(err): - // No gopls.mod, so we must build the workspace mod file automatically. - // Defensively ensure that the goplsModFH is nil as this controls automatic - // building of the workspace mod file. - goplsModFH = nil - default: - return nil, errors.Errorf("error getting gopls.mod: %w", err) - } - - sort.Slice(fhs, func(i, j int) bool { - return fhs[i].URI() < fhs[j].URI() - }) - var k string - for _, fh := range fhs { - k += fh.FileIdentity().String() - } - key := workspaceModuleKey(hashContents([]byte(k))) - h := s.generation.Bind(key, func(ctx context.Context, arg memoize.Arg) interface{} { - if goplsModFH != nil { - parsed, err := s.ParseMod(ctx, goplsModFH) - if err != nil { - return &workspaceModuleData{err: err} - } - return &workspaceModuleData{file: parsed.File} - } - s := arg.(*snapshot) - data := &workspaceModuleData{} - data.file, data.err = s.BuildWorkspaceModFile(ctx) - return data - }) - wsModule = &workspaceModuleHandle{ - handle: h, - } - s.mu.Lock() - defer s.mu.Unlock() - s.workspaceModuleHandle = wsModule - return s.workspaceModuleHandle, nil -} - -// BuildWorkspaceModFile generates a workspace module given the modules in the -// the workspace. It does not read gopls.mod. -func (s *snapshot) BuildWorkspaceModFile(ctx context.Context) (*modfile.File, error) { +// TODO(rfindley): move this to workspacemodule.go +func buildWorkspaceModFile(ctx context.Context, modFiles map[span.URI]struct{}, fs source.FileSource) (*modfile.File, error) { file := &modfile.File{} file.AddModuleStmt("gopls-workspace") - paths := make(map[string]*moduleRoot) - for _, mod := range s.modules { - fh, err := s.GetFile(ctx, mod.modURI) + paths := make(map[string]span.URI) + for modURI := range modFiles { + fh, err := fs.GetFile(ctx, modURI) if err != nil { return nil, err } - parsed, err := s.ParseMod(ctx, fh) + content, err := fh.Read() if err != nil { return nil, err } - if parsed.File == nil || parsed.File.Module == nil { - return nil, fmt.Errorf("no module declaration for %s", mod.modURI) + parsed, err := modfile.Parse(fh.URI().Filename(), content, nil) + if err != nil { + return nil, err } - path := parsed.File.Module.Mod.Path - paths[path] = mod + if file == nil || parsed.Module == nil { + return nil, fmt.Errorf("no module declaration for %s", modURI) + } + path := parsed.Module.Mod.Path + paths[path] = modURI // If the module's path includes a major version, we expect it to have // a matching major version. _, majorVersion, _ := module.SplitPathVersion(path) @@ -1646,24 +1446,28 @@ func (s *snapshot) BuildWorkspaceModFile(ctx context.Context) (*modfile.File, er } majorVersion = strings.TrimLeft(majorVersion, "/.") // handle gopkg.in versions file.AddNewRequire(path, source.WorkspaceModuleVersion(majorVersion), false) - if err := file.AddReplace(path, "", mod.rootURI.Filename(), ""); err != nil { + if err := file.AddReplace(path, "", dirURI(modURI).Filename(), ""); err != nil { return nil, err } } // Go back through all of the modules to handle any of their replace // statements. - for _, module := range s.modules { - fh, err := s.GetFile(ctx, module.modURI) + for modURI := range modFiles { + fh, err := fs.GetFile(ctx, modURI) if err != nil { return nil, err } - pmf, err := s.ParseMod(ctx, fh) + content, err := fh.Read() + if err != nil { + return nil, err + } + parsed, err := modfile.Parse(fh.URI().Filename(), content, nil) if err != nil { return nil, err } // If any of the workspace modules have replace directives, they need // to be reflected in the workspace module. - for _, rep := range pmf.File.Replace { + for _, rep := range parsed.Replace { // Don't replace any modules that are in our workspace--we should // always use the version in the workspace. if _, ok := paths[rep.Old.Path]; ok { @@ -1673,12 +1477,12 @@ func (s *snapshot) BuildWorkspaceModFile(ctx context.Context) (*modfile.File, er newVersion := rep.New.Version // If a replace points to a module in the workspace, make sure we // direct it to version of the module in the workspace. - if mod, ok := paths[rep.New.Path]; ok { - newPath = mod.rootURI.Filename() + if m, ok := paths[rep.New.Path]; ok { + newPath = dirURI(m).Filename() newVersion = "" } else if rep.New.Version == "" && !filepath.IsAbs(rep.New.Path) { // Make any relative paths absolute. - newPath = filepath.Join(module.rootURI.Filename(), rep.New.Path) + newPath = filepath.Join(dirURI(modURI).Filename(), rep.New.Path) } if err := file.AddReplace(rep.Old.Path, rep.Old.Version, newPath, newVersion); err != nil { return nil, err @@ -1687,23 +1491,3 @@ func (s *snapshot) BuildWorkspaceModFile(ctx context.Context) (*modfile.File, er } return file, nil } - -func getViewModule(ctx context.Context, viewRootURI, modURI span.URI, options *source.Options) *moduleRoot { - rootURI := span.URIFromPath(filepath.Dir(modURI.Filename())) - // If we are not in multi-module mode, check that the affected module is - // in the workspace root. - if !options.ExperimentalWorkspaceModule { - if span.CompareURI(rootURI, viewRootURI) != 0 { - return nil - } - } - sumURI := span.URIFromPath(sumFilename(modURI)) - if info, _ := os.Stat(sumURI.Filename()); info == nil { - sumURI = "" - } - return &moduleRoot{ - rootURI: rootURI, - modURI: modURI, - sumURI: sumURI, - } -} diff --git a/internal/lsp/cache/view.go b/internal/lsp/cache/view.go index 42017a853f..448273ab0c 100644 --- a/internal/lsp/cache/view.go +++ b/internal/lsp/cache/view.go @@ -151,11 +151,6 @@ type builtinPackageData struct { err error } -type moduleRoot struct { - rootURI span.URI - modURI, sumURI span.URI -} - // fileBase holds the common functionality for all files. // It is intended to be embedded in the file implementations type fileBase struct { @@ -183,7 +178,7 @@ func (v *View) ID() string { return v.id } // tempModFile creates a temporary go.mod file based on the contents of the // given go.mod file. It is the caller's responsibility to clean up the files // when they are done using them. -func tempModFile(modFh, sumFH source.FileHandle) (tmpURI span.URI, cleanup func(), err error) { +func tempModFile(modFh source.FileHandle, gosum []byte) (tmpURI span.URI, cleanup func(), err error) { filenameHash := hashContents([]byte(modFh.URI().Filename())) tmpMod, err := ioutil.TempFile("", fmt.Sprintf("go.%s.*.mod", filenameHash)) if err != nil { @@ -217,12 +212,8 @@ func tempModFile(modFh, sumFH source.FileHandle) (tmpURI span.URI, cleanup func( }() // Create an analogous go.sum, if one exists. - if sumFH != nil { - sumContents, err := sumFH.Read() - if err != nil { - return "", cleanup, err - } - if err := ioutil.WriteFile(tmpSumName, sumContents, 0655); err != nil { + if gosum != nil { + if err := ioutil.WriteFile(tmpSumName, gosum, 0655); err != nil { return "", cleanup, err } } @@ -452,14 +443,14 @@ func (v *View) BackgroundContext() context.Context { func (s *snapshot) IgnoredFile(uri span.URI) bool { filename := uri.Filename() var prefixes []string - if len(s.modules) == 0 { + if len(s.workspace.activeModFiles()) == 0 { for _, entry := range filepath.SplitList(s.view.gopath) { prefixes = append(prefixes, filepath.Join(entry, "src")) } } else { prefixes = append(prefixes, s.view.gomodcache) - for _, m := range s.modules { - prefixes = append(prefixes, m.rootURI.Filename()) + for m := range s.workspace.activeModFiles() { + prefixes = append(prefixes, dirURI(m).Filename()) } } for _, prefix := range prefixes { @@ -527,25 +518,23 @@ func (s *snapshot) initialize(ctx context.Context, firstAttempt bool) { Message: err.Error(), }) } - if len(s.modules) > 0 { - for _, mod := range s.modules { - fh, err := s.GetFile(ctx, mod.modURI) - if err != nil { - addError(mod.modURI, err) - continue - } - parsed, err := s.ParseMod(ctx, fh) - if err != nil { - addError(mod.modURI, err) - continue - } - if parsed.File == nil || parsed.File.Module == nil { - addError(mod.modURI, fmt.Errorf("no module path for %s", mod.modURI)) - continue - } - path := parsed.File.Module.Mod.Path - scopes = append(scopes, moduleLoadScope(path)) + for modURI := range s.workspace.activeModFiles() { + fh, err := s.GetFile(ctx, modURI) + if err != nil { + addError(modURI, err) + continue } + parsed, err := s.ParseMod(ctx, fh) + if err != nil { + addError(modURI, err) + continue + } + if parsed.File == nil || parsed.File.Module == nil { + addError(modURI, fmt.Errorf("no module path for %s", modURI)) + continue + } + path := parsed.File.Module.Mod.Path + scopes = append(scopes, moduleLoadScope(path)) } if len(scopes) == 0 { scopes = append(scopes, viewLoadScope("LOAD_VIEW")) @@ -568,7 +557,7 @@ func (s *snapshot) initialize(ctx context.Context, firstAttempt bool) { // invalidateContent invalidates the content of a Go file, // including any position and type information that depends on it. // It returns true if we were already tracking the given file, false otherwise. -func (v *View) invalidateContent(ctx context.Context, uris map[span.URI]source.VersionedFileHandle, forceReloadMetadata bool) (source.Snapshot, func()) { +func (v *View) invalidateContent(ctx context.Context, changes map[span.URI]*fileChange, forceReloadMetadata bool) (source.Snapshot, func()) { // Detach the context so that content invalidation cannot be canceled. ctx = xcontext.Detach(ctx) @@ -584,8 +573,9 @@ func (v *View) invalidateContent(ctx context.Context, uris map[span.URI]source.V defer v.snapshotMu.Unlock() oldSnapshot := v.snapshot + var reinitialize reinitializeView - v.snapshot, reinitialize = oldSnapshot.clone(ctx, uris, forceReloadMetadata) + v.snapshot, reinitialize = oldSnapshot.clone(ctx, changes, forceReloadMetadata) go oldSnapshot.generation.Destroy() if reinitialize == maybeReinit || reinitialize == definitelyReinit { @@ -686,7 +676,7 @@ func defaultCheckPathCase(path string) error { return nil } -func validBuildConfiguration(folder span.URI, ws *workspaceInformation, modules map[span.URI]*moduleRoot) bool { +func validBuildConfiguration(folder span.URI, ws *workspaceInformation, modFiles map[span.URI]struct{}) bool { // Since we only really understand the `go` command, if the user has a // different GOPACKAGESDRIVER, assume that their configuration is valid. if ws.hasGopackagesDriver { @@ -694,7 +684,7 @@ func validBuildConfiguration(folder span.URI, ws *workspaceInformation, modules } // Check if the user is working within a module or if we have found // multiple modules in the workspace. - if len(modules) > 0 { + if len(modFiles) > 0 { return true } // The user may have a multiple directories in their GOPATH. diff --git a/internal/lsp/cache/workspace.go b/internal/lsp/cache/workspace.go new file mode 100644 index 0000000000..71d3842329 --- /dev/null +++ b/internal/lsp/cache/workspace.go @@ -0,0 +1,409 @@ +// Copyright 2020 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package cache + +import ( + "context" + "os" + "path/filepath" + "sort" + "strings" + "sync" + + "golang.org/x/mod/modfile" + "golang.org/x/tools/internal/event" + "golang.org/x/tools/internal/lsp/source" + "golang.org/x/tools/internal/span" + "golang.org/x/tools/internal/xcontext" + errors "golang.org/x/xerrors" +) + +type workspaceSource int + +const ( + legacyWorkspace = iota + goplsModWorkspace + fileSystemWorkspace +) + +func (s workspaceSource) String() string { + switch s { + case legacyWorkspace: + return "legacy" + case goplsModWorkspace: + return "gopls.mod" + case fileSystemWorkspace: + return "file system" + default: + return "!(unknown module source)" + } +} + +// workspace tracks go.mod files in the workspace, along with the +// gopls.mod file, to provide support for multi-module workspaces. +// +// Specifically, it provides: +// - the set of modules contained within in the workspace root considered to +// be 'active' +// - the workspace modfile, to be used for the go command `-modfile` flag +// - the set of workspace directories +// +// This type is immutable (or rather, idempotent), so that it may be shared +// across multiple snapshots. +type workspace struct { + root span.URI + moduleSource workspaceSource + + // modFiles holds the active go.mod files. + modFiles map[span.URI]struct{} + + // The workspace module is lazily re-built once after being invalidated. + // buildMu+built guards this reconstruction. + // + // file and wsDirs may be non-nil even if built == false, if they were copied + // from the previous workspace module version. In this case, they will be + // preserved if building fails. + buildMu sync.Mutex + built bool + buildErr error + file *modfile.File + wsDirs map[span.URI]struct{} +} + +func newWorkspace(ctx context.Context, root span.URI, fs source.FileSource, experimental bool) (*workspace, error) { + if !experimental { + modFiles, err := getLegacyModules(ctx, root, fs) + if err != nil { + return nil, err + } + return &workspace{ + root: root, + modFiles: modFiles, + moduleSource: legacyWorkspace, + }, nil + } + goplsModFH, err := fs.GetFile(ctx, goplsModURI(root)) + if err != nil { + return nil, err + } + contents, err := goplsModFH.Read() + if err == nil { + file, modFiles, err := parseGoplsMod(root, goplsModFH.URI(), contents) + if err != nil { + return nil, err + } + return &workspace{ + root: root, + modFiles: modFiles, + file: file, + moduleSource: goplsModWorkspace, + }, nil + } + modFiles, err := findAllModules(ctx, root) + if err != nil { + return nil, err + } + return &workspace{ + root: root, + modFiles: modFiles, + moduleSource: fileSystemWorkspace, + }, nil +} + +func (wm *workspace) activeModFiles() map[span.URI]struct{} { + return wm.modFiles +} + +// modFile gets the workspace modfile associated with this workspace, +// computing it if it doesn't exist. +// +// A fileSource must be passed in to solve a chicken-egg problem: it is not +// correct to pass in the snapshot file source to newWorkspace when +// invalidating, because at the time these are called the snapshot is locked. +// So we must pass it in later on when actually using the modFile. +func (wm *workspace) modFile(ctx context.Context, fs source.FileSource) (*modfile.File, error) { + wm.build(ctx, fs) + return wm.file, wm.buildErr +} + +func (wm *workspace) build(ctx context.Context, fs source.FileSource) { + wm.buildMu.Lock() + defer wm.buildMu.Unlock() + + if wm.built { + return + } + // Building should never be cancelled. Since the workspace module is shared + // across multiple snapshots, doing so would put us in a bad state, and it + // would not be obvious to the user how to recover. + ctx = xcontext.Detach(ctx) + + // If our module source is not gopls.mod, try to build the workspace module + // from modules. Fall back on the pre-existing mod file if parsing fails. + if wm.moduleSource != goplsModWorkspace { + file, err := buildWorkspaceModFile(ctx, wm.modFiles, fs) + switch { + case err == nil: + wm.file = file + case wm.file != nil: + // Parsing failed, but we have a previous file version. + event.Error(ctx, "building workspace mod file", err) + default: + // No file to fall back on. + wm.buildErr = err + } + } + if wm.file != nil { + wm.wsDirs = map[span.URI]struct{}{ + wm.root: {}, + } + for _, r := range wm.file.Replace { + // We may be replacing a module with a different version, not a path + // on disk. + if r.New.Version != "" { + continue + } + wm.wsDirs[span.URIFromPath(r.New.Path)] = struct{}{} + } + } + // Ensure that there is always at least the root dir. + if len(wm.wsDirs) == 0 { + wm.wsDirs = map[span.URI]struct{}{ + wm.root: {}, + } + } + wm.built = true +} + +// dirs returns the workspace directories for the loaded modules. +func (wm *workspace) dirs(ctx context.Context, fs source.FileSource) []span.URI { + wm.build(ctx, fs) + var dirs []span.URI + for d := range wm.wsDirs { + dirs = append(dirs, d) + } + sort.Slice(dirs, func(i, j int) bool { + return span.CompareURI(dirs[i], dirs[j]) < 0 + }) + return dirs +} + +// invalidate returns a (possibly) new workspaceModule after invalidating +// changedURIs. If wm is still valid in the presence of changedURIs, it returns +// itself unmodified. +func (wm *workspace) invalidate(ctx context.Context, changes map[span.URI]*fileChange) (*workspace, bool) { + // Prevent races to wm.modFile or wm.wsDirs below, if wm has not yet been + // built. + wm.buildMu.Lock() + defer wm.buildMu.Unlock() + // Any gopls.mod change is processed first, followed by go.mod changes, as + // changes to gopls.mod may affect the set of active go.mod files. + var ( + // New values. We return a new workspace module if and only if modFiles is + // non-nil. + modFiles map[span.URI]struct{} + moduleSource = wm.moduleSource + modFile = wm.file + err error + ) + if wm.moduleSource == goplsModWorkspace { + // If we are currently reading the modfile from gopls.mod, we default to + // preserving it even if module metadata changes (which may be the case if + // a go.sum file changes). + modFile = wm.file + } + // First handle changes to the gopls.mod file. + if wm.moduleSource != legacyWorkspace { + // If gopls.mod has changed we need to either re-read it if it exists or + // walk the filesystem if it doesn't exist. + gmURI := goplsModURI(wm.root) + if change, ok := changes[gmURI]; ok { + if change.exists { + // Only invalidate if the gopls.mod actually parses. Otherwise, stick with the current gopls.mod + parsedFile, parsedModules, err := parseGoplsMod(wm.root, gmURI, change.content) + if err == nil { + modFile = parsedFile + moduleSource = goplsModWorkspace + modFiles = parsedModules + } else { + // Note that modFile is not invalidated here. + event.Error(ctx, "parsing gopls.mod", err) + } + } else { + // gopls.mod is deleted. search for modules again. + moduleSource = fileSystemWorkspace + modFiles, err = findAllModules(ctx, wm.root) + // the modFile is no longer valid. + if err != nil { + event.Error(ctx, "finding file system modules", err) + } + modFile = nil + } + } + } + + // Next, handle go.mod changes that could affect our set of tracked modules. + // If we're reading our tracked modules from the gopls.mod, there's nothing + // to do here. + if wm.moduleSource != goplsModWorkspace { + for uri, change := range changes { + // If a go.mod file has changed, we may need to update the set of active + // modules. + if !isGoMod(uri) { + continue + } + if wm.moduleSource == legacyWorkspace && !equalURI(modURI(wm.root), uri) { + // Legacy mode only considers a module a workspace root. + continue + } + if !isSubdirectory(wm.root.Filename(), uri.Filename()) { + // Otherwise, the module must be contained within the workspace root. + continue + } + if modFiles == nil { + modFiles = make(map[span.URI]struct{}) + for k := range wm.modFiles { + modFiles[k] = struct{}{} + } + } + if change.exists { + modFiles[uri] = struct{}{} + } else { + delete(modFiles, uri) + } + } + } + if modFiles != nil { + // Any change to modules triggers a new version. + return &workspace{ + root: wm.root, + moduleSource: moduleSource, + modFiles: modFiles, + file: modFile, + wsDirs: wm.wsDirs, + }, true + } + // No change. Just return wm, since it is immutable. + return wm, false +} + +func equalURI(left, right span.URI) bool { + return span.CompareURI(left, right) == 0 +} + +// goplsModURI returns the URI for the gopls.mod file contained in root. +func goplsModURI(root span.URI) span.URI { + return span.URIFromPath(filepath.Join(root.Filename(), "gopls.mod")) +} + +// modURI returns the URI for the go.mod file contained in root. +func modURI(root span.URI) span.URI { + return span.URIFromPath(filepath.Join(root.Filename(), "go.mod")) +} + +// isGoMod reports if uri is a go.mod file. +func isGoMod(uri span.URI) bool { + return filepath.Base(uri.Filename()) == "go.mod" +} + +// isGoMod reports if uri is a go.sum file. +func isGoSum(uri span.URI) bool { + return filepath.Base(uri.Filename()) == "go.sum" +} + +// fileExists reports if the file uri exists within source. +func fileExists(ctx context.Context, uri span.URI, source source.FileSource) (bool, error) { + fh, err := source.GetFile(ctx, uri) + if err != nil { + return false, err + } + return fileHandleExists(fh) +} + +// fileHandleExists reports if the file underlying fh actually exits. +func fileHandleExists(fh source.FileHandle) (bool, error) { + _, err := fh.Read() + if err == nil { + return true, nil + } + if os.IsNotExist(err) { + return false, nil + } + return false, err +} + +// TODO(rFindley): replace this (and similar) with a uripath package analogous +// to filepath. +func dirURI(uri span.URI) span.URI { + return span.URIFromPath(filepath.Dir(uri.Filename())) +} + +// getLegacyModules returns a module set containing at most the root module. +func getLegacyModules(ctx context.Context, root span.URI, fs source.FileSource) (map[span.URI]struct{}, error) { + uri := span.URIFromPath(filepath.Join(root.Filename(), "go.mod")) + modules := make(map[span.URI]struct{}) + exists, err := fileExists(ctx, uri, fs) + if err != nil { + return nil, err + } + if exists { + modules[uri] = struct{}{} + } + return modules, nil +} + +func parseGoplsMod(root, uri span.URI, contents []byte) (*modfile.File, map[span.URI]struct{}, error) { + modFile, err := modfile.Parse(uri.Filename(), contents, nil) + if err != nil { + return nil, nil, errors.Errorf("parsing gopls.mod: %w", err) + } + modFiles := make(map[span.URI]struct{}) + for _, replace := range modFile.Replace { + if replace.New.Version != "" { + return nil, nil, errors.Errorf("gopls.mod: replaced module %q@%q must not have version", replace.New.Path, replace.New.Version) + } + dirFP := filepath.FromSlash(replace.New.Path) + if !filepath.IsAbs(dirFP) { + dirFP = filepath.Join(root.Filename(), dirFP) + // The resulting modfile must use absolute paths, so that it can be + // written to a temp directory. + replace.New.Path = dirFP + } + modURI := span.URIFromPath(filepath.Join(dirFP, "go.mod")) + modFiles[modURI] = struct{}{} + } + return modFile, modFiles, nil +} + +// findAllModules recursively walks the root directory looking for go.mod +// files, returning the set of modules it discovers. +// TODO(rfindley): consider overlays. +func findAllModules(ctx context.Context, root span.URI) (map[span.URI]struct{}, error) { + // Walk the view's folder to find all modules in the view. + modFiles := make(map[span.URI]struct{}) + return modFiles, filepath.Walk(root.Filename(), func(path string, info os.FileInfo, err error) error { + if err != nil { + // Probably a permission error. Keep looking. + return filepath.SkipDir + } + // For any path that is not the workspace folder, check if the path + // would be ignored by the go command. Vendor directories also do not + // contain workspace modules. + if info.IsDir() && path != root.Filename() { + suffix := strings.TrimPrefix(path, root.Filename()) + switch { + case checkIgnored(suffix), + strings.Contains(filepath.ToSlash(suffix), "/vendor/"): + return filepath.SkipDir + } + } + // We're only interested in go.mod files. + uri := span.URIFromPath(path) + if isGoMod(uri) { + modFiles[uri] = struct{}{} + } + return nil + }) +} diff --git a/internal/lsp/cache/workspace_test.go b/internal/lsp/cache/workspace_test.go new file mode 100644 index 0000000000..7cd0ecee56 --- /dev/null +++ b/internal/lsp/cache/workspace_test.go @@ -0,0 +1,257 @@ +// Copyright 2020 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package cache + +import ( + "context" + "os" + "testing" + + "golang.org/x/tools/internal/lsp/fake" + "golang.org/x/tools/internal/lsp/source" + "golang.org/x/tools/internal/span" +) + +// osFileSource is a fileSource that just reads from the operating system. +type osFileSource struct{} + +func (s osFileSource) GetFile(ctx context.Context, uri span.URI) (source.FileHandle, error) { + fi, statErr := os.Stat(uri.Filename()) + if statErr != nil { + return &fileHandle{ + err: statErr, + uri: uri, + }, nil + } + fh, err := readFile(ctx, uri, fi.ModTime()) + if err != nil { + return nil, err + } + return fh, nil +} + +func TestWorkspaceModule(t *testing.T) { + tests := []struct { + desc string + initial string // txtar-encoded + legacyMode bool + initialSource workspaceSource + initialModules []string + initialDirs []string + updates map[string]string + finalSource workspaceSource + finalModules []string + finalDirs []string + }{ + { + desc: "legacy mode", + initial: ` +-- go.mod -- +module mod.com +-- a/go.mod -- +module moda.com`, + legacyMode: true, + initialModules: []string{"./go.mod"}, + initialSource: legacyWorkspace, + initialDirs: []string{"."}, + }, + { + desc: "nested module", + initial: ` +-- go.mod -- +module mod.com +-- a/go.mod -- +module moda.com`, + initialModules: []string{"./go.mod", "a/go.mod"}, + initialSource: fileSystemWorkspace, + initialDirs: []string{".", "a"}, + }, + { + desc: "removing module", + initial: ` +-- a/go.mod -- +module moda.com +-- b/go.mod -- +module modb.com`, + initialModules: []string{"a/go.mod", "b/go.mod"}, + initialSource: fileSystemWorkspace, + initialDirs: []string{".", "a", "b"}, + updates: map[string]string{ + "gopls.mod": `module gopls-workspace + +require moda.com v0.0.0-goplsworkspace +replace moda.com => $SANDBOX_WORKDIR/a`, + }, + finalModules: []string{"a/go.mod"}, + finalSource: goplsModWorkspace, + finalDirs: []string{".", "a"}, + }, + { + desc: "adding module", + initial: ` +-- gopls.mod -- +require moda.com v0.0.0-goplsworkspace +replace moda.com => $SANDBOX_WORKDIR/a +-- a/go.mod -- +module moda.com +-- b/go.mod -- +module modb.com`, + initialModules: []string{"a/go.mod"}, + initialSource: goplsModWorkspace, + initialDirs: []string{".", "a"}, + updates: map[string]string{ + "gopls.mod": `module gopls-workspace + +require moda.com v0.0.0-goplsworkspace +require modb.com v0.0.0-goplsworkspace + +replace moda.com => $SANDBOX_WORKDIR/a +replace modb.com => $SANDBOX_WORKDIR/b`, + }, + finalModules: []string{"a/go.mod", "b/go.mod"}, + finalSource: goplsModWorkspace, + finalDirs: []string{".", "a", "b"}, + }, + { + desc: "deleting gopls.mod", + initial: ` +-- gopls.mod -- +module gopls-workspace + +require moda.com v0.0.0-goplsworkspace +replace moda.com => $SANDBOX_WORKDIR/a +-- a/go.mod -- +module moda.com +-- b/go.mod -- +module modb.com`, + initialModules: []string{"a/go.mod"}, + initialSource: goplsModWorkspace, + initialDirs: []string{".", "a"}, + updates: map[string]string{ + "gopls.mod": "", + }, + finalModules: []string{"a/go.mod", "b/go.mod"}, + finalSource: fileSystemWorkspace, + finalDirs: []string{".", "a", "b"}, + }, + { + desc: "broken module parsing", + initial: ` +-- a/go.mod -- +module moda.com + +require gopls.test v0.0.0-goplsworkspace +replace gopls.test => ../../gopls.test // (this path shouldn't matter) +-- b/go.mod -- +module modb.com`, + initialModules: []string{"a/go.mod", "b/go.mod"}, + initialSource: fileSystemWorkspace, + initialDirs: []string{".", "a", "b", "../gopls.test"}, + updates: map[string]string{ + "a/go.mod": `modul moda.com + +require gopls.test v0.0.0-goplsworkspace +replace gopls.test => ../../gopls.test2`, + }, + finalModules: []string{"a/go.mod", "b/go.mod"}, + finalSource: fileSystemWorkspace, + // finalDirs should be unchanged: we should preserve dirs in the presence + // of a broken modfile. + finalDirs: []string{".", "a", "b", "../gopls.test"}, + }, + } + + for _, test := range tests { + t.Run(test.desc, func(t *testing.T) { + ctx := context.Background() + dir, err := fake.Tempdir(test.initial) + if err != nil { + t.Fatal(err) + } + defer os.RemoveAll(dir) + root := span.URIFromPath(dir) + + fs := osFileSource{} + wm, err := newWorkspace(ctx, root, fs, !test.legacyMode) + if err != nil { + t.Fatal(err) + } + rel := fake.RelativeTo(dir) + checkWorkspaceModule(t, rel, wm, test.initialSource, test.initialModules) + gotDirs := wm.dirs(ctx, fs) + checkWorkspaceDirs(t, rel, gotDirs, test.initialDirs) + if test.updates != nil { + changes := make(map[span.URI]*fileChange) + for k, v := range test.updates { + if v == "" { + // for convenience, use this to signal a deletion. TODO: more doc + err := os.Remove(rel.AbsPath(k)) + if err != nil { + t.Fatal(err) + } + } else { + fake.WriteFileData(k, []byte(v), rel) + } + uri := span.URIFromPath(rel.AbsPath(k)) + fh, err := fs.GetFile(ctx, uri) + if err != nil { + t.Fatal(err) + } + content, err := fh.Read() + changes[uri] = &fileChange{ + content: content, + exists: err == nil, + fileHandle: &closedFile{fh}, + } + } + wm, _ := wm.invalidate(ctx, changes) + checkWorkspaceModule(t, rel, wm, test.finalSource, test.finalModules) + gotDirs := wm.dirs(ctx, fs) + checkWorkspaceDirs(t, rel, gotDirs, test.finalDirs) + } + }) + } +} + +func checkWorkspaceModule(t *testing.T, rel fake.RelativeTo, got *workspace, wantSource workspaceSource, want []string) { + t.Helper() + if got.moduleSource != wantSource { + t.Errorf("module source = %v, want %v", got.moduleSource, wantSource) + } + modules := make(map[span.URI]struct{}) + for k := range got.activeModFiles() { + modules[k] = struct{}{} + } + for _, modPath := range want { + path := rel.AbsPath(modPath) + uri := span.URIFromPath(path) + if _, ok := modules[uri]; !ok { + t.Errorf("missing module %q", uri) + } + delete(modules, uri) + } + for remaining := range modules { + t.Errorf("unexpected module %q", remaining) + } +} + +func checkWorkspaceDirs(t *testing.T, rel fake.RelativeTo, got []span.URI, want []string) { + t.Helper() + gotM := make(map[span.URI]bool) + for _, dir := range got { + gotM[dir] = true + } + for _, dir := range want { + path := rel.AbsPath(dir) + uri := span.URIFromPath(path) + if !gotM[uri] { + t.Errorf("missing dir %q", uri) + } + delete(gotM, uri) + } + for remaining := range gotM { + t.Errorf("unexpected dir %q", remaining) + } +} diff --git a/internal/lsp/command.go b/internal/lsp/command.go index 6dd5045bba..d51a98b4c9 100644 --- a/internal/lsp/command.go +++ b/internal/lsp/command.go @@ -15,6 +15,7 @@ import ( "golang.org/x/tools/internal/event" "golang.org/x/tools/internal/gocommand" + "golang.org/x/tools/internal/lsp/cache" "golang.org/x/tools/internal/lsp/protocol" "golang.org/x/tools/internal/lsp/source" "golang.org/x/tools/internal/span" @@ -250,7 +251,7 @@ func (s *Server) runCommand(ctx context.Context, work *workDone, command *source } snapshot, release := v.Snapshot(ctx) defer release() - modFile, err := snapshot.BuildWorkspaceModFile(ctx) + modFile, err := cache.BuildGoplsMod(ctx, v.Folder(), snapshot) if err != nil { return errors.Errorf("getting workspace mod file: %w", err) } diff --git a/internal/lsp/fake/editor.go b/internal/lsp/fake/editor.go index dba8997696..b367293f16 100644 --- a/internal/lsp/fake/editor.go +++ b/internal/lsp/fake/editor.go @@ -522,6 +522,13 @@ func (e *Editor) EditBuffer(ctx context.Context, path string, edits []Edit) erro return e.editBufferLocked(ctx, path, edits) } +func (e *Editor) SetBufferContent(ctx context.Context, path, content string) error { + e.mu.Lock() + defer e.mu.Unlock() + lines := strings.Split(content, "\n") + return e.setBufferContentLocked(ctx, path, lines, nil) +} + // BufferText returns the content of the buffer with the given name. func (e *Editor) BufferText(name string) string { e.mu.Lock() @@ -542,24 +549,28 @@ func (e *Editor) editBufferLocked(ctx context.Context, path string, edits []Edit if !ok { return fmt.Errorf("unknown buffer %q", path) } - var ( - content = make([]string, len(buf.content)) - err error - evts []protocol.TextDocumentContentChangeEvent - ) + content := make([]string, len(buf.content)) copy(content, buf.content) - content, err = editContent(content, edits) + content, err := editContent(content, edits) if err != nil { return err } + return e.setBufferContentLocked(ctx, path, content, edits) +} +func (e *Editor) setBufferContentLocked(ctx context.Context, path string, content []string, fromEdits []Edit) error { + buf, ok := e.buffers[path] + if !ok { + return fmt.Errorf("unknown buffer %q", path) + } buf.content = content buf.version++ e.buffers[path] = buf // A simple heuristic: if there is only one edit, send it incrementally. // Otherwise, send the entire content. - if len(edits) == 1 { - evts = append(evts, edits[0].toProtocolChangeEvent()) + var evts []protocol.TextDocumentContentChangeEvent + if len(fromEdits) == 1 { + evts = append(evts, fromEdits[0].toProtocolChangeEvent()) } else { evts = append(evts, protocol.TextDocumentContentChangeEvent{ Text: buf.text(), diff --git a/internal/lsp/source/view.go b/internal/lsp/source/view.go index bdb388bfae..ccac54db9d 100644 --- a/internal/lsp/source/view.go +++ b/internal/lsp/source/view.go @@ -116,10 +116,6 @@ type Snapshot interface { // GoModForFile returns the URI of the go.mod file for the given URI. GoModForFile(ctx context.Context, uri span.URI) span.URI - // BuildWorkspaceModFile builds the contents of mod file to be used for - // multi-module workspace. - BuildWorkspaceModFile(ctx context.Context) (*modfile.File, error) - // BuiltinPackage returns information about the special builtin package. BuiltinPackage(ctx context.Context) (*BuiltinPackage, error)