From 434d569df75794e10a5cc02a37ba796737f4de5d Mon Sep 17 00:00:00 2001 From: Robert Findley Date: Tue, 15 Nov 2022 10:56:08 -0500 Subject: [PATCH] gopls/internal/lsp/regtest: improve documentation Start turning the regtest package documentation into more of a guide for new users. Along the way, move runner options into a separate options.go file, for discoverability. Change-Id: I18dec7c632df3e491d166a00959b9b5648d9ddf0 Reviewed-on: https://go-review.googlesource.com/c/tools/+/450677 Run-TryBot: Robert Findley gopls-CI: kokoro TryBot-Result: Gopher Robot Reviewed-by: Hyang-Ah Hana Kim --- gopls/internal/lsp/regtest/doc.go | 159 +++++++++++++++++++++++--- gopls/internal/lsp/regtest/options.go | 105 +++++++++++++++++ gopls/internal/lsp/regtest/runner.go | 102 +---------------- 3 files changed, 247 insertions(+), 119 deletions(-) create mode 100644 gopls/internal/lsp/regtest/options.go diff --git a/gopls/internal/lsp/regtest/doc.go b/gopls/internal/lsp/regtest/doc.go index 39eddd8dcf..4f4c7c020b 100644 --- a/gopls/internal/lsp/regtest/doc.go +++ b/gopls/internal/lsp/regtest/doc.go @@ -12,25 +12,146 @@ // // The regtest package provides an API for developers to express these types of // user interactions in ordinary Go tests, validate them, and run them in a -// variety of execution modes (see gopls/doc/daemon.md for more information on -// execution modes). This is achieved roughly as follows: -// - the Runner type starts and connects to a gopls instance for each -// configured execution mode. -// - the Env type provides a collection of resources to use in writing tests -// (for example a temporary working directory and fake text editor) -// - user interactions with these resources are scripted using test wrappers -// around the API provided by the golang.org/x/tools/gopls/internal/lsp/fake -// package. +// variety of execution modes. // -// Regressions are expressed in terms of Expectations, which at a high level -// are conditions that we expect to be met (or not to be met) at some point -// after performing the interactions in the test. This is necessary because the -// LSP is by construction asynchronous: both client and server can send -// each other notifications without formal acknowledgement that they have been -// fully processed. +// # Test package setup // -// Simple Expectations may be combined to match specific conditions reported by -// the user. In the example above, a regtest validating that the user-reported -// bug had been fixed would "expect" that the editor never displays the -// confusing diagnostic. +// The regression test package uses a couple of uncommon patterns to reduce +// boilerplate in test bodies. First, it is intended to be imported as "." so +// that helpers do not need to be qualified. Second, it requires some setup +// that is currently implemented in the regtest.Main function, which must be +// invoked by TestMain. Therefore, a minimal regtest testing package looks +// like this: +// +// package lsptests +// +// import ( +// "fmt" +// "testing" +// +// "golang.org/x/tools/gopls/internal/hooks" +// . "golang.org/x/tools/gopls/internal/lsp/regtest" +// ) +// +// func TestMain(m *testing.M) { +// Main(m, hooks.Options) +// } +// +// # Writing a simple regression test +// +// To run a regression test use the regtest.Run function, which accepts a +// txtar-encoded archive defining the initial workspace state. This function +// sets up the workspace in a temporary directory, creates a fake text editor, +// starts gopls, and initializes an LSP session. It then invokes the provided +// test function with an *Env handle encapsulating the newly created +// environment. Because gopls may be run in various modes (as a sidecar or +// daemon process, with different settings), the test runner may perform this +// process multiple times, re-running the test function each time with a new +// environment. +// +// func TestOpenFile(t *testing.T) { +// const files = ` +// -- go.mod -- +// module mod.com +// +// go 1.12 +// -- foo.go -- +// package foo +// ` +// Run(t, files, func(t *testing.T, env *Env) { +// env.OpenFile("foo.go") +// }) +// } +// +// # Configuring Regtest Execution +// +// The regtest package exposes several options that affect the setup process +// described above. To use these options, use the WithOptions function: +// +// WithOptions(opts...).Run(...) +// +// See options.go for a full list of available options. +// +// # Operating on editor state +// +// To operate on editor state within the test body, the Env type provides +// access to the workspace directory (Env.SandBox), text editor (Env.Editor), +// LSP server (Env.Server), and 'awaiter' (Env.Awaiter). +// +// In most cases, operations on these primitive building blocks of the +// regression test environment expect a Context (which should be a child of +// env.Ctx), and return an error. To avoid boilerplate, the Env exposes a set +// of wrappers in wrappers.go for use in scripting: +// +// env.CreateBuffer("c/c.go", "") +// env.EditBuffer("c/c.go", fake.Edit{ +// Text: `package c`, +// }) +// +// These wrappers thread through Env.Ctx, and call t.Fatal on any errors. +// +// # Expressing expectations +// +// The general pattern for a regression test is to script interactions with the +// fake editor and sandbox, and assert that gopls behaves correctly after each +// state change. Unfortunately, this is complicated by the fact that state +// changes are communicated to gopls via unidirectional client->server +// notifications (didOpen, didChange, etc.), and resulting gopls behavior such +// as diagnostics, logs, or messages is communicated back via server->client +// notifications. Therefore, within regression tests we must be able to say "do +// this, and then eventually gopls should do that". To achieve this, the +// regtest package provides a framework for expressing conditions that must +// eventually be met, in terms of the Expectation type. +// +// To express the assertion that "eventually gopls must meet these +// expectations", use env.Await(...): +// +// env.RegexpReplace("x/x.go", `package x`, `package main`) +// env.Await(env.DiagnosticAtRegexp("x/main.go", `fmt`)) +// +// Await evaluates the provided expectations atomically, whenever the client +// receives a state-changing notification from gopls. See expectation.go for a +// full list of available expectations. +// +// A fundamental problem with this model is that if gopls never meets the +// provided expectations, the test runner will hang until the test timeout +// (which defaults to 10m). There are two ways to work around this poor +// behavior: +// +// 1. Use a precondition to define precisely when we expect conditions to be +// met. Gopls provides the OnceMet(precondition, expectations...) pattern +// to express ("once this precondition is met, the following expectations +// must all hold"). To instrument preconditions, gopls uses verbose +// progress notifications to inform the client about ongoing work (see +// CompletedWork). The most common precondition is to wait for gopls to be +// done processing all change notifications, for which the regtest package +// provides the AfterChange helper. For example: +// +// // We expect diagnostics to be cleared after gopls is done processing the +// // didSave notification. +// env.SaveBuffer("a/go.mod") +// env.AfterChange(EmptyDiagnostics("a/go.mod")) +// +// 2. Set a shorter timeout during development, if you expect to be breaking +// tests. By setting the environment variable GOPLS_REGTEST_TIMEOUT=5s, +// regression tests will time out after 5 seconds. +// +// # Tips & Tricks +// +// Here are some tips and tricks for working with regression tests: +// +// 1. Set the environment variable GOPLS_REGTEST_TIMEOUT=5s during development. +// 2. Run tests with -short. This will only run regression tests in the +// default gopls execution mode. +// 3. Use capture groups to narrow regexp positions. All regular-expression +// based positions (such as DiagnosticAtRegexp) will match the position of +// the first capture group, if any are provided. This can be used to +// identify a specific position in the code for a pattern that may occur in +// multiple places. For example `var (mu) sync.Mutex` matches the position +// of "mu" within the variable declaration. +// 4. Read diagnostics into a variable to implement more complicated +// assertions about diagnostic state in the editor. To do this, use the +// pattern OnceMet(precondition, ReadDiagnostics("file.go", &d)) to capture +// the current diagnostics as soon as the precondition is met. This is +// preferable to accessing the diagnostics directly, as it avoids races. package regtest diff --git a/gopls/internal/lsp/regtest/options.go b/gopls/internal/lsp/regtest/options.go new file mode 100644 index 0000000000..3820e96b37 --- /dev/null +++ b/gopls/internal/lsp/regtest/options.go @@ -0,0 +1,105 @@ +// Copyright 2022 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package regtest + +import "golang.org/x/tools/gopls/internal/lsp/fake" + +type runConfig struct { + editor fake.EditorConfig + sandbox fake.SandboxConfig + modes Mode + skipHooks bool +} + +// A RunOption augments the behavior of the test runner. +type RunOption interface { + set(*runConfig) +} + +type optionSetter func(*runConfig) + +func (f optionSetter) set(opts *runConfig) { + f(opts) +} + +// ProxyFiles configures a file proxy using the given txtar-encoded string. +func ProxyFiles(txt string) RunOption { + return optionSetter(func(opts *runConfig) { + opts.sandbox.ProxyFiles = fake.UnpackTxt(txt) + }) +} + +// Modes configures the execution modes that the test should run in. +// +// By default, modes are configured by the test runner. If this option is set, +// it overrides the set of default modes and the test runs in exactly these +// modes. +func Modes(modes Mode) RunOption { + return optionSetter(func(opts *runConfig) { + if opts.modes != 0 { + panic("modes set more than once") + } + opts.modes = modes + }) +} + +// WindowsLineEndings configures the editor to use windows line endings. +func WindowsLineEndings() RunOption { + return optionSetter(func(opts *runConfig) { + opts.editor.WindowsLineEndings = true + }) +} + +// Settings is a RunOption that sets user-provided configuration for the LSP +// server. +// +// As a special case, the env setting must not be provided via Settings: use +// EnvVars instead. +type Settings map[string]interface{} + +func (s Settings) set(opts *runConfig) { + if opts.editor.Settings == nil { + opts.editor.Settings = make(map[string]interface{}) + } + for k, v := range s { + opts.editor.Settings[k] = v + } +} + +// WorkspaceFolders configures the workdir-relative workspace folders to send +// to the LSP server. By default the editor sends a single workspace folder +// corresponding to the workdir root. To explicitly configure no workspace +// folders, use WorkspaceFolders with no arguments. +func WorkspaceFolders(relFolders ...string) RunOption { + if len(relFolders) == 0 { + // Use an empty non-nil slice to signal explicitly no folders. + relFolders = []string{} + } + return optionSetter(func(opts *runConfig) { + opts.editor.WorkspaceFolders = relFolders + }) +} + +// EnvVars sets environment variables for the LSP session. When applying these +// variables to the session, the special string $SANDBOX_WORKDIR is replaced by +// the absolute path to the sandbox working directory. +type EnvVars map[string]string + +func (e EnvVars) set(opts *runConfig) { + if opts.editor.Env == nil { + opts.editor.Env = make(map[string]string) + } + for k, v := range e { + opts.editor.Env[k] = v + } +} + +// InGOPATH configures the workspace working directory to be GOPATH, rather +// than a separate working directory for use with modules. +func InGOPATH() RunOption { + return optionSetter(func(opts *runConfig) { + opts.sandbox.InGoPath = true + }) +} diff --git a/gopls/internal/lsp/regtest/runner.go b/gopls/internal/lsp/regtest/runner.go index effc8aae4b..c83318fbee 100644 --- a/gopls/internal/lsp/regtest/runner.go +++ b/gopls/internal/lsp/regtest/runner.go @@ -23,14 +23,14 @@ import ( exec "golang.org/x/sys/execabs" - "golang.org/x/tools/internal/jsonrpc2" - "golang.org/x/tools/internal/jsonrpc2/servertest" "golang.org/x/tools/gopls/internal/lsp/cache" "golang.org/x/tools/gopls/internal/lsp/debug" "golang.org/x/tools/gopls/internal/lsp/fake" "golang.org/x/tools/gopls/internal/lsp/lsprpc" "golang.org/x/tools/gopls/internal/lsp/protocol" "golang.org/x/tools/gopls/internal/lsp/source" + "golang.org/x/tools/internal/jsonrpc2" + "golang.org/x/tools/internal/jsonrpc2/servertest" "golang.org/x/tools/internal/memoize" "golang.org/x/tools/internal/testenv" "golang.org/x/tools/internal/xcontext" @@ -131,104 +131,6 @@ type Runner struct { cancelRemote func() } -type runConfig struct { - editor fake.EditorConfig - sandbox fake.SandboxConfig - modes Mode - skipHooks bool -} - -// A RunOption augments the behavior of the test runner. -type RunOption interface { - set(*runConfig) -} - -type optionSetter func(*runConfig) - -func (f optionSetter) set(opts *runConfig) { - f(opts) -} - -// ProxyFiles configures a file proxy using the given txtar-encoded string. -func ProxyFiles(txt string) RunOption { - return optionSetter(func(opts *runConfig) { - opts.sandbox.ProxyFiles = fake.UnpackTxt(txt) - }) -} - -// Modes configures the execution modes that the test should run in. -// -// By default, modes are configured by the test runner. If this option is set, -// it overrides the set of default modes and the test runs in exactly these -// modes. -func Modes(modes Mode) RunOption { - return optionSetter(func(opts *runConfig) { - if opts.modes != 0 { - panic("modes set more than once") - } - opts.modes = modes - }) -} - -// WindowsLineEndings configures the editor to use windows line endings. -func WindowsLineEndings() RunOption { - return optionSetter(func(opts *runConfig) { - opts.editor.WindowsLineEndings = true - }) -} - -// Settings is a RunOption that sets user-provided configuration for the LSP -// server. -// -// As a special case, the env setting must not be provided via Settings: use -// EnvVars instead. -type Settings map[string]interface{} - -func (s Settings) set(opts *runConfig) { - if opts.editor.Settings == nil { - opts.editor.Settings = make(map[string]interface{}) - } - for k, v := range s { - opts.editor.Settings[k] = v - } -} - -// WorkspaceFolders configures the workdir-relative workspace folders to send -// to the LSP server. By default the editor sends a single workspace folder -// corresponding to the workdir root. To explicitly configure no workspace -// folders, use WorkspaceFolders with no arguments. -func WorkspaceFolders(relFolders ...string) RunOption { - if len(relFolders) == 0 { - // Use an empty non-nil slice to signal explicitly no folders. - relFolders = []string{} - } - return optionSetter(func(opts *runConfig) { - opts.editor.WorkspaceFolders = relFolders - }) -} - -// EnvVars sets environment variables for the LSP session. When applying these -// variables to the session, the special string $SANDBOX_WORKDIR is replaced by -// the absolute path to the sandbox working directory. -type EnvVars map[string]string - -func (e EnvVars) set(opts *runConfig) { - if opts.editor.Env == nil { - opts.editor.Env = make(map[string]string) - } - for k, v := range e { - opts.editor.Env[k] = v - } -} - -// InGOPATH configures the workspace working directory to be GOPATH, rather -// than a separate working directory for use with modules. -func InGOPATH() RunOption { - return optionSetter(func(opts *runConfig) { - opts.sandbox.InGoPath = true - }) -} - type TestFunc func(t *testing.T, env *Env) // Run executes the test function in the default configured gopls execution