gopls: write out the raw usage text

This adds a small script and uses it to write all the help text
to files.
This will be used to show the diff of a collection of changes to
come.

For #41860

Change-Id: Iaef52365d421dd3c375d22ac5a1f7c5eb6b69ac2
Reviewed-on: https://go-review.googlesource.com/c/tools/+/367834
Trust: Ian Cottrell <iancottrell@google.com>
Run-TryBot: Ian Cottrell <iancottrell@google.com>
Reviewed-by: Robert Findley <rfindley@google.com>
gopls-CI: kokoro <noreply+kokoro@google.com>
TryBot-Result: Gopher Robot <gobot@golang.org>

internal/lsp/cmd/cmd.go

@@ -144,12 +144,14 @@ func (app *Application) Run(ctx context.Context, args ...string) error {
 	ctx = debug.WithInstance(ctx, app.wd, app.OCAgent)
 	app.Serve.app = app
 	if len(args) == 0 {
-		return tool.Run(ctx, &app.Serve, args)
+		s := flag.NewFlagSet(app.Name(), flag.ExitOnError)
+		return tool.Run(ctx, s, &app.Serve, args)
 	}
 	command, args := args[0], args[1:]
-	for _, c := range app.commands() {
+	for _, c := range app.Commands() {
 		if c.Name() == command {
-			return tool.Run(ctx, c, args)
+			s := flag.NewFlagSet(app.Name(), flag.ExitOnError)
+			return tool.Run(ctx, s, c, args)
 		}
 	}
 	return tool.CommandLineErrorf("Unknown command %v", command)
@@ -158,7 +160,7 @@ func (app *Application) Run(ctx context.Context, args ...string) error {
 // commands returns the set of commands supported by the gopls tool on the
 // command line.
 // The command is specified by the first non flag argument.
-func (app *Application) commands() []tool.Application {
+func (app *Application) Commands() []tool.Application {
 	var commands []tool.Application
 	commands = append(commands, app.mainCommands()...)
 	commands = append(commands, app.featureCommands()...)

internal/lsp/cmd/help_test.go

@@ -0,0 +1,55 @@
+// Copyright 2019 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package cmd_test
+
+import (
+	"bytes"
+	"context"
+	"flag"
+	"io/ioutil"
+	"path/filepath"
+	"testing"
+
+	"golang.org/x/tools/internal/lsp/cmd"
+	"golang.org/x/tools/internal/testenv"
+	"golang.org/x/tools/internal/tool"
+)
+
+var updateHelpFiles = flag.Bool("update-help-files", false, "Write out the help files instead of checking them")
+
+const appName = "gopls"
+
+func TestHelpFiles(t *testing.T) {
+	testenv.NeedsGoBuild(t) // This is a lie. We actually need the source code.
+	app := cmd.New(appName, "", nil, nil)
+	ctx := context.Background()
+	for _, page := range append(app.Commands(), app) {
+		t.Run(page.Name(), func(t *testing.T) {
+			var buf bytes.Buffer
+			s := flag.NewFlagSet(page.Name(), flag.ContinueOnError)
+			s.SetOutput(&buf)
+			tool.Run(ctx, s, page, []string{"-h"})
+			name := page.Name()
+			if name == appName {
+				name = "usage"
+			}
+			helpFile := filepath.Join("usage", name+".hlp")
+			got := buf.Bytes()
+			if *updateHelpFiles {
+				if err := ioutil.WriteFile(helpFile, got, 0666); err != nil {
+					t.Errorf("Failed writing %v: %v", helpFile, err)
+				}
+				return
+			}
+			expect, err := ioutil.ReadFile(helpFile)
+			switch {
+			case err != nil:
+				t.Errorf("Missing help file %q", helpFile)
+			case !bytes.Equal(expect, got):
+				t.Errorf("Help file %q did not match, got:\n%q\nwant:\n%q", helpFile, string(got), string(expect))
+			}
+		})
+	}
+}

internal/lsp/cmd/subcommands.go

@@ -33,7 +33,8 @@ func (s subcommands) Run(ctx context.Context, args ...string) error {
 	command, args := args[0], args[1:]
 	for _, c := range s {
 		if c.Name() == command {
-			return tool.Run(ctx, c, args)
+			s := flag.NewFlagSet(c.Name(), flag.ExitOnError)
+			return tool.Run(ctx, s, c, args)
 		}
 	}
 	return tool.CommandLineErrorf("unknown subcommand %v", command)

internal/lsp/cmd/test/cmdtest.go

@@ -8,6 +8,7 @@ package cmdtest
 import (
 	"bytes"
 	"context"
+	"flag"
 	"fmt"
 	"io"
 	"os"
@@ -137,7 +138,8 @@ func (r *runner) runGoplsCmd(t testing.TB, args ...string) (string, string) {
 	os.Stdout, os.Stderr = wStdout, wStderr
 	app := cmd.New("gopls-test", r.data.Config.Dir, r.data.Exported.Config.Env, r.options)
 	remote := r.remote
-	err = tool.Run(tests.Context(t),
+	s := flag.NewFlagSet(app.Name(), flag.ExitOnError)
+	err = tool.Run(tests.Context(t), s,
 		app,
 		append([]string{fmt.Sprintf("-remote=internal@%s", remote)}, args...))
 	if err != nil {

internal/lsp/cmd/usage/api-json.hlp

@@ -0,0 +1,3 @@
+print json describing gopls API
+
+Usage: api-json [flags] 

internal/lsp/cmd/usage/bug.hlp

@@ -0,0 +1,3 @@
+report a bug in gopls
+
+Usage: bug [flags] 

internal/lsp/cmd/usage/call_hierarchy.hlp

@@ -0,0 +1,9 @@
+display selected identifier's call hierarchy
+
+Usage: call_hierarchy [flags] <position>
+
+Example:
+
+  $ # 1-indexed location (:line:column or :#offset) of the target identifier
+  $ gopls call_hierarchy helper/helper.go:8:6
+  $ gopls call_hierarchy helper/helper.go:#53

internal/lsp/cmd/usage/check.hlp

@@ -0,0 +1,7 @@
+show diagnostic results for the specified file
+
+Usage: check [flags] <filename>
+
+Example: show the diagnostic results of this file:
+
+  $ gopls check internal/lsp/cmd/check.go

internal/lsp/cmd/usage/definition.hlp

@@ -0,0 +1,14 @@
+show declaration of selected identifier
+
+Usage: definition [flags] <position>
+
+Example: show the definition of the identifier at syntax at offset 44 in this file (flag.FlagSet):
+
+$ gopls definition internal/lsp/cmd/definition.go:44:47
+$ gopls definition internal/lsp/cmd/definition.go:#1270
+
+	gopls query definition flags are:
+  -json
+    	emit output in JSON format
+  -markdown
+    	support markdown in responses

internal/lsp/cmd/usage/fix.hlp

@@ -0,0 +1,12 @@
+apply suggested fixes
+
+Usage: fix [flags] <filename>
+
+Example: apply suggested fixes for this file:
+
+  $ gopls fix -w internal/lsp/cmd/check.go
+
+gopls fix flags are:
+  -a	apply all fixes, not just preferred fixes
+  -d	display diffs instead of rewriting files
+  -w	write result to (source) file instead of stdout

internal/lsp/cmd/usage/folding_ranges.hlp

@@ -0,0 +1,7 @@
+display selected file's folding ranges
+
+Usage: folding_ranges [flags] <file>
+
+Example:
+
+  $ gopls folding_ranges helper/helper.go

internal/lsp/cmd/usage/format.hlp

@@ -0,0 +1,14 @@
+format the code according to the go standard
+
+Usage: format [flags] <filerange>
+
+The arguments supplied may be simple file names, or ranges within files.
+
+Example: reformat this file:
+
+  $ gopls format -w internal/lsp/cmd/check.go
+
+	gopls format flags are:
+  -d	display diffs instead of rewriting files
+  -l	list files whose formatting differs from gofmt's
+  -w	write result to (source) file instead of stdout

internal/lsp/cmd/usage/highlight.hlp

@@ -0,0 +1,9 @@
+display selected identifier's highlights
+
+Usage: highlight [flags] <position>
+
+Example:
+
+  $ # 1-indexed location (:line:column or :#offset) of the target identifier
+  $ gopls highlight helper/helper.go:8:6
+  $ gopls highlight helper/helper.go:#53

internal/lsp/cmd/usage/implementation.hlp

@@ -0,0 +1,9 @@
+display selected identifier's implementation
+
+Usage: implementation [flags] <position>
+
+Example:
+
+  $ # 1-indexed location (:line:column or :#offset) of the target identifier
+  $ gopls implementation helper/helper.go:8:6
+  $ gopls implementation helper/helper.go:#53

internal/lsp/cmd/usage/imports.hlp

@@ -0,0 +1,11 @@
+updates import statements
+
+Usage: imports [flags] <filename>
+
+Example: update imports statements in a file:
+
+  $ gopls imports -w internal/lsp/cmd/check.go
+
+gopls imports flags are:
+  -d	display diffs instead of rewriting files
+  -w	write result to (source) file instead of stdout

internal/lsp/cmd/usage/inspect.hlp

@@ -0,0 +1,7 @@
+interact with the gopls daemon (deprecated: use 'remote')
+
+Usage: inspect [flags] <subcommand> [args...]
+
+subcommands:
+  sessions: print information about current gopls sessions
+  debug: start the debug server

internal/lsp/cmd/usage/licenses.hlp

@@ -0,0 +1,3 @@
+print licenses of included software
+
+Usage: licenses [flags] 

internal/lsp/cmd/usage/links.hlp

@@ -0,0 +1,11 @@
+list links in a file
+
+Usage: links [flags] <filename>
+
+Example: list links contained within a file:
+
+  $ gopls links internal/lsp/cmd/check.go
+
+gopls links flags are:
+  -json
+    	emit document links in JSON format

internal/lsp/cmd/usage/prepare_rename.hlp

@@ -0,0 +1,9 @@
+test validity of a rename operation at location
+
+Usage: prepare_rename [flags] <position>
+
+Example:
+
+	$ # 1-indexed location (:line:column or :#offset) of the target identifier
+	$ gopls prepare_rename helper/helper.go:8:6
+	$ gopls prepare_rename helper/helper.go:#53

internal/lsp/cmd/usage/references.hlp

@@ -0,0 +1,12 @@
+display selected identifier's references
+
+Usage: references [flags] <position>
+
+Example:
+
+  $ # 1-indexed location (:line:column or :#offset) of the target identifier
+  $ gopls references helper/helper.go:8:6
+  $ gopls references helper/helper.go:#53
+
+  gopls references flags are:
+  -d	include the declaration of the specified identifier in the results

internal/lsp/cmd/usage/remote.hlp

@@ -0,0 +1,7 @@
+interact with the gopls daemon
+
+Usage: remote [flags] <subcommand> [args...]
+
+subcommands:
+  sessions: print information about current gopls sessions
+  debug: start the debug server

internal/lsp/cmd/usage/rename.hlp

@@ -0,0 +1,15 @@
+rename selected identifier
+
+Usage: rename [flags] <position> <new name>
+
+Example:
+
+  $ # 1-based location (:line:column or :#position) of the thing to change
+  $ gopls rename helper/helper.go:8:6 Foo
+  $ gopls rename helper/helper.go:#53 Foo
+
+	gopls rename flags are:
+  -d	display diffs instead of rewriting files
+  -preserve
+    	preserve original files
+  -w	write result to (source) file instead of stdout

internal/lsp/cmd/usage/semtok.hlp

@@ -0,0 +1,7 @@
+show semantic tokens for the specified file
+
+Usage: semtok [flags] <filename>
+
+Example: show the semantic tokens for this file:
+
+  $ gopls semtok internal/lsp/cmd/semtok.go

internal/lsp/cmd/usage/serve.hlp

@@ -0,0 +1,28 @@
+run a server for Go code using the Language Server Protocol
+
+Usage: serve [flags] 
+
+The server communicates using JSONRPC2 on stdin and stdout, and is intended to be run directly as
+a child of an editor process.
+
+gopls server flags are:
+  -debug string
+    	serve debug information on the supplied address
+  -listen string
+    	address on which to listen for remote connections. If prefixed by 'unix;', the subsequent address is assumed to be a unix domain socket. Otherwise, TCP is used.
+  -listen.timeout duration
+    	when used with -listen, shut down the server when there are no connected clients for this duration
+  -logfile string
+    	filename to log to. if value is "auto", then logging to a default output file is enabled
+  -mode string
+    	no effect
+  -port int
+    	port on which to run gopls for debugging purposes
+  -remote.debug string
+    	when used with -remote=auto, the -debug value used to start the daemon
+  -remote.listen.timeout duration
+    	when used with -remote=auto, the -listen.timeout value used to start the daemon (default 1m0s)
+  -remote.logfile string
+    	when used with -remote=auto, the -logfile value used to start the daemon
+  -rpc.trace
+    	print the full rpc trace in lsp inspector format

internal/lsp/cmd/usage/signature.hlp

@@ -0,0 +1,9 @@
+display selected identifier's signature
+
+Usage: signature [flags] <position>
+
+Example:
+
+  $ # 1-indexed location (:line:column or :#offset) of the target identifier
+  $ gopls signature helper/helper.go:8:6
+  $ gopls signature helper/helper.go:#53

internal/lsp/cmd/usage/symbols.hlp

@@ -0,0 +1,6 @@
+display selected file's symbols
+
+Usage: symbols [flags] <file>
+
+Example:
+  $ gopls symbols helper/helper.go

internal/lsp/cmd/usage/usage.hlp

@@ -0,0 +1,73 @@
+The Go Language source tools.
+
+Usage: gopls [flags] <command> [command-flags] [command-args]
+
+gopls is a Go language server. It is typically used with an editor to provide
+language features. When no command is specified, gopls will default to the 'serve'
+command. The language features can also be accessed via the gopls command-line interface.
+
+Available commands are:
+
+main:
+  serve : run a server for Go code using the Language Server Protocol
+  version : print the gopls version information
+  bug : report a bug in gopls
+  api-json : print json describing gopls API
+  licenses : print licenses of included software
+
+features:
+  call_hierarchy : display selected identifier's call hierarchy
+  check : show diagnostic results for the specified file
+  definition : show declaration of selected identifier
+  folding_ranges : display selected file's folding ranges
+  format : format the code according to the go standard
+  highlight : display selected identifier's highlights
+  implementation : display selected identifier's implementation
+  imports : updates import statements
+  remote : interact with the gopls daemon
+  inspect : interact with the gopls daemon (deprecated: use 'remote')
+  links : list links in a file
+  prepare_rename : test validity of a rename operation at location
+  references : display selected identifier's references
+  rename : rename selected identifier
+  semtok : show semantic tokens for the specified file
+  signature : display selected identifier's signature
+  fix : apply suggested fixes
+  symbols : display selected file's symbols
+  workspace : manage the gopls workspace (experimental: under development)
+  workspace_symbol : search symbols in workspace
+
+gopls flags are:
+  -debug string
+    	serve debug information on the supplied address
+  -listen string
+    	address on which to listen for remote connections. If prefixed by 'unix;', the subsequent address is assumed to be a unix domain socket. Otherwise, TCP is used.
+  -listen.timeout duration
+    	when used with -listen, shut down the server when there are no connected clients for this duration
+  -logfile string
+    	filename to log to. if value is "auto", then logging to a default output file is enabled
+  -mode string
+    	no effect
+  -ocagent string
+    	the address of the ocagent (e.g. http://localhost:55678), or off (default "off")
+  -port int
+    	port on which to run gopls for debugging purposes
+  -profile.cpu string
+    	write CPU profile to this file
+  -profile.mem string
+    	write memory profile to this file
+  -profile.trace string
+    	write trace log to this file
+  -remote string
+    	forward all commands to a remote lsp specified by this flag. With no special prefix, this is assumed to be a TCP address. If prefixed by 'unix;', the subsequent address is assumed to be a unix domain socket. If 'auto', or prefixed by 'auto;', the remote address is automatically resolved based on the executing environment.
+  -remote.debug string
+    	when used with -remote=auto, the -debug value used to start the daemon
+  -remote.listen.timeout duration
+    	when used with -remote=auto, the -listen.timeout value used to start the daemon (default 1m0s)
+  -remote.logfile string
+    	when used with -remote=auto, the -logfile value used to start the daemon
+  -rpc.trace
+    	print the full rpc trace in lsp inspector format
+  -v	verbose output
+  -vv
+    	very verbose output

internal/lsp/cmd/usage/version.hlp

@@ -0,0 +1,3 @@
+print the gopls version information
+
+Usage: version [flags] 

internal/lsp/cmd/usage/workspace.hlp

@@ -0,0 +1,6 @@
+manage the gopls workspace (experimental: under development)
+
+Usage: workspace [flags] <subcommand> [args...]
+
+subcommands:
+  generate: generate a gopls.mod file for a workspace

internal/lsp/cmd/usage/workspace_symbol.hlp

@@ -0,0 +1,12 @@
+search symbols in workspace
+
+Usage: workspace_symbol [flags] <query>
+
+Example:
+
+  $ gopls workspace_symbol -matcher fuzzy 'wsymbols'
+
+gopls workspace_symbol flags are:
+  -matcher string
+    	specifies the type of matcher: fuzzy, caseSensitive, or caseInsensitive.
+    	The default is caseInsensitive.

internal/tool/tool.go

@@ -83,13 +83,7 @@ func CommandLineErrorf(message string, args ...interface{}) error {
 // application exits with an exit code of 2.
 func Main(ctx context.Context, app Application, args []string) {
 	s := flag.NewFlagSet(app.Name(), flag.ExitOnError)
-	s.Usage = func() {
-		fmt.Fprint(s.Output(), app.ShortHelp())
-		fmt.Fprintf(s.Output(), "\n\nUsage: %v [flags] %v\n", app.Name(), app.Usage())
-		app.DetailedHelp(s)
-	}
-	addFlags(s, reflect.StructField{}, reflect.ValueOf(app))
-	if err := Run(ctx, app, args); err != nil {
+	if err := Run(ctx, s, app, args); err != nil {
 		fmt.Fprintf(s.Output(), "%s: %v\n", app.Name(), err)
 		if _, printHelp := err.(commandLineError); printHelp {
 			s.Usage()
@@ -101,15 +95,16 @@ func Main(ctx context.Context, app Application, args []string) {
 // Run is the inner loop for Main; invoked by Main, recursively by
 // Run, and by various tests.  It runs the application and returns an
 // error.
-func Run(ctx context.Context, app Application, args []string) error {
-	s := flag.NewFlagSet(app.Name(), flag.ExitOnError)
+func Run(ctx context.Context, s *flag.FlagSet, app Application, args []string) error {
 	s.Usage = func() {
 		fmt.Fprint(s.Output(), app.ShortHelp())
 		fmt.Fprintf(s.Output(), "\n\nUsage: %v [flags] %v\n", app.Name(), app.Usage())
 		app.DetailedHelp(s)
 	}
 	p := addFlags(s, reflect.StructField{}, reflect.ValueOf(app))
-	s.Parse(args)
+	if err := s.Parse(args); err != nil {
+		return err
+	}
 
 	if p != nil && p.CPU != "" {
 		f, err := os.Create(p.CPU)
@@ -165,15 +160,15 @@ func addFlags(f *flag.FlagSet, field reflect.StructField, value reflect.Value) *
 	help := field.Tag.Get("help")
 	if !isFlag {
 		// not a flag, but it might be a struct with flags in it
-		if value.Elem().Kind() != reflect.Struct {
+		value = resolve(value.Elem())
+		if value.Kind() != reflect.Struct {
 			return nil
 		}
-		p, _ := value.Interface().(*Profile)
+		p, _ := value.Addr().Interface().(*Profile)
 		// go through all the fields of the struct
-		sv := value.Elem()
-		for i := 0; i < sv.Type().NumField(); i++ {
-			child := sv.Type().Field(i)
-			v := sv.Field(i)
+		for i := 0; i < value.Type().NumField(); i++ {
+			child := value.Type().Field(i)
+			v := value.Field(i)
 			// make sure we have a pointer
 			if v.Kind() != reflect.Ptr {
 				v = v.Addr()
@@ -209,3 +204,14 @@ func addFlags(f *flag.FlagSet, field reflect.StructField, value reflect.Value) *
 	}
 	return nil
 }
+
+func resolve(v reflect.Value) reflect.Value {
+	for {
+		switch v.Kind() {
+		case reflect.Interface, reflect.Ptr:
+			v = v.Elem()
+		default:
+			return v
+		}
+	}
+}