From d87e9bbada1ffc4e8a44c415bc9b5b8e0110b61b Mon Sep 17 00:00:00 2001 From: Sander van Harmelen Date: Thu, 15 Aug 2019 14:54:27 +0200 Subject: [PATCH] internal/lsp: limit concurrent file reads We need to limit the concurrent file reads to prevent `too many open file` errors. The first attempt to solve this was done in change 189437, but this change turned out to be incorrect. So this change reverts the changes made in change 189437 and instead adds an new semaphore around the `nativeFileHandle.Read` method. The limit is currently set to half of the lowest open file system limit of *nix, Windows and MacOS system (their respective limits are 1024, 16384, and 256). Change-Id: I27d741e3a3c36eb11bfa7457575f5d5831c06ad7 Reviewed-on: https://go-review.googlesource.com/c/tools/+/190417 Reviewed-by: Ian Cottrell Run-TryBot: Ian Cottrell TryBot-Result: Gobot Gobot --- internal/lsp/cache/external.go | 5 +++++ internal/lsp/cache/parse.go | 9 ++++----- 2 files changed, 9 insertions(+), 5 deletions(-) diff --git a/internal/lsp/cache/external.go b/internal/lsp/cache/external.go index b52899ba7f..7a938a2f4c 100644 --- a/internal/lsp/cache/external.go +++ b/internal/lsp/cache/external.go @@ -15,6 +15,9 @@ import ( "golang.org/x/tools/internal/span" ) +// ioLimit limits the number of parallel file reads per process. +var ioLimit = make(chan struct{}, 128) + // nativeFileSystem implements FileSystem reading from the normal os file system. type nativeFileSystem struct{} @@ -54,6 +57,8 @@ func (h *nativeFileHandle) Kind() source.FileKind { func (h *nativeFileHandle) Read(ctx context.Context) ([]byte, string, error) { ctx, done := trace.StartSpan(ctx, "cache.nativeFileHandle.Read", telemetry.File.Of(h.identity.URI.Filename())) defer done() + ioLimit <- struct{}{} + defer func() { <-ioLimit }() //TODO: this should fail if the version is not the same as the handle data, err := ioutil.ReadFile(h.identity.URI.Filename()) if err != nil { diff --git a/internal/lsp/cache/parse.go b/internal/lsp/cache/parse.go index 3da4a6f238..8e9def8740 100644 --- a/internal/lsp/cache/parse.go +++ b/internal/lsp/cache/parse.go @@ -20,8 +20,8 @@ import ( errors "golang.org/x/xerrors" ) -// Limits the number of parallel file reads per process. -var ioLimit = make(chan struct{}, 20) +// Limits the number of parallel parser calls per process. +var parseLimit = make(chan struct{}, 20) // parseKey uniquely identifies a parsed Go file. type parseKey struct { @@ -104,13 +104,12 @@ func parseGo(ctx context.Context, c *cache, fh source.FileHandle, mode source.Pa ctx, done := trace.StartSpan(ctx, "cache.parseGo", telemetry.File.Of(fh.Identity().URI.Filename())) defer done() - ioLimit <- struct{}{} buf, _, err := fh.Read(ctx) - <-ioLimit if err != nil { return nil, err } - + parseLimit <- struct{}{} + defer func() { <-parseLimit }() parserMode := parser.AllErrors | parser.ParseComments if mode == source.ParseHeader { parserMode = parser.ImportsOnly | parser.ParseComments