diff --git a/internal/imports/fix.go b/internal/imports/fix.go
index 6e17ce8f2b..83349a07da 100644
--- a/internal/imports/fix.go
+++ b/internal/imports/fix.go
@@ -643,6 +643,22 @@ func getCandidatePkgs(ctx context.Context, wrappedCallback *scanCallback, filena
 	return env.GetResolver().scan(ctx, scanFilter)
 }
 
+func PrimeCache(ctx context.Context, env *ProcessEnv) error {
+	// Fully scan the disk for directories, but don't actually read any Go files.
+	callback := &scanCallback{
+		rootFound: func(gopathwalk.Root) bool {
+			return true
+		},
+		dirFound: func(pkg *pkg) bool {
+			return false
+		},
+		packageNameLoaded: func(pkg *pkg) bool {
+			return false
+		},
+	}
+	return getCandidatePkgs(ctx, callback, "", "", env)
+}
+
 func candidateImportName(pkg *pkg) string {
 	if ImportPathToAssumedName(pkg.importPathShort) != pkg.packageName {
 		return pkg.packageName
@@ -737,6 +753,13 @@ type ProcessEnv struct {
 	resolver Resolver
 }
 
+// CopyConfig copies the env's configuration into a new env.
+func (e *ProcessEnv) CopyConfig() *ProcessEnv {
+	copy := *e
+	copy.resolver = nil
+	return &copy
+}
+
 func (e *ProcessEnv) env() []string {
 	env := os.Environ()
 	add := func(k, v string) {
diff --git a/internal/imports/fix_test.go b/internal/imports/fix_test.go
index df92e0a288..37785fd2f5 100644
--- a/internal/imports/fix_test.go
+++ b/internal/imports/fix_test.go
@@ -1660,8 +1660,7 @@ func (t *goimportTest) processNonModule(file string, contents []byte, opts *Opti
 		opts = &Options{Comments: true, TabIndent: true, TabWidth: 8}
 	}
 	// ProcessEnv is not safe for concurrent use. Make a copy.
-	env := *t.env
-	opts.Env = &env
+	opts.Env = t.env.CopyConfig()
 	return Process(file, contents, opts)
 }
 
diff --git a/internal/lsp/cache/view.go b/internal/lsp/cache/view.go
index 8e7ddf87e9..c27303e85d 100644
--- a/internal/lsp/cache/view.go
+++ b/internal/lsp/cache/view.go
@@ -11,6 +11,7 @@ import (
 	"fmt"
 	"go/ast"
 	"go/token"
+	stdlog "log"
 	"os"
 	"path/filepath"
 	"reflect"
@@ -25,7 +26,6 @@ import (
 	"golang.org/x/tools/internal/memoize"
 	"golang.org/x/tools/internal/span"
 	"golang.org/x/tools/internal/telemetry/log"
-	"golang.org/x/tools/internal/telemetry/tag"
 	"golang.org/x/tools/internal/xcontext"
 	errors "golang.org/x/xerrors"
 )
@@ -66,8 +66,9 @@ type view struct {
 	// TODO(suzmue): the state cached in the process env is specific to each view,
 	// however, there is state that can be shared between views that is not currently
 	// cached, like the module cache.
-	processEnv       *imports.ProcessEnv
-	cacheRefreshTime time.Time
+	processEnv           *imports.ProcessEnv
+	cacheRefreshDuration time.Duration
+	cacheRefreshTimer    *time.Timer
 
 	// modFileVersions stores the last seen versions of the module files that are used
 	// by processEnvs resolver.
@@ -306,29 +307,43 @@ func (v *view) RunProcessEnvFunc(ctx context.Context, fn func(*imports.Options)
 	if err := fn(opts); err != nil {
 		return err
 	}
-	if v.cacheRefreshTime.IsZero() {
-		v.cacheRefreshTime = time.Now()
-	}
 
 	// If applicable, store the file versions of the 'go.mod' files that are
 	// looked at by the resolver.
 	v.storeModFileVersions()
 
-	if time.Since(v.cacheRefreshTime) > 30*time.Second {
-		go func() {
-			v.mu.Lock()
-			defer v.mu.Unlock()
-
-			log.Print(context.Background(), "background imports cache refresh starting")
-			v.processEnv.GetResolver().ClearForNewScan()
-			// TODO(heschi): prime the cache
-			v.cacheRefreshTime = time.Now()
-			log.Print(context.Background(), "background refresh finished with err: ", tag.Of("err", nil))
-		}()
+	if v.cacheRefreshTimer == nil {
+		// Don't refresh more than twice per minute.
+		delay := 30 * time.Second
+		// Don't spend more than a couple percent of the time refreshing.
+		if adaptive := 50 * v.cacheRefreshDuration; adaptive > delay {
+			delay = adaptive
+		}
+		v.cacheRefreshTimer = time.AfterFunc(delay, v.refreshProcessEnv)
 	}
+
 	return nil
 }
 
+func (v *view) refreshProcessEnv() {
+	start := time.Now()
+
+	v.mu.Lock()
+	env := v.processEnv
+	env.GetResolver().ClearForNewScan()
+	v.mu.Unlock()
+
+	// We don't have a context handy to use for logging, so use the stdlib for now.
+	stdlog.Printf("background imports cache refresh starting")
+	err := imports.PrimeCache(context.Background(), env)
+	stdlog.Printf("background refresh finished after %v with err: %v", time.Since(start), err)
+
+	v.mu.Lock()
+	v.cacheRefreshDuration = time.Since(start)
+	v.cacheRefreshTimer = nil
+	v.mu.Unlock()
+}
+
 func (v *view) buildProcessEnv(ctx context.Context) (*imports.ProcessEnv, error) {
 	cfg := v.Config(ctx)
 	env := &imports.ProcessEnv{