diff --git a/internal/imports/fix.go b/internal/imports/fix.go
index 3d1f33ade7..dffee291bd 100644
--- a/internal/imports/fix.go
+++ b/internal/imports/fix.go
@@ -1175,18 +1175,20 @@ func (r *gopathResolver) scan(ctx context.Context, callback *scanCallback) error
 			p.relevance = MaxRelevance
 		}
 
-		if callback.dirFound(p) {
-			var err error
-			p.packageName, err = r.cache.CachePackageName(info)
-			if err != nil {
-				continue
-			}
+		if !callback.dirFound(p) {
+			continue
+		}
+		var err error
+		p.packageName, err = r.cache.CachePackageName(info)
+		if err != nil {
+			continue
 		}
 
-		if callback.packageNameLoaded(p) {
-			if _, exports, err := r.loadExports(ctx, p); err == nil {
-				callback.exportsLoaded(p, exports)
-			}
+		if !callback.packageNameLoaded(p) {
+			continue
+		}
+		if _, exports, err := r.loadExports(ctx, p); err == nil {
+			callback.exportsLoaded(p, exports)
 		}
 	}
 	return nil
diff --git a/internal/imports/mod.go b/internal/imports/mod.go
index a7bcdfe000..2ef55b9314 100644
--- a/internal/imports/mod.go
+++ b/internal/imports/mod.go
@@ -378,27 +378,27 @@ func (r *ModuleResolver) scan(ctx context.Context, callback *scanCallback) error
 		if scanned, err := info.reachedStatus(directoryScanned); !scanned || err != nil {
 			return
 		}
-
 		pkg, err := r.canonicalize(info)
 		if err != nil {
 			return
 		}
 
-		if callback.dirFound(pkg) {
-			var err error
-			pkg.packageName, err = r.cachePackageName(info)
-			if err != nil {
-				return
-			}
+		if !callback.dirFound(pkg) {
+			return
+		}
+		pkg.packageName, err = r.cachePackageName(info)
+		if err != nil {
+			return
 		}
 
-		if callback.packageNameLoaded(pkg) {
-			_, exports, err := r.loadExports(ctx, pkg)
-			if err != nil {
-				return
-			}
-			callback.exportsLoaded(pkg, exports)
+		if !callback.packageNameLoaded(pkg) {
+			return
 		}
+		_, exports, err := r.loadExports(ctx, pkg)
+		if err != nil {
+			return
+		}
+		callback.exportsLoaded(pkg, exports)
 	}
 
 	// Everything we already had is in the cache. Process it now, in hopes we