Cache glob results across --cache runs

Author: webpro

packages/knip/src/run.ts

@@ -10,6 +10,7 @@ import { ProjectPrincipal } from './ProjectPrincipal.ts';
 import watchReporter from './reporters/watch.ts';
 import type { MainOptions } from './util/create-options.ts';
 import { debugLogObject } from './util/debug.ts';
+import { flushGlobCache, initGlobCache } from './util/glob-cache.ts';
 import { getGitIgnoredHandler } from './util/glob-core.ts';
 import { getModuleSourcePathHandler } from './util/to-source-path.ts';
 import { getSessionHandler, type OnFileChange, type SessionHandler } from './util/watch.ts';
@@ -20,6 +21,8 @@ export const run = async (options: MainOptions) => {
   debugLogObject('*', 'Unresolved configuration', options);
   debugLogObject('*', 'Included issue types', options.includedIssueTypes);
 
+  if (options.isCache) initGlobCache(options.cacheLocation);
+
   const chief = new ConfigurationChief(options);
   const deputy = new DependencyDeputy(options);
   const streamer = new ConsoleStreamer(options);
@@ -100,6 +103,8 @@ export const run = async (options: MainOptions) => {
 
   if (!options.isWatch) streamer.clear();
 
+  if (options.isCache) flushGlobCache();
+
   return {
     results: {
       issues,

packages/knip/src/util/glob-cache.ts

@@ -0,0 +1,135 @@
+import { createHash } from 'node:crypto';
+import fs from 'node:fs';
+// oxlint-disable-next-line no-restricted-imports
+import path from 'node:path';
+import { deserialize, serialize } from 'node:v8';
+import { version } from '../version.ts';
+import { debugLog } from './debug.ts';
+import { isDirectory, isFile } from './fs.ts';
+import { dirname } from './path.ts';
+
+interface GlobCacheEntry {
+  paths: string[];
+  /** Absolute dir path → mtimeMs of dir at the time the glob ran */
+  dirMtimes: Record<string, number>;
+}
+
+const CACHE_FILENAME = `glob-${version}.cache`;
+
+let cacheFilePath: string | undefined;
+let cache: Map<string, GlobCacheEntry> | undefined;
+let isDirty = false;
+
+export const initGlobCache = (cacheLocation: string) => {
+  cacheFilePath = path.resolve(cacheLocation, CACHE_FILENAME);
+  if (isFile(cacheFilePath)) {
+    try {
+      cache = deserialize(fs.readFileSync(cacheFilePath));
+    } catch {
+      debugLog('*', `Error reading glob cache from ${cacheFilePath}`);
+      cache = new Map();
+    }
+  } else {
+    cache = new Map();
+  }
+};
+
+export const isGlobCacheEnabled = () => cache !== undefined;
+
+export const computeGlobCacheKey = (input: {
+  patterns: string[];
+  cwd: string;
+  dir: string;
+  gitignore: boolean;
+}): string => {
+  const h = createHash('sha1');
+  h.update(input.cwd);
+  h.update('\0');
+  h.update(input.dir);
+  h.update('\0');
+  h.update(input.gitignore ? '1' : '0');
+  h.update('\0');
+  for (const p of input.patterns) {
+    h.update(p);
+    h.update('\0');
+  }
+  return h.digest('base64url');
+};
+
+const validateEntry = (entry: GlobCacheEntry): boolean => {
+  for (const dir in entry.dirMtimes) {
+    try {
+      const stat = fs.statSync(dir);
+      if (stat.mtimeMs !== entry.dirMtimes[dir]) return false;
+    } catch {
+      return false;
+    }
+  }
+  return true;
+};
+
+export const getCachedGlob = (key: string): string[] | undefined => {
+  if (!cache) return undefined;
+  const entry = cache.get(key);
+  if (!entry) return undefined;
+  if (!validateEntry(entry)) {
+    cache.delete(key);
+    isDirty = true;
+    return undefined;
+  }
+  return entry.paths;
+};
+
+const captureDirMtimes = (paths: string[], baseDir: string): Record<string, number> => {
+  const dirs = new Set<string>();
+  // Always track the base dir to catch new top-level files/dirs
+  dirs.add(baseDir);
+  for (const p of paths) {
+    let d = dirname(p);
+    while (d.length >= baseDir.length) {
+      if (dirs.has(d)) break;
+      dirs.add(d);
+      const parent = dirname(d);
+      if (parent === d) break;
+      d = parent;
+    }
+  }
+  const result: Record<string, number> = {};
+  for (const d of dirs) {
+    try {
+      const stat = fs.statSync(d);
+      if (stat.isDirectory()) result[d] = stat.mtimeMs;
+    } catch {
+      // dir disappeared between glob and stat — skip
+    }
+  }
+  return result;
+};
+
+export const setCachedGlob = (key: string, paths: string[], baseDir: string): void => {
+  if (!cache) return;
+  cache.set(key, {
+    paths,
+    dirMtimes: captureDirMtimes(paths, baseDir),
+  });
+  isDirty = true;
+};
+
+export const clearGlobCache = (): void => {
+  if (cache) {
+    cache.clear();
+    isDirty = true;
+  }
+};
+
+export const flushGlobCache = (): void => {
+  if (!cache || !cacheFilePath || !isDirty) return;
+  try {
+    const dir = dirname(cacheFilePath);
+    if (!isDirectory(dir)) fs.mkdirSync(dir, { recursive: true });
+    fs.writeFileSync(cacheFilePath, serialize(cache));
+    isDirty = false;
+  } catch {
+    debugLog('*', `Error writing glob cache to ${cacheFilePath}`);
+  }
+};

packages/knip/src/util/glob.ts

@@ -1,5 +1,6 @@
 import fg from 'fast-glob';
 import { compact } from './array.ts';
+import { computeGlobCacheKey, getCachedGlob, isGlobCacheEnabled, setCachedGlob } from './glob-cache.ts';
 import { glob } from './glob-core.ts';
 import { timerify } from './Performance.ts';
 import { isAbsolute, join, relative } from './path.ts';
@@ -43,14 +44,25 @@ const defaultGlob = async ({ cwd, dir = cwd, patterns, gitignore = true, label }
   // Only negated patterns? Bail out.
   if (globPatterns[0].startsWith('!')) return [];
 
-  return glob(globPatterns, {
+  const cacheEnabled = isGlobCacheEnabled();
+  const cacheKey = cacheEnabled ? computeGlobCacheKey({ patterns: globPatterns, cwd, dir, gitignore }) : '';
+  if (cacheEnabled) {
+    const cached = getCachedGlob(cacheKey);
+    if (cached) return cached;
+  }
+
+  const paths = await glob(globPatterns, {
     cwd,
     dir,
     gitignore,
     absolute: true,
     dot: true,
     label,
   });
+
+  if (cacheEnabled && paths.length > 0) setCachedGlob(cacheKey, paths, dir);
+
+  return paths;
 };
 
 const syncGlob = ({ cwd, patterns }: { cwd?: string; patterns: string | string[] }) =>

packages/knip/src/util/watch.ts

@@ -12,6 +12,7 @@ import { isFile } from './fs.ts';
 import { updateImportMap } from './module-graph.ts';
 import { toAbsolute, toPosix, toRelative } from './path.ts';
 import { clearModuleResolutionCaches } from '../typescript/resolve-module-names.ts';
+import { clearGlobCache } from './glob-cache.ts';
 import { clearResolverCache } from './resolve.ts';
 
 export type OnFileChange = (options: { issues: Issues; duration?: number; mem?: number }) => void;
@@ -110,6 +111,7 @@ export const getSessionHandler = async (
 
     clearResolverCache();
     clearModuleResolutionCaches();
+    clearGlobCache();
     invalidateCache(graph);
 
     unreferencedFiles.clear();