fix: allow traversal of symlinks in glob

Closes #2130

This adds a `followSymlinks` option to the `glob()` function, and enables it when searching for entry points.

If `true`, and a symbolic link is encountered, a stat will be taken of the symlink's target. If a dir or file, the symlink is handled like any other dir or file.  If a symbolic link itself (symlink to a symlink), we take a stat of the next until we find a dir or file, then handle it.

There is a little bit of caching to avoid extra I/O, and protection from recursive symlinks.  However, it's possible (though unlikely) that the FS could cause a "max call stack exceeded" exception.  If someone runs into this, we can change the implementation to a loop instead of a recursive function.

Apologies for blasting the `do..while`.  I love a good `do` myself, but splitting out the lambda functions make it untenable.

Author: boneskull

src/lib/utils/entry-point.ts

@@ -203,7 +203,7 @@ export function getExpandedEntryPointsForPaths(
 function expandGlobs(inputFiles: string[]) {
     const base = getCommonDirectory(inputFiles);
     const result = inputFiles.flatMap((entry) =>
-        glob(entry, base, { includeDirectories: true })
+        glob(entry, base, { includeDirectories: true, followSymlinks: true })
     );
     return result;
 }

src/lib/utils/fs.ts

@@ -1,7 +1,7 @@
 import * as fs from "fs";
 import { promises as fsp } from "fs";
 import { Minimatch } from "minimatch";
-import { dirname, join } from "path";
+import { dirname, join, relative } from "path";
 
 /**
  * Get the longest directory path common to all files.
@@ -139,41 +139,92 @@ export function copySync(src: string, dest: string): void {
 export function glob(
     pattern: string,
     root: string,
-    options?: { includeDirectories?: boolean }
+    options: { includeDirectories?: boolean; followSymlinks?: boolean } = {}
 ): string[] {
     const result: string[] = [];
     const mini = new Minimatch(normalizePath(pattern));
     const dirs: string[][] = [normalizePath(root).split("/")];
+    // cache of real paths to avoid infinite recursion
+    const symlinkTargetsSeen: Set<string> = new Set();
+    // cache of fs.realpathSync results to avoid extra I/O
+    const realpathCache: Map<string, string> = new Map();
+    const { includeDirectories = false, followSymlinks = false } = options;
+
+    let dir = dirs.shift();
+
+    const handleFile = (path: string) => {
+        const childPath = [...dir!, path].join("/");
+        if (mini.match(childPath)) {
+            result.push(childPath);
+        }
+    };
+
+    const handleDirectory = (path: string) => {
+        const childPath = [...dir!, path];
+        if (
+            mini.set.some((row) =>
+                mini.matchOne(childPath, row, /* partial */ true)
+            )
+        ) {
+            dirs.push(childPath);
+        }
+    };
+
+    const handleSymlink = (path: string) => {
+        const childPath = [...dir!, path].join("/");
+        let realpath: string;
+        try {
+            realpath =
+                realpathCache.get(childPath) ?? fs.realpathSync(childPath);
+            realpathCache.set(childPath, realpath);
+        } catch {
+            return;
+        }
 
-    do {
-        const dir = dirs.shift()!;
+        if (symlinkTargetsSeen.has(realpath)) {
+            return;
+        }
+        symlinkTargetsSeen.add(realpath);
+
+        try {
+            const stats = fs.statSync(realpath);
+            if (stats.isDirectory()) {
+                handleDirectory(path);
+            } else if (stats.isFile()) {
+                handleFile(path);
+            } else if (stats.isSymbolicLink()) {
+                const dirpath = dir!.join("/");
+                if (dirpath === realpath) {
+                    // special case: real path of symlink is the directory we're currently traversing
+                    return;
+                }
+                const targetPath = relative(dirpath, realpath);
+                handleSymlink(targetPath);
+            } // everything else should be ignored
+        } catch (e) {
+            // invalid symbolic link; ignore
+        }
+    };
 
-        if (options?.includeDirectories && mini.match(dir.join("/"))) {
+    while (dir) {
+        if (includeDirectories && mini.match(dir.join("/"))) {
             result.push(dir.join("/"));
         }
 
         for (const child of fs.readdirSync(dir.join("/"), {
             withFileTypes: true,
         })) {
             if (child.isFile()) {
-                const childPath = [...dir, child.name].join("/");
-                if (mini.match(childPath)) {
-                    result.push(childPath);
-                }
-            }
-
-            if (child.isDirectory() && child.name !== "node_modules") {
-                const childPath = dir.concat(child.name);
-                if (
-                    mini.set.some((row) =>
-                        mini.matchOne(childPath, row, /* partial */ true)
-                    )
-                ) {
-                    dirs.push(childPath);
-                }
+                handleFile(child.name);
+            } else if (child.isDirectory() && child.name !== "node_modules") {
+                handleDirectory(child.name);
+            } else if (followSymlinks && child.isSymbolicLink()) {
+                handleSymlink(child.name);
             }
         }
-    } while (dirs.length);
+
+        dir = dirs.shift();
+    }
 
     return result;
 }

src/test/utils/fs.test.ts

@@ -1,6 +1,8 @@
+import * as fs from "fs";
+import { createServer } from "net";
 import { Project, tempdirProject } from "@typestrong/fs-fixture-builder";
-import { deepStrictEqual as equal } from "assert";
-import { basename } from "path";
+import { AssertionError, deepStrictEqual as equal } from "assert";
+import { basename, dirname, resolve } from "path";
 import { glob } from "../../lib/utils/fs";
 
 describe("fs.ts", () => {
@@ -37,5 +39,87 @@ describe("fs.ts", () => {
                 ["test.ts", "test2.ts"]
             );
         });
+
+        describe("when 'followSymlinks' option is true", () => {
+            it("should navigate symlinked directories", () => {
+                const target = dirname(fix.dir("a").addFile("test.ts").path);
+                fix.write();
+                fs.symlinkSync(target, resolve(fix.cwd, "b"), "junction");
+                equal(
+                    glob(`${fix.cwd}/b/*.ts`, fix.cwd, {
+                        followSymlinks: true,
+                    }).map((f) => basename(f)),
+                    ["test.ts"]
+                );
+            });
+
+            it("should navigate recursive symlinked directories only once", () => {
+                fix.addFile("test.ts");
+                fix.write();
+                fs.symlinkSync(
+                    fix.cwd,
+                    resolve(fix.cwd, "recursive"),
+                    "junction"
+                );
+                equal(
+                    glob(`${fix.cwd}/**/*.ts`, fix.cwd, {
+                        followSymlinks: true,
+                    }).map((f) => basename(f)),
+                    ["test.ts", "test.ts"]
+                );
+            });
+
+            it("should handle symlinked files", function () {
+                const { path } = fix.addFile("test.ts");
+                fix.write();
+                try {
+                    fs.symlinkSync(
+                        path,
+                        resolve(dirname(path), "test-2.ts"),
+                        "file"
+                    );
+                } catch (err) {
+                    // on windows, you need elevated permissions to create a file symlink.
+                    // maybe we have them! maybe we don't!
+                    if (
+                        (err as NodeJS.ErrnoException).code === "EPERM" &&
+                        process.platform === "win32"
+                    ) {
+                        return this.skip();
+                    }
+                }
+                equal(
+                    glob(`${fix.cwd}/**/*.ts`, fix.cwd, {
+                        followSymlinks: true,
+                    }).map((f) => basename(f)),
+                    ["test-2.ts", "test.ts"]
+                );
+            });
+        });
+
+        it("should ignore anything that is not a file, symbolic link, or directory", function (done) {
+            // Use unix socket for example, because that's easiest to create.
+            // Skip on Windows because it doesn't support unix sockets
+            if (process.platform === "win32") {
+                return this.skip();
+            }
+            fix.write();
+
+            const sockServer = createServer()
+                .unref()
+                .listen(resolve(fix.cwd, "socket.sock"))
+                .once("listening", () => {
+                    let err: AssertionError | null = null;
+                    try {
+                        equal(glob(`${fix.cwd}/*.sock`, fix.cwd), []);
+                    } catch (e) {
+                        err = e as AssertionError;
+                    } finally {
+                        sockServer.close(() => {
+                            done(err);
+                        });
+                    }
+                });
+        });
     });
 });