diff --git a/.gitignore b/.gitignore index 0bdc3221..f2fe9304 100644 --- a/.gitignore +++ b/.gitignore @@ -5,3 +5,5 @@ nimcache/ nimblecache/ htmldocs/ + +/build diff --git a/src/fusion/filewalks.nim b/src/fusion/filewalks.nim new file mode 100644 index 00000000..128b97af --- /dev/null +++ b/src/fusion/filewalks.nim @@ -0,0 +1,126 @@ +#[ +## Design rationale +* an intermediate `WalkOpt` is used to allow easier proc forwarding +* a yieldFilter, regex match etc isn't needed because caller can filter at + call site, without loss of generality, unlike `follow`; this simplifies the API. + +## Future work: +* provide a way to do error reporting, which is tricky because iteration cannot be resumed +]# + +import std/[os, algorithm, deques, macros] + +type + PathEntrySub* = object + kind*: PathComponent + path*: string + PathEntry* = object + kind*: PathComponent + path*: string + ## absolute or relative path with respect to walked dir + depth*: int + ## depth with respect to WalkOpt.dir (which is at depth 0) + epilogue*: bool + WalkMode* = enum + dfs ## depth first search + bfs ## breadth first search + FollowCallback* = proc(entry: PathEntry): bool + SortCmpCallback* = proc(x, y: PathEntrySub): int + WalkOpt* = object + dir*: string ## root of walk + relative: bool ## when true, paths are returned relative to `dir`. Otherwise they start with `dir` + checkDir: bool ## if true, raises `OSError` when `dir` can't be listed. Deeper + ## directories do not cause `OSError`, and currently no error reporting is done for those. + walkMode: WalkMode ## controls how paths are returned + includeRoot: bool ## whether to include root `dir` + includeEpilogue: bool + ## when false, yields: someDir, + ## when true, yields: someDir, , someDir: each dir is + ## yielded a 2nd time. This is useful in applications that aggregate data over dirs. + followSymlinks: bool ## whether to follow symlinks + follow: FollowCallback + ## if not `nil`, `walkPath` visits `entry` if `follow(entry) == true`. + sortCmp: SortCmpCallback + ## if not `nil`, immediate children of a dir are sorted using `sortCmp` + +macro ctor(obj: untyped, a: varargs[untyped]): untyped = + ##[ + Generates an object constructor call from a list of fields. + ]## + # xxx expose in some `fusion/macros` + runnableExamples: + type Foo = object + a, b: int + doAssert Foo.ctor(a,b) == Foo(a: a, b: b) + result = nnkObjConstr.newTree(obj) + for ai in a: result.add nnkExprColonExpr.newTree(ai, ai) + +proc initWalkOpt*( + dir: string, relative = false, checkDir = true, walkMode = dfs, + includeRoot = false, includeEpilogue = false, followSymlinks = false, + follow: FollowCallback = nil, sortCmp: SortCmpCallback = nil): WalkOpt = + WalkOpt.ctor(dir, relative, checkDir, walkMode, includeRoot, includeEpilogue, followSymlinks, follow, sortCmp) + +iterator walkPathsOpt*(opt: WalkOpt): PathEntry = + ##[ + Recursively walks `dir`. + This is more flexible than `os.walkDirRec`. + ]## + runnableExamples: + import os,sugar + if false: # see also `tfilewalks.nim` + # list hidden files of depth <= 2 + 1 in your home. + for e in walkPaths(getHomeDir(), follow = a=>a.path.isHidden and a.depth <= 2): + if e.kind in {pcFile, pcLinkToFile}: echo e.path + + var entry = PathEntry(depth: 0, path: ".") + when nimvm: entry.kind = pcDir + else: # xxx support `symlinkExists` in nimvm + entry.kind = if symlinkExists(opt.dir): pcLinkToDir else: pcDir + var stack = initDeque[PathEntry]() + + var checkDir = opt.checkDir + if dirExists(opt.dir): + stack.addLast entry + elif checkDir: + raise newException(OSError, "invalid root dir: " & opt.dir) + + var dirsLevel: seq[PathEntrySub] + while stack.len > 0: + let current = if opt.walkMode == dfs: stack.popLast() else: stack.popFirst() + entry.epilogue = current.epilogue + entry.depth = current.depth + entry.kind = current.kind + entry.path = if opt.relative: current.path else: opt.dir / current.path + normalizePath(entry.path) # pending https://github.com/timotheecour/Nim/issues/343 + + if opt.includeRoot or current.depth > 0: + yield entry # single `yield` to avoid code bloat + + let isSort = opt.sortCmp != nil + if (current.kind == pcDir or current.kind == pcLinkToDir and opt.followSymlinks) and not current.epilogue: + if opt.follow == nil or opt.follow(current): + if isSort: + dirsLevel.setLen 0 + if opt.includeEpilogue: + stack.addLast PathEntry(depth: current.depth, path: current.path, kind: current.kind, epilogue: true) + # `checkDir` is still needed here in first iteration because things could + # fail for reasons other than `not dirExists`. + for k, p in walkDir(opt.dir / current.path, relative = true, checkDir = checkDir): + if isSort: + dirsLevel.add PathEntrySub(kind: k, path: p) + else: + stack.addLast PathEntry(depth: current.depth + 1, path: current.path / p, kind: k) + checkDir = false + # We only check top-level dir, otherwise if a subdir is invalid (eg. wrong + # permissions), it'll abort iteration and there would be no way to resume iteration. + if isSort: + sort(dirsLevel, opt.sortCmp) + for i in 0.. 0 + let a2 = dir / a + if a.endsWith("/"): + createDir(a2) + else: + createDir(a2.parentDir) + writeFile(a2, "") diff --git a/tests/lib/tfusion/paths.nim b/tests/lib/tfusion/paths.nim new file mode 100644 index 00000000..4894e82d --- /dev/null +++ b/tests/lib/tfusion/paths.nim @@ -0,0 +1,8 @@ +import std/os + +const + fusionRoot* = currentSourcePath.parentDir.parentDir.parentDir.parentDir + buildDir* = fusionRoot / "build" + nimbleFile* = fusionRoot / "fusion.nimble" + +static: doAssert nimbleFile.fileExists # sanity check diff --git a/tests/tfilewalks.nim b/tests/tfilewalks.nim new file mode 100644 index 00000000..b4d76552 --- /dev/null +++ b/tests/tfilewalks.nim @@ -0,0 +1,73 @@ +import std/[os,sequtils,sugar] +from tfusion/paths import buildDir + +const dir = buildDir/"tfilewalks" + +when defined(fusionTfilewalksTesting): + import std/[sugar,os,sequtils,algorithm] + from std/private/globs import nativeToUnixPath + import fusion/filewalks + + proc processAux[T](a: T): seq[string] = + a.mapIt(it.path.nativeToUnixPath) + + proc process[T](a: T): seq[string] = + a.processAux.sorted + + proc test() = + block: # follow + # filter by pcFile + doAssert toSeq(walkPaths(dir, follow = a=>a.path.lastPathPart != "d1b", relative = true)) + .filterIt(it.kind == pcFile).process == @["d1/d1a/f2.txt", "d1/d1a/f3", "d1/f1.txt", "f5"] + # filter by pcDir + doAssert toSeq(walkPaths(dir, relative = true)) + .filterIt(it.kind == pcDir).process == @["d1", "d1/d1a", "d1/d1a/d1a1", "d1/d1b", "d1/d1b/d1b1", "d2"] + + block: # includeRoot + doAssert toSeq(walkPaths(dir, relative = true, includeRoot = true)) + .filterIt(it.kind == pcDir).process == @[".", "d1", "d1/d1a", "d1/d1a/d1a1", "d1/d1b", "d1/d1b/d1b1", "d2"] + + block: # checkDir + doAssertRaises(OSError): discard toSeq(walkPaths("nonexistant")) + doAssertRaises(OSError): discard toSeq(walkPaths("f5")) + doAssert toSeq(walkPaths("nonexistant", checkDir = false)) == @[] + + # sortCmp + proc mySort(a, b: PathEntrySub): int = cmp(a.path, b.path) + doAssert toSeq(walkPaths(dir, relative = true, sortCmp = mySort)).processAux == + @["d1", "d1/d1a", "d1/d1a/d1a1", "d1/d1a/f2.txt", "d1/d1a/f3", "d1/d1b", "d1/d1b/d1b1", "d1/d1b/d1b1/f4", "d1/f1.txt", "d2", "f5"] + + # bfs + doAssert toSeq(walkPaths(dir, relative = true, sortCmp = mySort, walkMode = bfs)).processAux == + @["d1", "d2", "f5", "d1/d1a", "d1/d1b", "d1/f1.txt", "d1/d1a/d1a1", "d1/d1a/f2.txt", "d1/d1a/f3", "d1/d1b/d1b1", "d1/d1b/d1b1/f4"] + + # includeEpilogue + doAssert toSeq(walkPaths(dir, relative = true, sortCmp = mySort, includeEpilogue = true, includeRoot = true)).processAux == + @[".", "d1", "d1/d1a", "d1/d1a/d1a1", "d1/d1a/d1a1", "d1/d1a/f2.txt", "d1/d1a/f3", "d1/d1a", "d1/d1b", "d1/d1b/d1b1", "d1/d1b/d1b1/f4", "d1/d1b/d1b1", "d1/d1b", "d1/f1.txt", "d1", "d2", "d2", "f5", "."] + + when (NimMajor, NimMinor, NimPatch) >= (1, 5, 1): + static: test() + test() +else: + from tfusion/osutils import genTestPaths + import std/[strformat,strutils] + proc main() = + defer: removeDir(dir) + let paths = """ +d1/f1.txt +d1/d1a/f2.txt +d1/d1a/f3 +d1/d1a/d1a1/ +d1/d1b/d1b1/f4 +d2/ +f5 +""".splitLines.filter(a=>a.len>0) + genTestPaths(dir, paths) + const nim = getCurrentCompilerExe() + const input = currentSourcePath() + let cmd = &"{nim} r -d:fusionTfilewalksTesting {input}" + when (NimMajor, NimMinor, NimPatch) >= (1, 4, 0): + # for `nativeToUnixPath` + let status = execShellCmd(cmd) + doAssert status == 0 + main()