update koch so the nightlies will be able to bundle Nimble

- Previously, `winrelease` called `bundleNimbleExe`, which tries to clone
  nimble to `dist/nimble` unconditionally.
- Since nightlies source tarball bundles nimble, git couldn't clone
  because `dist/nimble` was already there.
  This leads to 1.0.x couldn't be built with nimble bundled.
- The fix backports the logic from Nim >= 1.2.8 which can check if
  `dist/nimble` was bundled by the source archive.
  With this fix we can re-enable bundling of nimble to the source
  archive for 1.0.x.

Author: narimiran

koch.nim

@@ -27,6 +27,7 @@ import
   os, strutils, parseopt, osproc, streams
 
 import tools / kochdocs
+import tools / deps
 
 const VersionAsString = system.NimVersion
 
@@ -115,52 +116,25 @@ proc copyExe(source, dest: string) =
 
 const
   compileNimInst = "tools/niminst/niminst"
+  distDir = "dist"
 
 proc csource(args: string) =
   nimexec(("cc $1 -r $3 --var:version=$2 --var:mingw=none csource " &
            "--main:compiler/nim.nim compiler/installer.ini $1") %
        [args, VersionAsString, compileNimInst])
 
 proc bundleC2nim(args: string) =
-  if not dirExists("dist/c2nim/.git"):
-    exec("git clone https://github.com/nim-lang/c2nim.git dist/c2nim")
+  cloneDependency(distDir, "https://github.com/nim-lang/c2nim.git")
   nimCompile("dist/c2nim/c2nim",
              options = "--noNimblePath --path:. " & args)
 
 proc bundleNimbleExe(latest: bool, args: string) =
-  if not dirExists("dist/nimble/.git"):
-    exec("git clone https://github.com/nim-lang/nimble.git dist/nimble")
-  if not latest:
-    withDir("dist/nimble"):
-      exec("git fetch")
-      exec("git checkout " & NimbleStableCommit)
+  let commit = if latest: "HEAD" else: NimbleStableCommit
+  cloneDependency(distDir, "https://github.com/nim-lang/nimble.git",
+                  commit = commit, allowBundled = true)
   # installer.ini expects it under $nim/bin
   nimCompile("dist/nimble/src/nimble.nim",
-             options = "-d:release --nilseqs:on " & args)
-
-proc buildNimble(latest: bool, args: string) =
-  # if koch is used for a tar.xz, build the dist/nimble we shipped
-  # with the tarball:
-  var installDir = "dist/nimble"
-  if not latest and dirExists(installDir) and not dirExists("dist/nimble/.git"):
-    discard "don't do the git dance"
-  else:
-    if not dirExists("dist/nimble/.git"):
-      if dirExists(installDir):
-        var id = 0
-        while dirExists("dist/nimble" & $id):
-          inc id
-        installDir = "dist/nimble" & $id
-      exec("git clone https://github.com/nim-lang/nimble.git " & installDir)
-    withDir(installDir):
-      if latest:
-        exec("git checkout -f master")
-        exec("git pull")
-      else:
-        exec("git fetch")
-        exec("git checkout " & NimbleStableCommit)
-  nimCompile(installDir / "src/nimble.nim",
-             options = "--noNimblePath --nilseqs:on -d:release " & args)
+             options = "-d:release --noNimblePath --nilseqs:on " & args)
 
 proc bundleNimsuggest(args: string) =
   nimCompileFold("Compile nimsuggest", "nimsuggest/nimsuggest.nim",
@@ -318,10 +292,10 @@ proc boot(args: string) =
     # in order to use less memory, we split the build into two steps:
     # --compileOnly produces a $project.json file and does not run GCC/Clang.
     # jsonbuild then uses the $project.json file to build the Nim binary.
-    exec "$# $# $# $# --nimcache:$# --compileOnly compiler" / "nim.nim" %
-      [nimi, bootOptions, extraOption, args, smartNimcache]
-    exec "$# jsonscript $# --nimcache:$# compiler" / "nim.nim" %
-      [nimi, args, smartNimcache]
+    exec "$# $# $# --nimcache:$# $# --compileOnly compiler" / "nim.nim" %
+      [nimi, bootOptions, extraOption, smartNimcache, args]
+    exec "$# jsonscript --nimcache:$# $# compiler" / "nim.nim" %
+      [nimi, smartNimcache, args]
 
     if sameFileContent(output, i.thVersion):
       copyExe(output, finalDest)
@@ -479,7 +453,7 @@ proc runCI(cmd: string) =
   when defined(posix): # appveyor (on windows) didn't run this
     kochExecFold("Boot", "boot")
   # boot without -d:nimHasLibFFI to make sure this still works
-  kochExecFold("Boot in release mode", "boot -d:release -d:danger")
+  kochExecFold("Boot in release mode", "boot -d:release")
 
   ## build nimble early on to enable remainder to depend on it if needed
   kochExecFold("Build Nimble", "nimble")
@@ -491,7 +465,7 @@ proc runCI(cmd: string) =
   if getEnv("NIM_TEST_PACKAGES", "false") == "true":
     execFold("Test selected Nimble packages", "nim c -r testament/testament cat nimble-packages")
   else:
-    buildTools() # altenatively, kochExec "tools --toolsNoNimble"
+    buildTools()
 
     ## run tests
     execFold("Test nimscript", "nim e tests/test_nimscript.nims")
@@ -628,13 +602,13 @@ when isMainModule:
       of "temp": temp(op.cmdLineRest)
       of "xtemp": xtemp(op.cmdLineRest)
       of "wintools": bundleWinTools(op.cmdLineRest)
-      of "nimble": buildNimble(latest, op.cmdLineRest)
+      of "nimble": bundleNimbleExe(latest, op.cmdLineRest)
       of "nimsuggest": bundleNimsuggest(op.cmdLineRest)
       of "toolsnonimble":
         buildTools(op.cmdLineRest)
       of "tools":
         buildTools(op.cmdLineRest)
-        buildNimble(latest, op.cmdLineRest)
+        bundleNimbleExe(latest, op.cmdLineRest)
       of "pushcsource", "pushcsources": pushCsources()
       of "valgrind": valgrind(op.cmdLineRest)
       of "c2nim": bundleC2nim(op.cmdLineRest)

tools/deps.nim

@@ -0,0 +1,43 @@
+import os, uri, strformat, osproc, strutils
+
+proc exec(cmd: string) =
+  echo "deps.cmd: " & cmd
+  let status = execShellCmd(cmd)
+  doAssert status == 0, cmd
+
+proc execEx(cmd: string): tuple[output: TaintedString, exitCode: int] =
+  echo "deps.cmd: " & cmd
+  execCmdEx(cmd, {poStdErrToStdOut, poUsePath, poEvalCommand})
+
+proc isGitRepo(dir: string): bool =
+  # This command is used to get the relative path to the root of the repository.
+  # Using this, we can verify whether a folder is a git repository by checking
+  # whether the command success and if the output is empty.
+  let (output, status) = execEx fmt"git -C {quoteShell(dir)} rev-parse --show-cdup"
+  # On Windows there will be a trailing newline on success, remove it.
+  # The value of a successful call typically won't have a whitespace (it's
+  # usually a series of ../), so we know that it's safe to unconditionally
+  # remove trailing whitespaces from the result.
+  result = status == 0 and output.strip() == ""
+
+const commitHead* = "HEAD"
+
+proc cloneDependency*(destDirBase: string, url: string, commit = commitHead,
+                      appendRepoName = true, allowBundled = false) =
+  let destDirBase = destDirBase.absolutePath
+  let p = url.parseUri.path
+  let name = p.splitFile.name
+  var destDir = destDirBase
+  if appendRepoName: destDir = destDir / name
+  let destDir2 = destDir.quoteShell
+  if not dirExists(destDir):
+    # note: old code used `destDir / .git` but that wouldn't prevent git clone
+    # from failing
+    exec fmt"git clone -q {url} {destDir2}"
+  if isGitRepo(destDir):
+    exec fmt"git -C {destDir2} fetch -q"
+    exec fmt"git -C {destDir2} checkout -q {commit}"
+  elif allowBundled:
+    discard "this dependency was bundled with Nim, don't do anything"
+  else:
+    quit "FAILURE: " & destdir & " already exists but is not a git repo"