enable seeding of GLOBAL_RNG when running tests

Author: rfourquet

base/interactiveutil.jl

@@ -697,19 +697,23 @@ end
 
 """
     Base.runtests(tests=["all"], numcores=ceil(Int, Sys.CPU_CORES / 2);
-                  exit_on_error=false)
+                  exit_on_error=false, [seed])
 
 Run the Julia unit tests listed in `tests`, which can be either a string or an array of
 strings, using `numcores` processors. If `exit_on_error` is `false`, when one test
 fails, all remaining tests in other files will still be run; they are otherwise discarded,
 when `exit_on_error == true`.
+If a seed is provided via the keyword argument, it is used to seed the
+global RNG in the context where the tests are run; otherwise the seed is chosen randomly.
 """
 function runtests(tests = ["all"], numcores = ceil(Int, Sys.CPU_CORES / 2);
-                  exit_on_error=false)
+                  exit_on_error=false,
+                  seed::Union{BitInteger,Void}=nothing)
     if isa(tests,AbstractString)
         tests = split(tests)
     end
     exit_on_error && push!(tests, "--exit-on-error")
+    seed != nothing && push!(tests, "--seed=0x$(hex(seed % UInt128))") # cast to UInt128 to avoid a minus sign
     ENV2 = copy(ENV)
     ENV2["JULIA_CPU_CORES"] = "$numcores"
     try

test/choosetests.jl

@@ -2,22 +2,29 @@
 
 @doc """
 
-`tests, net_on, exit_on_error = choosetests(choices)` selects a set of tests to be
+`tests, net_on, exit_on_error, seed = choosetests(choices)` selects a set of tests to be
 run. `choices` should be a vector of test names; if empty or set to
 `["all"]`, all tests are selected.
 
 This function also supports "test collections": specifically, "linalg"
  refers to collections of tests in the correspondingly-named
 directories.
 
-Upon return, `tests` is a vector of fully-expanded test names,
-`net_on` is true if networking is available (required for some tests),
-and `exit_on_error` is true if an error in one test should cancel
-remaining tests to be run (otherwise, all tests are run unconditionally).
-
-Two options can be passed to `choosetests` by including a special token
-in the `choices` argument: "--skip", which makes all tests coming after
-be skipped, and "--exit-on-error" which sets the value of `exit_on_error`.
+Upon return:
+  - `tests` is a vector of fully-expanded test names,
+  - `net_on` is true if networking is available (required for some tests),
+  - `exit_on_error` is true if an error in one test should cancel
+    remaining tests to be run (otherwise, all tests are run unconditionally),
+  - `seed` is a seed which will be used to initialize the global RNG for each
+    test to be run.
+
+Three options can be passed to `choosetests` by including a special token
+in the `choices` argument:
+   - "--skip", which makes all tests coming after be skipped,
+   - "--exit-on-error" which sets the value of `exit_on_error`,
+   - "--seed=SEED", which sets the value of `seed` to `SEED`
+     (parsed as an `UInt128`); `seed` is otherwise initialized randomly.
+     This option can be used to reproduce failed tests.
 """ ->
 function choosetests(choices = [])
     testnames = [
@@ -58,13 +65,16 @@ function choosetests(choices = [])
     tests = []
     skip_tests = []
     exit_on_error = false
+    seed = rand(RandomDevice(), UInt128)
 
     for (i, t) in enumerate(choices)
         if t == "--skip"
             skip_tests = choices[i + 1:end]
             break
         elseif t == "--exit-on-error"
             exit_on_error = true
+        elseif startswith(t, "--seed=")
+            seed = parse(UInt128, t[8:end])
         else
             push!(tests, t)
         end
@@ -177,5 +187,5 @@ function choosetests(choices = [])
 
     filter!(x -> !(x in skip_tests), tests)
 
-    tests, net_on, exit_on_error
+    tests, net_on, exit_on_error, seed
 end

test/runtests.jl

@@ -4,7 +4,7 @@ using Test
 include("choosetests.jl")
 include("testenv.jl")
 
-tests, net_on, exit_on_error = choosetests(ARGS)
+tests, net_on, exit_on_error, seed = choosetests(ARGS)
 tests = unique(tests)
 
 const max_worker_rss = if haskey(ENV, "JULIA_TEST_MAXRSS_MB")
@@ -55,7 +55,7 @@ cd(dirname(@__FILE__)) do
                     local resp
                     wrkr = p
                     try
-                        resp = remotecall_fetch(runtests, wrkr, test)
+                        resp = remotecall_fetch(runtests, wrkr, test; seed=seed)
                     catch e
                         resp = [e]
                     end
@@ -105,7 +105,7 @@ cd(dirname(@__FILE__)) do
         n > 1 && print("\tFrom worker 1:\t")
         local resp
         try
-            resp = eval(Expr(:call, () -> runtests(t))) # runtests is defined by the include above
+            resp = eval(Expr(:call, () -> runtests(t, seed=seed))) # runtests is defined by the include above
         catch e
             resp = [e]
         end
@@ -188,7 +188,8 @@ cd(dirname(@__FILE__)) do
     else
         println("    \033[31;1mFAILURE\033[0m\n")
         skipped > 0 &&
-            println("$skipped test", skipped > 1 ? "s were" : " was", " skipped due to failure.\n")
+            println("$skipped test", skipped > 1 ? "s were" : " was", " skipped due to failure.")
+        println("The global RNG seed was 0x$(hex(seed)).\n")
         Test.print_test_errors(o_ts)
         throw(Test.FallbackTestSetException("Test run finished with errors"))
     end

test/testdefs.jl

@@ -2,7 +2,7 @@
 
 using Test
 
-function runtests(name, isolate=true)
+function runtests(name, isolate=true; seed=nothing)
     old_print_setting = Test.TESTSET_PRINT_ENABLE[]
     Test.TESTSET_PRINT_ENABLE[] = false
     try
@@ -17,6 +17,7 @@ function runtests(name, isolate=true)
         @eval(m, using Test)
         ex = quote
             @timed @testset $"$name" begin
+                srand($seed)
                 include($"$name.jl")
             end
         end