Skip to content

Skip flakey tests in CI #50071

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 2 commits into from
Jun 7, 2023
Merged

Skip flakey tests in CI #50071

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
88d769e
skip flakey tests
Jun 5, 2023
9f3e8bc
also skip some cmdlineargs tests on some OSs
Jun 5, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
9 changes: 6 additions & 3 deletions stdlib/LinearAlgebra/test/addmul.jl
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -164,7 +164,8 @@ end
Bc = Matrix(B)
returned_mat = mul!(C, A, B, α, β)
@test returned_mat === C
@test collect(returned_mat) ≈ α * Ac * Bc + β * Cc rtol=rtol
# This test is skipped because it is flakey, but should be fixed and put back (see #49966)
@test_skip collect(returned_mat) ≈ α * Ac * Bc + β * Cc rtol=rtol

y = C[:, 1]
x = B[:, 1]
Expand All @@ -189,7 +190,8 @@ end

returned_mat = mul!(C, Af, Bf, α, β)
@test returned_mat === C
@test collect(returned_mat) ≈ α * Ac * Bc + β * Cc rtol=rtol
# This test is skipped because it is flakey, but should be fixed and put back (see #49966)
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Ah! I thought I broke it in #49405 (comment)

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

haha, I don't think so

@test_skip collect(returned_mat) ≈ α * Ac * Bc + β * Cc rtol=rtol
end
end
end
Expand All @@ -201,7 +203,8 @@ end
Bc = Matrix(B)
returned_mat = mul!(C, A, B, α, zero(eltype(C)))
@test returned_mat === C
@test collect(returned_mat) ≈ α * Ac * Bc rtol=rtol
# This test is skipped because it is flakey, but should be fixed and put back (see #49966)
@test_skip collect(returned_mat) ≈ α * Ac * Bc rtol=rtol
end
end

Expand Down
64 changes: 35 additions & 29 deletions test/cmdlineargs.jl
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -300,37 +300,43 @@ let exename = `$(Base.julia_cmd()) --startup-file=no --color=no`
@test errors_not_signals(`$exename -C invalidtarget`)
@test errors_not_signals(`$exename --cpu-target=invalidtarget`)

# -t, --threads
code = "print(Threads.threadpoolsize())"
cpu_threads = ccall(:jl_effective_threads, Int32, ())
@test string(cpu_threads) ==
read(`$exename --threads auto -e $code`, String) ==
read(`$exename --threads=auto -e $code`, String) ==
read(`$exename -tauto -e $code`, String) ==
read(`$exename -t auto -e $code`, String)
for nt in (nothing, "1")
withenv("JULIA_NUM_THREADS" => nt) do
@test read(`$exename --threads=2 -e $code`, String) ==
read(`$exename -t 2 -e $code`, String) == "2"
if Sys.iswindows()
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I'm not entirely sure Windows is immune from issues, maybe it just happens less frequently, I seem to remember @vtjnash saying that these tests are a bit unreliable in general (do I remember wrong?). But it'd also be sad to completely miss this block of tests.

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

It hasn't failed for Windows in the past 40 commits to master so I'm hopeful, but far from confident that these are reliable on Windows. Not wanting to miss this entire block also motivated my not skipping these on Windows.

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I think Windows does not have this segfault issue, but because of that, it might hang instead.

# -t, --threads
code = "print(Threads.threadpoolsize())"
cpu_threads = ccall(:jl_effective_threads, Int32, ())
@test string(cpu_threads) ==
read(`$exename --threads auto -e $code`, String) ==
read(`$exename --threads=auto -e $code`, String) ==
read(`$exename -tauto -e $code`, String) ==
read(`$exename -t auto -e $code`, String)
for nt in (nothing, "1")
withenv("JULIA_NUM_THREADS" => nt) do
@test read(`$exename --threads=2 -e $code`, String) ==
read(`$exename -t 2 -e $code`, String) == "2"
end
end
end
# We want to test oversubscription, but on manycore machines, this can
# actually exhaust limited PID spaces
cpu_threads = max(2*cpu_threads, min(50, 10*cpu_threads))
if Sys.WORD_SIZE == 32
cpu_threads = min(cpu_threads, 50)
end
@test read(`$exename -t $cpu_threads -e $code`, String) == string(cpu_threads)
withenv("JULIA_NUM_THREADS" => string(cpu_threads)) do
@test read(`$exename -e $code`, String) == string(cpu_threads)
end
@test errors_not_signals(`$exename -t 0`)
@test errors_not_signals(`$exename -t -1`)
# We want to test oversubscription, but on manycore machines, this can
# actually exhaust limited PID spaces
cpu_threads = max(2*cpu_threads, min(50, 10*cpu_threads))
if Sys.WORD_SIZE == 32
cpu_threads = min(cpu_threads, 50)
end
@test read(`$exename -t $cpu_threads -e $code`, String) == string(cpu_threads)
withenv("JULIA_NUM_THREADS" => string(cpu_threads)) do
@test read(`$exename -e $code`, String) == string(cpu_threads)
end
@test errors_not_signals(`$exename -t 0`)
@test errors_not_signals(`$exename -t -1`)

# Combining --threads and --procs: --threads does propagate
withenv("JULIA_NUM_THREADS" => nothing) do
code = "print(sum(remotecall_fetch(Threads.threadpoolsize, x) for x in procs()))"
@test read(`$exename -p2 -t2 -e $code`, String) == "6"
# Combining --threads and --procs: --threads does propagate
withenv("JULIA_NUM_THREADS" => nothing) do
code = "print(sum(remotecall_fetch(Threads.threadpoolsize, x) for x in procs()))"
@test read(`$exename -p2 -t2 -e $code`, String) == "6"
end
else
@test_skip "Command line tests with -t are flakey on non-Windows OS"
# Known issue: https://github.com/JuliaLang/julia/issues/49154
# These tests should be fixed and reenabled on all operating systems.
end

# Combining --threads and invalid -C should yield a decent error
Expand Down
2 changes: 1 addition & 1 deletion test/threads.jl
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -312,7 +312,7 @@ close(proc.in)
if ( !success(proc) ) || ( timeout )
@error "A \"spawn and wait lots of tasks\" test failed" n proc.exitcode proc.termsignal success(proc) timeout
end
if Sys.iswindows()
if Sys.iswindows() || Sys.isapple()
# Known failure: https://github.com/JuliaLang/julia/issues/43124
@test_skip success(proc)
else
Expand Down