fix rem2pi for non-finite arguments (#46163)

(cherry picked from commit 73c1eeb)

Author: oscardssmith

base/math.jl

@@ -989,6 +989,8 @@ julia> rem2pi(7pi/4, RoundDown)
 """
 function rem2pi end
 function rem2pi(x::Float64, ::RoundingMode{:Nearest})
+    isfinite(x) || return NaN
+
     abs(x) < pi && return x
 
     n,y = rem_pio2_kernel(x)
@@ -1012,6 +1014,8 @@ function rem2pi(x::Float64, ::RoundingMode{:Nearest})
     end
 end
 function rem2pi(x::Float64, ::RoundingMode{:ToZero})
+    isfinite(x) || return NaN
+
     ax = abs(x)
     ax <= 2*Float64(pi,RoundDown) && return x
 
@@ -1037,6 +1041,8 @@ function rem2pi(x::Float64, ::RoundingMode{:ToZero})
     copysign(z,x)
 end
 function rem2pi(x::Float64, ::RoundingMode{:Down})
+    isfinite(x) || return NaN
+
     if x < pi4o2_h
         if x >= 0
             return x
@@ -1066,6 +1072,8 @@ function rem2pi(x::Float64, ::RoundingMode{:Down})
     end
 end
 function rem2pi(x::Float64, ::RoundingMode{:Up})
+    isfinite(x) || return NaN
+
     if x > -pi4o2_h
         if x <= 0
             return x

test/numbers.jl

@@ -2530,6 +2530,15 @@ end
     @test rem2pi(T(-8), RoundUp)      ≈ -8+2pi
 end
 
+@testset "PR #36420 $T" for T in (Float16, Float32, Float64)
+    for r in (RoundToZero, RoundNearest, RoundDown, RoundUp)
+        for x in (Inf, -Inf, NaN, -NaN)
+            @test isnan(rem2pi(T(x), r))
+            @test rem2pi(T(x), r) isa T
+        end
+    end
+end
+
 import Base.^
 struct PR20530; end
 struct PR20889; x; end