REPL: Query libuv for virtual terminal support

The query-and-restore logic here was pretty flawed:
  - it had no way to guarantee when the "default" mode is available to
    query, so it could easily save a "bad" mode
  - it did not ensure / check whether the "default" mode is compatible
    with the output generated by REPL (esp. ASCII escape codes / color)
  - it persisted the "default" mode from pre-compilation to runtime,
    causing #56073

`ENABLE_VIRTUAL_TERMINAL_PROCESSING` is the only flag that we're
agnostic about. It was added relatively recently (Windows 10 version
1511), and `libuv` has support to emulate its behavior when it's not
available natively.

Otherwise this PR resets ENABLE_PROCESSED_OUTPUT and
ENABLE_WRAP_AT_EOL_OUTPUT always, since we output ASCII control
sequences unconditionally.

Resolves #56073.

Author: topolarity

stdlib/REPL/src/LineEdit.jl

@@ -1638,29 +1638,35 @@ end
 # not leave the console mode in a corrupt state.
 # FIXME: remove when pseudo-tty are implemented for child processes
 if Sys.iswindows()
-function _console_mode()
-    hOutput = ccall(:GetStdHandle, stdcall, Ptr{Cvoid}, (UInt32,), -11 % UInt32) # STD_OUTPUT_HANDLE
-    dwMode = Ref{UInt32}()
-    ccall(:GetConsoleMode, stdcall, Int32, (Ref{Cvoid}, Ref{UInt32}), hOutput, dwMode)
-    return dwMode[]
-end
-const default_console_mode_ref = Ref{UInt32}()
-const default_console_mode_assigned = Ref(false)
-function get_default_console_mode()
-    if default_console_mode_assigned[] == false
-        default_console_mode_assigned[] = true
-        default_console_mode_ref[] = _console_mode()
-    end
-    return default_console_mode_ref[]
-end
+
+#= Get/SetConsoleMode flags =#
+const ENABLE_PROCESSED_OUTPUT            = UInt32(0x0001)
+const ENABLE_WRAP_AT_EOL_OUTPUT          = UInt32(0x0002)
+const ENABLE_VIRTUAL_TERMINAL_PROCESSING = UInt32(0x0004)
+const DISABLE_NEWLINE_AUTO_RETURN        = UInt32(0x0008)
+const ENABLE_LVB_GRID_WORLDWIDE          = UInt32(0x0010)
+
+#= libuv flags =#
+const UV_TTY_SUPPORTED = 0
+const UV_TTY_UNSUPPORTED = 1
+
 function _reset_console_mode()
-    mode = _console_mode()
-    if mode !== get_default_console_mode()
-        hOutput = ccall(:GetStdHandle, stdcall, Ptr{Cvoid}, (UInt32,), -11 % UInt32) # STD_OUTPUT_HANDLE
-        ccall(:SetConsoleMode, stdcall, Int32, (Ptr{Cvoid}, UInt32), hOutput, default_console_mode_ref[])
+    # Query libuv to see whether it expects the console to support virtual
+    # terminal sequences.
+    vterm_state = Ref{CInt}()
+    ccall(:uv_tty_get_vterm_state, Cint, (Ref{Cint},), vterm_state)
+
+    mode::UInt32 = ENABLE_PROCESSED_OUTPUT | ENABLE_WRAP_AT_EOL_OUTPUT
+    if vterm_state[] == UV_TTY_SUPPORTED
+        mode |= ENABLE_VIRTUAL_TERMINAL_PROCESSING
     end
-    nothing
+
+    hOutput = ccall(:GetStdHandle, stdcall, Ptr{Cvoid}, (UInt32,), -11 % UInt32) # STD_OUTPUT_HANDLE
+    hOutput == Ptr{Cvoid}(-1) && return nothing # error
+    ccall(:SetConsoleMode, stdcall, Int32, (Ptr{Cvoid}, UInt32), hOutput, mode)
+    return nothing
 end
+
 end
 
 # returns the width of the written prompt