8352906: stdout/err.encoding on Windows set by incorrect Win32 call

Reviewed-by: bpb, alanb

Author: naotoj

src/java.base/windows/native/libjava/java_props_md.c

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 1998, 2024, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 1998, 2025, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -45,7 +45,7 @@
 #endif
 
 typedef void (WINAPI *PGNSI)(LPSYSTEM_INFO);
-static boolean SetupI18nProps(LCID lcid, char** language, char** script, char** country,
+static BOOL SetupI18nProps(LCID lcid, char** language, char** script, char** country,
                char** variant, char** encoding);
 
 #define PROPSIZE 9      // eight-letter + null terminator
@@ -128,21 +128,30 @@ getEncodingInternal(LCID lcid)
     return ret;
 }
 
-static char* getConsoleEncoding()
+static char* getConsoleEncoding(BOOL output)
 {
     size_t buflen = 16;
     char* buf = malloc(buflen);
     int cp;
     if (buf == NULL) {
         return NULL;
     }
-    cp = GetConsoleCP();
-    if (cp >= 874 && cp <= 950)
+    if (output) {
+        cp = GetConsoleOutputCP();
+    } else {
+        cp = GetConsoleCP();
+    }
+    if (cp >= 874 && cp <= 950) {
         snprintf(buf, buflen, "ms%d", cp);
-    else if (cp == 65001)
+    } else if (cp == 65001) {
         snprintf(buf, buflen, "UTF-8");
-    else
+    } else if (cp == 0) {
+        // Failed to get the console code page
+        free(buf);
+        buf = NULL;
+    } else {
         snprintf(buf, buflen, "cp%d", cp);
+    }
     return buf;
 }
 
@@ -221,7 +230,7 @@ getHomeFromShell32()
     return u_path;
 }
 
-static boolean
+static BOOL
 haveMMX(void)
 {
     return IsProcessorFeaturePresent(PF_MMX_INSTRUCTIONS_AVAILABLE);
@@ -251,7 +260,7 @@ cpu_isalist(void)
     return NULL;
 }
 
-static boolean
+static BOOL
 SetupI18nProps(LCID lcid, char** language, char** script, char** country,
                char** variant, char** encoding) {
     /* script */
@@ -343,8 +352,8 @@ GetJavaProperties(JNIEnv* env)
     /* OS properties */
     {
         char buf[100];
-        boolean is_workstation;
-        boolean is_64bit;
+        BOOL is_workstation;
+        BOOL is_64bit;
         DWORD platformId;
         {
             OSVERSIONINFOEX ver;
@@ -677,15 +686,15 @@ GetJavaProperties(JNIEnv* env)
             hStdOutErr = GetStdHandle(STD_OUTPUT_HANDLE);
             if (hStdOutErr != INVALID_HANDLE_VALUE &&
                 GetFileType(hStdOutErr) == FILE_TYPE_CHAR) {
-                sprops.stdout_encoding = getConsoleEncoding();
+                sprops.stdout_encoding = getConsoleEncoding(TRUE);
             }
             hStdOutErr = GetStdHandle(STD_ERROR_HANDLE);
             if (hStdOutErr != INVALID_HANDLE_VALUE &&
                 GetFileType(hStdOutErr) == FILE_TYPE_CHAR) {
                 if (sprops.stdout_encoding != NULL)
                     sprops.stderr_encoding = sprops.stdout_encoding;
                 else
-                    sprops.stderr_encoding = getConsoleEncoding();
+                    sprops.stderr_encoding = getConsoleEncoding(TRUE);
             }
         }
     }