Set sys.stdout.encoding properly.

Always set LC_CTYPE on interpreter startup.
Add device_encoding function.

Author: loewis

Lib/io.py

@@ -971,8 +971,13 @@ def __init__(self, buffer, encoding=None, newline=None):
         if newline not in (None, "\n", "\r\n"):
             raise ValueError("illegal newline value: %r" % (newline,))
         if encoding is None:
-            # XXX This is questionable
-            encoding = sys.getfilesystemencoding() or "latin-1"
+            try:
+                encoding = os.device_encoding(buffer.fileno())
+            except AttributeError:
+                pass
+            if encoding is None:
+                import locale
+                encoding = locale.getpreferredencoding()
 
         self.buffer = buffer
         self._encoding = encoding

Modules/_localemodule.c

@@ -1,5 +1,5 @@
 /***********************************************************
-Copyright (C) 1997, 2002, 2003 Martin von Loewis
+Copyright (C) 1997, 2002, 2003, 2007 Martin von Loewis
 
 Permission to use, copy, modify, and distribute this software and its
 documentation for any purpose and without fee is hereby granted,
@@ -562,7 +562,8 @@ PyLocale_nl_langinfo(PyObject* self, PyObject* args)
             /* Check NULL as a workaround for GNU libc's returning NULL
                instead of an empty string for nl_langinfo(ERA).  */
             const char *result = nl_langinfo(item);
-            return PyString_FromString(result != NULL ? result : "");
+            /* XXX may have to convert this to wcs first. */
+            return PyUnicode_FromString(result != NULL ? result : "");
         }
     PyErr_SetString(PyExc_ValueError, "unsupported langinfo constant");
     return NULL;

Modules/posixmodule.c

@@ -92,6 +92,10 @@ corresponding Unix manual entries for more information on calls.");
 #include <sys/loadavg.h>
 #endif
 
+#ifdef HAVE_LANGINFO_H
+#include <langinfo.h>
+#endif
+
 /* Various compilers have only certain posix functions */
 /* XXX Gosh I wish these were all moved into pyconfig.h */
 #if defined(PYCC_VACPP) && defined(PYOS_OS2)
@@ -6581,6 +6585,43 @@ win32_urandom(PyObject *self, PyObject *args)
 }
 #endif
 
+PyDoc_STRVAR(device_encoding__doc__,
+"device_encoding(fd) -> str\n\n\
+Return a string describing the encoding of the device\n\
+if the output is a terminal; else return None.");
+
+static PyObject *
+device_encoding(PyObject *self, PyObject *args)
+{
+	int fd;
+	if (!PyArg_ParseTuple(args, "i:device_encoding", &fd))
+		return NULL;
+	if (!isatty(fd)) {
+		Py_INCREF(Py_None);
+		return Py_None;
+	}
+#if defined(MS_WINDOWS) || defined(MS_WIN64)
+	if (fd == 0) {
+		char buf[100];
+		sprintf(buf, "cp%d", GetConsoleCP());
+		return PyUnicode_FromString(buf);
+	}
+	if (fd == 1 || fd == 2) {
+		char buf[100];
+		sprintf(buf, "cp%d", GetConsoleOutputCP());
+		return PyUnicode_FromString(buf);
+	}
+#elif defined(CODESET)
+	{
+		char *codeset = nl_langinfo(CODESET);
+		if (codeset)
+			return PyUnicode_FromString(codeset);
+	}
+#endif
+	Py_INCREF(Py_None);
+	return Py_None;
+}
+
 #ifdef __VMS
 /* Use openssl random routine */
 #include <openssl/rand.h>
@@ -6793,6 +6834,7 @@ static PyMethodDef posix_methods[] = {
 #endif /* HAVE_TCSETPGRP */
 	{"open",	posix_open, METH_VARARGS, posix_open__doc__},
 	{"close",	posix_close, METH_VARARGS, posix_close__doc__},
+	{"device_encoding", device_encoding, METH_VARARGS, device_encoding__doc__},
 	{"dup",		posix_dup, METH_VARARGS, posix_dup__doc__},
 	{"dup2",	posix_dup2, METH_VARARGS, posix_dup2__doc__},
 	{"lseek",	posix_lseek, METH_VARARGS, posix_lseek__doc__},

Python/pythonrun.c

@@ -154,14 +154,20 @@ Py_InitializeEx(int install_sigs)
 	char *p;
 #if defined(HAVE_LANGINFO_H) && defined(CODESET)
 	char *codeset;
-	char *saved_locale;
 #endif
 	extern void _Py_ReadyTypes(void);
 
 	if (initialized)
 		return;
 	initialized = 1;
 
+#ifdef HAVE_SETLOCALE
+	/* Set up the LC_CTYPE locale, so we can obtain
+	   the locale's charset without having to switch
+	   locales. */
+	setlocale(LC_CTYPE, "");
+#endif
+
 	if ((p = Py_GETENV("PYTHONDEBUG")) && *p != '\0')
 		Py_DebugFlag = add_flag(Py_DebugFlag, p);
 	if ((p = Py_GETENV("PYTHONVERBOSE")) && *p != '\0')
@@ -254,8 +260,6 @@ Py_InitializeEx(int install_sigs)
 	   initialized by other means. Also set the encoding of
 	   stdin and stdout if these are terminals.  */
 
-	saved_locale = strdup(setlocale(LC_CTYPE, NULL));
-	setlocale(LC_CTYPE, "");
 	codeset = nl_langinfo(CODESET);
 	if (codeset && *codeset) {
 		PyObject *enc = PyCodec_Encoder(codeset);
@@ -268,8 +272,6 @@ Py_InitializeEx(int install_sigs)
 		}
 	} else
 		codeset = NULL;
-	setlocale(LC_CTYPE, saved_locale);
-	free(saved_locale);
 
 	if (codeset) {
 		if (!Py_FileSystemDefaultEncoding)