bpo-42260: Main init modify sys.flags in-place (GH-23150)

When Py_Initialize() is called twice, the second call now updates
more sys attributes for the configuration, rather than only sys.argv.

* Rename _PySys_InitMain() to _PySys_UpdateConfig().
* _PySys_UpdateConfig() now modifies sys.flags in-place, instead of
  creating a new flags object.
* Remove old commented sys.flags flags (unbuffered and skip_first).
* Add private _PySys_GetObject() function.
* When Py_Initialize(), Py_InitializeFromConfig() and

Author: vstinner

Include/internal/pycore_pylifecycle.h

@@ -44,7 +44,7 @@ extern PyStatus _PySys_Create(
     PyObject **sysmod_p);
 extern PyStatus _PySys_ReadPreinitWarnOptions(PyWideStringList *options);
 extern PyStatus _PySys_ReadPreinitXOptions(PyConfig *config);
-extern int _PySys_InitMain(PyThreadState *tstate);
+extern int _PySys_UpdateConfig(PyThreadState *tstate);
 extern PyStatus _PyExc_Init(PyThreadState *tstate);
 extern PyStatus _PyErr_Init(void);
 extern PyStatus _PyBuiltins_AddExceptions(PyObject * bltinmod);

Misc/NEWS.d/next/C API/2020-11-04-16-31-55.bpo-42260.CmgHtF.rst

@@ -0,0 +1,3 @@
+When :c:func:`Py_Initialize` is called twice, the second call now updates
+more :mod:`sys` attributes for the configuration, rather than only
+:data:`sys.argv`. Patch by Victor Stinner.

Python/pylifecycle.c

@@ -949,19 +949,10 @@ pyinit_core(_PyRuntimeState *runtime,
    configuration. Example of bpo-34008: Py_Main() called after
    Py_Initialize(). */
 static PyStatus
-_Py_ReconfigureMainInterpreter(PyThreadState *tstate)
+pyinit_main_reconfigure(PyThreadState *tstate)
 {
-    const PyConfig *config = _PyInterpreterState_GetConfig(tstate->interp);
-
-    PyObject *argv = _PyWideStringList_AsList(&config->argv);
-    if (argv == NULL) {
-        return _PyStatus_NO_MEMORY(); \
-    }
-
-    int res = PyDict_SetItemString(tstate->interp->sysdict, "argv", argv);
-    Py_DECREF(argv);
-    if (res < 0) {
-        return _PyStatus_ERR("fail to set sys.argv");
+    if (_PySys_UpdateConfig(tstate) < 0) {
+        return _PyStatus_ERR("fail to update sys for the new conf");
     }
     return _PyStatus_OK();
 }
@@ -995,7 +986,7 @@ init_interp_main(PyThreadState *tstate)
         }
     }
 
-    if (_PySys_InitMain(tstate) < 0) {
+    if (_PySys_UpdateConfig(tstate) < 0) {
         return _PyStatus_ERR("can't finish initializing sys");
     }
 
@@ -1100,7 +1091,7 @@ pyinit_main(PyThreadState *tstate)
     }
 
     if (interp->runtime->initialized) {
-        return _Py_ReconfigureMainInterpreter(tstate);
+        return pyinit_main_reconfigure(tstate);
     }
 
     PyStatus status = init_interp_main(tstate);
@@ -1111,19 +1102,6 @@ pyinit_main(PyThreadState *tstate)
 }
 
 
-PyStatus
-_Py_InitializeMain(void)
-{
-    PyStatus status = _PyRuntime_Initialize();
-    if (_PyStatus_EXCEPTION(status)) {
-        return status;
-    }
-    _PyRuntimeState *runtime = &_PyRuntime;
-    PyThreadState *tstate = _PyRuntimeState_GetThreadState(runtime);
-    return pyinit_main(tstate);
-}
-
-
 PyStatus
 Py_InitializeFromConfig(const PyConfig *config)
 {
@@ -1191,6 +1169,19 @@ Py_Initialize(void)
 }
 
 
+PyStatus
+_Py_InitializeMain(void)
+{
+    PyStatus status = _PyRuntime_Initialize();
+    if (_PyStatus_EXCEPTION(status)) {
+        return status;
+    }
+    _PyRuntimeState *runtime = &_PyRuntime;
+    PyThreadState *tstate = _PyRuntimeState_GetThreadState(runtime);
+    return pyinit_main(tstate);
+}
+
+
 static void
 finalize_modules_delete_special(PyThreadState *tstate, int verbose)
 {

Python/sysmodule.c

@@ -84,17 +84,24 @@ _PySys_GetObjectId(_Py_Identifier *key)
     return sys_get_object_id(tstate, key);
 }
 
+static PyObject *
+_PySys_GetObject(PyThreadState *tstate, const char *name)
+{
+    PyObject *sysdict = tstate->interp->sysdict;
+    if (sysdict == NULL) {
+        return NULL;
+    }
+    return _PyDict_GetItemStringWithError(sysdict, name);
+}
+
 PyObject *
 PySys_GetObject(const char *name)
 {
     PyThreadState *tstate = _PyThreadState_GET();
-    PyObject *sd = tstate->interp->sysdict;
-    if (sd == NULL) {
-        return NULL;
-    }
+
     PyObject *exc_type, *exc_value, *exc_tb;
     _PyErr_Fetch(tstate, &exc_type, &exc_value, &exc_tb);
-    PyObject *value = _PyDict_GetItemStringWithError(sd, name);
+    PyObject *value = _PySys_GetObject(tstate, name);
     /* XXX Suppress a new exception if it was raised and restore
      * the old one. */
     _PyErr_Restore(tstate, exc_type, exc_value, exc_tb);
@@ -2464,8 +2471,6 @@ static PyStructSequence_Field flags_fields[] = {
     {"no_site",                 "-S"},
     {"ignore_environment",      "-E"},
     {"verbose",                 "-v"},
-    /* {"unbuffered",                   "-u"}, */
-    /* {"skip_first",                   "-x"}, */
     {"bytes_warning",           "-b"},
     {"quiet",                   "-q"},
     {"hash_randomization",      "-R"},
@@ -2482,21 +2487,27 @@ static PyStructSequence_Desc flags_desc = {
     15
 };
 
-static PyObject*
-make_flags(PyThreadState *tstate)
+static int
+set_flags_from_config(PyObject *flags, PyThreadState *tstate)
 {
     PyInterpreterState *interp = tstate->interp;
     const PyPreConfig *preconfig = &interp->runtime->preconfig;
     const PyConfig *config = _PyInterpreterState_GetConfig(interp);
 
-    PyObject *seq = PyStructSequence_New(&FlagsType);
-    if (seq == NULL) {
-        return NULL;
-    }
-
-    int pos = 0;
-#define SetFlag(flag) \
-    PyStructSequence_SET_ITEM(seq, pos++, PyLong_FromLong(flag))
+    // _PySys_UpdateConfig() modifies sys.flags in-place:
+    // Py_XDECREF() is needed in this case.
+    Py_ssize_t pos = 0;
+#define SetFlagObj(expr) \
+    do { \
+        PyObject *value = (expr); \
+        if (value == NULL) { \
+            return -1; \
+        } \
+        Py_XDECREF(PyStructSequence_GET_ITEM(flags, pos)); \
+        PyStructSequence_SET_ITEM(flags, pos, value); \
+        pos++; \
+    } while (0)
+#define SetFlag(expr) SetFlagObj(PyLong_FromLong(expr))
 
     SetFlag(config->parser_debug);
     SetFlag(config->inspect);
@@ -2507,23 +2518,34 @@ make_flags(PyThreadState *tstate)
     SetFlag(!config->site_import);
     SetFlag(!config->use_environment);
     SetFlag(config->verbose);
-    /* SetFlag(saw_unbuffered_flag); */
-    /* SetFlag(skipfirstline); */
     SetFlag(config->bytes_warning);
     SetFlag(config->quiet);
     SetFlag(config->use_hash_seed == 0 || config->hash_seed != 0);
     SetFlag(config->isolated);
-    PyStructSequence_SET_ITEM(seq, pos++, PyBool_FromLong(config->dev_mode));
+    SetFlagObj(PyBool_FromLong(config->dev_mode));
     SetFlag(preconfig->utf8_mode);
+#undef SetFlagObj
 #undef SetFlag
+    return 0;
+}
 
-    if (_PyErr_Occurred(tstate)) {
-        Py_DECREF(seq);
+
+static PyObject*
+make_flags(PyThreadState *tstate)
+{
+    PyObject *flags = PyStructSequence_New(&FlagsType);
+    if (flags == NULL) {
         return NULL;
     }
-    return seq;
+
+    if (set_flags_from_config(flags, tstate) < 0) {
+        Py_DECREF(flags);
+        return NULL;
+    }
+    return flags;
 }
 
+
 PyDoc_STRVAR(version_info__doc__,
 "sys.version_info\n\
 \n\
@@ -2767,14 +2789,23 @@ _PySys_InitCore(PyThreadState *tstate, PyObject *sysdict)
     /* implementation */
     SET_SYS("implementation", make_impl_info(version_info));
 
-    /* flags */
+    // sys.flags: updated in-place later by _PySys_UpdateConfig()
     if (FlagsType.tp_name == 0) {
         if (PyStructSequence_InitType2(&FlagsType, &flags_desc) < 0) {
             goto type_init_failed;
         }
     }
-    /* Set flags to their default values (updated by _PySys_InitMain()) */
     SET_SYS("flags", make_flags(tstate));
+    /* prevent user from creating new instances */
+    FlagsType.tp_init = NULL;
+    FlagsType.tp_new = NULL;
+    res = PyDict_DelItemString(FlagsType.tp_dict, "__new__");
+    if (res < 0) {
+        if (!_PyErr_ExceptionMatches(tstate, PyExc_KeyError)) {
+            goto err_occurred;
+        }
+        _PyErr_Clear(tstate);
+    }
 
 #if defined(MS_WINDOWS)
     /* getwindowsversion */
@@ -2876,8 +2907,10 @@ sys_create_xoptions_dict(const PyConfig *config)
 }
 
 
+// Update sys attributes for a new PyConfig configuration.
+// This function also adds attributes that _PySys_InitCore() didn't add.
 int
-_PySys_InitMain(PyThreadState *tstate)
+_PySys_UpdateConfig(PyThreadState *tstate)
 {
     PyObject *sysdict = tstate->interp->sysdict;
     const PyConfig *config = _PyInterpreterState_GetConfig(tstate->interp);
@@ -2914,28 +2947,16 @@ _PySys_InitMain(PyThreadState *tstate)
 #undef COPY_LIST
 #undef SET_SYS_FROM_WSTR
 
-
-    /* Set flags to their final values */
-    SET_SYS("flags", make_flags(tstate));
-    /* prevent user from creating new instances */
-    FlagsType.tp_init = NULL;
-    FlagsType.tp_new = NULL;
-    res = PyDict_DelItemString(FlagsType.tp_dict, "__new__");
-    if (res < 0) {
-        if (!_PyErr_ExceptionMatches(tstate, PyExc_KeyError)) {
-            return res;
-        }
-        _PyErr_Clear(tstate);
+    // sys.flags
+    PyObject *flags = _PySys_GetObject(tstate, "flags");  // borrowed ref
+    if (flags == NULL) {
+        return -1;
     }
-
-    SET_SYS("dont_write_bytecode", PyBool_FromLong(!config->write_bytecode));
-
-    if (get_warnoptions(tstate) == NULL) {
+    if (set_flags_from_config(flags, tstate) < 0) {
         return -1;
     }
 
-    if (get_xoptions(tstate) == NULL)
-        return -1;
+    SET_SYS("dont_write_bytecode", PyBool_FromLong(!config->write_bytecode));
 
     if (_PyErr_Occurred(tstate)) {
         goto err_occurred;
@@ -2977,8 +2998,8 @@ _PySys_SetPreliminaryStderr(PyObject *sysdict)
 }
 
 
-/* Create sys module without all attributes: _PySys_InitMain() should be called
-   later to add remaining attributes. */
+/* Create sys module without all attributes.
+   _PySys_UpdateConfig() should be called later to add remaining attributes. */
 PyStatus
 _PySys_Create(PyThreadState *tstate, PyObject **sysmod_p)
 {