blender
/
blender
129
398
Fork 572
Code Issues 6.2k Pull Requests 746 Projects 19 Wiki Activity

PyAPI: Use PyPreConfig & PyConfig for Python initialization 

Use Python 3.8's API for setting the initial configuration.

This replaces a mix of our logic and direct calls to the Python API
and has no user visible changes.

Using the Python API makes the logic easier to follow and provides
utilities such as `PyConfig_SetBytesArgv`
that wasn't available in previous releases.

Note that this uses Python's utf8/wchar_t conversions,
which used to cause problems (see T31506).

Since `Py_UTF8Mode` was set, the systems locale isn't used for decoding,
allowing us to use Python's utility functions that call
`Py_DecodeLocale` internally.

Ref D10382
This commit is contained in:
Campbell Barton 2021-02-12 08:08:17 +11:00
parent aa43e2ec29
commit cafd6b519c
Notes: blender-bot 2023-02-14 00:28:11 +01:00 
Referenced by commit 06212759bc, Fix missing NULL check on macOS when system-python isn't found
Referenced by issue #95410, Regression: Blender text editor lost support for dead keys
Referenced by issue #70861, Use 'PyPreConfig' for initialization (Python 3.8)
3 changed files with 131 additions and 116 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

34
source/blender/python/generic/py_capi_utils.c
View File

@ -879,40 +879,6 @@ void PyC_MainModule_Restore(PyObject *main_mod)
Py_XDECREF(main_mod);
}
/** \} */
/* -------------------------------------------------------------------- */
/** \name #Py_SetPythonHome Wrapper
* \{ */
/**
* - Must be called before #Py_Initialize.
* - Expects output of `BKE_appdir_folder_id(BLENDER_PYTHON, NULL)`.
* - Note that the `PYTHONPATH` environment variable isn't reliable, see T31506.
* Use #Py_SetPythonHome instead.
*/
void PyC_SetHomePath(const char *py_path_bundle)
{
# ifdef __APPLE__
/* OSX allow file/directory names to contain : character (represented as / in the Finder)
* but current Python lib (release 3.1.1) doesn't handle these correctly */
if (strchr(py_path_bundle, ':')) {
fprintf(stderr,
"Warning! Blender application is located in a path containing ':' or '/' chars\n"
"This may make python import function fail\n");
}
# endif
/* Set the environment path. */
wchar_t py_path_bundle_wchar[1024];
/* Can't use `mbstowcs` on linux gives bug: T23018. */
BLI_strncpy_wchar_from_utf8(
py_path_bundle_wchar, py_path_bundle, ARRAY_SIZE(py_path_bundle_wchar));
Py_SetPythonHome(py_path_bundle_wchar);
}
bool PyC_IsInterpreterActive(void)
{
/* instead of PyThreadState_Get, which calls Py_FatalError */

2
source/blender/python/generic/py_capi_utils.h
View File

@ -88,8 +88,6 @@ bool PyC_NameSpace_ImportArray(PyObject *py_dict, const char *imports[]);
void PyC_MainModule_Backup(PyObject **r_main_mod);
void PyC_MainModule_Restore(PyObject *main_mod);
void PyC_SetHomePath(const char *py_path_bundle);
bool PyC_IsInterpreterActive(void);
void *PyC_RNA_AsPointer(PyObject *value, const char *type_name);

211
source/blender/python/intern/bpy_interface.c
View File

@ -304,100 +304,150 @@ static struct _inittab bpy_internal_modules[] = {
{NULL, NULL},
};
/**
* Convenience function for #BPY_python_start.
*
* These should happen so rarely that having comprehensive errors isn't needed.
* For example if `sys.argv` fails to allocate memory.
*
* Show an error just to avoid silent failure in the unlikely event something goes wrong,
* in this case a developer will need to track down the root cause.
*/
static void pystatus_exit_on_error(PyStatus status)
{
if (UNLIKELY(PyStatus_Exception(status))) {
fputs("Internal error initializing Python!\n", stderr);
/* This calls `exit`. */
Py_ExitStatusException(status);
}
}
/* call BPY_context_set first */
void BPY_python_start(bContext *C, int argc, const char **argv)
{
#ifndef WITH_PYTHON_MODULE
PyThreadState *py_tstate = NULL;
/* Needed for Python's initialization for portable Python installations.
* We could use #Py_SetPath, but this overrides Python's internal logic
* for calculating it's own module search paths.
*
* `sys.executable` is overwritten after initialization to the Python binary. */
/* #PyPreConfig (early-configuration). */
{
const char *program_path = BKE_appdir_program_path();
wchar_t program_path_wchar[FILE_MAX];
BLI_strncpy_wchar_from_utf8(program_path_wchar, program_path, ARRAY_SIZE(program_path_wchar));
Py_SetProgramName(program_path_wchar);
PyPreConfig preconfig;
PyStatus status;
if (py_use_system_env) {
PyPreConfig_InitPythonConfig(&preconfig);
}
else {
/* Only use the systems environment variables and site when explicitly requested.
* Since an incorrect 'PYTHONPATH' causes difficult to debug errors, see: T72807.
* An alternative to setting `preconfig.use_environment = 0` */
PyPreConfig_InitIsolatedConfig(&preconfig);
}
/* Force `utf-8` on all platforms, since this is what's used for Blender's internal strings,
* providing consistent encoding behavior across all Blender installations.
*
* This also uses the `surrogateescape` error handler ensures any unexpected bytes are escaped
* instead of raising an error.
*
* Without this `sys.getfilesystemencoding()` and `sys.stdout` for example may be set to ASCII
* or some other encoding - where printing some `utf-8` values will raise an error.
*
* This can cause scripts to fail entirely on some systems.
*
* This assignment is the equivalent of enabling the `PYTHONUTF8` environment variable.
* See `PEP-540` for details on exactly what this changes. */
preconfig.utf8_mode = true;
/* Note that there is no reason to call #Py_PreInitializeFromBytesArgs here
* as this is only used so that command line arguments can be handled by Python itself,
* not for setting `sys.argv` (handled below). */
status = Py_PreInitialize(&preconfig);
pystatus_exit_on_error(status);
}
/* must run before python initializes */
/* Must run before python initializes, but after #PyPreConfig. */
PyImport_ExtendInittab(bpy_internal_modules);
/* Allow to use our own included Python. `py_path_bundle` may be NULL. */
/* #PyConfig (initialize Python). */
{
const char *py_path_bundle = BKE_appdir_folder_id(BLENDER_SYSTEM_PYTHON, NULL);
if (py_path_bundle != NULL) {
PyC_SetHomePath(py_path_bundle);
PyConfig config;
PyStatus status;
bool has_python_executable = false;
PyConfig_InitPythonConfig(&config);
/* Suppress error messages when calculating the module search path.
* While harmless, it's noisy. */
config.pathconfig_warnings = 0;
/* When using the system's Python, allow the site-directory as well. */
config.user_site_directory = py_use_system_env;
/* While `sys.argv` is set, we don't want Python to interpret it. */
config.parse_argv = 0;
status = PyConfig_SetBytesArgv(&config, argc, (char *const *)argv);
pystatus_exit_on_error(status);
/* Needed for Python's initialization for portable Python installations.
* We could use #Py_SetPath, but this overrides Python's internal logic
* for calculating it's own module search paths.
*
* `sys.executable` is overwritten after initialization to the Python binary. */
{
const char *program_path = BKE_appdir_program_path();
status = PyConfig_SetBytesString(&config, &config.program_name, program_path);
pystatus_exit_on_error(status);
}
else {
/* Common enough to use the system Python on Linux/Unix, warn on other systems. */
# if defined(__APPLE__) || defined(_WIN32)
fprintf(stderr,
"Bundled Python not found and is expected on this platform "
"(the 'install' target may have not been built)\n");
/* Setting the program name is important so the 'multiprocessing' module
* can launch new Python instances. */
{
char program_path[FILE_MAX];
if (BKE_appdir_program_python_search(
program_path, sizeof(program_path), PY_MAJOR_VERSION, PY_MINOR_VERSION)) {
status = PyConfig_SetBytesString(&config, &config.executable, program_path);
pystatus_exit_on_error(status);
has_python_executable = true;
}
else {
/* Set to `sys.executable = None` below (we can't do before Python is initialized). */
fprintf(stderr,
"Unable to find the python binary, "
"the multiprocessing module may not be functional!\n");
}
}
/* Allow to use our own included Python. `py_path_bundle` may be NULL. */
{
const char *py_path_bundle = BKE_appdir_folder_id(BLENDER_SYSTEM_PYTHON, NULL);
# ifdef __APPLE__
/* OSX allow file/directory names to contain : character (represented as / in the Finder)
* but current Python lib (release 3.1.1) doesn't handle these correctly */
if (strchr(py_path_bundle, ':')) {
fprintf(stderr,
"Warning! Blender application is located in a path containing ':' or '/' chars\n"
"This may make python import function fail\n");
}
# endif
if (py_path_bundle != NULL) {
status = PyConfig_SetBytesString(&config, &config.home, py_path_bundle);
pystatus_exit_on_error(status);
}
else {
/* Common enough to use the system Python on Linux/Unix, warn on other systems. */
# if defined(__APPLE__) || defined(_WIN32)
fprintf(stderr,
"Bundled Python not found and is expected on this platform "
"(the 'install' target may have not been built)\n");
# endif
}
}
}
/* Force `utf-8` on all platforms, since this is what's used for Blender's internal strings,
* providing consistent encoding behavior across all Blender installations.
*
* This also uses the `surrogateescape` error handler ensures any unexpected bytes are escaped
* instead of raising an error.
*
* Without this `sys.getfilesystemencoding()` and `sys.stdout` for example may be set to ASCII
* or some other encoding - where printing some `utf-8` values will raise an error.
*
* This can cause scripts to fail entirely on some systems.
*
* This assignment is the equivalent of enabling the `PYTHONUTF8` environment variable.
* See `PEP-540` for details on exactly what this changes. */
Py_UTF8Mode = 1;
/* Initialize Python (also acquires lock). */
status = Py_InitializeFromConfig(&config);
pystatus_exit_on_error(status);
/* Suppress error messages when calculating the module search path.
* While harmless, it's noisy. */
Py_FrozenFlag = 1;
/* Only use the systems environment variables and site when explicitly requested.
* Since an incorrect 'PYTHONPATH' causes difficult to debug errors, see: T72807. */
Py_IgnoreEnvironmentFlag = !py_use_system_env;
Py_NoUserSiteDirectory = !py_use_system_env;
/* Initialize Python (also acquires lock). */
Py_Initialize();
/* We could convert to #wchar_t then pass to #PySys_SetArgv (or use #PyConfig in Python 3.8+).
* However this risks introducing subtle changes in encoding that are hard to track down.
*
* So rely on #PyC_UnicodeFromByte since it's a tried & true way of getting paths
* that include non `utf-8` compatible characters, see: T20021. */
{
PyObject *py_argv = PyList_New(argc);
for (int i = 0; i < argc; i++) {
PyList_SET_ITEM(py_argv, i, PyC_UnicodeFromByte(argv[i]));
}
PySys_SetObject("argv", py_argv);
Py_DECREF(py_argv);
}
/* Setting the program name is important so the 'multiprocessing' module
* can launch new Python instances. */
{
const char *sys_variable = "executable";
char program_path[FILE_MAX];
if (BKE_appdir_program_python_search(
program_path, sizeof(program_path), PY_MAJOR_VERSION, PY_MINOR_VERSION)) {
PyObject *py_program_path = PyC_UnicodeFromByte(program_path);
PySys_SetObject(sys_variable, py_program_path);
Py_DECREF(py_program_path);
}
else {
fprintf(stderr,
"Unable to find the python binary, "
"the multiprocessing module may not be functional!\n");
PySys_SetObject(sys_variable, Py_None);
if (!has_python_executable) {
PySys_SetObject("executable", Py_None);
}
}
@ -447,8 +497,9 @@ void BPY_python_start(bContext *C, int argc, const char **argv)
/* py module runs atexit when bpy is freed */
BPY_atexit_register(); /* this can init any time */
py_tstate = PyGILState_GetThisThreadState();
PyEval_ReleaseThread(py_tstate);
/* Free the lock acquired (implicitly) when Python is initialized. */
PyEval_ReleaseThread(PyGILState_GetThisThreadState());
#endif
#ifdef WITH_PYTHON_MODULE