From aa75de8bafded86176c30cc361475af47f485c03 Mon Sep 17 00:00:00 2001 From: ahgamut <41098605+ahgamut@users.noreply.github.com> Date: Sun, 24 Apr 2022 05:18:39 +0530 Subject: [PATCH] load C extensions of external packages correctly At present, all C extension modules loaded in the APE Python are built-in modules -- they are compiled into the interpreter instead of being separate shared objects in some folder. In the Python3.6 stdlib, the modules usually follow a naming convention like: X imports a module called _X, which is built from Modules/_X.c. (example: json imports _json, which is built from Modules/_json.c). However, external Python packages follow a different naming convention: X imports a module called ._Y, which is available as a shared object located in the same directory as X. (example: markupsafe imports from ._speedups, expecting a _speedups.so to be available within the markupsafe folder). This change makes the interpreter load the required module as a builtin (if present) if the requisite shared object/python file is not found. The benefit of this is that the external packages can keep the same naming convention for their C extensions, and once built, the APE will load those extensions without error. --- third_party/python/Lib/importlib/_bootstrap.py | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/third_party/python/Lib/importlib/_bootstrap.py b/third_party/python/Lib/importlib/_bootstrap.py index e2343dd43..20807d5d2 100644 --- a/third_party/python/Lib/importlib/_bootstrap.py +++ b/third_party/python/Lib/importlib/_bootstrap.py @@ -949,6 +949,19 @@ def _find_and_load_unlocked(name, import_): msg = (_ERR_MSG + '; {!r} is not a package').format(name, parent) raise ModuleNotFoundError(msg, name=name) from None spec = _find_spec(name, path) + if spec is None and name in sys.builtin_module_names: + # If this module is a C extension, the interpreter + # expects it to be a shared object located in path, + # and returns spec is None because it was not found. + # + # however, if it is a C extension, we can check if it + # is available using sys.builtin_module_names, + # because the APE is statically compiled. + # + # if the module is present as a builtin, we call + # BuiltinImporter with the full name (and no path) + # to create the module spec correctly. + spec = BuiltinImporter.find_spec(name) if spec is None: raise ModuleNotFoundError(_ERR_MSG.format(name), name=name) else: