Add sys.meta_path entry for APE zip store (#425)

third_party/python/Include/import.h

@@ -73,6 +73,8 @@ PyObject * PyImport_GetImporter(PyObject *path);
 PyObject * PyImport_Import(PyObject *name);
 PyObject * PyImport_ReloadModule(PyObject *m);
 void PyImport_Cleanup(void);
+void _PyImportLookupTables_Init(void);
+void _PyImportLookupTables_Cleanup(void);
 #if !defined(Py_LIMITED_API) || Py_LIMITED_API+0 >= 0x03030000
 int PyImport_ImportFrozenModuleObject(
     PyObject *name

third_party/python/Lib/importlib/_bootstrap.py

@@ -226,7 +226,7 @@ def _verbose_message(message, *args, verbosity=1):
 def _requires_builtin(fxn):
     """Decorator to verify the named module is built-in."""
     def _requires_builtin_wrapper(self, fullname):
-        if fullname not in BUILTIN_MODULE_NAMES:
+        if not _imp.is_builtin(fullname):
             raise ImportError('{!r} is not a built-in module'.format(fullname),
                               name=fullname)
         return fxn(self, fullname)
@@ -631,7 +631,7 @@ class BuiltinImporter:
     def find_spec(cls, fullname, path=None, target=None):
         if path is not None:
             return None
-        if fullname in BUILTIN_MODULE_NAMES:
+        if _imp.is_builtin(fullname):
             return spec_from_loader(fullname, cls, origin='built-in')
         else:
             return None
@@ -651,7 +651,7 @@ class BuiltinImporter:
     @classmethod
     def create_module(self, spec):
         """Create a built-in module"""
-        if spec.name not in BUILTIN_MODULE_NAMES:
+        if not _imp.is_builtin(spec.name):
             raise ImportError('{!r} is not a built-in module'.format(spec.name),
                               name=spec.name)
         return _call_with_frames_removed(_imp.create_builtin, spec)
@@ -871,7 +871,7 @@ def _find_and_load_unlocked(name, import_):
             msg = (_ERR_MSG + '; {!r} is not a package').format(name, parent)
             raise ModuleNotFoundError(msg, name=name) from None
     spec = _find_spec(name, path)
-    if spec is None and name in BUILTIN_MODULE_NAMES:
+    if spec is None and _imp.is_builtin(name):
         # If this module is a C extension, the interpreter
         # expects it to be a shared object located in path,
         # and returns spec is None because it was not found.
@@ -1033,6 +1033,21 @@ def _builtin_from_name(name):
         raise ImportError('no built-in module named ' + name)
     return _load_unlocked(spec)
 
+def _get_builtin_spec(name):
+    # called from CosmoImporter in import.c
+    return ModuleSpec(name, BuiltinImporter, origin="built-in", is_package=False)
+
+def _get_frozen_spec(name, is_package):
+    # called from CosmoImporter in import.c
+    return ModuleSpec(name, FrozenImporter, origin="frozen", is_package=is_package)
+
+def _get_zipstore_spec(name, loader, origin, is_package):
+    # called from CosmoImporter in import.c
+    spec = ModuleSpec(name, loader, origin=origin, is_package=is_package)
+    spec.has_location = True
+    if is_package:
+        spec.submodule_search_locations = [origin.rpartition("/")[0]]
+    return spec
 
 def _setup(sys_module, _imp_module):
     """Setup importlib by importing needed built-in modules and injecting them
@@ -1042,16 +1057,15 @@ def _setup(sys_module, _imp_module):
     modules, those two modules must be explicitly passed in.
 
     """
-    global _imp, sys, BUILTIN_MODULE_NAMES
+    global _imp, sys
     _imp = _imp_module
     sys = sys_module
-    BUILTIN_MODULE_NAMES = frozenset(sys.builtin_module_names)
 
     # Set up the spec for existing builtin/frozen modules.
     module_type = type(sys)
     for name, module in sys.modules.items():
         if isinstance(module, module_type):
-            if name in BUILTIN_MODULE_NAMES:
+            if _imp.is_builtin(name):
                 loader = BuiltinImporter
             elif _imp.is_frozen(name):
                 loader = FrozenImporter
@@ -1072,8 +1086,9 @@ def _install(sys_module, _imp_module):
     """Install importlib as the implementation of import."""
     _setup(sys_module, _imp_module)
 
-    sys.meta_path.append(BuiltinImporter)
-    sys.meta_path.append(FrozenImporter)
+    sys.meta_path.append(_imp_module.CosmoImporter)
+    # sys.meta_path.append(BuiltinImporter)
+    # sys.meta_path.append(FrozenImporter)
 
     global _bootstrap_external
     import _frozen_importlib_external

third_party/python/Lib/importlib/_bootstrap_external.py

@@ -1346,10 +1346,10 @@ def _get_supported_file_loaders():
 
     Each item is a tuple (loader, suffixes).
     """
-    extensions = ExtensionFileLoader, _imp.extension_suffixes()
+    # extensions = ExtensionFileLoader, _imp.extension_suffixes()
     source = SourceFileLoader, SOURCE_SUFFIXES
     bytecode = SourcelessFileLoader, BYTECODE_SUFFIXES
-    return [bytecode, extensions, source]
+    return [source, bytecode] #, extensions]
 
 def _setup(_bootstrap_module):
     """Setup the path-based importers for importlib by importing needed

third_party/python/Lib/io.py

@@ -92,8 +92,6 @@ for klass in (StringIO, TextIOWrapper):
 del klass
 
 try:
-    from _io import _WindowsConsoleIO
-except ImportError:
+    RawIOBase.register(_io._WindowsConsoleIO)
+except AttributeError:
     pass
-else:
-    RawIOBase.register(_WindowsConsoleIO)

third_party/python/Lib/ntpath.py

@@ -20,6 +20,7 @@ devnull = 'nul'
 import os
 import sys
 import stat
+import posix
 import genericpath
 from genericpath import *
 
@@ -276,8 +277,8 @@ def lexists(path):
 # common case: drive letter roots. The alternative which uses GetVolumePathName
 # fails if the drive letter is the result of a SUBST.
 try:
-    from posix import _getvolumepathname
-except ImportError:
+    _getvolumepathname = posix._getvolumepathname
+except AttributeError:
     _getvolumepathname = None
 def ismount(path):
     """Test whether a path is a mount point (a drive root, the root of a
@@ -534,9 +535,9 @@ def _abspath_fallback(path):
 
 # Return an absolute path.
 try:
-    from posix import _getfullpathname
+    _getfullpathname = posix._getfullpathname
 
-except ImportError: # not running on Windows - mock up something sensible
+except AttributeError: # not running on Windows - mock up something sensible
     abspath = _abspath_fallback
 
 else:  # use native Windows method on Windows
@@ -664,7 +665,7 @@ try:
     # GetFinalPathNameByHandle is available starting with Windows 6.0.
     # Windows XP and non-Windows OS'es will mock _getfinalpathname.
     if sys.getwindowsversion()[:2] >= (6, 0):
-        from posix import _getfinalpathname
+        _getfinalpathname = posix._getfinalpathname
     else:
         raise ImportError
 except (AttributeError, ImportError, OSError):
@@ -681,7 +682,7 @@ try:
     # attribute to tell whether or not the path is a directory.
     # This is overkill on Windows - just pass the path to GetFileAttributes
     # and check the attribute from there.
-    from posix import _isdir as isdir
-except ImportError:
+    isdir = posix._isdir
+except AttributeError:
     # Use genericpath.isdir as imported above.
     pass

third_party/python/Lib/test/test_cmd_line.py

@@ -483,6 +483,10 @@ class CmdLineTest(unittest.TestCase):
             with open(fake, "w") as f:
                 f.write("raise RuntimeError('isolated mode test')\n")
             with open(main, "w") as f:
+                f.write("import sys\n")
+                f.write("import _imp\n")
+                f.write("if sys.meta_path[0] == _imp.CosmoImporter:\n")
+                f.write("\tsys.meta_path.pop(0)\n")
                 f.write("import uuid\n")
                 f.write("print('ok')\n")
             self.assertRaises(subprocess.CalledProcessError,

third_party/python/Python/finalize.c

@@ -96,6 +96,7 @@ Py_FinalizeEx(void)
 #endif
     /* Destroy all modules */
     PyImport_Cleanup();
+    _PyImportLookupTables_Cleanup();
 
     /* Flush sys.stdout and sys.stderr (again, in case more was printed) */
     if (_Py_FlushStdFiles() < 0) {

third_party/python/Python/import.c

@@ -4,11 +4,13 @@
 │ Python 3                                                                     │
 │ https://docs.python.org/3/license.html                                       │
 ╚─────────────────────────────────────────────────────────────────────────────*/
+#include "libc/alg/alg.h"
 #include "libc/bits/bits.h"
 #include "libc/calls/calls.h"
 #include "libc/calls/struct/stat.h"
 #include "libc/calls/struct/stat.macros.h"
 #include "libc/fmt/conv.h"
+#include "libc/macros.internal.h"
 #include "libc/runtime/gc.h"
 #include "libc/x/x.h"
 #include "libc/sysv/consts/o.h"
@@ -82,9 +84,6 @@ static PyObject *extensions = NULL;
 
 static PyObject *initstr = NULL;
 
-static struct stat stinfo;
-_Py_IDENTIFIER(__builtins__);
-_Py_IDENTIFIER(_load_module_shim);
 /*[clinic input]
 module _imp
 [clinic start generated code]*/
@@ -93,6 +92,27 @@ module _imp
 #include "third_party/python/Python/clinic/import.inc"
 
 /* Initialize things */
+typedef struct {
+  const char *name;
+  union {
+    const struct _inittab *tab;
+    const struct _frozen *frz;
+  };
+} initentry;
+
+typedef struct {
+  size_t n;
+  initentry *entries;
+} Lookup;
+
+static Lookup Builtins_Lookup = {.n = 0, .entries = NULL};
+static Lookup Frozens_Lookup = {.n = 0, .entries = NULL};
+
+static int cmp_initentry(const void *_x, const void *_y) {
+    const initentry *x = _x;
+    const initentry *y = _y;
+    return strcmp(x->name, y->name);
+}
 
 void
 _PyImport_Init(void)
@@ -106,6 +126,41 @@ _PyImport_Init(void)
         Py_FatalError("Can't backup builtins dict");
 }
 
+void _PyImportLookupTables_Init(void) {
+    size_t i, n;
+    if (Builtins_Lookup.entries == NULL) {
+        for(n=0; PyImport_Inittab[n].name; n++);
+        Builtins_Lookup.n = n;
+        Builtins_Lookup.entries = malloc(sizeof(initentry) * n);
+        for(i=0; i < n; i++) {
+            Builtins_Lookup.entries[i].name = PyImport_Inittab[i].name;
+            Builtins_Lookup.entries[i].tab = &(PyImport_Inittab[i]);
+        }
+        qsort(Builtins_Lookup.entries, Builtins_Lookup.n, sizeof(initentry), cmp_initentry);
+    }
+    if (Frozens_Lookup.entries == NULL) {
+        for(n=0; PyImport_FrozenModules[n].name; n++);
+        Frozens_Lookup.n = n;
+        Frozens_Lookup.entries = malloc(sizeof(initentry) * n);
+        for(i=0; i<n; i++) {
+            Frozens_Lookup.entries[i].name = PyImport_FrozenModules[i].name;
+            Frozens_Lookup.entries[i].frz = &(PyImport_FrozenModules[i]);
+        }
+        qsort(Frozens_Lookup.entries, Frozens_Lookup.n, sizeof(initentry), cmp_initentry);
+    }
+}
+
+void _PyImportLookupTables_Cleanup(void) {
+    if (Builtins_Lookup.entries != NULL) {
+        free(Builtins_Lookup.entries);
+        Builtins_Lookup.entries = NULL;
+    }
+    if (Frozens_Lookup.entries != NULL) {
+        free(Frozens_Lookup.entries);
+        Frozens_Lookup.entries = NULL;
+    }
+}
+
 void
 _PyImportHooks_Init(void)
 {
@@ -536,7 +591,6 @@ PyImport_Cleanup(void)
 
     /* Once more */
     _PyGC_CollectNoFail();
-
 #undef CLEAR_MODULE
 #undef STORE_MODULE_WEAKREF
 }
@@ -807,6 +861,7 @@ PyImport_ExecCodeModuleWithPathnames(const char *name, PyObject *co,
                                      const char *pathname,
                                      const char *cpathname)
 {
+    struct stat stinfo;
     PyObject *m = NULL;
     PyObject *nameobj, *pathobj = NULL, *cpathobj = NULL, *external= NULL;
 
@@ -997,14 +1052,16 @@ static const struct _frozen * find_frozen(PyObject *);
 static int
 is_builtin(PyObject *name)
 {
-    int i;
-    for (i = 0; PyImport_Inittab[i].name != NULL; i++) {
-        if (_PyUnicode_EqualToASCIIString(name, PyImport_Inittab[i].name)) {
-            if (PyImport_Inittab[i].initfunc == NULL)
-                return -1;
-            else
-                return 1;
-        }
+    initentry key;
+    initentry *res;
+    key.name = PyUnicode_AsUTF8(name);
+    key.tab = NULL;
+    if(!name || !key.name)
+        return 0;
+    res = bsearch(&key, Builtins_Lookup.entries, Builtins_Lookup.n, sizeof(initentry), cmp_initentry);
+    if (res) {
+        if (res->tab->initfunc == NULL) return -1;
+        return 1;
     }
     return 0;
 }
@@ -1095,6 +1152,8 @@ _imp_create_builtin(PyObject *module, PyObject *spec)
 /*[clinic end generated code: output=ace7ff22271e6f39 input=37f966f890384e47]*/
 {
     struct _inittab *p;
+    initentry key;
+    initentry *res;
     PyObject *name;
     char *namestr;
     PyObject *mod;
@@ -1117,38 +1176,41 @@ _imp_create_builtin(PyObject *module, PyObject *spec)
         return NULL;
     }
 
-    for (p = PyImport_Inittab; p->name != NULL; p++) {
+    key.name = namestr;
+    key.tab = NULL;
+    res = bsearch(&key, Builtins_Lookup.entries, Builtins_Lookup.n, sizeof(initentry), cmp_initentry);
+
+    if (res != NULL) {
+        p = res->tab;
         PyModuleDef *def;
-        if (_PyUnicode_EqualToASCIIString(name, p->name)) {
-            if (p->initfunc == NULL) {
-                /* Cannot re-init internal module ("sys" or "builtins") */
-                mod = PyImport_AddModule(namestr);
-                Py_DECREF(name);
-                return mod;
-            }
-            mod = (*p->initfunc)();
-            if (mod == NULL) {
+        if (p->initfunc == NULL) {
+            /* Cannot re-init internal module ("sys" or "builtins") */
+            mod = PyImport_AddModule(namestr);
+            Py_DECREF(name);
+            return mod;
+        }
+        mod = (*p->initfunc)();
+        if (mod == NULL) {
+            Py_DECREF(name);
+            return NULL;
+        }
+        if (PyObject_TypeCheck(mod, &PyModuleDef_Type)) {
+            Py_DECREF(name);
+            return PyModule_FromDefAndSpec((PyModuleDef*)mod, spec);
+        } else {
+            /* Remember pointer to module init function. */
+            def = PyModule_GetDef(mod);
+            if (def == NULL) {
                 Py_DECREF(name);
                 return NULL;
             }
-            if (PyObject_TypeCheck(mod, &PyModuleDef_Type)) {
+            def->m_base.m_init = p->initfunc;
+            if (_PyImport_FixupExtensionObject(mod, name, name) < 0) {
                 Py_DECREF(name);
-                return PyModule_FromDefAndSpec((PyModuleDef*)mod, spec);
-            } else {
-                /* Remember pointer to module init function. */
-                def = PyModule_GetDef(mod);
-                if (def == NULL) {
-                    Py_DECREF(name);
-                    return NULL;
-                }
-                def->m_base.m_init = p->initfunc;
-                if (_PyImport_FixupExtensionObject(mod, name, name) < 0) {
-                    Py_DECREF(name);
-                    return NULL;
-                }
-                Py_DECREF(name);
-                return mod;
+                return NULL;
             }
+            Py_DECREF(name);
+            return mod;
         }
     }
     Py_DECREF(name);
@@ -1161,18 +1223,20 @@ _imp_create_builtin(PyObject *module, PyObject *spec)
 static const struct _frozen *
 find_frozen(PyObject *name)
 {
-    const struct _frozen *p;
+    initentry key;
+    initentry *res;
 
     if (name == NULL)
         return NULL;
 
-    for (p = PyImport_FrozenModules; ; p++) {
-        if (p->name == NULL)
-            return NULL;
-        if (_PyUnicode_EqualToASCIIString(name, p->name))
-            break;
+    key.name = PyUnicode_AsUTF8(name);
+    key.frz = NULL;
+
+    res = bsearch(&key, Frozens_Lookup.entries, Frozens_Lookup.n, sizeof(initentry), cmp_initentry);
+    if (res && res->frz->name != NULL) {
+        return res->frz;
     }
-    return p;
+    return NULL;
 }
 
 static PyObject *
@@ -2057,6 +2121,7 @@ dump buffer
 /*[clinic end generated code: output=da39a3ee5e6b4b0d input=524ce2e021e4eba6]*/
 
 static PyObject *_check_path_mode(const char *path, uint32_t mode) {
+  struct stat stinfo;
   if (stat(path, &stinfo)) Py_RETURN_FALSE;
   if ((stinfo.st_mode & S_IFMT) == mode) Py_RETURN_TRUE;
   Py_RETURN_FALSE;
@@ -2092,6 +2157,7 @@ static PyObject *_imp_path_isdir(PyObject *module, PyObject *arg) {
 PyDoc_STRVAR(_imp_path_isdir_doc, "check if path is dir");
 
 static PyObject *_imp_calc_mode(PyObject *module, PyObject *arg) {
+  struct stat stinfo;
   Py_ssize_t n;
   const char *path;
   if (!PyArg_Parse(arg, "s#:_calc_mode", &path, &n)) return 0;
@@ -2101,6 +2167,7 @@ static PyObject *_imp_calc_mode(PyObject *module, PyObject *arg) {
 PyDoc_STRVAR(_imp_calc_mode_doc, "return stat.st_mode of path");
 
 static PyObject *_imp_calc_mtime_and_size(PyObject *module, PyObject *arg) {
+  struct stat stinfo;
   Py_ssize_t n;
   const char *path;
   if (!PyArg_Parse(arg, "z#:_calc_mtime_and_size", &path, &n)) return 0;
@@ -2145,17 +2212,24 @@ static PyObject *_imp_write_atomic(PyObject *module, PyObject **args,
   Py_buffer data = {NULL, NULL};
   uint32_t mode = 0666;
   int fd;
+  int failure = 0;
   if (!_PyArg_ParseStack(args, nargs, "s#y*|I:_write_atomic", &path, &n, &data,
                          &mode))
-    return 0;
+    goto end;
   mode &= 0666;
-  if ((fd = open(path, O_EXCL | O_CREAT | O_WRONLY, mode)) == -1 ||
-      write(fd, data.buf, data.len) == -1) {
-    PyErr_Format(PyExc_OSError, "");
-    if (data.obj) PyBuffer_Release(&data);
-    return 0;
+  if ((fd = open(path, O_CREAT | O_WRONLY | O_TRUNC, mode)) == -1) {
+    failure = 1;
+    PyErr_Format(PyExc_OSError, "failed to create file: %s\n", path);
+    goto end;
   }
+  if (write(fd, data.buf, data.len) == -1) {
+    failure = 1;
+    PyErr_Format(PyExc_OSError, "failed to write to file: %s\n", path);
+    goto end;
+  }
+end:
   if (data.obj) PyBuffer_Release(&data);
+  if (failure) return 0;
   Py_RETURN_NONE;
 }
 PyDoc_STRVAR(_imp_write_atomic_doc, "atomic write to a file");
@@ -2290,6 +2364,7 @@ static PyObject *_imp_cache_from_source(PyObject *module, PyObject **args, Py_ss
 PyDoc_STRVAR(_imp_cache_from_source_doc, "given a .py filename, return .pyc");
 
 static PyObject *_imp_source_from_cache(PyObject *module, PyObject *arg) {
+  struct stat stinfo;
   char *path = NULL;
   Py_ssize_t pathlen = 0;
   if (!PyArg_Parse(PyOS_FSPath(arg), "z#:source_from_cache", &path, &pathlen))
@@ -2309,10 +2384,12 @@ static PyObject *_imp_source_from_cache(PyObject *module, PyObject *arg) {
 PyDoc_STRVAR(_imp_source_from_cache_doc, "given a .pyc filename, return .py");
 
 typedef struct {
-  PyObject_HEAD char *name;
+  PyObject_HEAD
+  char *name;
   char *path;
   Py_ssize_t namelen;
   Py_ssize_t pathlen;
+  Py_ssize_t present;
 } SourcelessFileLoader;
 
 static PyTypeObject SourcelessFileLoaderType;
@@ -2327,6 +2404,7 @@ static SourcelessFileLoader *SFLObject_new(PyObject *cls, PyObject *args,
   obj->path = NULL;
   obj->namelen = 0;
   obj->pathlen = 0;
+  obj->present = 0;
   return obj;
 }
 
@@ -2366,6 +2444,7 @@ static int SFLObject_init(SourcelessFileLoader *self, PyObject *args,
       // TODO: should this be via PyMem_RawMalloc?
       self->name = strndup(name, namelen);
       self->path = strndup(path, pathlen);
+      self->present = 0;
   }
   return result;
 }
@@ -2401,6 +2480,7 @@ static PyObject *SFLObject_get_source(SourcelessFileLoader *self,
 }
 
 static PyObject *SFLObject_get_code(SourcelessFileLoader *self, PyObject *arg) {
+  struct stat stinfo;
   char bytecode_header[12] = {0};
   int32_t magic = 0;
   size_t headerlen;
@@ -2420,7 +2500,8 @@ static PyObject *SFLObject_get_code(SourcelessFileLoader *self, PyObject *arg) {
                  self->name, name);
     goto exit;
   }
-  if (stat(self->path, &stinfo) || !(fp = fopen(self->path, "rb"))) {
+  self->present = self->present || !stat(self->path, &stinfo);
+  if (!self->present || !(fp = fopen(self->path, "rb"))) {
     PyErr_Format(PyExc_ImportError, "%s does not exist\n", self->path);
     goto exit;
   }
@@ -2439,20 +2520,24 @@ static PyObject *SFLObject_get_code(SourcelessFileLoader *self, PyObject *arg) {
                  "reached EOF while reading timestamp in %s\n", name);
     goto exit;
   }
-  if (headerlen < 12 || stinfo.st_size <= headerlen) {
+  if (headerlen < 12) {
     PyErr_Format(PyExc_ImportError, "reached EOF while size of source in %s\n",
                  name);
     goto exit;
   }
   // return _compile_bytecode(bytes_data, name=fullname, bytecode_path=path)
+  /* since we don't have the stat call sometimes, we need
+   * a different way to load the remaining bytes into file
+   */
+  /*
   rawlen = stinfo.st_size - headerlen;
   rawbuf = PyMem_RawMalloc(rawlen);
   if (rawlen != fread(rawbuf, sizeof(char), rawlen, fp)) {
     PyErr_Format(PyExc_ImportError, "reached EOF while size of source in %s\n",
                  name);
     goto exit;
-  }
-  if (!(res = PyMarshal_ReadObjectFromString(rawbuf, rawlen))) goto exit;
+  }*/
+  if (!(res = PyMarshal_ReadObjectFromFile(fp))) goto exit;
 exit:
   if (rawbuf) PyMem_RawFree(rawbuf);
   if (fp) fclose(fp);
@@ -2460,6 +2545,7 @@ exit:
 }
 
 static PyObject *SFLObject_get_data(SourcelessFileLoader *self, PyObject *arg) {
+  struct stat stinfo;
   char *name = NULL;
   char *data = NULL;
   size_t datalen = 0;
@@ -2491,6 +2577,7 @@ static PyObject *SFLObject_get_filename(SourcelessFileLoader *self,
 
 static PyObject *SFLObject_load_module(SourcelessFileLoader *self,
                                        PyObject **args, Py_ssize_t nargs) {
+  _Py_IDENTIFIER(_load_module_shim);
   char *name = NULL;
   PyObject *bootstrap = NULL;
   PyObject *fullname = NULL;
@@ -2523,6 +2610,7 @@ static PyObject *SFLObject_create_module(SourcelessFileLoader *self,
 
 static PyObject *SFLObject_exec_module(SourcelessFileLoader *self,
                                        PyObject *arg) {
+  _Py_IDENTIFIER(__builtins__);
   PyObject *module = NULL;
   PyObject *name = NULL;
   PyObject *code = NULL;
@@ -2598,10 +2686,8 @@ static PyMethodDef SFLObject_methods[] = {
 };
 
 static PyTypeObject SourcelessFileLoaderType = {
-    /* The ob_type field must be initialized in the module init function
-     * to be portable to Windows without using C++. */
-    PyVarObject_HEAD_INIT(NULL, 0).tp_name =
-        "_imp.SourcelessFileLoader",                      /*tp_name*/
+    PyVarObject_HEAD_INIT(NULL, 0)
+    .tp_name = "_imp.SourcelessFileLoader",               /*tp_name*/
     .tp_basicsize = sizeof(SourcelessFileLoader),         /*tp_basicsize*/
     .tp_dealloc = (destructor)SFLObject_dealloc,          /*tp_dealloc*/
     .tp_hash = (hashfunc)SFLObject_hash,                  /*tp_hash*/
@@ -2612,6 +2698,145 @@ static PyTypeObject SourcelessFileLoaderType = {
     .tp_new = (newfunc)SFLObject_new,                     /*tp_new*/
 };
 
+typedef struct {
+  PyObject_HEAD
+} CosmoImporter;
+
+static PyTypeObject CosmoImporterType;
+#define CosmoImporterCheck(o) (Py_TYPE(o) == &CosmoImporterType)
+
+static PyObject *CosmoImporter_find_spec(PyObject *cls, PyObject **args,
+                                         Py_ssize_t nargs, PyObject *kwargs) {
+  static const char *const _keywords[] = {"fullname", "path", "target", NULL};
+  static _PyArg_Parser _parser = {"U|OO", _keywords, 0};
+  _Py_IDENTIFIER(_get_builtin_spec);
+  _Py_IDENTIFIER(_get_frozen_spec);
+  _Py_IDENTIFIER(_get_zipstore_spec);
+
+  PyObject *fullname = NULL;
+  PyObject *path = NULL;
+  /* path is a LIST! it contains strings similar to those in sys.path,
+   * ie folders that are likely to contain a particular file.
+   * during startup the expected scenario is checking the ZIP store
+   * of the APE, so we ignore path and let these slower cases to
+   * handled by the importer objects already provided by Python. */
+  PyObject *target = NULL;
+  PyInterpreterState *interp = PyThreadState_GET()->interp;
+
+  const struct _frozen *p = NULL;
+
+  static const char basepath[] = "/zip/.python/";
+  const char *cname = NULL;
+  Py_ssize_t cnamelen = 0;
+
+  char *newpath = NULL;
+  Py_ssize_t newpathsize = 0;
+  Py_ssize_t newpathlen = 0;
+  Py_ssize_t i = 0;
+
+  SourcelessFileLoader *loader = NULL;
+  PyObject *origin = NULL;
+  int inside_zip = 0;
+  int is_package = 0;
+  int is_available = 0;
+
+  struct stat stinfo;
+
+  if (!_PyArg_ParseStackAndKeywords(args, nargs, kwargs, &_parser, &fullname,
+                                    &path, &target)) {
+    return NULL;
+  }
+
+  if (fullname == NULL) {
+    PyErr_SetString(PyExc_ImportError, "fullname not provided\n");
+    return NULL;
+  }
+
+  if ((!path || path == Py_None)) {
+    /* we do some of BuiltinImporter's work */
+    if (is_builtin(fullname) == 1) {
+      return _PyObject_CallMethodIdObjArgs(
+          interp->importlib, &PyId__get_builtin_spec, fullname, NULL);
+    }
+    /* we do some of FrozenImporter's work */
+    else if ((p = find_frozen(fullname)) != NULL) {
+      return _PyObject_CallMethodIdObjArgs(interp->importlib,
+                                           &PyId__get_frozen_spec, fullname,
+                                           PyBool_FromLong(p->size < 0), NULL);
+    }
+  }
+
+  if (!PyArg_Parse(fullname, "z#:find_spec", &cname, &cnamelen)) return 0;
+  /* before checking within the zip store,
+   * we can check cname here to skip any values
+   * of cname that we know for sure won't be there,
+   * because worst case is two failed stat calls here
+   */
+
+  newpathsize = sizeof(basepath) + cnamelen + sizeof("/__init__.pyc") + 1;
+  newpath = _gc(malloc(newpathsize));
+  bzero(newpath, newpathsize);
+  /* performing a memccpy sequence equivalent to:
+   * snprintf(newpath, newpathsize, "/zip/.python/%s.pyc", cname); */
+  memccpy(newpath, basepath, '\0', newpathsize);
+  memccpy(newpath + sizeof(basepath) - 1, cname, '\0',
+          newpathsize - sizeof(basepath));
+  memccpy(newpath + sizeof(basepath) + cnamelen - 1, ".pyc", '\0',
+          newpathsize - (sizeof(basepath) + cnamelen));
+
+  /* if cname part of newpath has '.' (e.g. encodings.utf_8) convert them to '/'
+   */
+  for (i = sizeof(basepath); i < sizeof(basepath) + cnamelen - 1; i++) {
+    if (newpath[i] == '.') newpath[i] = '/';
+  }
+
+  is_available = inside_zip || !stat(newpath, &stinfo);
+  if (is_package || !is_available) {
+    memccpy(newpath + sizeof(basepath) + cnamelen - 1, "/__init__.pyc", '\0',
+            newpathsize);
+    is_available = is_available || !stat(newpath, &stinfo);
+    is_package = 1;
+  }
+
+  if (is_available) {
+    newpathlen = strlen(newpath);
+    loader = SFLObject_new(NULL, NULL, NULL);
+    origin = PyUnicode_FromStringAndSize(newpath, newpathlen);
+    if (loader == NULL || origin == NULL) {
+      return NULL;
+    }
+    loader->name = strdup(cname);
+    loader->namelen = cnamelen;
+    loader->path = strdup(newpath);
+    loader->pathlen = newpathlen;
+    loader->present = 1; /* this means we avoid atleast one stat call (the one
+                            in SFLObject_get_code) */
+    return _PyObject_CallMethodIdObjArgs(interp->importlib,
+                                         &PyId__get_zipstore_spec, fullname,
+                                         (PyObject *)loader, (PyObject *)origin,
+                                         PyBool_FromLong(is_package), NULL);
+  }
+
+  Py_RETURN_NONE;
+}
+
+static PyMethodDef CosmoImporter_methods[] = {
+    {"find_spec", (PyCFunction)CosmoImporter_find_spec,
+     METH_FASTCALL | METH_KEYWORDS | METH_CLASS, PyDoc_STR("")},
+    {NULL, NULL}  // sentinel
+};
+
+static PyTypeObject CosmoImporterType = {
+    PyVarObject_HEAD_INIT(NULL, 0)
+    .tp_name = "_imp.CosmoImporter", /* tp_name */
+    .tp_dealloc = 0,
+    .tp_basicsize = sizeof(CosmoImporter),                /* tp_basicsize */
+    .tp_flags = Py_TPFLAGS_DEFAULT,                       /* tp_flags */
+    .tp_methods = CosmoImporter_methods,                  /* tp_methods */
+    .tp_init = 0,
+    .tp_new = 0,
+};
+
 PyDoc_STRVAR(doc_imp,
 "(Extremely) low-level import machinery bits as used by importlib and imp.");
 
@@ -2673,8 +2898,16 @@ PyInit_imp(void)
 
     if (PyType_Ready(&SourcelessFileLoaderType) < 0)
         goto failure;
-    PyModule_AddObject(m, "SourcelessFileLoader", (PyObject*)&SourcelessFileLoaderType);
-
+    if (PyModule_AddObject(m, "SourcelessFileLoader", (PyObject*)&SourcelessFileLoaderType) < 0)
+        goto failure;
+    if (PyType_Ready(&CosmoImporterType) < 0)
+        goto failure;
+    if (PyModule_AddObject(m, "CosmoImporter", (PyObject*)&CosmoImporterType) < 0)
+        goto failure;
+    /* test_atexit segfaults without the below incref, but
+     * I'm not supposed to Py_INCREF a static PyTypeObject, so
+     * what's going on? */
+    Py_INCREF(&CosmoImporterType);
     return m;
   failure:
     Py_XDECREF(m);

third_party/python/Python/pylifecycle.c

@@ -248,6 +248,7 @@ _Py_InitializeEx_Private(int install_sigs, int install_importlib)
     PySys_SetObject("__stderr__", pstderr);
     Py_DECREF(pstderr);
 
+    _PyImportLookupTables_Init();
     _PyImport_Init();
 
     _PyImportHooks_Init();

third_party/python/python.mk

@@ -441,6 +441,7 @@ THIRD_PARTY_PYTHON_STAGE1_A_SRCS =					\
 
 THIRD_PARTY_PYTHON_STAGE1_A_DIRECTDEPS =				\
 	DSP_SCALE							\
+	LIBC_ALG							\
 	LIBC_BITS							\
 	LIBC_CALLS							\
 	LIBC_FMT							\