From 60eb34509b7faf933a11088acf4a4da9e5c689a5 Mon Sep 17 00:00:00 2001 From: ahgamut Date: Thu, 23 Feb 2023 10:08:42 +0530 Subject: [PATCH] quick addition of cosmo pthreads to python.com - enable WITH_THREAD and _POSIX_THREADS - add headers everywhere - breaks only two tests (faulthandler and signal) - disabled terminal completion because it causes segfaults for some reason (probably could not get the current thread) --- .../python/Lib/importlib/_bootstrap.py | 40 ++++---- .../Lib/importlib/_bootstrap_external.py | 2 +- third_party/python/Lib/threading.py | 4 +- third_party/python/Modules/_bz2module.c | 1 + third_party/python/Modules/_io/bufferedio.c | 2 + third_party/python/Modules/_threadmodule.c | 92 ++++++++++++++++++- third_party/python/Modules/socketmodule.c | 1 + third_party/python/Python/ceval.c | 1 + third_party/python/Python/ceval_gil.inc | 4 + third_party/python/Python/finalize.c | 6 ++ third_party/python/Python/pylifecycle.c | 5 +- third_party/python/Python/thread.c | 11 ++- third_party/python/pyconfig.h | 8 +- third_party/python/python.c | 18 ++++ third_party/python/python.mk | 3 + third_party/python/runpythonmodule.c | 2 +- 16 files changed, 166 insertions(+), 34 deletions(-) diff --git a/third_party/python/Lib/importlib/_bootstrap.py b/third_party/python/Lib/importlib/_bootstrap.py index 53391aaed..a72b26714 100644 --- a/third_party/python/Lib/importlib/_bootstrap.py +++ b/third_party/python/Lib/importlib/_bootstrap.py @@ -520,31 +520,29 @@ def _module_repr_from_spec(spec): else: return ''.format(spec.name, spec.origin) - -# Used by importlib.reload() and _load_module_shim(). def _exec(spec, module): """Execute the spec's specified module in an existing module's namespace.""" name = spec.name - if sys.modules.get(name) is not module: - msg = 'module {!r} not in sys.modules'.format(name) - raise ImportError(msg, name=name) - if spec.loader is None: - if spec.submodule_search_locations is None: - raise ImportError('missing loader', name=spec.name) - # namespace package + with _ModuleLockManager(name): + if sys.modules.get(name) is not module: + msg = 'module {!r} not in sys.modules'.format(name) + raise ImportError(msg, name=name) + if spec.loader is None: + if spec.submodule_search_locations is None: + raise ImportError('missing loader', name=spec.name) + # namespace package + _init_module_attrs(spec, module, override=True) + return module _init_module_attrs(spec, module, override=True) - return module - _init_module_attrs(spec, module, override=True) - if not hasattr(spec.loader, 'exec_module'): - # (issue19713) Once BuiltinImporter and ExtensionFileLoader - # have exec_module() implemented, we can add a deprecation - # warning here. - spec.loader.load_module(name) - else: - spec.loader.exec_module(module) + if not hasattr(spec.loader, 'exec_module'): + # (issue19713) Once BuiltinImporter and ExtensionFileLoader + # have exec_module() implemented, we can add a deprecation + # warning here. + spec.loader.load_module(name) + else: + spec.loader.exec_module(module) return sys.modules[name] - def _load_backward_compatible(spec): # (issue19713) Once BuiltinImporter and ExtensionFileLoader # have exec_module() implemented, we can add a deprecation @@ -606,8 +604,8 @@ def _load(spec): clobbered. """ - return _load_unlocked(spec) - + with _ModuleLockManager(spec.name): + return _load_unlocked(spec) # Loaders ##################################################################### diff --git a/third_party/python/Lib/importlib/_bootstrap_external.py b/third_party/python/Lib/importlib/_bootstrap_external.py index f8ab20b54..d5edc8eab 100644 --- a/third_party/python/Lib/importlib/_bootstrap_external.py +++ b/third_party/python/Lib/importlib/_bootstrap_external.py @@ -1388,6 +1388,7 @@ def _setup(_bootstrap_module): "builtins", "marshal", "posix", + "_thread", "_weakref", ): self_mod_dict[name] = sys.modules.get( @@ -1400,7 +1401,6 @@ def _setup(_bootstrap_module): self_mod_dict["_os"] = sys.modules.get(builtin_os, builtin_from_name(builtin_os)) self_mod_dict["path_sep"] = path_separators[0] self_mod_dict["path_separators"] = "".join(path_separators) - self_mod_dict["_thread"] = None # Constants EXTENSION_SUFFIXES.extend(_imp.extension_suffixes()) diff --git a/third_party/python/Lib/threading.py b/third_party/python/Lib/threading.py index f93be3022..ecbaa6973 100644 --- a/third_party/python/Lib/threading.py +++ b/third_party/python/Lib/threading.py @@ -1323,7 +1323,7 @@ def enumerate(): return list(_active.values()) + list(_limbo.values()) -from _dummy_thread import stack_size +from _thread import stack_size # Create the main thread object, # and make it available for the interpreter @@ -1371,7 +1371,7 @@ def main_thread(): # module, or from the python fallback try: - from _dummy_thread import _local as local + from _thread import _local as local except ImportError: from _threading_local import local diff --git a/third_party/python/Modules/_bz2module.c b/third_party/python/Modules/_bz2module.c index 164d10f00..939583633 100644 --- a/third_party/python/Modules/_bz2module.c +++ b/third_party/python/Modules/_bz2module.c @@ -15,6 +15,7 @@ #include "third_party/python/Include/pyerrors.h" #include "third_party/python/Include/pymacro.h" #include "third_party/python/Include/pymem.h" +#include "third_party/python/Include/pythread.h" #include "third_party/python/Include/structmember.h" #include "third_party/python/Include/yoink.h" /* clang-format off */ diff --git a/third_party/python/Modules/_io/bufferedio.c b/third_party/python/Modules/_io/bufferedio.c index e3f4c27fc..c357dfd91 100644 --- a/third_party/python/Modules/_io/bufferedio.c +++ b/third_party/python/Modules/_io/bufferedio.c @@ -9,6 +9,7 @@ #include "third_party/python/Include/abstract.h" #include "third_party/python/Include/boolobject.h" #include "third_party/python/Include/bytesobject.h" +#include "third_party/python/Include/ceval.h" #include "third_party/python/Include/descrobject.h" #include "third_party/python/Include/longobject.h" #include "third_party/python/Include/memoryobject.h" @@ -16,6 +17,7 @@ #include "third_party/python/Include/object.h" #include "third_party/python/Include/objimpl.h" #include "third_party/python/Include/pyerrors.h" +#include "third_party/python/Include/pylifecycle.h" #include "third_party/python/Include/pymacro.h" #include "third_party/python/Include/pymem.h" #include "third_party/python/Include/pythread.h" diff --git a/third_party/python/Modules/_threadmodule.c b/third_party/python/Modules/_threadmodule.c index d3b09b39c..79c27565c 100644 --- a/third_party/python/Modules/_threadmodule.c +++ b/third_party/python/Modules/_threadmodule.c @@ -4,8 +4,91 @@ │ Python 3 │ │ https://docs.python.org/3/license.html │ ╚─────────────────────────────────────────────────────────────────────────────*/ +#include "libc/assert.h" +#include "libc/calls/calls.h" +#include "libc/calls/internal.h" +#include "libc/calls/makedev.h" +#include "libc/calls/struct/dirent.h" +#include "libc/calls/struct/iovec.h" +#include "libc/calls/struct/rusage.h" +#include "libc/calls/struct/sched_param.h" +#include "libc/calls/struct/stat.macros.h" +#include "libc/calls/struct/statvfs.h" +#include "libc/calls/struct/timespec.h" +#include "libc/calls/struct/timeval.h" +#include "libc/calls/struct/tms.h" +#include "libc/calls/struct/utsname.h" +#include "libc/calls/struct/winsize.h" +#include "libc/calls/syscall-sysv.internal.h" +#include "libc/calls/sysparam.h" +#include "libc/calls/termios.h" +#include "libc/calls/weirdtypes.h" +#include "libc/dce.h" +#include "libc/errno.h" +#include "libc/log/log.h" +#include "libc/mem/alg.h" +#include "libc/mem/gc.internal.h" +#include "libc/nt/createfile.h" +#include "libc/nt/dll.h" +#include "libc/nt/enum/creationdisposition.h" +#include "libc/nt/enum/fileflagandattributes.h" +#include "libc/nt/enum/sw.h" +#include "libc/nt/files.h" +#include "libc/nt/runtime.h" +#include "libc/runtime/dlfcn.h" +#include "libc/runtime/pathconf.h" +#include "libc/runtime/sysconf.h" +#include "libc/sock/sendfile.internal.h" +#include "libc/sock/sock.h" +#include "libc/stdio/stdio.h" +#include "libc/sysv/consts/at.h" +#include "libc/sysv/consts/dt.h" +#include "libc/sysv/consts/ex.h" +#include "libc/sysv/consts/f.h" +#include "libc/sysv/consts/grnd.h" +#include "libc/sysv/consts/o.h" +#include "libc/sysv/consts/posix.h" +#include "libc/sysv/consts/prio.h" +#include "libc/sysv/consts/s.h" +#include "libc/sysv/consts/sched.h" +#include "libc/sysv/consts/seek.h" +#include "libc/sysv/consts/sf.h" +#include "libc/sysv/consts/sicode.h" +#include "libc/sysv/consts/st.h" +#include "libc/sysv/consts/w.h" +#include "libc/sysv/consts/waitid.h" +#include "libc/sysv/errfuns.h" +#include "libc/time/struct/utimbuf.h" +#include "libc/time/time.h" +#include "libc/x/x.h" +#include "third_party/musl/lockf.h" +#include "third_party/musl/passwd.h" +#include "third_party/python/Include/abstract.h" +#include "third_party/python/Include/boolobject.h" +#include "third_party/python/Include/ceval.h" +#include "third_party/python/Include/dictobject.h" +#include "third_party/python/Include/fileobject.h" +#include "third_party/python/Include/fileutils.h" +#include "third_party/python/Include/floatobject.h" +#include "third_party/python/Include/import.h" +#include "third_party/python/Include/intrcheck.h" +#include "third_party/python/Include/longobject.h" +#include "third_party/python/Include/modsupport.h" +#include "third_party/python/Include/objimpl.h" +#include "third_party/python/Include/osmodule.h" +#include "third_party/python/Include/pyerrors.h" +#include "third_party/python/Include/pylifecycle.h" +#include "third_party/python/Include/pymacro.h" +#include "third_party/python/Include/pymem.h" +#include "third_party/python/Include/pythread.h" +#include "third_party/python/Include/pytime.h" #include "third_party/python/Include/structmember.h" +#include "third_party/python/Include/structseq.h" +#include "third_party/python/Include/sysmodule.h" +#include "third_party/python/Include/warnings.h" +#include "third_party/python/Include/weakrefobject.h" #include "third_party/python/Include/yoink.h" +#include "third_party/python/pyconfig.h" /* clang-format off */ PYTHON_PROVIDE("_thread"); @@ -1029,7 +1112,7 @@ t_bootstrap(void *boot_raw) else { PyObject *file; PyObject *exc, *value, *tb; - PySys_WriteStderr( + PySys_FormatStderr( "Unhandled exception in thread started by "); PyErr_Fetch(&exc, &value, &tb); file = _PySys_GetObjectId(&PyId_stderr); @@ -1037,7 +1120,7 @@ t_bootstrap(void *boot_raw) PyFile_WriteObject(boot->func, file, 0); else PyObject_Print(boot->func, stderr, 0); - PySys_WriteStderr("\n"); + PySys_FormatStderr("\n"); PyErr_Restore(exc, value, tb); PyErr_PrintEx(0); } @@ -1436,3 +1519,8 @@ PyInit__thread(void) PyThread_init_thread(); return m; } + +_Section(".rodata.pytab.1") const struct _inittab _PyImport_Inittab__thread = { + "_thread", + PyInit__thread, +}; diff --git a/third_party/python/Modules/socketmodule.c b/third_party/python/Modules/socketmodule.c index e95f5d79e..1ce7030c2 100644 --- a/third_party/python/Modules/socketmodule.c +++ b/third_party/python/Modules/socketmodule.c @@ -50,6 +50,7 @@ #include "third_party/python/Include/pyerrors.h" #include "third_party/python/Include/pymacro.h" #include "third_party/python/Include/pymem.h" +#include "third_party/python/Include/pythread.h" #include "third_party/python/Include/structmember.h" #include "third_party/python/Include/tupleobject.h" #include "third_party/python/Include/warnings.h" diff --git a/third_party/python/Python/ceval.c b/third_party/python/Python/ceval.c index 4458fe5bb..9f4653ff3 100644 --- a/third_party/python/Python/ceval.c +++ b/third_party/python/Python/ceval.c @@ -5,6 +5,7 @@ │ https://docs.python.org/3/license.html │ ╚─────────────────────────────────────────────────────────────────────────────*/ #define PY_LOCAL_AGGRESSIVE +#include "libc/errno.h" #include "libc/intrin/likely.h" #include "third_party/python/Include/abstract.h" #include "third_party/python/Include/boolobject.h" diff --git a/third_party/python/Python/ceval_gil.inc b/third_party/python/Python/ceval_gil.inc index 383dc0cb9..ef58897ea 100644 --- a/third_party/python/Python/ceval_gil.inc +++ b/third_party/python/Python/ceval_gil.inc @@ -4,6 +4,10 @@ │ Python 3 │ │ https://docs.python.org/3/license.html │ ╚─────────────────────────────────────────────────────────────────────────────*/ +#include "libc/calls/struct/timespec.h" +#include "libc/calls/struct/timeval.h" +#include "libc/thread/thread2.h" +#include "third_party/python/Include/pylifecycle.h" #include "third_party/python/Python/condvar.h" /* clang-format off */ diff --git a/third_party/python/Python/finalize.c b/third_party/python/Python/finalize.c index 81d45ad99..82ac66151 100644 --- a/third_party/python/Python/finalize.c +++ b/third_party/python/Python/finalize.c @@ -32,6 +32,12 @@ Locking: as above. */ +#ifdef WITH_THREAD +extern void wait_for_thread_shutdown(void); +extern void _PyGILState_Init(PyInterpreterState *, PyThreadState *); +extern void _PyGILState_Fini(void); +#endif /* WITH_THREAD */ + int Py_FinalizeEx(void) { diff --git a/third_party/python/Python/pylifecycle.c b/third_party/python/Python/pylifecycle.c index ca43dfd2e..69c8140f3 100644 --- a/third_party/python/Python/pylifecycle.c +++ b/third_party/python/Python/pylifecycle.c @@ -19,6 +19,7 @@ #include "third_party/python/Include/abstract.h" #include "third_party/python/Include/ast.h" #include "third_party/python/Include/boolobject.h" +#include "third_party/python/Include/ceval.h" #include "third_party/python/Include/code.h" #include "third_party/python/Include/codecs.h" #include "third_party/python/Include/cosmo.h" @@ -58,7 +59,7 @@ _Py_IDENTIFIER(stdout); _Py_IDENTIFIER(stderr); /* Forward */ -static void wait_for_thread_shutdown(void); +void wait_for_thread_shutdown(void); #ifdef WITH_THREAD extern void _PyGILState_Init(PyInterpreterState *, PyThreadState *); extern void _PyGILState_Fini(void); @@ -470,7 +471,7 @@ Py_EndInterpreter(PyThreadState *tstate) the threading module was imported in the first place. The shutdown routine will wait until all non-daemon "threading" threads have completed. */ -static void +void wait_for_thread_shutdown(void) { #ifdef WITH_THREAD diff --git a/third_party/python/Python/thread.c b/third_party/python/Python/thread.c index cfa9c32af..fc2b8b269 100644 --- a/third_party/python/Python/thread.c +++ b/third_party/python/Python/thread.c @@ -5,6 +5,11 @@ │ https://docs.python.org/3/license.html │ ╚─────────────────────────────────────────────────────────────────────────────*/ #include "libc/assert.h" +#include "libc/calls/struct/timespec.h" +#include "libc/calls/struct/timeval.h" +#include "libc/errno.h" +#include "libc/thread/thread2.h" +#include "third_party/python/Include/dynamic_annotations.h" #include "third_party/python/Include/pyerrors.h" #include "third_party/python/Include/pymacro.h" #include "third_party/python/Include/pymem.h" @@ -77,18 +82,18 @@ static size_t _pythread_stacksize = 0; #ifdef _POSIX_THREADS #define PYTHREAD_NAME "pthread" -#include "thread_pthread.inc" +#include "third_party/python/Python/thread_pthread.inc" #endif #ifdef NT_THREADS #define PYTHREAD_NAME "nt" -#include "thread_nt.inc" + // #include "thread_nt.inc" #endif /* #ifdef FOOBAR_THREADS -#include "thread_foobar.inc" +// #include "thread_foobar.inc" #endif */ diff --git a/third_party/python/pyconfig.h b/third_party/python/pyconfig.h index 87dd476a0..9dda213b2 100644 --- a/third_party/python/pyconfig.h +++ b/third_party/python/pyconfig.h @@ -492,7 +492,9 @@ #endif /* Define if you want to compile in rudimentary thread support */ -/* #undef WITH_THREAD */ +#ifndef WITH_THREAD +#define WITH_THREAD 1 +#endif /* Define if you want pymalloc to be disabled when running under valgrind */ /* #undef WITH_VALGRIND */ @@ -508,7 +510,9 @@ /* #undef _POSIX_1_SOURCE */ /* Define if you have POSIX threads, and your system does not define that. */ -/* #undef _POSIX_THREADS */ +#ifndef _POSIX_THREADS +#define _POSIX_THREADS +#endif /* #define _Py_MEMORY_SANITIZER */ diff --git a/third_party/python/python.c b/third_party/python/python.c index feab2bbda..bb5b3046d 100644 --- a/third_party/python/python.c +++ b/third_party/python/python.c @@ -167,6 +167,7 @@ STATIC_YOINK("_PyImport_Inittab__stat"); STATIC_YOINK("_PyImport_Inittab__struct"); STATIC_YOINK("_PyImport_Inittab__symtable"); STATIC_YOINK("_PyImport_Inittab__testcapi"); +STATIC_YOINK("_PyImport_Inittab__thread"); STATIC_YOINK("_PyImport_Inittab__tracemalloc"); STATIC_YOINK("_PyImport_Inittab_array"); STATIC_YOINK("_PyImport_Inittab_atexit"); @@ -495,6 +496,23 @@ PYTHON_YOINK("nntplib"); PYTHON_YOINK("asdl"); #ifdef WITH_THREAD +PYTHON_YOINK("_thread"); +PYTHON_YOINK("_thread.LockType"); +PYTHON_YOINK("_thread.RLock"); +PYTHON_YOINK("_thread.TIMEOUT_MAX"); +PYTHON_YOINK("_thread._count"); +PYTHON_YOINK("_thread._local"); +PYTHON_YOINK("_thread._set_sentinel"); +PYTHON_YOINK("_thread.allocate"); +PYTHON_YOINK("_thread.allocate_lock"); +PYTHON_YOINK("_thread.error"); +PYTHON_YOINK("_thread.exit"); +PYTHON_YOINK("_thread.exit_thread"); +PYTHON_YOINK("_thread.get_ident"); +PYTHON_YOINK("_thread.interrupt_main"); +PYTHON_YOINK("_thread.stack_size"); +PYTHON_YOINK("_thread.start_new"); +PYTHON_YOINK("_thread.start_new_thread"); PYTHON_YOINK("asynchat"); PYTHON_YOINK("asyncore"); PYTHON_YOINK("asyncio"); diff --git a/third_party/python/python.mk b/third_party/python/python.mk index ab6285855..7b4ffebdf 100644 --- a/third_party/python/python.mk +++ b/third_party/python/python.mk @@ -291,6 +291,7 @@ THIRD_PARTY_PYTHON_INCS = \ third_party/python/Modules/clinic/audioop.inc THIRD_PARTY_PYTHON_STAGE1_A_SRCS = \ + third_party/python/Modules/_threadmodule.c \ third_party/python/Modules/_tracemalloc.c \ third_party/python/Modules/faulthandler.c \ third_party/python/Objects/abstract.c \ @@ -408,6 +409,7 @@ THIRD_PARTY_PYTHON_STAGE1_A_SRCS = \ third_party/python/Python/symtable.c \ third_party/python/Parser/listnode.c \ third_party/python/Python/sysmodule.c \ + third_party/python/Python/thread.c \ third_party/python/Python/traceback.c \ third_party/python/Modules/unicodedata_3.2.0.c \ third_party/python/Modules/unicodedata_bidirectionalnames.c \ @@ -449,6 +451,7 @@ THIRD_PARTY_PYTHON_STAGE1_A_DIRECTDEPS = \ LIBC_NEXGEN32E \ LIBC_NT_KERNEL32 \ LIBC_RUNTIME \ + LIBC_THREAD \ LIBC_STDIO \ LIBC_STR \ LIBC_STUBS \ diff --git a/third_party/python/runpythonmodule.c b/third_party/python/runpythonmodule.c index d40daeb01..7131d5bd2 100644 --- a/third_party/python/runpythonmodule.c +++ b/third_party/python/runpythonmodule.c @@ -215,7 +215,7 @@ Complete(const char *p, linenoiseCompletions *c) static void TerminalCompletion(const char *p, linenoiseCompletions *c) { - Complete(p, c); + // Complete(p, c); if (PyErr_Occurred()) { PyErr_Clear(); }