mirrors/cosmopolitan
1
0
Fork 0
mirror of https://github.com/jart/cosmopolitan.git synced 2025-05-24 14:22:28 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 1

Improve Python and Linenoise

Browse source 
This change reinvents all the GNU Readline features I discovered that I
couldn't live without, e.g. UTF-8, CTRL-R search and CTRL-Y yanking. It
now feels just as good in terms of user interface from the subconscious
workflow perspective. It's real nice to finally have an embeddable line
reader that's actually good with a 30 kb footprint and a bsd-2 license.

This change adds a directory to the examples folder, explaining how the
new Python compiler may be used.  Some of the bugs with Python binaries
have been addressed but overall it's still a work in progress.
This commit is contained in:
Justine Tunney 2021-09-11 22:30:37 -07:00
parent ad52387b74
commit 51904e2687
35 changed files with 3541 additions and 8587 deletions
Download patch file Download diff file Expand all files Collapse all files

21
third_party/python/Lib/_bootlocale.py vendored
View file

@ -1,21 +1,2 @@
"""A minimal subset of the locale module used at interpreter startup
(imported by the _io module), in order to reduce startup time.
Don't import directly from third-party code; use the `locale` module instead!
"""
import sys
import _locale
def getpreferredencoding(do_setlocale=True):
assert not do_setlocale
result = _locale.nl_langinfo(_locale.CODESET)
if not result and sys.platform in ('darwin', 'cosmo'):
# nl_langinfo can return an empty string
# when the setting has an invalid value.
# Default to UTF-8 in that case because
# UTF-8 is the default charset on OSX and
# returning nothing will crash the
# interpreter.
result = 'UTF-8'
return result
return 'UTF-8'

5
third_party/python/Lib/re.py vendored
View file

@ -123,10 +123,7 @@ import enum
import sre_compile
import sre_parse
import functools
try:
import _locale
except ImportError:
_locale = None
import _locale
# public symbols
__all__ = [

2
third_party/python/Programs/hello.c vendored
View file

@ -1,2 +0,0 @@
#define LAUNCH "hello"
#include "third_party/python/Programs/launch.c"

2
third_party/python/Programs/httpserver.c vendored
View file

@ -1,2 +0,0 @@
#define LAUNCH "http.server"
#include "third_party/python/Programs/launch.c"

143
third_party/python/Programs/launch.c vendored
View file

@ -1,143 +0,0 @@
/*-*- mode:c;indent-tabs-mode:nil;c-basic-offset:4;tab-width:8;coding:utf-8 -*-│
│vi: set net ft=c ts=4 sts=4 sw=4 fenc=utf-8 :vi│
╞══════════════════════════════════════════════════════════════════════════════╡
│ Python 3 │
│ https://docs.python.org/3/license.html │
╚─────────────────────────────────────────────────────────────────────────────*/
#include "libc/bits/bits.h"
#include "libc/bits/safemacros.internal.h"
#include "libc/calls/calls.h"
#include "libc/dce.h"
#include "libc/errno.h"
#include "libc/log/check.h"
#include "libc/log/log.h"
#include "libc/mem/mem.h"
#include "libc/nexgen32e/rdtsc.h"
#include "libc/runtime/gc.internal.h"
#include "libc/runtime/runtime.h"
#include "libc/runtime/symbols.internal.h"
#include "libc/stdio/stdio.h"
#include "libc/str/str.h"
#include "libc/sysv/consts/fileno.h"
#include "libc/sysv/consts/sig.h"
#include "libc/unicode/locale.h"
#include "libc/x/x.h"
#include "libc/zip.h"
#include "third_party/linenoise/linenoise.h"
#include "third_party/python/Include/abstract.h"
#include "third_party/python/Include/ceval.h"
#include "third_party/python/Include/dictobject.h"
#include "third_party/python/Include/fileutils.h"
#include "third_party/python/Include/funcobject.h"
#include "third_party/python/Include/import.h"
#include "third_party/python/Include/listobject.h"
#include "third_party/python/Include/modsupport.h"
#include "third_party/python/Include/moduleobject.h"
#include "third_party/python/Include/object.h"
#include "third_party/python/Include/pydebug.h"
#include "third_party/python/Include/pyerrors.h"
#include "third_party/python/Include/pylifecycle.h"
#include "third_party/python/Include/pymem.h"
#include "third_party/python/Include/pyport.h"
#include "third_party/python/Include/pythonrun.h"
#include "third_party/python/Include/sysmodule.h"
#include "third_party/python/Include/unicodeobject.h"
#include "third_party/python/Include/yoink.h"
/* clang-format off */
#define _L(x) L##x
#define L(x) _L(x)
PYTHON_YOINK(LAUNCH);
PYTHON_YOINK("_bootlocale");
PYTHON_YOINK("_locale");
PYTHON_YOINK("encodings.aliases");
PYTHON_YOINK("encodings.latin_1");
PYTHON_YOINK("encodings.utf_8");
PYTHON_YOINK("launchpy");
const struct _frozen *PyImport_FrozenModules = _PyImport_FrozenModules;
struct _inittab *PyImport_Inittab = _PyImport_Inittab;
static int LaunchModule(wchar_t *modname)
{
PyObject *module, *runpy, *runmodule, *runargs, *result;
runpy = PyImport_ImportModule("launchpy");
if (runpy == NULL) {
fprintf(stderr, "Could not import launchpy module\n");
PyErr_Print();
return -1;
}
runmodule = PyObject_GetAttrString(runpy, "run_module_as_main");
if (runmodule == NULL) {
fprintf(stderr, "Could not access launchpy.run_module_as_main\n");
PyErr_Print();
Py_DECREF(runpy);
return -1;
}
module = PyUnicode_FromWideChar(modname, wcslen(modname));
if (module == NULL) {
fprintf(stderr, "Could not convert module name to unicode\n");
PyErr_Print();
Py_DECREF(runpy);
Py_DECREF(runmodule);
return -1;
}
runargs = Py_BuildValue("(O)", module);
if (runargs == NULL) {
fprintf(stderr,
"Could not create arguments for runpy._run_module_as_main\n");
PyErr_Print();
Py_DECREF(runpy);
Py_DECREF(runmodule);
Py_DECREF(module);
return -1;
}
result = PyObject_Call(runmodule, runargs, NULL);
if (result == NULL) {
PyErr_Print();
}
Py_DECREF(runpy);
Py_DECREF(runmodule);
Py_DECREF(module);
Py_DECREF(runargs);
if (result == NULL) {
return -1;
}
Py_DECREF(result);
return 0;
}
int
main(int argc, char **argv)
{
int i, res;
char *oldloc;
wchar_t **argv_copy;
wchar_t **argv_copy2;
_PyMem_SetupAllocators("malloc");
argv_copy = gc(malloc(sizeof(wchar_t*) * (argc+1)));
argv_copy2 = gc(malloc(sizeof(wchar_t*) * (argc+1)));
oldloc = _PyMem_RawStrdup(setlocale(LC_ALL, NULL));
setlocale(LC_ALL, "");
for (i = 0; i < argc; i++) {
argv_copy2[i] = argv_copy[i] = gc(utf8toutf32(argv[i], -1, 0));
}
argv_copy2[argc] = argv_copy[argc] = NULL;
setlocale(LC_ALL, oldloc);
PyMem_RawFree(oldloc);
_PyRandom_Init();
Py_FrozenFlag++;
Py_NoSiteFlag++;
/* Py_VerboseFlag++; */
Py_NoUserSiteDirectory++;
Py_IgnoreEnvironmentFlag++;
Py_DontWriteBytecodeFlag++;
Py_Initialize();
Py_LimitedPath();
PySys_SetArgvEx(argc, argv_copy, 0);
res = LaunchModule(L(LAUNCH)) != 0;
_PyMem_SetupAllocators("malloc");
return res;
}

2
third_party/python/Programs/repl.c vendored
View file

@ -153,7 +153,7 @@ TerminalReadline(FILE *sys_stdin, FILE *sys_stdout, const char *prompt)
PyOS_sighandler_t saint;
saint = PyOS_setsig(SIGINT, OnKeyboardInterrupt);
if (setjmp(jbuf)) {
linenoiseDisableRawMode(STDIN_FILENO);
linenoiseDisableRawMode();
PyOS_setsig(SIGINT, saint);
return NULL;
}

103
third_party/python/launch.c vendored Normal file
View file

@ -0,0 +1,103 @@
/*-*- mode:c;indent-tabs-mode:nil;c-basic-offset:4;tab-width:8;coding:utf-8 -*-│
│vi: set net ft=c ts=4 sts=4 sw=4 fenc=utf-8 :vi│
╞══════════════════════════════════════════════════════════════════════════════╡
│ Python 3 │
│ https://docs.python.org/3/license.html │
╚─────────────────────────────────────────────────────────────────────────────*/
#include "libc/x/x.h"
#include "third_party/python/Include/abstract.h"
#include "third_party/python/Include/import.h"
#include "third_party/python/Include/modsupport.h"
#include "third_party/python/Include/object.h"
#include "third_party/python/Include/pydebug.h"
#include "third_party/python/Include/pylifecycle.h"
#include "third_party/python/Include/pymem.h"
#include "third_party/python/Include/pythonrun.h"
#include "third_party/python/Include/sysmodule.h"
#include "third_party/python/Include/unicodeobject.h"
#include "third_party/python/Include/yoink.h"
/* clang-format off */
STATIC_YOINK("zip_uri_support");
PYTHON_YOINK("_bootlocale");
PYTHON_YOINK("_locale");
PYTHON_YOINK("encodings.aliases");
PYTHON_YOINK("encodings.latin_1");
PYTHON_YOINK("encodings.utf_8");
PYTHON_YOINK("launchpy");
extern char kLaunchPythonModuleName[]; /* generated by pyobj.com */
const struct _frozen *PyImport_FrozenModules = _PyImport_FrozenModules;
struct _inittab *PyImport_Inittab = _PyImport_Inittab;
int
LaunchPythonModule(const char *name)
{
PyObject *mod, *runpy, *runmodule, *runargs, *result;
if (!(runpy = PyImport_ImportModule("launchpy"))) {
PyErr_Print();
return 123;
}
if (!(runmodule = PyObject_GetAttrString(runpy, "run_module_as_main"))) {
PyErr_Print();
Py_DECREF(runpy);
return 122;
}
if (!(mod = PyUnicode_DecodeUTF8Stateful(name, strlen(name), 0, 0))) {
PyErr_Print();
Py_DECREF(runpy);
Py_DECREF(runmodule);
return 121;
}
if (!(runargs = Py_BuildValue("(O)", mod))) {
PyErr_Print();
Py_DECREF(runpy);
Py_DECREF(runmodule);
Py_DECREF(mod);
return 119;
}
if (!(result = PyObject_Call(runmodule, runargs, NULL))) {
PyErr_Print();
}
Py_DECREF(runpy);
Py_DECREF(runmodule);
Py_DECREF(mod);
Py_DECREF(runargs);
if (!result) return 118;
Py_DECREF(result);
return 0;
}
int
main(int argc, char *argv[])
{
size_t n;
int i, sts;
wchar_t *w;
PyObject *a, *s;
Py_FrozenFlag++;
Py_NoSiteFlag++;
/* Py_VerboseFlag++; */
Py_NoUserSiteDirectory++;
Py_IgnoreEnvironmentFlag++;
Py_DontWriteBytecodeFlag++;
#if defined(Py_DEBUG) || defined(USE_TRACEMALLOC)
_PyMem_SetupAllocators(Py_GETENV("PYTHONMALLOC"));
#else
_PyMem_SetupAllocators(0);
#endif
_PyRandom_Init();
Py_Initialize();
Py_LimitedPath();
if (!(a = PyList_New(argc))) return 127;
for (i = 0; i < argc; ++i) {
if (!(w = utf8toutf32(argv[i], -1, &n))) return 126;
if (!(s = PyUnicode_FromWideChar(w, n))) return 125;
PyList_SetItem(a, i, s);
free(w);
}
if (PySys_SetObject("argv", a)) return 124;
sts = LaunchPythonModule(kLaunchPythonModuleName);
if (Py_FinalizeEx() < 0) sts = 120;
return sts;
}

6
third_party/python/pyconfig.h vendored
View file

@ -985,8 +985,10 @@
/* Define to printf format modifier for Py_ssize_t */
#define PY_FORMAT_SIZE_T "z"
/* Define if you want to build an interpreter with many run-time checks. */
/* #define Py_DEBUG 1 */
#ifdef MODE_DBG
#define Py_DEBUG 1
#define USE_TRACEMALLOC 1
#endif
/* Defined if Python is built as a shared library. */
/* #undef Py_ENABLE_SHARED */

41
third_party/python/pyobj.c vendored
View file

@ -64,12 +64,13 @@ OVERVIEW\n\
FLAGS\n\
\n\
-o PATH output elf object file\n\
-P STR prefix fake directory in zip\n\
-P STR prefix fake zip directory (default .python)\n\
-C INT strip directory components from src in zip\n\
-O0 don't optimize [default]\n\
-O1 remove debug statements\n\
-O2 remove debug statements and docstrings\n\
-B binary only (don't include .py file)\n\
-m insert executable launch.c yoink\n\
-0 zip uncompressed\n\
-n do nothing\n\
-h help\n\
@ -212,6 +213,7 @@ static struct stat st;
static PyObject *code;
static PyObject *marsh;
static bool nocompress;
static bool insertlauncher;
static uint64_t image_base;
static int strip_components;
static struct ElfWriter *elf;
@ -224,7 +226,8 @@ GetOpts(int argc, char *argv[])
{
int opt;
image_base = IMAGE_BASE_VIRTUAL;
while ((opt = getopt(argc, argv, "hn0Bb:O:o:C:P:")) != -1) {
path_prefix = ".python";
while ((opt = getopt(argc, argv, "hnm0Bb:O:o:C:P:")) != -1) {
switch (opt) {
case 'B':
binonly = true;
@ -232,6 +235,9 @@ GetOpts(int argc, char *argv[])
case '0':
nocompress = true;
break;
case 'm':
insertlauncher = true;
break;
case 'o':
outpath = optarg;
break;
@ -289,7 +295,7 @@ GetZipFile(void)
const char *zipfile;
zipfile = pyfile;
zipfile = StripComponents(zipfile, strip_components);
if (path_prefix) {
if (*path_prefix) {
zipfile = gc(xjoinpaths(path_prefix, zipfile));
}
return strdup(zipfile);
@ -418,9 +424,10 @@ Analyze(const char *modname, PyObject *code, struct Interner *globals)
PyObject *co_code, *co_names, *co_consts, *name, *cnst, *iter, *item;
mod = 0;
istry = rel = 0;
co_code = PyObject_GetAttrString(code, "co_code");
co_names = PyObject_GetAttrString(code, "co_names");
co_consts = PyObject_GetAttrString(code, "co_consts");
assert(PyCode_Check(code));
co_code = ((PyCodeObject *)code)->co_code;
co_names = ((PyCodeObject *)code)->co_names;
co_consts = ((PyCodeObject *)code)->co_consts;
n = PyBytes_GET_SIZE(co_code);
p = PyBytes_AS_STRING(co_code);
for (a = i = 0; i + 2 <= n; i += 2) {
@ -490,8 +497,6 @@ Analyze(const char *modname, PyObject *code, struct Interner *globals)
}
a = 0;
}
Py_DECREF(co_names);
Py_DECREF(co_code);
free(mod);
iter = PyObject_GetIter(co_consts);
while ((item = PyIter_Next(iter))) {
@ -501,7 +506,6 @@ Analyze(const char *modname, PyObject *code, struct Interner *globals)
Py_DECREF(item);
}
Py_DECREF(iter);
Py_DECREF(co_consts);
}
static void
@ -521,6 +525,7 @@ AnalyzeModule(const char *modname)
static int
Objectify(void)
{
size_t n;
bool ispkg;
char header[12];
size_t pysize, pycsize, marsize;
@ -568,14 +573,28 @@ Objectify(void)
elfwriter_startsection(elf, ".yoink", SHT_PROGBITS,
SHF_ALLOC | SHF_EXECINSTR);
AnalyzeModule(modname);
if (path_prefix && !IsDot()) {
if (*path_prefix && !IsDot()) {
elfwriter_yoink(elf, gc(xstrcat(path_prefix, "/")), STB_GLOBAL);
}
if (strchr(modname, '.')) {
Yoink(gc(GetParent()), STB_GLOBAL);
}
elfwriter_yoink(elf, "__zip_start", STB_GLOBAL);
if (insertlauncher) {
elfwriter_yoink(elf, "LaunchPythonModule", STB_GLOBAL);
}
elfwriter_finishsection(elf);
if (insertlauncher) {
n = strlen(modname) + 1;
elfwriter_align(elf, 1, 0);
elfwriter_startsection(elf, ".rodata.str1.1", SHT_PROGBITS,
SHF_ALLOC | SHF_MERGE | SHF_STRINGS);
memcpy(elfwriter_reserve(elf, n), modname, n);
elfwriter_appendsym(elf, "kLaunchPythonModuleName",
ELF64_ST_INFO(STB_GLOBAL, STT_OBJECT),
STV_DEFAULT, 0, n);
elfwriter_commit(elf, n);
elfwriter_finishsection(elf);
}
elfwriter_close(elf);
freeinterner(yoinked);
return 0;

33
third_party/python/python.mk vendored
View file

@ -27,8 +27,7 @@ THIRD_PARTY_PYTHON_COMS = \
o/$(MODE)/third_party/python/pyobj.com \
o/$(MODE)/third_party/python/pycomp.com \
o/$(MODE)/third_party/python/repl.com \
o/$(MODE)/third_party/python/hello.com \
o/$(MODE)/third_party/python/httpserver.com \
o/$(MODE)/third_party/python/Lib/hello.com \
o/$(MODE)/third_party/python/pythontester.com
THIRD_PARTY_PYTHON_CHECKS = \
@ -408,6 +407,7 @@ THIRD_PARTY_PYTHON_STAGE1_A_SRCS = \
third_party/python/Python/traceback.c
THIRD_PARTY_PYTHON_STAGE2_A_SRCS = \
third_party/python/launch.c \
third_party/python/Modules/_hashmbedtls.c \
third_party/python/Objects/fromfd.c \
third_party/python/Modules/_bisectmodule.c \
@ -1874,7 +1874,6 @@ THIRD_PARTY_PYTHON_STAGE2_A_DEPS = \
o/$(MODE)/third_party/python/pyobj.com.dbg: \
$(THIRD_PARTY_PYTHON_STAGE1) \
$(THIRD_PARTY_PYTHON_STAGE1_A).pkg \
o/$(MODE)/third_party/python/pyobj.o \
$(CRT) \
$(APE)
@ -1882,7 +1881,6 @@ o/$(MODE)/third_party/python/pyobj.com.dbg: \
o/$(MODE)/third_party/python/pycomp.com.dbg: \
$(THIRD_PARTY_PYTHON_STAGE1) \
$(THIRD_PARTY_PYTHON_STAGE1_A).pkg \
o/$(MODE)/third_party/python/pycomp.o \
$(CRT) \
$(APE)
@ -1890,7 +1888,6 @@ o/$(MODE)/third_party/python/pycomp.com.dbg: \
o/$(MODE)/third_party/python/freeze.com.dbg: \
$(THIRD_PARTY_PYTHON_STAGE1) \
$(THIRD_PARTY_PYTHON_STAGE1_A).pkg \
o/$(MODE)/third_party/python/Programs/freeze.o \
$(CRT) \
$(APE)
@ -1898,9 +1895,7 @@ o/$(MODE)/third_party/python/freeze.com.dbg: \
o/$(MODE)/third_party/python/python.com.dbg: \
$(THIRD_PARTY_PYTHON_STAGE1) \
$(THIRD_PARTY_PYTHON_STAGE1_A).pkg \
$(THIRD_PARTY_PYTHON_STAGE2) \
$(THIRD_PARTY_PYTHON_STAGE2_A).pkg \
o/$(MODE)/third_party/python/Programs/python.o \
$(CRT) \
$(APE)
@ -1908,39 +1903,23 @@ o/$(MODE)/third_party/python/python.com.dbg: \
o/$(MODE)/third_party/python/repl.com.dbg: \
$(THIRD_PARTY_PYTHON_STAGE1) \
$(THIRD_PARTY_PYTHON_STAGE1_A).pkg \
$(THIRD_PARTY_PYTHON_STAGE2) \
$(THIRD_PARTY_PYTHON_STAGE2_A).pkg \
o/$(MODE)/third_party/python/Programs/repl.o \
$(CRT) \
$(APE)
@$(APELINK)
o/$(MODE)/third_party/python/httpserver.com.dbg: \
o/$(MODE)/third_party/python/Lib/hello.com.dbg: \
$(THIRD_PARTY_PYTHON_STAGE1) \
$(THIRD_PARTY_PYTHON_STAGE1_A).pkg \
$(THIRD_PARTY_PYTHON_STAGE2) \
$(THIRD_PARTY_PYTHON_STAGE2_A).pkg \
o/$(MODE)/third_party/python/Programs/httpserver.o \
$(CRT) \
$(APE)
@$(APELINK)
o/$(MODE)/third_party/python/hello.com.dbg: \
$(THIRD_PARTY_PYTHON_STAGE1) \
$(THIRD_PARTY_PYTHON_STAGE1_A).pkg \
$(THIRD_PARTY_PYTHON_STAGE2) \
$(THIRD_PARTY_PYTHON_STAGE2_A).pkg \
o/$(MODE)/third_party/python/Programs/hello.o \
o/$(MODE)/third_party/python/Lib/hello.o \
$(CRT) \
$(APE)
@$(APELINK)
o/$(MODE)/third_party/python/pythontester.com.dbg: \
$(THIRD_PARTY_PYTHON_STAGE1) \
$(THIRD_PARTY_PYTHON_STAGE1_A).pkg \
$(THIRD_PARTY_PYTHON_STAGE2) \
$(THIRD_PARTY_PYTHON_STAGE2_A).pkg \
o/$(MODE)/third_party/python/Programs/pythontester.o \
$(CRT) \
$(APE)
@ -2025,6 +2004,7 @@ o/$(MODE)/third_party/python/Modules/faulthandler.o: \
$(THIRD_PARTY_PYTHON_STDLIB_PYS_OBJS): PYFLAGS += -P.python -C3
$(THIRD_PARTY_PYTHON_STDLIB_DATA_OBJS): ZIPOBJ_FLAGS += -P.python -C3
o/$(MODE)/third_party/python/Lib/hello.o: PYFLAGS += -m
o/$(MODE)/third_party/python/Python/ceval.o: QUOTA = -M512m
o/$(MODE)/third_party/python/Objects/unicodeobject.o: QUOTA += -C16
@ -2050,11 +2030,8 @@ THIRD_PARTY_PYTHON_SRCS = \
third_party/python/pyobj.c \
third_party/python/pycomp.c \
third_party/python/Programs/repl.c \
third_party/python/Programs/hello.c \
third_party/python/Programs/launch.c \
third_party/python/Programs/freeze.c \
third_party/python/Programs/python.c \
third_party/python/Programs/httpserver.c \
third_party/python/Programs/pythontester.c
#$(THIRD_PARTY_PYTHON_OBJS): \