Add some Python 3.7 backports (#306)

* make.com now uses stack size of 2mb
* optimize _PyCFunction_FastCallKeywords
* backport python@cpython/7fc252adfbedece75f2330bcfdadbf84dee7836f
This commit is contained in:
Gautham 2021-10-30 11:24:14 +05:30 committed by GitHub
parent 903cc38c37
commit d7ff346b52
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
9 changed files with 700 additions and 504 deletions

View file

@ -25,7 +25,7 @@
#include "third_party/make/lib/error.h"
#include "third_party/make/lib/exitfail.h"
#include "third_party/make/src/gettext.h"
#include "third_party/make/lib/gettext.h"
#define _(msgid) gettext (msgid)
void

View file

@ -26,6 +26,7 @@ this program. If not, see <http://www.gnu.org/licenses/>. */
#include "third_party/make/src/getopt.h"
#include "libc/sysv/consts/sa.h"
#include "libc/runtime/stack.h"
#include "third_party/gdtoa/gdtoa.h"
#include <assert.h>
@ -1057,6 +1058,7 @@ int
main (int argc, char **argv, char **envp)
#endif
{
STATIC_STACK_SIZE(0x00200000); // 2mb stack
static char *stdin_nm = 0;
int makefile_status = MAKE_SUCCESS;
struct goaldep *read_files;

View file

@ -16,8 +16,12 @@ PyObject *PyObject_Call(PyObject *callable_object, PyObject *args,
PyObject *_PyStack_AsTuple(PyObject **stack, Py_ssize_t nargs);
PyObject *_PyStack_AsDict(PyObject **values, PyObject *kwnames);
int _PyStack_UnpackDict(PyObject **args, Py_ssize_t nargs, PyObject *kwargs,
PyObject ***p_stack, PyObject **p_kwnames);
PyObject ** _PyStack_UnpackDict(
PyObject **args,
Py_ssize_t nargs,
PyObject *kwargs,
PyObject **kwnames,
PyObject *func);
PyObject *_PyObject_FastCallDict(PyObject *func, PyObject **args,
Py_ssize_t nargs, PyObject *kwargs);

View file

@ -108,6 +108,7 @@ PyObject * PyDict_Copy(PyObject *mp);
int PyDict_Contains(PyObject *mp, PyObject *key);
#ifndef Py_LIMITED_API
int _PyDict_Contains(PyObject *mp, PyObject *key, Py_hash_t hash);
#define PyDict_GET_SIZE(mp) (assert(PyDict_Check(mp)),((PyDictObject *)mp)->ma_used)
PyObject * _PyDict_NewPresized(Py_ssize_t minused);
void _PyDict_MaybeUntrack(PyObject *mp);
int _PyDict_HasOnlyStringKeys(PyObject *mp);

View file

@ -23,6 +23,14 @@ PyObject * _Py_VaBuildValue_SizeT(const char *, va_list);
#endif /* !Py_LIMITED_API */
#endif
#define _Py_VaBuildStack _Py_VaBuildStack_SizeT
PyObject ** _Py_VaBuildStack_SizeT(
PyObject **small_stack,
Py_ssize_t small_stack_len,
const char *format,
va_list va,
Py_ssize_t *p_nargs);
/* Due to a glitch in 3.2, the _SizeT versions weren't exported from the DLL. */
#if !defined(PY_SSIZE_T_CLEAN) || !defined(Py_LIMITED_API) || Py_LIMITED_API+0 >= 0x03030000
int PyArg_Parse(PyObject *, const char *, ...);

View file

@ -1,5 +1,4 @@
import unittest
import sys
from io import StringIO
from test import support
@ -129,91 +128,5 @@ class TestPrint(unittest.TestCase):
raise RuntimeError
self.assertRaises(RuntimeError, print, 1, file=noflush(), flush=True)
class TestPy2MigrationHint(unittest.TestCase):
"""Test that correct hint is produced analogous to Python3 syntax,
if print statement is executed as in Python 2.
"""
def test_normal_string(self):
python2_print_str = 'print "Hello World"'
with self.assertRaises(SyntaxError) as context:
exec(python2_print_str)
self.assertIn('print("Hello World")', str(context.exception))
def test_string_with_soft_space(self):
python2_print_str = 'print "Hello World",'
with self.assertRaises(SyntaxError) as context:
exec(python2_print_str)
self.assertIn('print("Hello World", end=" ")', str(context.exception))
def test_string_with_excessive_whitespace(self):
python2_print_str = 'print "Hello World", '
with self.assertRaises(SyntaxError) as context:
exec(python2_print_str)
self.assertIn('print("Hello World", end=" ")', str(context.exception))
def test_string_with_leading_whitespace(self):
python2_print_str = '''if 1:
print "Hello World"
'''
with self.assertRaises(SyntaxError) as context:
exec(python2_print_str)
self.assertIn('print("Hello World")', str(context.exception))
# bpo-32685: Suggestions for print statement should be proper when
# it is in the same line as the header of a compound statement
# and/or followed by a semicolon
def test_string_with_semicolon(self):
python2_print_str = 'print p;'
with self.assertRaises(SyntaxError) as context:
exec(python2_print_str)
self.assertIn('print(p)', str(context.exception))
def test_string_in_loop_on_same_line(self):
python2_print_str = 'for i in s: print i'
with self.assertRaises(SyntaxError) as context:
exec(python2_print_str)
self.assertIn('print(i)', str(context.exception))
def test_stream_redirection_hint_for_py2_migration(self):
# Test correct hint produced for Py2 redirection syntax
with self.assertRaises(TypeError) as context:
print >> sys.stderr, "message"
self.assertIn('Did you mean "print(<message>, '
'file=<output_stream>)"?', str(context.exception))
# Test correct hint is produced in the case where RHS implements
# __rrshift__ but returns NotImplemented
with self.assertRaises(TypeError) as context:
print >> 42
self.assertIn('Did you mean "print(<message>, '
'file=<output_stream>)"?', str(context.exception))
# Test stream redirection hint is specific to print
with self.assertRaises(TypeError) as context:
max >> sys.stderr
self.assertNotIn('Did you mean ', str(context.exception))
# Test stream redirection hint is specific to rshift
with self.assertRaises(TypeError) as context:
print << sys.stderr
self.assertNotIn('Did you mean', str(context.exception))
# Ensure right operand implementing rrshift still works
class OverrideRRShift:
def __rrshift__(self, lhs):
return 42 # Force result independent of LHS
self.assertEqual(print >> OverrideRRShift(), 42)
if __name__ == "__main__":
unittest.main()

File diff suppressed because it is too large Load diff

View file

@ -100,8 +100,9 @@ PyCFunction_Call(PyObject *func, PyObject *args, PyObject *kwds)
Py_ssize_t size;
int flags;
assert(kwds == NULL || PyDict_Check(kwds));
/* PyCFunction_Call() must not be called with an exception set,
because it may clear it (directly or indirectly) and so the
because it can clear it (directly or indirectly) and so the
caller loses its exception */
assert(!PyErr_Occurred());
@ -116,7 +117,7 @@ PyCFunction_Call(PyObject *func, PyObject *args, PyObject *kwds)
res = _PyCFunction_FastCallDict(func, stack, nargs, kwds);
}
else {
if (kwds != NULL && PyDict_Size(kwds) != 0) {
if (kwds != NULL && PyDict_GET_SIZE(kwds) != 0) {
PyErr_Format(PyExc_TypeError, "%.200s() takes no keyword arguments",
f->m_ml->ml_name);
return NULL;
@ -167,14 +168,14 @@ PyObject *
_PyCFunction_FastCallDict(PyObject *func_obj, PyObject **args, Py_ssize_t nargs,
PyObject *kwargs)
{
PyCFunctionObject *func = (PyCFunctionObject*)func_obj;
PyCFunction meth = PyCFunction_GET_FUNCTION(func);
PyObject *self = PyCFunction_GET_SELF(func);
PyCFunctionObject *func;
PyCFunction meth;
PyObject *self;
PyObject *result;
int flags;
assert(PyCFunction_Check(func));
assert(func != NULL);
assert(func_obj != NULL);
assert(PyCFunction_Check(func_obj));
assert(nargs >= 0);
assert(nargs == 0 || args != NULL);
assert(kwargs == NULL || PyDict_Check(kwargs));
@ -184,21 +185,21 @@ _PyCFunction_FastCallDict(PyObject *func_obj, PyObject **args, Py_ssize_t nargs,
caller loses its exception */
assert(!PyErr_Occurred());
func = (PyCFunctionObject*)func_obj;
meth = PyCFunction_GET_FUNCTION(func);
self = PyCFunction_GET_SELF(func);
flags = PyCFunction_GET_FLAGS(func) & ~(METH_CLASS | METH_STATIC | METH_COEXIST);
switch (flags)
{
case METH_NOARGS:
if (kwargs != NULL && PyDict_Size(kwargs) != 0) {
PyErr_Format(PyExc_TypeError, "%.200s() takes no keyword arguments",
func->m_ml->ml_name);
return NULL;
if (nargs != 0) {
goto no_keyword_error;
}
if (nargs != 0) {
PyErr_Format(PyExc_TypeError,
"%.200s() takes no arguments (%zd given)",
func->m_ml->ml_name, nargs);
if (kwargs != NULL && PyDict_GET_SIZE(kwargs) != 0) {
PyErr_Format(PyExc_TypeError, "%.200s() takes no keyword arguments",
func->m_ml->ml_name);
return NULL;
}
@ -206,12 +207,6 @@ _PyCFunction_FastCallDict(PyObject *func_obj, PyObject **args, Py_ssize_t nargs,
break;
case METH_O:
if (kwargs != NULL && PyDict_Size(kwargs) != 0) {
PyErr_Format(PyExc_TypeError, "%.200s() takes no keyword arguments",
func->m_ml->ml_name);
return NULL;
}
if (nargs != 1) {
PyErr_Format(PyExc_TypeError,
"%.200s() takes exactly one argument (%zd given)",
@ -219,29 +214,27 @@ _PyCFunction_FastCallDict(PyObject *func_obj, PyObject **args, Py_ssize_t nargs,
return NULL;
}
if (kwargs != NULL && PyDict_GET_SIZE(kwargs) != 0) {
goto no_keyword_error;
}
result = (*meth) (self, args[0]);
break;
case METH_VARARGS:
case METH_VARARGS | METH_KEYWORDS:
{
/* Slow-path: create a temporary tuple */
Py_ssize_t i;
PyObject *item, *tuple;
/* Slow-path: create a temporary tuple for positional arguments */
PyObject *tuple;
if (!(flags & METH_KEYWORDS) && kwargs != NULL && PyDict_Size(kwargs) != 0) {
PyErr_Format(PyExc_TypeError,
"%.200s() takes no keyword arguments",
func->m_ml->ml_name);
return NULL;
if (!(flags & METH_KEYWORDS)
&& kwargs != NULL && PyDict_GET_SIZE(kwargs) != 0) {
goto no_keyword_error;
}
/* [jart] inlined _PyStack_AsTuple b/c profiling */
if (!(tuple = PyTuple_New(nargs))) return 0;
for (i = 0; i < nargs; i++) {
item = args[i];
Py_INCREF(item);
PyTuple_SET_ITEM(tuple, i, item);
tuple = _PyStack_AsTuple(args, nargs);
if (tuple == NULL) {
return NULL;
}
if (flags & METH_KEYWORDS) {
@ -260,7 +253,8 @@ _PyCFunction_FastCallDict(PyObject *func_obj, PyObject **args, Py_ssize_t nargs,
PyObject *kwnames;
_PyCFunctionFast fastmeth = (_PyCFunctionFast)meth;
if (_PyStack_UnpackDict(args, nargs, kwargs, &stack, &kwnames) < 0) {
stack = _PyStack_UnpackDict(args, nargs, kwargs, &kwnames, func_obj);
if (stack == NULL) {
return NULL;
}
@ -282,35 +276,134 @@ _PyCFunction_FastCallDict(PyObject *func_obj, PyObject **args, Py_ssize_t nargs,
result = _Py_CheckFunctionResult(func_obj, result, NULL);
return result;
no_keyword_error:
PyErr_Format(PyExc_TypeError,
"%.200s() takes no arguments (%zd given)",
func->m_ml->ml_name, nargs);
return NULL;
}
PyObject *
_PyCFunction_FastCallKeywords(PyObject *func, PyObject **stack,
_PyCFunction_FastCallKeywords(PyObject *func_obj, PyObject **args,
Py_ssize_t nargs, PyObject *kwnames)
{
PyObject *kwdict, *result;
PyCFunctionObject *func;
PyCFunction meth;
PyObject *self, *result;
Py_ssize_t nkwargs = (kwnames == NULL) ? 0 : PyTuple_GET_SIZE(kwnames);
int flags;
assert(PyCFunction_Check(func));
assert(func_obj != NULL);
assert(PyCFunction_Check(func_obj));
assert(nargs >= 0);
assert(kwnames == NULL || PyTuple_CheckExact(kwnames));
assert((nargs == 0 && nkwargs == 0) || stack != NULL);
assert((nargs == 0 && nkwargs == 0) || args != NULL);
/* kwnames must only contains str strings, no subclass, and all keys must
be unique */
if (nkwargs > 0) {
kwdict = _PyStack_AsDict(stack + nargs, kwnames);
if (kwdict == NULL) {
/* _PyCFunction_FastCallKeywords() must not be called with an exception
set, because it can clear it (directly or indirectly) and so the caller
loses its exception */
assert(!PyErr_Occurred());
func = (PyCFunctionObject*)func_obj;
meth = PyCFunction_GET_FUNCTION(func);
self = PyCFunction_GET_SELF(func);
flags = PyCFunction_GET_FLAGS(func) & ~(METH_CLASS | METH_STATIC | METH_COEXIST);
switch (flags)
{
case METH_NOARGS:
if (nargs != 0) {
PyErr_Format(PyExc_TypeError,
"%.200s() takes no arguments (%zd given)",
func->m_ml->ml_name, nargs);
return NULL;
}
}
else {
kwdict = NULL;
if (nkwargs) {
goto no_keyword_error;
}
result = (*meth) (self, NULL);
break;
case METH_O:
if (nargs != 1) {
PyErr_Format(PyExc_TypeError,
"%.200s() takes exactly one argument (%zd given)",
func->m_ml->ml_name, nargs);
return NULL;
}
if (nkwargs) {
goto no_keyword_error;
}
result = (*meth) (self, args[0]);
break;
case METH_FASTCALL:
/* Fast-path: avoid temporary dict to pass keyword arguments */
result = ((_PyCFunctionFast)meth) (self, args, nargs, kwnames);
break;
case METH_VARARGS:
case METH_VARARGS | METH_KEYWORDS:
{
/* Slow-path: create a temporary tuple for positional arguments
and a temporary dict for keyword arguments */
PyObject *argtuple;
if (!(flags & METH_KEYWORDS) && nkwargs) {
goto no_keyword_error;
}
argtuple = _PyStack_AsTuple(args, nargs);
if (argtuple == NULL) {
return NULL;
}
if (flags & METH_KEYWORDS) {
PyObject *kwdict;
if (nkwargs > 0) {
kwdict = _PyStack_AsDict(args + nargs, kwnames);
if (kwdict == NULL) {
Py_DECREF(argtuple);
return NULL;
}
}
else {
kwdict = NULL;
}
result = (*(PyCFunctionWithKeywords)meth) (self, argtuple, kwdict);
Py_XDECREF(kwdict);
}
else {
result = (*meth) (self, argtuple);
}
Py_DECREF(argtuple);
break;
}
result = _PyCFunction_FastCallDict(func, stack, nargs, kwdict);
Py_XDECREF(kwdict);
default:
PyErr_SetString(PyExc_SystemError,
"Bad call flags in _PyCFunction_FastCallKeywords. "
"METH_OLDARGS is no longer supported!");
return NULL;
}
result = _Py_CheckFunctionResult(func_obj, result, NULL);
return result;
no_keyword_error:
PyErr_Format(PyExc_TypeError,
"%.200s() takes no keyword arguments",
func->m_ml->ml_name);
return NULL;
}
/* Methods (the standard built-in methods, that is) */
@ -337,13 +430,16 @@ meth_dealloc(PyCFunctionObject *m)
static PyObject *
meth_reduce(PyCFunctionObject *m)
{
PyObject *builtins;
PyObject *getattr;
_Py_IDENTIFIER(getattr);
if (m->m_self == NULL || PyModule_Check(m->m_self))
return PyUnicode_FromString(m->m_ml->ml_name);
return Py_BuildValue("N(Os)", _PyEval_GetBuiltinId(&PyId_getattr),
m->m_self, m->m_ml->ml_name);
builtins = PyEval_GetBuiltins();
getattr = _PyDict_GetItemId(builtins, &PyId_getattr);
return Py_BuildValue("O(Os)", getattr, m->m_self, m->m_ml->ml_name);
}
static PyMethodDef meth_methods[] = {

View file

@ -13,25 +13,25 @@
#include "third_party/python/Include/longobject.h"
#include "third_party/python/Include/object.h"
#include "third_party/python/Include/pyerrors.h"
#include "third_party/python/Include/pymem.h"
#include "third_party/python/Include/tupleobject.h"
/* clang-format off */
/* Module support implementation */
#define FLAG_SIZE_T 1
typedef double va_double;
static PyObject *va_build_value(const char *, va_list, int);
static PyObject **va_build_stack(PyObject **small_stack, Py_ssize_t small_stack_len, const char *, va_list, int, Py_ssize_t*);
/* Package context -- the full module name for package imports */
char *_Py_PackageContext = NULL;
const char *_Py_PackageContext = NULL;
/* Helper for mkvalue() to scan the length of a format */
static int
countformat(const char *format, int endchar)
static Py_ssize_t
countformat(const char *format, char endchar)
{
int count = 0;
Py_ssize_t count = 0;
int level = 0;
while (level > 0 || *format != endchar) {
switch (*format) {
@ -43,8 +43,9 @@ countformat(const char *format, int endchar)
case '(':
case '[':
case '{':
if (level == 0)
if (level == 0) {
count++;
}
level++;
break;
case ')':
@ -60,8 +61,9 @@ countformat(const char *format, int endchar)
case '\t':
break;
default:
if (level == 0)
if (level == 0) {
count++;
}
}
format++;
}
@ -72,17 +74,18 @@ countformat(const char *format, int endchar)
/* Generic function to create a value -- the inverse of getargs() */
/* After an original idea and first implementation by Steven Miale */
static PyObject *do_mktuple(const char**, va_list *, int, int, int);
static PyObject *do_mklist(const char**, va_list *, int, int, int);
static PyObject *do_mkdict(const char**, va_list *, int, int, int);
static PyObject *do_mktuple(const char**, va_list *, char, Py_ssize_t, int);
static int do_mkstack(PyObject **, const char**, va_list *, char, Py_ssize_t, int);
static PyObject *do_mklist(const char**, va_list *, char, Py_ssize_t, int);
static PyObject *do_mkdict(const char**, va_list *, char, Py_ssize_t, int);
static PyObject *do_mkvalue(const char**, va_list *, int);
static void
do_ignore(const char **p_format, va_list *p_va, int endchar, int n, int flags)
do_ignore(const char **p_format, va_list *p_va, char endchar, Py_ssize_t n, int flags)
{
PyObject *v;
int i;
Py_ssize_t i;
assert(PyErr_Occurred());
v = PyTuple_New(n);
for (i = 0; i < n; i++) {
@ -106,15 +109,16 @@ do_ignore(const char **p_format, va_list *p_va, int endchar, int n, int flags)
"Unmatched paren in format");
return;
}
if (endchar)
if (endchar) {
++*p_format;
}
}
static PyObject *
do_mkdict(const char **p_format, va_list *p_va, int endchar, int n, int flags)
do_mkdict(const char **p_format, va_list *p_va, char endchar, Py_ssize_t n, int flags)
{
PyObject *d;
int i;
Py_ssize_t i;
if (n < 0)
return NULL;
if (n % 2) {
@ -161,10 +165,10 @@ do_mkdict(const char **p_format, va_list *p_va, int endchar, int n, int flags)
}
static PyObject *
do_mklist(const char **p_format, va_list *p_va, int endchar, int n, int flags)
do_mklist(const char **p_format, va_list *p_va, char endchar, Py_ssize_t n, int flags)
{
PyObject *v;
int i;
Py_ssize_t i;
if (n < 0)
return NULL;
/* Note that we can't bail immediately on error as this will leak
@ -194,11 +198,48 @@ do_mklist(const char **p_format, va_list *p_va, int endchar, int n, int flags)
return v;
}
static int
do_mkstack(PyObject **stack, const char **p_format, va_list *p_va,
char endchar, Py_ssize_t n, int flags)
{
Py_ssize_t i;
if (n < 0) {
return -1;
}
/* Note that we can't bail immediately on error as this will leak
refcounts on any 'N' arguments. */
for (i = 0; i < n; i++) {
PyObject *w = do_mkvalue(p_format, p_va, flags);
if (w == NULL) {
do_ignore(p_format, p_va, endchar, n - i - 1, flags);
goto error;
}
stack[i] = w;
}
if (**p_format != endchar) {
PyErr_SetString(PyExc_SystemError,
"Unmatched paren in format");
goto error;
}
if (endchar) {
++*p_format;
}
return 0;
error:
n = i;
for (i=0; i < n; i++) {
Py_DECREF(stack[i]);
}
return -1;
}
static PyObject *
do_mktuple(const char **p_format, va_list *p_va, int endchar, int n, int flags)
do_mktuple(const char **p_format, va_list *p_va, char endchar, Py_ssize_t n, int flags)
{
PyObject *v;
int i;
Py_ssize_t i;
if (n < 0)
return NULL;
/* Note that we can't bail immediately on error as this will leak
@ -302,7 +343,7 @@ do_mkvalue(const char **p_format, va_list *p_va, int flags)
else {
if (n < 0)
n = wcslen(u);
v = PyUnicode_FromUnicode(u, n);
v = PyUnicode_FromWideChar(u, n);
}
return v;
}
@ -480,7 +521,7 @@ static PyObject *
va_build_value(const char *format, va_list va, int flags)
{
const char *f = format;
int n = countformat(f, '\0');
Py_ssize_t n = countformat(f, '\0');
va_list lva;
PyObject *retval;
@ -500,9 +541,68 @@ va_build_value(const char *format, va_list va, int flags)
return retval;
}
PyObject **
_Py_VaBuildStack(PyObject **small_stack, Py_ssize_t small_stack_len,
const char *format, va_list va, Py_ssize_t *p_nargs)
{
return va_build_stack(small_stack, small_stack_len, format, va, 0, p_nargs);
}
PyObject **
_Py_VaBuildStack_SizeT(PyObject **small_stack, Py_ssize_t small_stack_len,
const char *format, va_list va, Py_ssize_t *p_nargs)
{
return va_build_stack(small_stack, small_stack_len, format, va, FLAG_SIZE_T, p_nargs);
}
static PyObject **
va_build_stack(PyObject **small_stack, Py_ssize_t small_stack_len,
const char *format, va_list va, int flags, Py_ssize_t *p_nargs)
{
const char *f;
Py_ssize_t n;
va_list lva;
PyObject **stack;
int res;
n = countformat(format, '\0');
if (n < 0) {
*p_nargs = 0;
return NULL;
}
if (n == 0) {
*p_nargs = 0;
return small_stack;
}
if (n <= small_stack_len) {
stack = small_stack;
}
else {
stack = PyMem_Malloc(n * sizeof(stack[0]));
if (stack == NULL) {
PyErr_NoMemory();
return NULL;
}
}
va_copy(lva, va);
f = format;
res = do_mkstack(stack, &f, &lva, '\0', n, flags);
va_end(lva);
if (res < 0) {
return NULL;
}
*p_nargs = n;
return stack;
}
PyObject *
PyEval_CallFunction(PyObject *obj, const char *format, ...)
PyEval_CallFunction(PyObject *callable, const char *format, ...)
{
va_list vargs;
PyObject *args;
@ -516,7 +616,7 @@ PyEval_CallFunction(PyObject *obj, const char *format, ...)
if (args == NULL)
return NULL;
res = PyEval_CallObject(obj, args);
res = PyEval_CallObject(callable, args);
Py_DECREF(args);
return res;
@ -524,14 +624,14 @@ PyEval_CallFunction(PyObject *obj, const char *format, ...)
PyObject *
PyEval_CallMethod(PyObject *obj, const char *methodname, const char *format, ...)
PyEval_CallMethod(PyObject *obj, const char *name, const char *format, ...)
{
va_list vargs;
PyObject *meth;
PyObject *args;
PyObject *res;
meth = PyObject_GetAttrString(obj, methodname);
meth = PyObject_GetAttrString(obj, name);
if (meth == NULL)
return NULL;