Fix Pyston speedups (#281)

We remove (i.e. hide behind a debug ifdef) the recursion checking methods,
and the memory hooks and memory allocator methods. ASAN mode has no
PYMALLOC, so we need a macro. Fix build break with des.c stack allocation.
This commit is contained in:
Gautham 2021-10-02 13:58:51 +05:30 committed by GitHub
parent 2fe8571010
commit 57f0eed382
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
42 changed files with 260 additions and 63 deletions

View file

@ -58,6 +58,10 @@ o/$(MODE)/third_party/mbedtls/everest.o: \
OVERRIDE_CFLAGS += \
-O3
o/$(MODE)/third_party/mbedtls/des.o: \
OVERRIDE_CFLAGS += \
-DSTACK_FRAME_UNLIMITED
o/$(MODE)/third_party/mbedtls/bigmul4.o \
o/$(MODE)/third_party/mbedtls/bigmul6.o: \
OVERRIDE_CFLAGS += \

View file

@ -35,7 +35,7 @@ PyObject *_PyObject_FastCallKeywords(PyObject *func, PyObject **args,
PyObject *_PyObject_Call_Prepend(PyObject *func, PyObject *obj, PyObject *args,
PyObject *kwargs);
#ifdef USE_CHECKFUNCRESULT
#if IsModeDbg()
PyObject *_Py_CheckFunctionResult(PyObject *func, PyObject *result,
const char *where);
#else

View file

@ -77,6 +77,7 @@ int Py_MakePendingCalls(void);
void Py_SetRecursionLimit(int);
int Py_GetRecursionLimit(void);
#if IsModeDbg()
#define Py_EnterRecursiveCall(where) \
(_Py_MakeRecCheck(PyThreadState_GET()->recursion_depth) && \
_Py_CheckRecursiveCall(where))
@ -84,6 +85,10 @@ int Py_GetRecursionLimit(void);
do{ if(_Py_MakeEndRecCheck(PyThreadState_GET()->recursion_depth)) \
PyThreadState_GET()->overflowed = 0; \
} while(0)
#else
#define Py_EnterRecursiveCall(where) (0)
#define Py_LeaveRecursiveCall(where) ((void)0)
#endif
int _Py_CheckRecursiveCall(const char *where);
extern int _Py_CheckRecursionLimit;

View file

@ -17,8 +17,10 @@ void * PyObject_Realloc(void *, size_t);
void PyObject_Free(void *);
#ifndef Py_LIMITED_API
#if IsModeDbg()
/* This function returns the number of allocated memory blocks, regardless of size */
Py_ssize_t _Py_GetAllocatedBlocks(void);
#endif
#endif /* !Py_LIMITED_API */
/* Macros */
@ -117,6 +119,7 @@ PyVarObject * _PyObject_NewVar(PyTypeObject *, Py_ssize_t);
constructor you would start directly with PyObject_Init/InitVar
*/
#if IsModeDbg()
#ifndef Py_LIMITED_API
typedef struct {
/* user context passed as the first argument to the 2 functions */
@ -135,7 +138,7 @@ void PyObject_GetArenaAllocator(PyObjectArenaAllocator *allocator);
/* Set the arena allocator. */
void PyObject_SetArenaAllocator(PyObjectArenaAllocator *allocator);
#endif
#endif
/*
* Garbage Collection Support

View file

@ -316,7 +316,7 @@ void PyErr_BadInternalCall(void);
void _PyErr_BadInternalCall(const char *filename, int lineno);
/* Mask the old API with a call to the new API for code compiled under
Python 2.0: */
#ifdef USE_BADINTERNALCALL
#if IsModeDbg()
#define PyErr_BadInternalCall() _PyErr_BadInternalCall(__FILE__, __LINE__)
#else
#define PyErr_BadInternalCall()

View file

@ -64,13 +64,15 @@ PyObject* _PyTraceMalloc_GetTraceback(
_PyTraceMalloc_domain_t domain,
uintptr_t ptr);
#ifdef USE_TRACEMALLOC
int _PyMem_IsFreed(void *ptr, size_t size);
#if !IsModeDbg()
#define _PyTraceMalloc_Track(domain, ptr, size) (-2)
#define _PyTraceMalloc_Untrack(domain, ptr) (-2)
#define _PyTraceMalloc_GetTraceback(domain, ptr) (&_Py_NoneStruct)
#define _PyMem_IsFreed(ptr, size) (0)
#endif
int _PyMem_IsFreed(void *ptr, size_t size);
#endif /* !defined(Py_LIMITED_API) */
@ -172,6 +174,7 @@ char * _PyMem_Strdup(const char *str);
#define PyMem_Del PyMem_Free
#define PyMem_DEL PyMem_FREE
#if IsModeDbg()
#ifndef Py_LIMITED_API
typedef enum {
/* PyMem_RawMalloc(), PyMem_RawRealloc() and PyMem_RawFree() */
@ -237,6 +240,7 @@ void PyMem_SetAllocator(PyMemAllocatorDomain domain,
The function does nothing if Python is not compiled is debug mode. */
void PyMem_SetupDebugHooks(void);
#endif
#endif
COSMOPOLITAN_C_END_
#endif /* !Py_PYMEM_H */

View file

@ -2,6 +2,7 @@ import unittest
from ctypes import *
from ctypes.test import need_symbol
import _ctypes_test
import cosmo
dll = CDLL(_ctypes_test.__file__)
@ -190,6 +191,7 @@ class BasicWrapTestCase(unittest.TestCase):
self.assertEqual((s8i.a, s8i.b, s8i.c, s8i.d, s8i.e, s8i.f, s8i.g, s8i.h),
(9*2, 8*3, 7*4, 6*5, 5*6, 4*7, 3*8, 2*9))
@unittest.skipUnless(cosmo.MODE == "dbg", "no recursion checking")
def test_recursive_as_param(self):
from ctypes import c_int

View file

@ -4,6 +4,8 @@ Tests common to list and UserList.UserList
import sys
import os
import unittest
import cosmo
from functools import cmp_to_key
from test import support, seq_tests
@ -53,6 +55,7 @@ class CommonTest(seq_tests.CommonTest):
self.assertEqual(str(a2), "[0, 1, 2, [...], 3]")
self.assertEqual(repr(a2), "[0, 1, 2, [...], 3]")
@unittest.skipUnless(cosmo.MODE == "dbg", "disabled recursion checking")
def test_repr_deep(self):
a = self.type2test([])
for i in range(sys.getrecursionlimit() + 100):

View file

@ -2,6 +2,7 @@
import unittest
import collections
import sys
import cosmo
class BasicTestMappingProtocol(unittest.TestCase):
@ -620,6 +621,7 @@ class TestHashMappingProtocol(TestMappingProtocol):
d = self._full_mapping({1: BadRepr()})
self.assertRaises(Exc, repr, d)
@unittest.skipUnless(cosmo.MODE == "dbg", "disabled recursion checking")
def test_repr_deep(self):
d = self._empty_mapping()
for i in range(sys.getrecursionlimit() + 100):

View file

@ -1943,8 +1943,7 @@ class AbstractPickleTests(unittest.TestCase):
self.assertEqual(y._reduce_called, 1)
@no_tracing
@unittest.skipIf(cosmo.MODE in ("asan", "dbg"),
"extremely slow in asan mode")
@unittest.skipIf(True, "disabled recursion checking + slow in asan, dbg")
def test_bad_getattr(self):
# Issue #3514: crash when there is an infinite loop in __getattr__
x = BadGetattr()

View file

@ -1,6 +1,7 @@
# Run the _testcapi module tests (tests for the Python/C API): by defn,
# these are all functions _testcapi exports whose name begins with 'test_'.
import cosmo
import os
import pickle
import random
@ -179,6 +180,7 @@ class CAPITest(unittest.TestCase):
o @= m1
self.assertEqual(o, ("matmul", 42, m1))
@unittest.skipUnless(cosmo.MODE == "dbg", "disabled recursion check")
def test_return_null_without_error(self):
# Issue #23571: A function must not return NULL without setting an
# error
@ -207,6 +209,7 @@ class CAPITest(unittest.TestCase):
'return_null_without_error.* '
'returned NULL without setting an error')
@unittest.skipUnless(cosmo.MODE == "dbg", "disabled recursion check")
def test_return_result_with_error(self):
# Issue #23571: A function must not return a result with an error set
if Py_DEBUG:
@ -242,6 +245,7 @@ class CAPITest(unittest.TestCase):
def test_buildvalue_N(self):
_testcapi.test_buildvalue_N()
@unittest.skipUnless(cosmo.MODE == "dbg", "disabled memory hooks")
def test_set_nomemory(self):
code = """if 1:
import _testcapi
@ -510,6 +514,7 @@ class Test_testcapi(unittest.TestCase):
if name.startswith('test_') and not name.endswith('_code'))
@unittest.skipUnless(cosmo.MODE == "dbg", "disabled memory debugging")
class PyMemDebugTests(unittest.TestCase):
PYTHONMALLOC = 'debug'
# '0x04c06e0' or '04C06E0'

View file

@ -1,6 +1,7 @@
"Test the functionality of Python classes implementing operators."
import unittest
import cosmo
testmeths = [
@ -490,6 +491,7 @@ class ClassTests(unittest.TestCase):
self.assertRaises(TypeError, hash, C2())
@unittest.skipUnless(cosmo.MODE == "dbg", "disabled recursion checking")
def testSFBug532646(self):
# Test for SF bug 532646

View file

@ -1,5 +1,6 @@
"""Unit tests for the copy module."""
import cosmo
import copy
import copyreg
import weakref
@ -372,6 +373,7 @@ class TestCopy(unittest.TestCase):
x = []
x.append(x)
y = copy.deepcopy(x)
if cosmo.MODE == "dbg": # requires recursion checking
for op in comparisons:
self.assertRaises(RecursionError, op, y, x)
self.assertIsNot(y, x)
@ -399,6 +401,7 @@ class TestCopy(unittest.TestCase):
x = ([],)
x[0].append(x)
y = copy.deepcopy(x)
if cosmo.MODE == "dbg": # requires recursion checking
for op in comparisons:
self.assertRaises(RecursionError, op, y, x)
self.assertIsNot(y, x)
@ -418,6 +421,7 @@ class TestCopy(unittest.TestCase):
y = copy.deepcopy(x)
for op in order_comparisons:
self.assertRaises(TypeError, op, y, x)
if cosmo.MODE == "dbg": # requires recursion checking
for op in equality_comparisons:
self.assertRaises(RecursionError, op, y, x)
self.assertIsNot(y, x)

View file

@ -5,6 +5,7 @@ import itertools
import math
import pickle
import sys
import cosmo
import types
import unittest
import warnings
@ -3469,6 +3470,7 @@ order (MRO) for bases """
list.__init__(a, sequence=[0, 1, 2])
self.assertEqual(a, [0, 1, 2])
@unittest.skipUnless(cosmo.MODE == "dbg", "disabled recursion checking")
def test_recursive_call(self):
# Testing recursive __call__() by setting to instance of class...
class A(object):
@ -4494,6 +4496,7 @@ order (MRO) for bases """
with self.assertRaises(TypeError):
str.__add__(fake_str, "abc")
@unittest.skipUnless(cosmo.MODE == "dbg", "disabled recursion checking")
def test_repr_as_str(self):
# Issue #11603: crash or infinite loop when rebinding __str__ as
# __repr__.

View file

@ -469,6 +469,7 @@ class DictTest(unittest.TestCase):
d = {1: BadRepr()}
self.assertRaises(Exc, repr, d)
@unittest.skipUnless(cosmo.MODE == "dbg", "disabled recursion checking")
def test_repr_deep(self):
d = {}
for i in range(sys.getrecursionlimit() + 100):
@ -1222,10 +1223,11 @@ class CAPITest(unittest.TestCase):
self.assertEqual(dict_getitem_knownhash(d, 'z', hash('z')), 3)
# # TODO: Did this break? What did this do?
# (likely related to disabling BadInternalCall in #264)
# # not a dict
# # find the APE compilation mode, run this test in dbg only #
# if cosmo.MODE == "dbg":
# self.assertRaises(SystemError, dict_getitem_knownhash, [], 1, hash(1))
if cosmo.MODE == "dbg":
self.assertRaises(SystemError, dict_getitem_knownhash, [], 1, hash(1))
# key does not exist
self.assertRaises(KeyError, dict_getitem_knownhash, {}, 1, hash(1))

View file

@ -2,6 +2,7 @@ import collections
import copy
import pickle
import sys
import cosmo
import unittest
class DictSetTest(unittest.TestCase):
@ -213,6 +214,7 @@ class DictSetTest(unittest.TestCase):
# Again.
self.assertIsInstance(r, str)
@unittest.skipUnless(cosmo.MODE == "dbg", "disabled recursion checking")
def test_deeply_nested_repr(self):
d = {}
for i in range(sys.getrecursionlimit() + 100):

View file

@ -515,6 +515,7 @@ class ExceptionTests(unittest.TestCase):
self.assertEqual(x.fancy_arg, 42)
@no_tracing
@unittest.skipUnless(cosmo.MODE == "dbg", "disabled recursion checking")
def testInfiniteRecursion(self):
def f():
return f()
@ -928,6 +929,7 @@ class ExceptionTests(unittest.TestCase):
else:
self.fail("Should have raised KeyError")
if cosmo.MODE == "dbg":
def g():
try:
return g()
@ -937,6 +939,7 @@ class ExceptionTests(unittest.TestCase):
self.assertTrue(isinstance(v, RecursionError), type(v))
self.assertIn("maximum recursion depth exceeded", str(v))
@unittest.skipUnless(cosmo.MODE == "dbg", "disabled recursion checking")
@cpython_only
def test_recursion_normalizing_exception(self):
# Issue #22898.
@ -1014,6 +1017,7 @@ class ExceptionTests(unittest.TestCase):
b'while normalizing an exception', err)
self.assertIn(b'Done.', out)
@unittest.skipUnless(cosmo.MODE == "dbg", "disabled recursion checking")
@cpython_only
def test_recursion_normalizing_with_no_memory(self):
# Issue #30697. Test that in the abort that occurs when there is no
@ -1119,6 +1123,7 @@ class ExceptionTests(unittest.TestCase):
self.assertEqual(wr(), None)
@no_tracing
@unittest.skipUnless(cosmo.MODE == "dbg", "disabled recursion checking")
def test_recursion_error_cleanup(self):
# Same test as above, but with "recursion exceeded" errors
class C:
@ -1208,6 +1213,7 @@ class ExceptionTests(unittest.TestCase):
self.assertIn("test message", report)
self.assertTrue(report.endswith("\n"))
@unittest.skipUnless(cosmo.MODE == "dbg", "disabled memory hooks")
@cpython_only
def test_memory_error_in_PyErr_PrintEx(self):
code = """if 1:

View file

@ -177,6 +177,7 @@ class AutoFileTests:
finally:
os.close(fd)
@unittest.skipUnless(cosmo.MODE == "dbg", "disabled recursion checking")
def testRecursiveRepr(self):
# Issue #25455
with swap_attr(self.f, 'name', self.f):

View file

@ -217,6 +217,7 @@ class TestPartial:
[f'{name}({capture!r}, {args_repr}, {kwargs_repr})'
for kwargs_repr in kwargs_reprs])
@unittest.skipUnless(cosmo.MODE == "dbg", "disabled recursion checking")
def test_recursive_repr(self):
if self.partial in (c_functools.partial, py_functools.partial):
name = 'functools.partial'
@ -329,6 +330,7 @@ class TestPartial:
def test_recursive_pickle(self):
with self.AllowPickle():
if cosmo.MODE == "dbg":
f = self.partial(capture)
f.__setstate__((f, (), {}, {}))
try:

View file

@ -28,6 +28,7 @@ import pickle
import random
import signal
import sys
import cosmo
import time
import unittest
import warnings
@ -1099,6 +1100,7 @@ class CommonBufferedTests:
raw.name = b"dummy"
self.assertEqual(repr(b), "<%s name=b'dummy'>" % clsname)
@unittest.skipUnless(cosmo.MODE == "dbg", "disabled recursion checking")
def test_recursive_repr(self):
# Issue #25455
raw = self.MockRawIO()
@ -2540,6 +2542,7 @@ class TextIOWrapperTest(unittest.TestCase):
t.buffer.detach()
repr(t) # Should not raise an exception
@unittest.skipUnless(cosmo.MODE == "dbg", "disabled recursion checking")
def test_recursive_repr(self):
# Issue #25455
raw = self.BytesIO()

View file

@ -4,6 +4,7 @@
import unittest
import sys
import cosmo
@ -257,11 +258,13 @@ class TestIsInstanceIsSubclass(unittest.TestCase):
self.assertEqual(True, issubclass(int, (int, (float, int))))
self.assertEqual(True, issubclass(str, (str, (Child, NewChild, str))))
@unittest.skipUnless(cosmo.MODE == "dbg", "disabled recursion checking")
def test_subclass_recursion_limit(self):
# make sure that issubclass raises RecursionError before the C stack is
# blown
self.assertRaises(RecursionError, blowstack, issubclass, str, str)
@unittest.skipUnless(cosmo.MODE == "dbg", "disabled recursion checking")
def test_isinstance_recursion_limit(self):
# make sure that issubclass raises RecursionError before the C stack is
# blown

View file

@ -1,4 +1,6 @@
from test.test_json import PyTest, CTest
import unittest
import cosmo
class JSONTestObject:
@ -65,6 +67,7 @@ class TestRecursion:
self.fail("didn't raise ValueError on default recursion")
@unittest.skipUnless(cosmo.MODE == "dbg", "disabled recursion checking")
def test_highly_nested_objects_decoding(self):
# test that loading highly-nested objects doesn't segfault when C
# accelerations are used. See #12017
@ -75,6 +78,7 @@ class TestRecursion:
with self.assertRaises(RecursionError):
self.loads('[' * 100000 + '1' + ']' * 100000)
@unittest.skipUnless(cosmo.MODE == "dbg", "disabled recursion checking")
def test_highly_nested_objects_encoding(self):
# See #12051
l, d = [], {}
@ -85,6 +89,7 @@ class TestRecursion:
with self.assertRaises(RecursionError):
self.dumps(d)
@unittest.skipUnless(cosmo.MODE == "dbg", "disabled recursion checking")
def test_endless_recursion(self):
# See #12051
class EndlessJSONEncoder(self.json.JSONEncoder):

View file

@ -1,5 +1,6 @@
# Copyright (C) 2003-2013 Python Software Foundation
import cosmo
import struct
import unittest
import plistlib
@ -813,6 +814,7 @@ class TestBinaryPlistlib(unittest.TestCase):
b = plistlib.loads(plistlib.dumps(a, fmt=plistlib.FMT_BINARY))
self.assertIs(b['x'], b)
@unittest.skipUnless(cosmo.MODE == "dbg", "disabled recursion checking")
def test_deep_nesting(self):
for N in [300, 100000]:
chunks = [b'\xa1' + (i + 1).to_bytes(4, 'big') for i in range(N)]

View file

@ -1,6 +1,7 @@
"""Test the interactive interpreter."""
import sys
import cosmo
import os
import unittest
import subprocess
@ -36,6 +37,7 @@ def spawn_repl(*args, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, **kw):
class TestInteractiveInterpreter(unittest.TestCase):
@unittest.skipUnless(cosmo.MODE == "dbg", "disabled memory hooks")
@cpython_only
def test_no_memory(self):
# Issue #30696: Fix the interactive interpreter looping endlessly when

View file

@ -1,6 +1,7 @@
# Tests for rich comparisons
import unittest
import cosmo
from test import support
import operator
@ -220,6 +221,7 @@ class MiscTest(unittest.TestCase):
for func in (do, operator.not_):
self.assertRaises(Exc, func, Bad())
@unittest.skipUnless(cosmo.MODE == "dbg", "disabled recursion checking")
@support.no_tracing
def test_recursion(self):
# Check that comparison for recursive objects fails gracefully

View file

@ -3,6 +3,7 @@ import unittest
import os
import os.path
import sys
import cosmo
import re
import tempfile
import importlib, importlib.machinery, importlib.util
@ -723,6 +724,7 @@ class RunPathTestCase(unittest.TestCase, CodeExecutionMixin):
self._check_import_error(zip_name, msg)
@no_tracing
@unittest.skipUnless(cosmo.MODE == "dbg", "disabled recursion checking")
def test_main_recursion_error(self):
with temp_dir() as script_dir, temp_dir() as dummy_dir:
mod_name = '__main__'

View file

@ -1,6 +1,7 @@
import unittest, test.support
from test.support.script_helper import assert_python_ok, assert_python_failure
import sys, io, os
import cosmo
import struct
import subprocess
import textwrap
@ -190,6 +191,7 @@ class SysModuleTest(unittest.TestCase):
finally:
sys.setswitchinterval(orig)
@unittest.skipUnless(cosmo.MODE == "dbg", "disabled recursion checking")
def test_recursionlimit(self):
self.assertRaises(TypeError, sys.getrecursionlimit, 42)
oldlimit = sys.getrecursionlimit()
@ -199,6 +201,7 @@ class SysModuleTest(unittest.TestCase):
self.assertEqual(sys.getrecursionlimit(), 10000)
sys.setrecursionlimit(oldlimit)
@unittest.skipUnless(cosmo.MODE == "dbg", "disabled recursion checking")
def test_recursionlimit_recovery(self):
if hasattr(sys, 'gettrace') and sys.gettrace():
self.skipTest('fatal error if run with a trace function')
@ -222,6 +225,7 @@ class SysModuleTest(unittest.TestCase):
finally:
sys.setrecursionlimit(oldlimit)
@unittest.skipUnless(cosmo.MODE == "dbg", "disabled recursion checking")
@test.support.cpython_only
def test_setrecursionlimit_recursion_depth(self):
# Issue #25274: Setting a low recursion limit must be blocked if the
@ -257,6 +261,7 @@ class SysModuleTest(unittest.TestCase):
finally:
sys.setrecursionlimit(oldlimit)
@unittest.skipUnless(cosmo.MODE == "dbg", "disabled recursion checking")
def test_recursionlimit_fatalerror(self):
# A fatal error occurs if a second recursion limit is hit when recovering
# from a first one.

View file

@ -9,6 +9,7 @@ from test.support.script_helper import assert_python_ok, assert_python_failure
import random
import sys
import cosmo
_thread = import_module('_thread')
threading = import_module('threading')
import time
@ -882,6 +883,7 @@ class ThreadingExceptionTests(BaseTestCase):
lock = threading.Lock()
self.assertRaises(RuntimeError, lock.release)
@unittest.skipUnless(cosmo.MODE == "dbg", "disabled recursion checking")
@unittest.skipUnless(sys.platform == 'darwin' and test.support.python_is_optimized(),
'test macosx problem')
def test_recursion_limit(self):

View file

@ -4,6 +4,7 @@ from collections import namedtuple
from io import StringIO
import linecache
import sys
import cosmo
import unittest
import re
from test import support
@ -300,6 +301,7 @@ class TracebackFormatTests(unittest.TestCase):
])
# issue 26823 - Shrink recursive tracebacks
@unittest.skipUnless(cosmo.MODE == "dbg", "disabled recursion checking")
def _check_recursive_traceback_display(self, render_exc):
# Always show full diffs when this test fails
# Note that rearranging things may require adjusting

View file

@ -1,6 +1,8 @@
import contextlib
import os
import sys
import cosmo
if cosmo.MODE == "dbg":
import tracemalloc
import unittest
from unittest.mock import patch
@ -87,6 +89,7 @@ def traceback_filename(filename):
return traceback_lineno(filename, 0)
@unittest.skipUnless(cosmo.MODE == "dbg", "requires APE debug build")
class TestTracemallocEnabled(unittest.TestCase):
def setUp(self):
if tracemalloc.is_tracing():
@ -297,6 +300,7 @@ class TestTracemallocEnabled(unittest.TestCase):
self.assertEqual(exitcode, 0)
@unittest.skipUnless(cosmo.MODE == "dbg", "requires APE debug build")
class TestSnapshot(unittest.TestCase):
maxDiff = 4000
@ -591,6 +595,7 @@ class TestSnapshot(unittest.TestCase):
[])
@unittest.skipUnless(cosmo.MODE == "dbg", "requires APE debug build")
class TestFilters(unittest.TestCase):
maxDiff = 2048
@ -802,6 +807,7 @@ class TestFilters(unittest.TestCase):
self.assertFalse(f._match_traceback(unknown))
@unittest.skipUnless(cosmo.MODE == "dbg", "requires APE debug build")
class TestCommandLine(unittest.TestCase):
def test_env_var_disabled_by_default(self):
# not tracing by default
@ -874,6 +880,7 @@ class TestCommandLine(unittest.TestCase):
assert_python_ok('-X', 'tracemalloc', '-c', code)
@unittest.skipUnless(cosmo.MODE == "dbg", "requires APE debug build")
@unittest.skipIf(_testcapi is None, 'need _testcapi')
class TestCAPI(unittest.TestCase):
maxDiff = 80 * 20

View file

@ -1,6 +1,7 @@
from contextlib import contextmanager
import linecache
import os
import cosmo
from io import StringIO
import re
import sys
@ -922,6 +923,7 @@ class CWarningsDisplayTests(WarningsDisplayTests, unittest.TestCase):
class PyWarningsDisplayTests(WarningsDisplayTests, unittest.TestCase):
module = py_warnings
@unittest.skipUnless(cosmo.MODE == "dbg", "requires APE debug build")
def test_tracemalloc(self):
self.addCleanup(support.unlink, support.TESTFN)

View file

@ -5,6 +5,7 @@
# For this purpose, the module-level "ET" symbol is temporarily
# monkey-patched when running the "test_xml_etree_c" test suite.
import cosmo
import copy
import html
import io
@ -1920,6 +1921,7 @@ class BadElementTest(ElementTestCase, unittest.TestCase):
e.extend([ET.Element('bar')])
self.assertRaises(ValueError, e.remove, X('baz'))
@unittest.skipUnless(cosmo.MODE == "dbg", "disabled recursion checking")
def test_recursive_repr(self):
# Issue #25455
e = ET.Element('foo')

View file

@ -4,6 +4,7 @@ import binascii
import pickle
import random
import sys
import cosmo
import zlib
from test.support import bigmemtest, _1G, _4G
@ -707,6 +708,7 @@ class CompressObjectTestCase(BaseCompressTestCase, unittest.TestCase):
# Memory use of the following functions takes into account overallocation
@unittest.skipUnless(cosmo.MODE == "dbg", "disabled tracemalloc")
@bigmemtest(size=_1G + 1024 * 1024, memuse=3)
def test_big_compress_buffer(self, size):
import tracemalloc

View file

@ -3488,6 +3488,7 @@ test_pymem_alloc0(PyObject *self)
Py_RETURN_NONE;
}
#if IsModeDbg()
typedef struct {
PyMemAllocatorEx alloc;
@ -3780,6 +3781,7 @@ remove_mem_hooks(PyObject *self)
fm_remove_hooks();
Py_RETURN_NONE;
}
#endif
PyDoc_STRVAR(docstring_empty,
""
@ -4234,6 +4236,7 @@ test_PyTime_AsMicroseconds(PyObject *self, PyObject *args)
return _PyTime_AsNanosecondsObject(ms);
}
#if IsModeDbg()
static PyObject*
get_recursion_depth(PyObject *self, PyObject *args)
{
@ -4242,6 +4245,7 @@ get_recursion_depth(PyObject *self, PyObject *args)
/* subtract one to ignore the frame of the get_recursion_depth() call */
return PyLong_FromLong(tstate->recursion_depth - 1);
}
#endif
static PyObject*
pymem_buffer_overflow(PyObject *self, PyObject *args)
@ -4656,6 +4660,7 @@ static PyMethodDef TestMethods[] = {
{"create_cfunction", create_cfunction, METH_NOARGS},
{"test_pymem_alloc0",
(PyCFunction)test_pymem_alloc0, METH_NOARGS},
#if IsModeDbg()
{"test_pymem_setrawallocators",
(PyCFunction)test_pymem_setrawallocators, METH_NOARGS},
{"test_pymem_setallocators",
@ -4666,6 +4671,7 @@ static PyMethodDef TestMethods[] = {
PyDoc_STR("set_nomemory(start:int, stop:int = 0)")},
{"remove_mem_hooks", (PyCFunction)remove_mem_hooks, METH_NOARGS,
PyDoc_STR("Remove memory hooks.")},
#endif
{"no_docstring",
(PyCFunction)test_with_docstring, METH_NOARGS},
{"docstring_empty",
@ -4723,6 +4729,7 @@ static PyMethodDef TestMethods[] = {
#endif
{"PyTime_AsMilliseconds", test_PyTime_AsMilliseconds, METH_VARARGS},
{"PyTime_AsMicroseconds", test_PyTime_AsMicroseconds, METH_VARARGS},
#if IsModeDbg()
{"get_recursion_depth", get_recursion_depth, METH_NOARGS},
{"pymem_buffer_overflow", pymem_buffer_overflow, METH_NOARGS},
{"pymem_api_misuse", pymem_api_misuse, METH_NOARGS},
@ -4731,6 +4738,7 @@ static PyMethodDef TestMethods[] = {
{"tracemalloc_track", tracemalloc_track, METH_VARARGS},
{"tracemalloc_untrack", tracemalloc_untrack, METH_VARARGS},
{"tracemalloc_get_traceback", tracemalloc_get_traceback, METH_VARARGS},
#endif
{"dict_get_version", dict_get_version, METH_VARARGS},
{"pyobject_fastcall", test_pyobject_fastcall, METH_VARARGS},
{"pyobject_fastcalldict", test_pyobject_fastcalldict, METH_VARARGS},

View file

@ -41,6 +41,7 @@ PYTHON_PROVIDE("_tracemalloc.is_tracing");
PYTHON_PROVIDE("_tracemalloc.start");
PYTHON_PROVIDE("_tracemalloc.stop");
#if IsModeDbg()
/* Trace memory blocks allocated by PyMem_RawMalloc() */
#define TRACE_RAW_MALLOC
@ -1686,30 +1687,6 @@ static PyMethodDef module_methods[] = {
PyDoc_STRVAR(module_doc,
"Debug module to trace memory blocks allocated by Python.");
static struct PyModuleDef module_def = {
PyModuleDef_HEAD_INIT,
"_tracemalloc",
module_doc,
0, /* non-negative size to be able to unload the module */
module_methods,
NULL,
};
PyMODINIT_FUNC
PyInit__tracemalloc(void)
{
PyObject *m;
m = PyModule_Create(&module_def);
if (m == NULL)
return NULL;
if (tracemalloc_init() < 0)
return NULL;
return m;
}
static int
parse_sys_xoptions(PyObject *value)
{
@ -1862,6 +1839,41 @@ PyObject*
return traceback_to_pyobject(traceback, NULL);
}
#endif
static struct PyModuleDef module_def = {
PyModuleDef_HEAD_INIT,
"_tracemalloc",
#ifdef USETRACEMALLOC
module_doc,
#else
NULL,
#endif
0, /* non-negative size to be able to unload the module */
#ifdef USETRACEMALLOC
module_methods,
#else
NULL,
#endif
NULL,
};
PyMODINIT_FUNC
PyInit__tracemalloc(void)
{
PyObject *m;
m = PyModule_Create(&module_def);
if (m == NULL)
return NULL;
#ifdef USETRACEMALLOC
if (tracemalloc_init() < 0)
return NULL;
#endif
return m;
}
_Section(".rodata.pytab.1") const struct _inittab _PyImport_Inittab__tracemalloc = {
"_tracemalloc",

View file

@ -424,12 +424,14 @@ Py_Main(int argc, wchar_t **argv)
}
}
#if IsModeDbg()
opt = Py_GETENV("PYTHONMALLOC");
if (_PyMem_SetupAllocators(opt) < 0) {
fprintf(stderr,
"Error in PYTHONMALLOC: unknown allocator \"%s\"!\n", opt);
exit(1);
}
#endif
_PyRandom_Init();

View file

@ -29,6 +29,7 @@
#define uint unsigned int /* assuming >= 16 bits */
/* Forward declaration */
#if IsModeDbg()
static void *_PyMem_DebugRawMalloc(void *, size_t);
static void *_PyMem_DebugRawCalloc(void *, size_t, size_t);
static void *_PyMem_DebugRawRealloc(void *, void *, size_t);
@ -37,6 +38,7 @@ static void *_PyMem_DebugMalloc(void *, size_t);
static void *_PyMem_DebugCalloc(void *, size_t, size_t);
static void *_PyMem_DebugRealloc(void *, void *, size_t);
static void _PyMem_DebugFree(void *, void *);
#endif
static void _PyObject_DebugDumpAddress(const void *);
static void _PyMem_DebugCheckAddress(char, const void *);
@ -82,6 +84,13 @@ static void* _PyObject_Malloc(void *ctx, size_t size);
static void* _PyObject_Calloc(void *ctx, size_t nelem, size_t elsize);
static void _PyObject_Free(void *ctx, void *p);
static void* _PyObject_Realloc(void *ctx, void *ptr, size_t size);
#else
/* in MODE=asan, no pymalloc, so use macro */
#define _PyObject_Malloc(ctx, size) _PyMem_RawMalloc((ctx), (size))
#define _PyObject_Calloc(ctx, nelem, elsize) _PyMem_RawCalloc((ctx), (nelem), (elsize))
#define _PyObject_Realloc(ctx, ptr, size) _PyMem_RawRealloc((ctx), (ptr), (size))
#define _PyObject_Free(ctx, p) _PyMem_RawFree((ctx), (p))
#endif
static inline void *
@ -207,6 +216,7 @@ _PyObject_ArenaFree(void *ctx, void *ptr, size_t size)
}
#endif
#if IsModeDbg()
#define PYRAW_FUNCS _PyMem_RawMalloc, _PyMem_RawCalloc, _PyMem_RawRealloc, _PyMem_RawFree
#ifdef WITH_PYMALLOC
# define PYOBJ_FUNCS _PyObject_Malloc, _PyObject_Calloc, _PyObject_Realloc, _PyObject_Free
@ -428,6 +438,7 @@ PyObject_SetArenaAllocator(PyObjectArenaAllocator *allocator)
{
_PyObject_Arena = *allocator;
}
#endif
void *
PyMem_RawMalloc(size_t size)
@ -440,7 +451,11 @@ PyMem_RawMalloc(size_t size)
*/
if (size > (size_t)PY_SSIZE_T_MAX)
return NULL;
#if IsModeDbg()
return _PyMem_Raw.malloc(_PyMem_Raw.ctx, size);
#else
return _PyMem_RawMalloc(NULL, size);
#endif
}
void *
@ -449,7 +464,11 @@ PyMem_RawCalloc(size_t nelem, size_t elsize)
/* see PyMem_RawMalloc() */
if (elsize != 0 && nelem > (size_t)PY_SSIZE_T_MAX / elsize)
return NULL;
#if IsModeDbg()
return _PyMem_Raw.calloc(_PyMem_Raw.ctx, nelem, elsize);
#else
return _PyMem_RawCalloc(NULL, nelem, elsize);
#endif
}
void*
@ -458,12 +477,20 @@ PyMem_RawRealloc(void *ptr, size_t new_size)
/* see PyMem_RawMalloc() */
if (new_size > (size_t)PY_SSIZE_T_MAX)
return NULL;
#if IsModeDbg()
return _PyMem_Raw.realloc(_PyMem_Raw.ctx, ptr, new_size);
#else
return _PyMem_RawRealloc(NULL, ptr, new_size);
#endif
}
void PyMem_RawFree(void *ptr)
{
#if IsModeDbg()
_PyMem_Raw.free(_PyMem_Raw.ctx, ptr);
#else
_PyMem_RawFree(NULL, ptr);
#endif
}
void *
@ -472,7 +499,11 @@ PyMem_Malloc(size_t size)
/* see PyMem_RawMalloc() */
if (size > (size_t)PY_SSIZE_T_MAX)
return NULL;
#if IsModeDbg()
return _PyMem.malloc(_PyMem.ctx, size);
#else
return _PyObject_Malloc(NULL, size);
#endif
}
void *
@ -481,7 +512,11 @@ PyMem_Calloc(size_t nelem, size_t elsize)
/* see PyMem_RawMalloc() */
if (elsize != 0 && nelem > (size_t)PY_SSIZE_T_MAX / elsize)
return NULL;
#if IsModeDbg()
return _PyMem.calloc(_PyMem.ctx, nelem, elsize);
#else
return _PyObject_Calloc(NULL, nelem, elsize);
#endif
}
void *
@ -490,13 +525,21 @@ PyMem_Realloc(void *ptr, size_t new_size)
/* see PyMem_RawMalloc() */
if (new_size > (size_t)PY_SSIZE_T_MAX)
return NULL;
#if IsModeDbg()
return _PyMem.realloc(_PyMem.ctx, ptr, new_size);
#else
return _PyObject_Realloc(NULL, ptr, new_size);
#endif
}
void
(PyMem_Free)(void *ptr)
{
#if IsModeDbg()
_PyMem.free(_PyMem.ctx, ptr);
#else
return _PyObject_Free(NULL, ptr);
#endif
}
char *
@ -531,7 +574,11 @@ void *
/* see PyMem_RawMalloc() */
if (size > (size_t)PY_SSIZE_T_MAX)
return NULL;
#if IsModeDbg()
return _PyObject.malloc(_PyObject.ctx, size);
#else
return _PyObject_Malloc(NULL, size);
#endif
}
void *
@ -540,7 +587,11 @@ PyObject_Calloc(size_t nelem, size_t elsize)
/* see PyMem_RawMalloc() */
if (elsize != 0 && nelem > (size_t)PY_SSIZE_T_MAX / elsize)
return NULL;
#if IsModeDbg()
return _PyObject.calloc(_PyObject.ctx, nelem, elsize);
#else
return _PyObject_Calloc(NULL, nelem, elsize);
#endif
}
void *
@ -549,13 +600,21 @@ PyObject_Realloc(void *ptr, size_t new_size)
/* see PyMem_RawMalloc() */
if (new_size > (size_t)PY_SSIZE_T_MAX)
return NULL;
#if IsModeDbg()
return _PyObject.realloc(_PyObject.ctx, ptr, new_size);
#else
return _PyObject_Realloc(NULL, ptr, new_size);
#endif
}
void
PyObject_Free(void *ptr)
{
#if IsModeDbg()
_PyObject.free(_PyObject.ctx, ptr);
#else
return _PyObject_Free(NULL, ptr);
#endif
}
@ -1137,7 +1196,11 @@ new_arena(void)
arenaobj = unused_arena_objects;
unused_arena_objects = arenaobj->nextarena;
assert(arenaobj->address == 0);
#if IsModeDbg()
address = _PyObject_Arena.alloc(_PyObject_Arena.ctx, ARENA_SIZE);
#else
address = _PyObject_ArenaMmap(NULL, ARENA_SIZE);
#endif
if (address == NULL) {
/* The allocation failed: return NULL after putting the
* arenaobj back.
@ -1615,8 +1678,13 @@ _PyObject_Free(void *ctx, void *p)
unused_arena_objects = ao;
/* Free the entire arena. */
#if IsModeDbg()
_PyObject_Arena.free(_PyObject_Arena.ctx,
(void *)ao->address, ARENA_SIZE);
#else
_PyObject_ArenaMunmap(NULL,
(void *)ao->address, ARENA_SIZE);
#endif
ao->address = 0; /* mark unassociated */
--narenas_currently_allocated;
@ -1853,6 +1921,7 @@ write_size_t(void *p, size_t n)
WRITE64BE((char *)p, n);
}
#if IsModeDbg()
/* Let S = sizeof(size_t). The debug malloc asks for 4*S extra bytes and
fills them with useful stuff, here calling the underlying malloc's result p:
@ -1941,7 +2010,7 @@ _PyMem_DebugRawCalloc(void *ctx, size_t nelem, size_t elsize)
Python memory allocators which fills the memory with DEADBYTE (0xDB) when
memory is deallocated. */
int
_PyMem_IsFreed(void *ptr, size_t size)
(_PyMem_IsFreed)(void *ptr, size_t size)
{
unsigned char *bytes = ptr;
for (size_t i=0; i < size; i++) {
@ -2222,11 +2291,12 @@ _PyObject_DebugDumpAddress(const void *p)
fputc('\n', stderr);
fflush(stderr);
#ifdef USE_TRACEMALLOC
#if IsModeDbg()
PYTHON_YOINK("_tracemalloc");
_PyMem_DumpTraceback(fileno(stderr), p);
#endif
}
#endif
static size_t
@ -2417,8 +2487,10 @@ _PyObject_DebugMallocStats(FILE *out)
quantization += p * ((POOL_SIZE - POOL_OVERHEAD) % size);
}
fputc('\n', out);
#if IsModeDbg()
if (_PyMem_DebugEnabled())
(void)printone(out, "# times object malloc called", serialno);
#endif
(void)printone(out, "# arenas allocated total", ntimes_arena_allocated);
(void)printone(out, "# arenas reclaimed", ntimes_arena_allocated - narenas);
(void)printone(out, "# arenas highwater mark", narenas_highwater);

View file

@ -121,7 +121,7 @@ Py_FinalizeEx(void)
_PyGC_CollectIfEnabled();
#endif
#ifdef MODE_DBG
#if IsModeDbg()
/* Disable tracemalloc after all Python objects have been destroyed,
so it is possible to use tracemalloc in objects destructor. */
_PyTraceMalloc_Fini();
@ -219,11 +219,13 @@ Py_FinalizeEx(void)
_Py_PrintReferenceAddresses(stderr);
#endif /* Py_TRACE_REFS */
#ifdef WITH_PYMALLOC
#if IsModeDbg()
if (_PyMem_PymallocEnabled()) {
char *opt = Py_GETENV("PYTHONMALLOCSTATS");
if (opt != NULL && *opt != '\0')
_PyObject_DebugMallocStats(stderr);
}
#endif
#endif
_Py_CallLlExitFuncs();

View file

@ -274,7 +274,7 @@ _Py_InitializeEx_Private(int install_sigs, int install_importlib)
if (install_sigs)
_Py_InitSigs(); /* Signal handling stuff, including initintr() */
#ifdef USE_TRACEMALLOC
#if IsModeDbg()
if (_PyTraceMalloc_Init() < 0)
Py_FatalError("Py_Initialize: can't initialize tracemalloc");
#endif

View file

@ -91,7 +91,9 @@ PYTHON_PROVIDE("sys.float_info");
PYTHON_PROVIDE("sys.float_repr_style");
PYTHON_PROVIDE("sys.get_asyncgen_hooks");
PYTHON_PROVIDE("sys.get_coroutine_wrapper");
#if IsModeDbg()
PYTHON_PROVIDE("sys.getallocatedblocks");
#endif
PYTHON_PROVIDE("sys.getcheckinterval");
PYTHON_PROVIDE("sys.getdefaultencoding");
PYTHON_PROVIDE("sys.getdlopenflags");
@ -1262,6 +1264,7 @@ one higher than you might expect, because it includes the (temporary)\n\
reference as an argument to getrefcount()."
);
#if IsModeDbg()
static PyObject *
sys_getallocatedblocks(PyObject *self)
{
@ -1274,6 +1277,7 @@ PyDoc_STRVAR(getallocatedblocks_doc,
Return the number of memory blocks currently allocated, regardless of their\n\
size."
);
#endif
#ifdef COUNT_ALLOCS
static PyObject *
@ -1382,10 +1386,12 @@ static PyObject *
sys_debugmallocstats(PyObject *self, PyObject *args)
{
#ifdef WITH_PYMALLOC
#if IsModeDbg()
if (_PyMem_PymallocEnabled()) {
_PyObject_DebugMallocStats(stderr);
fputc('\n', stderr);
}
#endif
#endif
_PyObject_DebugTypeStats(stderr);
@ -1455,8 +1461,10 @@ static PyMethodDef sys_methods[] = {
{"getdlopenflags", (PyCFunction)sys_getdlopenflags, METH_NOARGS,
getdlopenflags_doc},
#endif
#if IsModeDbg()
{"getallocatedblocks", (PyCFunction)sys_getallocatedblocks, METH_NOARGS,
getallocatedblocks_doc},
#endif
#ifdef COUNT_ALLOCS
{"getcounts", (PyCFunction)sys_getcounts, METH_NOARGS},
#endif

View file

@ -573,9 +573,8 @@
#define OPENSSL_NO_COMP 1
#define HAVE_LANGINFO_H 1
#ifdef MODE_DBG
#if IsModeDbg()
#define Py_DEBUG 1
#define USE_TRACEMALLOC 1
#endif
/* #define FAST_LOOPS 1 /\* froot loops *\/ */

View file

@ -280,8 +280,10 @@ RunPythonModule(int argc, char **argv)
linenoiseSetHintsCallback(TerminalHint);
linenoiseSetFreeHintsCallback(free);
#if IsModeDbg()
/* Force malloc() allocator to bootstrap Python */
_PyMem_SetupAllocators("malloc");
#endif
argv_copy = (wchar_t **)PyMem_RawMalloc(sizeof(wchar_t*) * (argc+1));
argv_copy2 = (wchar_t **)PyMem_RawMalloc(sizeof(wchar_t*) * (argc+1));
@ -324,9 +326,11 @@ RunPythonModule(int argc, char **argv)
res = Py_Main(argc, argv_copy);
#if IsModeDbg()
/* Force again malloc() allocator to release memory blocks allocated
before Py_Main() */
_PyMem_SetupAllocators("malloc");
#endif
for (i = 0; i < argc; i++) {
PyMem_RawFree(argv_copy2[i]);