Fix Pyston speedups (#281)

We remove (i.e. hide behind a debug ifdef) the recursion checking methods,
and the memory hooks and memory allocator methods. ASAN mode has no
PYMALLOC, so we need a macro. Fix build break with des.c stack allocation.
This commit is contained in:
Gautham 2021-10-02 13:58:51 +05:30 committed by GitHub
parent 2fe8571010
commit 57f0eed382
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
42 changed files with 260 additions and 63 deletions

View file

@ -58,6 +58,10 @@ o/$(MODE)/third_party/mbedtls/everest.o: \
OVERRIDE_CFLAGS += \ OVERRIDE_CFLAGS += \
-O3 -O3
o/$(MODE)/third_party/mbedtls/des.o: \
OVERRIDE_CFLAGS += \
-DSTACK_FRAME_UNLIMITED
o/$(MODE)/third_party/mbedtls/bigmul4.o \ o/$(MODE)/third_party/mbedtls/bigmul4.o \
o/$(MODE)/third_party/mbedtls/bigmul6.o: \ o/$(MODE)/third_party/mbedtls/bigmul6.o: \
OVERRIDE_CFLAGS += \ OVERRIDE_CFLAGS += \

View file

@ -35,7 +35,7 @@ PyObject *_PyObject_FastCallKeywords(PyObject *func, PyObject **args,
PyObject *_PyObject_Call_Prepend(PyObject *func, PyObject *obj, PyObject *args, PyObject *_PyObject_Call_Prepend(PyObject *func, PyObject *obj, PyObject *args,
PyObject *kwargs); PyObject *kwargs);
#ifdef USE_CHECKFUNCRESULT #if IsModeDbg()
PyObject *_Py_CheckFunctionResult(PyObject *func, PyObject *result, PyObject *_Py_CheckFunctionResult(PyObject *func, PyObject *result,
const char *where); const char *where);
#else #else

View file

@ -77,6 +77,7 @@ int Py_MakePendingCalls(void);
void Py_SetRecursionLimit(int); void Py_SetRecursionLimit(int);
int Py_GetRecursionLimit(void); int Py_GetRecursionLimit(void);
#if IsModeDbg()
#define Py_EnterRecursiveCall(where) \ #define Py_EnterRecursiveCall(where) \
(_Py_MakeRecCheck(PyThreadState_GET()->recursion_depth) && \ (_Py_MakeRecCheck(PyThreadState_GET()->recursion_depth) && \
_Py_CheckRecursiveCall(where)) _Py_CheckRecursiveCall(where))
@ -84,6 +85,10 @@ int Py_GetRecursionLimit(void);
do{ if(_Py_MakeEndRecCheck(PyThreadState_GET()->recursion_depth)) \ do{ if(_Py_MakeEndRecCheck(PyThreadState_GET()->recursion_depth)) \
PyThreadState_GET()->overflowed = 0; \ PyThreadState_GET()->overflowed = 0; \
} while(0) } while(0)
#else
#define Py_EnterRecursiveCall(where) (0)
#define Py_LeaveRecursiveCall(where) ((void)0)
#endif
int _Py_CheckRecursiveCall(const char *where); int _Py_CheckRecursiveCall(const char *where);
extern int _Py_CheckRecursionLimit; extern int _Py_CheckRecursionLimit;

View file

@ -17,8 +17,10 @@ void * PyObject_Realloc(void *, size_t);
void PyObject_Free(void *); void PyObject_Free(void *);
#ifndef Py_LIMITED_API #ifndef Py_LIMITED_API
#if IsModeDbg()
/* This function returns the number of allocated memory blocks, regardless of size */ /* This function returns the number of allocated memory blocks, regardless of size */
Py_ssize_t _Py_GetAllocatedBlocks(void); Py_ssize_t _Py_GetAllocatedBlocks(void);
#endif
#endif /* !Py_LIMITED_API */ #endif /* !Py_LIMITED_API */
/* Macros */ /* Macros */
@ -117,6 +119,7 @@ PyVarObject * _PyObject_NewVar(PyTypeObject *, Py_ssize_t);
constructor you would start directly with PyObject_Init/InitVar constructor you would start directly with PyObject_Init/InitVar
*/ */
#if IsModeDbg()
#ifndef Py_LIMITED_API #ifndef Py_LIMITED_API
typedef struct { typedef struct {
/* user context passed as the first argument to the 2 functions */ /* user context passed as the first argument to the 2 functions */
@ -135,7 +138,7 @@ void PyObject_GetArenaAllocator(PyObjectArenaAllocator *allocator);
/* Set the arena allocator. */ /* Set the arena allocator. */
void PyObject_SetArenaAllocator(PyObjectArenaAllocator *allocator); void PyObject_SetArenaAllocator(PyObjectArenaAllocator *allocator);
#endif #endif
#endif
/* /*
* Garbage Collection Support * Garbage Collection Support

View file

@ -316,7 +316,7 @@ void PyErr_BadInternalCall(void);
void _PyErr_BadInternalCall(const char *filename, int lineno); void _PyErr_BadInternalCall(const char *filename, int lineno);
/* Mask the old API with a call to the new API for code compiled under /* Mask the old API with a call to the new API for code compiled under
Python 2.0: */ Python 2.0: */
#ifdef USE_BADINTERNALCALL #if IsModeDbg()
#define PyErr_BadInternalCall() _PyErr_BadInternalCall(__FILE__, __LINE__) #define PyErr_BadInternalCall() _PyErr_BadInternalCall(__FILE__, __LINE__)
#else #else
#define PyErr_BadInternalCall() #define PyErr_BadInternalCall()

View file

@ -64,13 +64,15 @@ PyObject* _PyTraceMalloc_GetTraceback(
_PyTraceMalloc_domain_t domain, _PyTraceMalloc_domain_t domain,
uintptr_t ptr); uintptr_t ptr);
#ifdef USE_TRACEMALLOC int _PyMem_IsFreed(void *ptr, size_t size);
#if !IsModeDbg()
#define _PyTraceMalloc_Track(domain, ptr, size) (-2) #define _PyTraceMalloc_Track(domain, ptr, size) (-2)
#define _PyTraceMalloc_Untrack(domain, ptr) (-2) #define _PyTraceMalloc_Untrack(domain, ptr) (-2)
#define _PyTraceMalloc_GetTraceback(domain, ptr) (&_Py_NoneStruct) #define _PyTraceMalloc_GetTraceback(domain, ptr) (&_Py_NoneStruct)
#define _PyMem_IsFreed(ptr, size) (0)
#endif #endif
int _PyMem_IsFreed(void *ptr, size_t size);
#endif /* !defined(Py_LIMITED_API) */ #endif /* !defined(Py_LIMITED_API) */
@ -172,6 +174,7 @@ char * _PyMem_Strdup(const char *str);
#define PyMem_Del PyMem_Free #define PyMem_Del PyMem_Free
#define PyMem_DEL PyMem_FREE #define PyMem_DEL PyMem_FREE
#if IsModeDbg()
#ifndef Py_LIMITED_API #ifndef Py_LIMITED_API
typedef enum { typedef enum {
/* PyMem_RawMalloc(), PyMem_RawRealloc() and PyMem_RawFree() */ /* PyMem_RawMalloc(), PyMem_RawRealloc() and PyMem_RawFree() */
@ -237,6 +240,7 @@ void PyMem_SetAllocator(PyMemAllocatorDomain domain,
The function does nothing if Python is not compiled is debug mode. */ The function does nothing if Python is not compiled is debug mode. */
void PyMem_SetupDebugHooks(void); void PyMem_SetupDebugHooks(void);
#endif #endif
#endif
COSMOPOLITAN_C_END_ COSMOPOLITAN_C_END_
#endif /* !Py_PYMEM_H */ #endif /* !Py_PYMEM_H */

View file

@ -2,6 +2,7 @@ import unittest
from ctypes import * from ctypes import *
from ctypes.test import need_symbol from ctypes.test import need_symbol
import _ctypes_test import _ctypes_test
import cosmo
dll = CDLL(_ctypes_test.__file__) dll = CDLL(_ctypes_test.__file__)
@ -190,6 +191,7 @@ class BasicWrapTestCase(unittest.TestCase):
self.assertEqual((s8i.a, s8i.b, s8i.c, s8i.d, s8i.e, s8i.f, s8i.g, s8i.h), self.assertEqual((s8i.a, s8i.b, s8i.c, s8i.d, s8i.e, s8i.f, s8i.g, s8i.h),
(9*2, 8*3, 7*4, 6*5, 5*6, 4*7, 3*8, 2*9)) (9*2, 8*3, 7*4, 6*5, 5*6, 4*7, 3*8, 2*9))
@unittest.skipUnless(cosmo.MODE == "dbg", "no recursion checking")
def test_recursive_as_param(self): def test_recursive_as_param(self):
from ctypes import c_int from ctypes import c_int

View file

@ -4,6 +4,8 @@ Tests common to list and UserList.UserList
import sys import sys
import os import os
import unittest
import cosmo
from functools import cmp_to_key from functools import cmp_to_key
from test import support, seq_tests from test import support, seq_tests
@ -53,6 +55,7 @@ class CommonTest(seq_tests.CommonTest):
self.assertEqual(str(a2), "[0, 1, 2, [...], 3]") self.assertEqual(str(a2), "[0, 1, 2, [...], 3]")
self.assertEqual(repr(a2), "[0, 1, 2, [...], 3]") self.assertEqual(repr(a2), "[0, 1, 2, [...], 3]")
@unittest.skipUnless(cosmo.MODE == "dbg", "disabled recursion checking")
def test_repr_deep(self): def test_repr_deep(self):
a = self.type2test([]) a = self.type2test([])
for i in range(sys.getrecursionlimit() + 100): for i in range(sys.getrecursionlimit() + 100):

View file

@ -2,6 +2,7 @@
import unittest import unittest
import collections import collections
import sys import sys
import cosmo
class BasicTestMappingProtocol(unittest.TestCase): class BasicTestMappingProtocol(unittest.TestCase):
@ -620,6 +621,7 @@ class TestHashMappingProtocol(TestMappingProtocol):
d = self._full_mapping({1: BadRepr()}) d = self._full_mapping({1: BadRepr()})
self.assertRaises(Exc, repr, d) self.assertRaises(Exc, repr, d)
@unittest.skipUnless(cosmo.MODE == "dbg", "disabled recursion checking")
def test_repr_deep(self): def test_repr_deep(self):
d = self._empty_mapping() d = self._empty_mapping()
for i in range(sys.getrecursionlimit() + 100): for i in range(sys.getrecursionlimit() + 100):

View file

@ -1943,8 +1943,7 @@ class AbstractPickleTests(unittest.TestCase):
self.assertEqual(y._reduce_called, 1) self.assertEqual(y._reduce_called, 1)
@no_tracing @no_tracing
@unittest.skipIf(cosmo.MODE in ("asan", "dbg"), @unittest.skipIf(True, "disabled recursion checking + slow in asan, dbg")
"extremely slow in asan mode")
def test_bad_getattr(self): def test_bad_getattr(self):
# Issue #3514: crash when there is an infinite loop in __getattr__ # Issue #3514: crash when there is an infinite loop in __getattr__
x = BadGetattr() x = BadGetattr()

View file

@ -1,6 +1,7 @@
# Run the _testcapi module tests (tests for the Python/C API): by defn, # Run the _testcapi module tests (tests for the Python/C API): by defn,
# these are all functions _testcapi exports whose name begins with 'test_'. # these are all functions _testcapi exports whose name begins with 'test_'.
import cosmo
import os import os
import pickle import pickle
import random import random
@ -179,6 +180,7 @@ class CAPITest(unittest.TestCase):
o @= m1 o @= m1
self.assertEqual(o, ("matmul", 42, m1)) self.assertEqual(o, ("matmul", 42, m1))
@unittest.skipUnless(cosmo.MODE == "dbg", "disabled recursion check")
def test_return_null_without_error(self): def test_return_null_without_error(self):
# Issue #23571: A function must not return NULL without setting an # Issue #23571: A function must not return NULL without setting an
# error # error
@ -207,6 +209,7 @@ class CAPITest(unittest.TestCase):
'return_null_without_error.* ' 'return_null_without_error.* '
'returned NULL without setting an error') 'returned NULL without setting an error')
@unittest.skipUnless(cosmo.MODE == "dbg", "disabled recursion check")
def test_return_result_with_error(self): def test_return_result_with_error(self):
# Issue #23571: A function must not return a result with an error set # Issue #23571: A function must not return a result with an error set
if Py_DEBUG: if Py_DEBUG:
@ -242,6 +245,7 @@ class CAPITest(unittest.TestCase):
def test_buildvalue_N(self): def test_buildvalue_N(self):
_testcapi.test_buildvalue_N() _testcapi.test_buildvalue_N()
@unittest.skipUnless(cosmo.MODE == "dbg", "disabled memory hooks")
def test_set_nomemory(self): def test_set_nomemory(self):
code = """if 1: code = """if 1:
import _testcapi import _testcapi
@ -510,6 +514,7 @@ class Test_testcapi(unittest.TestCase):
if name.startswith('test_') and not name.endswith('_code')) if name.startswith('test_') and not name.endswith('_code'))
@unittest.skipUnless(cosmo.MODE == "dbg", "disabled memory debugging")
class PyMemDebugTests(unittest.TestCase): class PyMemDebugTests(unittest.TestCase):
PYTHONMALLOC = 'debug' PYTHONMALLOC = 'debug'
# '0x04c06e0' or '04C06E0' # '0x04c06e0' or '04C06E0'

View file

@ -1,6 +1,7 @@
"Test the functionality of Python classes implementing operators." "Test the functionality of Python classes implementing operators."
import unittest import unittest
import cosmo
testmeths = [ testmeths = [
@ -490,6 +491,7 @@ class ClassTests(unittest.TestCase):
self.assertRaises(TypeError, hash, C2()) self.assertRaises(TypeError, hash, C2())
@unittest.skipUnless(cosmo.MODE == "dbg", "disabled recursion checking")
def testSFBug532646(self): def testSFBug532646(self):
# Test for SF bug 532646 # Test for SF bug 532646

View file

@ -1,5 +1,6 @@
"""Unit tests for the copy module.""" """Unit tests for the copy module."""
import cosmo
import copy import copy
import copyreg import copyreg
import weakref import weakref
@ -372,6 +373,7 @@ class TestCopy(unittest.TestCase):
x = [] x = []
x.append(x) x.append(x)
y = copy.deepcopy(x) y = copy.deepcopy(x)
if cosmo.MODE == "dbg": # requires recursion checking
for op in comparisons: for op in comparisons:
self.assertRaises(RecursionError, op, y, x) self.assertRaises(RecursionError, op, y, x)
self.assertIsNot(y, x) self.assertIsNot(y, x)
@ -399,6 +401,7 @@ class TestCopy(unittest.TestCase):
x = ([],) x = ([],)
x[0].append(x) x[0].append(x)
y = copy.deepcopy(x) y = copy.deepcopy(x)
if cosmo.MODE == "dbg": # requires recursion checking
for op in comparisons: for op in comparisons:
self.assertRaises(RecursionError, op, y, x) self.assertRaises(RecursionError, op, y, x)
self.assertIsNot(y, x) self.assertIsNot(y, x)
@ -418,6 +421,7 @@ class TestCopy(unittest.TestCase):
y = copy.deepcopy(x) y = copy.deepcopy(x)
for op in order_comparisons: for op in order_comparisons:
self.assertRaises(TypeError, op, y, x) self.assertRaises(TypeError, op, y, x)
if cosmo.MODE == "dbg": # requires recursion checking
for op in equality_comparisons: for op in equality_comparisons:
self.assertRaises(RecursionError, op, y, x) self.assertRaises(RecursionError, op, y, x)
self.assertIsNot(y, x) self.assertIsNot(y, x)

View file

@ -5,6 +5,7 @@ import itertools
import math import math
import pickle import pickle
import sys import sys
import cosmo
import types import types
import unittest import unittest
import warnings import warnings
@ -3469,6 +3470,7 @@ order (MRO) for bases """
list.__init__(a, sequence=[0, 1, 2]) list.__init__(a, sequence=[0, 1, 2])
self.assertEqual(a, [0, 1, 2]) self.assertEqual(a, [0, 1, 2])
@unittest.skipUnless(cosmo.MODE == "dbg", "disabled recursion checking")
def test_recursive_call(self): def test_recursive_call(self):
# Testing recursive __call__() by setting to instance of class... # Testing recursive __call__() by setting to instance of class...
class A(object): class A(object):
@ -4494,6 +4496,7 @@ order (MRO) for bases """
with self.assertRaises(TypeError): with self.assertRaises(TypeError):
str.__add__(fake_str, "abc") str.__add__(fake_str, "abc")
@unittest.skipUnless(cosmo.MODE == "dbg", "disabled recursion checking")
def test_repr_as_str(self): def test_repr_as_str(self):
# Issue #11603: crash or infinite loop when rebinding __str__ as # Issue #11603: crash or infinite loop when rebinding __str__ as
# __repr__. # __repr__.

View file

@ -469,6 +469,7 @@ class DictTest(unittest.TestCase):
d = {1: BadRepr()} d = {1: BadRepr()}
self.assertRaises(Exc, repr, d) self.assertRaises(Exc, repr, d)
@unittest.skipUnless(cosmo.MODE == "dbg", "disabled recursion checking")
def test_repr_deep(self): def test_repr_deep(self):
d = {} d = {}
for i in range(sys.getrecursionlimit() + 100): for i in range(sys.getrecursionlimit() + 100):
@ -1222,10 +1223,11 @@ class CAPITest(unittest.TestCase):
self.assertEqual(dict_getitem_knownhash(d, 'z', hash('z')), 3) self.assertEqual(dict_getitem_knownhash(d, 'z', hash('z')), 3)
# # TODO: Did this break? What did this do? # # TODO: Did this break? What did this do?
# (likely related to disabling BadInternalCall in #264)
# # not a dict # # not a dict
# # find the APE compilation mode, run this test in dbg only # # # find the APE compilation mode, run this test in dbg only #
# if cosmo.MODE == "dbg": if cosmo.MODE == "dbg":
# self.assertRaises(SystemError, dict_getitem_knownhash, [], 1, hash(1)) self.assertRaises(SystemError, dict_getitem_knownhash, [], 1, hash(1))
# key does not exist # key does not exist
self.assertRaises(KeyError, dict_getitem_knownhash, {}, 1, hash(1)) self.assertRaises(KeyError, dict_getitem_knownhash, {}, 1, hash(1))

View file

@ -2,6 +2,7 @@ import collections
import copy import copy
import pickle import pickle
import sys import sys
import cosmo
import unittest import unittest
class DictSetTest(unittest.TestCase): class DictSetTest(unittest.TestCase):
@ -213,6 +214,7 @@ class DictSetTest(unittest.TestCase):
# Again. # Again.
self.assertIsInstance(r, str) self.assertIsInstance(r, str)
@unittest.skipUnless(cosmo.MODE == "dbg", "disabled recursion checking")
def test_deeply_nested_repr(self): def test_deeply_nested_repr(self):
d = {} d = {}
for i in range(sys.getrecursionlimit() + 100): for i in range(sys.getrecursionlimit() + 100):

View file

@ -515,6 +515,7 @@ class ExceptionTests(unittest.TestCase):
self.assertEqual(x.fancy_arg, 42) self.assertEqual(x.fancy_arg, 42)
@no_tracing @no_tracing
@unittest.skipUnless(cosmo.MODE == "dbg", "disabled recursion checking")
def testInfiniteRecursion(self): def testInfiniteRecursion(self):
def f(): def f():
return f() return f()
@ -928,6 +929,7 @@ class ExceptionTests(unittest.TestCase):
else: else:
self.fail("Should have raised KeyError") self.fail("Should have raised KeyError")
if cosmo.MODE == "dbg":
def g(): def g():
try: try:
return g() return g()
@ -937,6 +939,7 @@ class ExceptionTests(unittest.TestCase):
self.assertTrue(isinstance(v, RecursionError), type(v)) self.assertTrue(isinstance(v, RecursionError), type(v))
self.assertIn("maximum recursion depth exceeded", str(v)) self.assertIn("maximum recursion depth exceeded", str(v))
@unittest.skipUnless(cosmo.MODE == "dbg", "disabled recursion checking")
@cpython_only @cpython_only
def test_recursion_normalizing_exception(self): def test_recursion_normalizing_exception(self):
# Issue #22898. # Issue #22898.
@ -1014,6 +1017,7 @@ class ExceptionTests(unittest.TestCase):
b'while normalizing an exception', err) b'while normalizing an exception', err)
self.assertIn(b'Done.', out) self.assertIn(b'Done.', out)
@unittest.skipUnless(cosmo.MODE == "dbg", "disabled recursion checking")
@cpython_only @cpython_only
def test_recursion_normalizing_with_no_memory(self): def test_recursion_normalizing_with_no_memory(self):
# Issue #30697. Test that in the abort that occurs when there is no # Issue #30697. Test that in the abort that occurs when there is no
@ -1119,6 +1123,7 @@ class ExceptionTests(unittest.TestCase):
self.assertEqual(wr(), None) self.assertEqual(wr(), None)
@no_tracing @no_tracing
@unittest.skipUnless(cosmo.MODE == "dbg", "disabled recursion checking")
def test_recursion_error_cleanup(self): def test_recursion_error_cleanup(self):
# Same test as above, but with "recursion exceeded" errors # Same test as above, but with "recursion exceeded" errors
class C: class C:
@ -1208,6 +1213,7 @@ class ExceptionTests(unittest.TestCase):
self.assertIn("test message", report) self.assertIn("test message", report)
self.assertTrue(report.endswith("\n")) self.assertTrue(report.endswith("\n"))
@unittest.skipUnless(cosmo.MODE == "dbg", "disabled memory hooks")
@cpython_only @cpython_only
def test_memory_error_in_PyErr_PrintEx(self): def test_memory_error_in_PyErr_PrintEx(self):
code = """if 1: code = """if 1:

View file

@ -177,6 +177,7 @@ class AutoFileTests:
finally: finally:
os.close(fd) os.close(fd)
@unittest.skipUnless(cosmo.MODE == "dbg", "disabled recursion checking")
def testRecursiveRepr(self): def testRecursiveRepr(self):
# Issue #25455 # Issue #25455
with swap_attr(self.f, 'name', self.f): with swap_attr(self.f, 'name', self.f):

View file

@ -217,6 +217,7 @@ class TestPartial:
[f'{name}({capture!r}, {args_repr}, {kwargs_repr})' [f'{name}({capture!r}, {args_repr}, {kwargs_repr})'
for kwargs_repr in kwargs_reprs]) for kwargs_repr in kwargs_reprs])
@unittest.skipUnless(cosmo.MODE == "dbg", "disabled recursion checking")
def test_recursive_repr(self): def test_recursive_repr(self):
if self.partial in (c_functools.partial, py_functools.partial): if self.partial in (c_functools.partial, py_functools.partial):
name = 'functools.partial' name = 'functools.partial'
@ -329,6 +330,7 @@ class TestPartial:
def test_recursive_pickle(self): def test_recursive_pickle(self):
with self.AllowPickle(): with self.AllowPickle():
if cosmo.MODE == "dbg":
f = self.partial(capture) f = self.partial(capture)
f.__setstate__((f, (), {}, {})) f.__setstate__((f, (), {}, {}))
try: try:

View file

@ -28,6 +28,7 @@ import pickle
import random import random
import signal import signal
import sys import sys
import cosmo
import time import time
import unittest import unittest
import warnings import warnings
@ -1099,6 +1100,7 @@ class CommonBufferedTests:
raw.name = b"dummy" raw.name = b"dummy"
self.assertEqual(repr(b), "<%s name=b'dummy'>" % clsname) self.assertEqual(repr(b), "<%s name=b'dummy'>" % clsname)
@unittest.skipUnless(cosmo.MODE == "dbg", "disabled recursion checking")
def test_recursive_repr(self): def test_recursive_repr(self):
# Issue #25455 # Issue #25455
raw = self.MockRawIO() raw = self.MockRawIO()
@ -2540,6 +2542,7 @@ class TextIOWrapperTest(unittest.TestCase):
t.buffer.detach() t.buffer.detach()
repr(t) # Should not raise an exception repr(t) # Should not raise an exception
@unittest.skipUnless(cosmo.MODE == "dbg", "disabled recursion checking")
def test_recursive_repr(self): def test_recursive_repr(self):
# Issue #25455 # Issue #25455
raw = self.BytesIO() raw = self.BytesIO()

View file

@ -4,6 +4,7 @@
import unittest import unittest
import sys import sys
import cosmo
@ -257,11 +258,13 @@ class TestIsInstanceIsSubclass(unittest.TestCase):
self.assertEqual(True, issubclass(int, (int, (float, int)))) self.assertEqual(True, issubclass(int, (int, (float, int))))
self.assertEqual(True, issubclass(str, (str, (Child, NewChild, str)))) self.assertEqual(True, issubclass(str, (str, (Child, NewChild, str))))
@unittest.skipUnless(cosmo.MODE == "dbg", "disabled recursion checking")
def test_subclass_recursion_limit(self): def test_subclass_recursion_limit(self):
# make sure that issubclass raises RecursionError before the C stack is # make sure that issubclass raises RecursionError before the C stack is
# blown # blown
self.assertRaises(RecursionError, blowstack, issubclass, str, str) self.assertRaises(RecursionError, blowstack, issubclass, str, str)
@unittest.skipUnless(cosmo.MODE == "dbg", "disabled recursion checking")
def test_isinstance_recursion_limit(self): def test_isinstance_recursion_limit(self):
# make sure that issubclass raises RecursionError before the C stack is # make sure that issubclass raises RecursionError before the C stack is
# blown # blown

View file

@ -1,4 +1,6 @@
from test.test_json import PyTest, CTest from test.test_json import PyTest, CTest
import unittest
import cosmo
class JSONTestObject: class JSONTestObject:
@ -65,6 +67,7 @@ class TestRecursion:
self.fail("didn't raise ValueError on default recursion") self.fail("didn't raise ValueError on default recursion")
@unittest.skipUnless(cosmo.MODE == "dbg", "disabled recursion checking")
def test_highly_nested_objects_decoding(self): def test_highly_nested_objects_decoding(self):
# test that loading highly-nested objects doesn't segfault when C # test that loading highly-nested objects doesn't segfault when C
# accelerations are used. See #12017 # accelerations are used. See #12017
@ -75,6 +78,7 @@ class TestRecursion:
with self.assertRaises(RecursionError): with self.assertRaises(RecursionError):
self.loads('[' * 100000 + '1' + ']' * 100000) self.loads('[' * 100000 + '1' + ']' * 100000)
@unittest.skipUnless(cosmo.MODE == "dbg", "disabled recursion checking")
def test_highly_nested_objects_encoding(self): def test_highly_nested_objects_encoding(self):
# See #12051 # See #12051
l, d = [], {} l, d = [], {}
@ -85,6 +89,7 @@ class TestRecursion:
with self.assertRaises(RecursionError): with self.assertRaises(RecursionError):
self.dumps(d) self.dumps(d)
@unittest.skipUnless(cosmo.MODE == "dbg", "disabled recursion checking")
def test_endless_recursion(self): def test_endless_recursion(self):
# See #12051 # See #12051
class EndlessJSONEncoder(self.json.JSONEncoder): class EndlessJSONEncoder(self.json.JSONEncoder):

View file

@ -1,5 +1,6 @@
# Copyright (C) 2003-2013 Python Software Foundation # Copyright (C) 2003-2013 Python Software Foundation
import cosmo
import struct import struct
import unittest import unittest
import plistlib import plistlib
@ -813,6 +814,7 @@ class TestBinaryPlistlib(unittest.TestCase):
b = plistlib.loads(plistlib.dumps(a, fmt=plistlib.FMT_BINARY)) b = plistlib.loads(plistlib.dumps(a, fmt=plistlib.FMT_BINARY))
self.assertIs(b['x'], b) self.assertIs(b['x'], b)
@unittest.skipUnless(cosmo.MODE == "dbg", "disabled recursion checking")
def test_deep_nesting(self): def test_deep_nesting(self):
for N in [300, 100000]: for N in [300, 100000]:
chunks = [b'\xa1' + (i + 1).to_bytes(4, 'big') for i in range(N)] chunks = [b'\xa1' + (i + 1).to_bytes(4, 'big') for i in range(N)]

View file

@ -1,6 +1,7 @@
"""Test the interactive interpreter.""" """Test the interactive interpreter."""
import sys import sys
import cosmo
import os import os
import unittest import unittest
import subprocess import subprocess
@ -36,6 +37,7 @@ def spawn_repl(*args, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, **kw):
class TestInteractiveInterpreter(unittest.TestCase): class TestInteractiveInterpreter(unittest.TestCase):
@unittest.skipUnless(cosmo.MODE == "dbg", "disabled memory hooks")
@cpython_only @cpython_only
def test_no_memory(self): def test_no_memory(self):
# Issue #30696: Fix the interactive interpreter looping endlessly when # Issue #30696: Fix the interactive interpreter looping endlessly when

View file

@ -1,6 +1,7 @@
# Tests for rich comparisons # Tests for rich comparisons
import unittest import unittest
import cosmo
from test import support from test import support
import operator import operator
@ -220,6 +221,7 @@ class MiscTest(unittest.TestCase):
for func in (do, operator.not_): for func in (do, operator.not_):
self.assertRaises(Exc, func, Bad()) self.assertRaises(Exc, func, Bad())
@unittest.skipUnless(cosmo.MODE == "dbg", "disabled recursion checking")
@support.no_tracing @support.no_tracing
def test_recursion(self): def test_recursion(self):
# Check that comparison for recursive objects fails gracefully # Check that comparison for recursive objects fails gracefully

View file

@ -3,6 +3,7 @@ import unittest
import os import os
import os.path import os.path
import sys import sys
import cosmo
import re import re
import tempfile import tempfile
import importlib, importlib.machinery, importlib.util import importlib, importlib.machinery, importlib.util
@ -723,6 +724,7 @@ class RunPathTestCase(unittest.TestCase, CodeExecutionMixin):
self._check_import_error(zip_name, msg) self._check_import_error(zip_name, msg)
@no_tracing @no_tracing
@unittest.skipUnless(cosmo.MODE == "dbg", "disabled recursion checking")
def test_main_recursion_error(self): def test_main_recursion_error(self):
with temp_dir() as script_dir, temp_dir() as dummy_dir: with temp_dir() as script_dir, temp_dir() as dummy_dir:
mod_name = '__main__' mod_name = '__main__'

View file

@ -1,6 +1,7 @@
import unittest, test.support import unittest, test.support
from test.support.script_helper import assert_python_ok, assert_python_failure from test.support.script_helper import assert_python_ok, assert_python_failure
import sys, io, os import sys, io, os
import cosmo
import struct import struct
import subprocess import subprocess
import textwrap import textwrap
@ -190,6 +191,7 @@ class SysModuleTest(unittest.TestCase):
finally: finally:
sys.setswitchinterval(orig) sys.setswitchinterval(orig)
@unittest.skipUnless(cosmo.MODE == "dbg", "disabled recursion checking")
def test_recursionlimit(self): def test_recursionlimit(self):
self.assertRaises(TypeError, sys.getrecursionlimit, 42) self.assertRaises(TypeError, sys.getrecursionlimit, 42)
oldlimit = sys.getrecursionlimit() oldlimit = sys.getrecursionlimit()
@ -199,6 +201,7 @@ class SysModuleTest(unittest.TestCase):
self.assertEqual(sys.getrecursionlimit(), 10000) self.assertEqual(sys.getrecursionlimit(), 10000)
sys.setrecursionlimit(oldlimit) sys.setrecursionlimit(oldlimit)
@unittest.skipUnless(cosmo.MODE == "dbg", "disabled recursion checking")
def test_recursionlimit_recovery(self): def test_recursionlimit_recovery(self):
if hasattr(sys, 'gettrace') and sys.gettrace(): if hasattr(sys, 'gettrace') and sys.gettrace():
self.skipTest('fatal error if run with a trace function') self.skipTest('fatal error if run with a trace function')
@ -222,6 +225,7 @@ class SysModuleTest(unittest.TestCase):
finally: finally:
sys.setrecursionlimit(oldlimit) sys.setrecursionlimit(oldlimit)
@unittest.skipUnless(cosmo.MODE == "dbg", "disabled recursion checking")
@test.support.cpython_only @test.support.cpython_only
def test_setrecursionlimit_recursion_depth(self): def test_setrecursionlimit_recursion_depth(self):
# Issue #25274: Setting a low recursion limit must be blocked if the # Issue #25274: Setting a low recursion limit must be blocked if the
@ -257,6 +261,7 @@ class SysModuleTest(unittest.TestCase):
finally: finally:
sys.setrecursionlimit(oldlimit) sys.setrecursionlimit(oldlimit)
@unittest.skipUnless(cosmo.MODE == "dbg", "disabled recursion checking")
def test_recursionlimit_fatalerror(self): def test_recursionlimit_fatalerror(self):
# A fatal error occurs if a second recursion limit is hit when recovering # A fatal error occurs if a second recursion limit is hit when recovering
# from a first one. # from a first one.

View file

@ -9,6 +9,7 @@ from test.support.script_helper import assert_python_ok, assert_python_failure
import random import random
import sys import sys
import cosmo
_thread = import_module('_thread') _thread = import_module('_thread')
threading = import_module('threading') threading = import_module('threading')
import time import time
@ -882,6 +883,7 @@ class ThreadingExceptionTests(BaseTestCase):
lock = threading.Lock() lock = threading.Lock()
self.assertRaises(RuntimeError, lock.release) self.assertRaises(RuntimeError, lock.release)
@unittest.skipUnless(cosmo.MODE == "dbg", "disabled recursion checking")
@unittest.skipUnless(sys.platform == 'darwin' and test.support.python_is_optimized(), @unittest.skipUnless(sys.platform == 'darwin' and test.support.python_is_optimized(),
'test macosx problem') 'test macosx problem')
def test_recursion_limit(self): def test_recursion_limit(self):

View file

@ -4,6 +4,7 @@ from collections import namedtuple
from io import StringIO from io import StringIO
import linecache import linecache
import sys import sys
import cosmo
import unittest import unittest
import re import re
from test import support from test import support
@ -300,6 +301,7 @@ class TracebackFormatTests(unittest.TestCase):
]) ])
# issue 26823 - Shrink recursive tracebacks # issue 26823 - Shrink recursive tracebacks
@unittest.skipUnless(cosmo.MODE == "dbg", "disabled recursion checking")
def _check_recursive_traceback_display(self, render_exc): def _check_recursive_traceback_display(self, render_exc):
# Always show full diffs when this test fails # Always show full diffs when this test fails
# Note that rearranging things may require adjusting # Note that rearranging things may require adjusting

View file

@ -1,7 +1,9 @@
import contextlib import contextlib
import os import os
import sys import sys
import tracemalloc import cosmo
if cosmo.MODE == "dbg":
import tracemalloc
import unittest import unittest
from unittest.mock import patch from unittest.mock import patch
from test.support.script_helper import (assert_python_ok, assert_python_failure, from test.support.script_helper import (assert_python_ok, assert_python_failure,
@ -87,6 +89,7 @@ def traceback_filename(filename):
return traceback_lineno(filename, 0) return traceback_lineno(filename, 0)
@unittest.skipUnless(cosmo.MODE == "dbg", "requires APE debug build")
class TestTracemallocEnabled(unittest.TestCase): class TestTracemallocEnabled(unittest.TestCase):
def setUp(self): def setUp(self):
if tracemalloc.is_tracing(): if tracemalloc.is_tracing():
@ -297,6 +300,7 @@ class TestTracemallocEnabled(unittest.TestCase):
self.assertEqual(exitcode, 0) self.assertEqual(exitcode, 0)
@unittest.skipUnless(cosmo.MODE == "dbg", "requires APE debug build")
class TestSnapshot(unittest.TestCase): class TestSnapshot(unittest.TestCase):
maxDiff = 4000 maxDiff = 4000
@ -591,6 +595,7 @@ class TestSnapshot(unittest.TestCase):
[]) [])
@unittest.skipUnless(cosmo.MODE == "dbg", "requires APE debug build")
class TestFilters(unittest.TestCase): class TestFilters(unittest.TestCase):
maxDiff = 2048 maxDiff = 2048
@ -802,6 +807,7 @@ class TestFilters(unittest.TestCase):
self.assertFalse(f._match_traceback(unknown)) self.assertFalse(f._match_traceback(unknown))
@unittest.skipUnless(cosmo.MODE == "dbg", "requires APE debug build")
class TestCommandLine(unittest.TestCase): class TestCommandLine(unittest.TestCase):
def test_env_var_disabled_by_default(self): def test_env_var_disabled_by_default(self):
# not tracing by default # not tracing by default
@ -874,6 +880,7 @@ class TestCommandLine(unittest.TestCase):
assert_python_ok('-X', 'tracemalloc', '-c', code) assert_python_ok('-X', 'tracemalloc', '-c', code)
@unittest.skipUnless(cosmo.MODE == "dbg", "requires APE debug build")
@unittest.skipIf(_testcapi is None, 'need _testcapi') @unittest.skipIf(_testcapi is None, 'need _testcapi')
class TestCAPI(unittest.TestCase): class TestCAPI(unittest.TestCase):
maxDiff = 80 * 20 maxDiff = 80 * 20

View file

@ -1,6 +1,7 @@
from contextlib import contextmanager from contextlib import contextmanager
import linecache import linecache
import os import os
import cosmo
from io import StringIO from io import StringIO
import re import re
import sys import sys
@ -922,6 +923,7 @@ class CWarningsDisplayTests(WarningsDisplayTests, unittest.TestCase):
class PyWarningsDisplayTests(WarningsDisplayTests, unittest.TestCase): class PyWarningsDisplayTests(WarningsDisplayTests, unittest.TestCase):
module = py_warnings module = py_warnings
@unittest.skipUnless(cosmo.MODE == "dbg", "requires APE debug build")
def test_tracemalloc(self): def test_tracemalloc(self):
self.addCleanup(support.unlink, support.TESTFN) self.addCleanup(support.unlink, support.TESTFN)

View file

@ -5,6 +5,7 @@
# For this purpose, the module-level "ET" symbol is temporarily # For this purpose, the module-level "ET" symbol is temporarily
# monkey-patched when running the "test_xml_etree_c" test suite. # monkey-patched when running the "test_xml_etree_c" test suite.
import cosmo
import copy import copy
import html import html
import io import io
@ -1920,6 +1921,7 @@ class BadElementTest(ElementTestCase, unittest.TestCase):
e.extend([ET.Element('bar')]) e.extend([ET.Element('bar')])
self.assertRaises(ValueError, e.remove, X('baz')) self.assertRaises(ValueError, e.remove, X('baz'))
@unittest.skipUnless(cosmo.MODE == "dbg", "disabled recursion checking")
def test_recursive_repr(self): def test_recursive_repr(self):
# Issue #25455 # Issue #25455
e = ET.Element('foo') e = ET.Element('foo')

View file

@ -4,6 +4,7 @@ import binascii
import pickle import pickle
import random import random
import sys import sys
import cosmo
import zlib import zlib
from test.support import bigmemtest, _1G, _4G from test.support import bigmemtest, _1G, _4G
@ -707,6 +708,7 @@ class CompressObjectTestCase(BaseCompressTestCase, unittest.TestCase):
# Memory use of the following functions takes into account overallocation # Memory use of the following functions takes into account overallocation
@unittest.skipUnless(cosmo.MODE == "dbg", "disabled tracemalloc")
@bigmemtest(size=_1G + 1024 * 1024, memuse=3) @bigmemtest(size=_1G + 1024 * 1024, memuse=3)
def test_big_compress_buffer(self, size): def test_big_compress_buffer(self, size):
import tracemalloc import tracemalloc

View file

@ -3488,6 +3488,7 @@ test_pymem_alloc0(PyObject *self)
Py_RETURN_NONE; Py_RETURN_NONE;
} }
#if IsModeDbg()
typedef struct { typedef struct {
PyMemAllocatorEx alloc; PyMemAllocatorEx alloc;
@ -3780,6 +3781,7 @@ remove_mem_hooks(PyObject *self)
fm_remove_hooks(); fm_remove_hooks();
Py_RETURN_NONE; Py_RETURN_NONE;
} }
#endif
PyDoc_STRVAR(docstring_empty, PyDoc_STRVAR(docstring_empty,
"" ""
@ -4234,6 +4236,7 @@ test_PyTime_AsMicroseconds(PyObject *self, PyObject *args)
return _PyTime_AsNanosecondsObject(ms); return _PyTime_AsNanosecondsObject(ms);
} }
#if IsModeDbg()
static PyObject* static PyObject*
get_recursion_depth(PyObject *self, PyObject *args) get_recursion_depth(PyObject *self, PyObject *args)
{ {
@ -4242,6 +4245,7 @@ get_recursion_depth(PyObject *self, PyObject *args)
/* subtract one to ignore the frame of the get_recursion_depth() call */ /* subtract one to ignore the frame of the get_recursion_depth() call */
return PyLong_FromLong(tstate->recursion_depth - 1); return PyLong_FromLong(tstate->recursion_depth - 1);
} }
#endif
static PyObject* static PyObject*
pymem_buffer_overflow(PyObject *self, PyObject *args) pymem_buffer_overflow(PyObject *self, PyObject *args)
@ -4656,6 +4660,7 @@ static PyMethodDef TestMethods[] = {
{"create_cfunction", create_cfunction, METH_NOARGS}, {"create_cfunction", create_cfunction, METH_NOARGS},
{"test_pymem_alloc0", {"test_pymem_alloc0",
(PyCFunction)test_pymem_alloc0, METH_NOARGS}, (PyCFunction)test_pymem_alloc0, METH_NOARGS},
#if IsModeDbg()
{"test_pymem_setrawallocators", {"test_pymem_setrawallocators",
(PyCFunction)test_pymem_setrawallocators, METH_NOARGS}, (PyCFunction)test_pymem_setrawallocators, METH_NOARGS},
{"test_pymem_setallocators", {"test_pymem_setallocators",
@ -4666,6 +4671,7 @@ static PyMethodDef TestMethods[] = {
PyDoc_STR("set_nomemory(start:int, stop:int = 0)")}, PyDoc_STR("set_nomemory(start:int, stop:int = 0)")},
{"remove_mem_hooks", (PyCFunction)remove_mem_hooks, METH_NOARGS, {"remove_mem_hooks", (PyCFunction)remove_mem_hooks, METH_NOARGS,
PyDoc_STR("Remove memory hooks.")}, PyDoc_STR("Remove memory hooks.")},
#endif
{"no_docstring", {"no_docstring",
(PyCFunction)test_with_docstring, METH_NOARGS}, (PyCFunction)test_with_docstring, METH_NOARGS},
{"docstring_empty", {"docstring_empty",
@ -4723,6 +4729,7 @@ static PyMethodDef TestMethods[] = {
#endif #endif
{"PyTime_AsMilliseconds", test_PyTime_AsMilliseconds, METH_VARARGS}, {"PyTime_AsMilliseconds", test_PyTime_AsMilliseconds, METH_VARARGS},
{"PyTime_AsMicroseconds", test_PyTime_AsMicroseconds, METH_VARARGS}, {"PyTime_AsMicroseconds", test_PyTime_AsMicroseconds, METH_VARARGS},
#if IsModeDbg()
{"get_recursion_depth", get_recursion_depth, METH_NOARGS}, {"get_recursion_depth", get_recursion_depth, METH_NOARGS},
{"pymem_buffer_overflow", pymem_buffer_overflow, METH_NOARGS}, {"pymem_buffer_overflow", pymem_buffer_overflow, METH_NOARGS},
{"pymem_api_misuse", pymem_api_misuse, METH_NOARGS}, {"pymem_api_misuse", pymem_api_misuse, METH_NOARGS},
@ -4731,6 +4738,7 @@ static PyMethodDef TestMethods[] = {
{"tracemalloc_track", tracemalloc_track, METH_VARARGS}, {"tracemalloc_track", tracemalloc_track, METH_VARARGS},
{"tracemalloc_untrack", tracemalloc_untrack, METH_VARARGS}, {"tracemalloc_untrack", tracemalloc_untrack, METH_VARARGS},
{"tracemalloc_get_traceback", tracemalloc_get_traceback, METH_VARARGS}, {"tracemalloc_get_traceback", tracemalloc_get_traceback, METH_VARARGS},
#endif
{"dict_get_version", dict_get_version, METH_VARARGS}, {"dict_get_version", dict_get_version, METH_VARARGS},
{"pyobject_fastcall", test_pyobject_fastcall, METH_VARARGS}, {"pyobject_fastcall", test_pyobject_fastcall, METH_VARARGS},
{"pyobject_fastcalldict", test_pyobject_fastcalldict, METH_VARARGS}, {"pyobject_fastcalldict", test_pyobject_fastcalldict, METH_VARARGS},

View file

@ -41,6 +41,7 @@ PYTHON_PROVIDE("_tracemalloc.is_tracing");
PYTHON_PROVIDE("_tracemalloc.start"); PYTHON_PROVIDE("_tracemalloc.start");
PYTHON_PROVIDE("_tracemalloc.stop"); PYTHON_PROVIDE("_tracemalloc.stop");
#if IsModeDbg()
/* Trace memory blocks allocated by PyMem_RawMalloc() */ /* Trace memory blocks allocated by PyMem_RawMalloc() */
#define TRACE_RAW_MALLOC #define TRACE_RAW_MALLOC
@ -1686,30 +1687,6 @@ static PyMethodDef module_methods[] = {
PyDoc_STRVAR(module_doc, PyDoc_STRVAR(module_doc,
"Debug module to trace memory blocks allocated by Python."); "Debug module to trace memory blocks allocated by Python.");
static struct PyModuleDef module_def = {
PyModuleDef_HEAD_INIT,
"_tracemalloc",
module_doc,
0, /* non-negative size to be able to unload the module */
module_methods,
NULL,
};
PyMODINIT_FUNC
PyInit__tracemalloc(void)
{
PyObject *m;
m = PyModule_Create(&module_def);
if (m == NULL)
return NULL;
if (tracemalloc_init() < 0)
return NULL;
return m;
}
static int static int
parse_sys_xoptions(PyObject *value) parse_sys_xoptions(PyObject *value)
{ {
@ -1862,6 +1839,41 @@ PyObject*
return traceback_to_pyobject(traceback, NULL); return traceback_to_pyobject(traceback, NULL);
} }
#endif
static struct PyModuleDef module_def = {
PyModuleDef_HEAD_INIT,
"_tracemalloc",
#ifdef USETRACEMALLOC
module_doc,
#else
NULL,
#endif
0, /* non-negative size to be able to unload the module */
#ifdef USETRACEMALLOC
module_methods,
#else
NULL,
#endif
NULL,
};
PyMODINIT_FUNC
PyInit__tracemalloc(void)
{
PyObject *m;
m = PyModule_Create(&module_def);
if (m == NULL)
return NULL;
#ifdef USETRACEMALLOC
if (tracemalloc_init() < 0)
return NULL;
#endif
return m;
}
_Section(".rodata.pytab.1") const struct _inittab _PyImport_Inittab__tracemalloc = { _Section(".rodata.pytab.1") const struct _inittab _PyImport_Inittab__tracemalloc = {
"_tracemalloc", "_tracemalloc",

View file

@ -424,12 +424,14 @@ Py_Main(int argc, wchar_t **argv)
} }
} }
#if IsModeDbg()
opt = Py_GETENV("PYTHONMALLOC"); opt = Py_GETENV("PYTHONMALLOC");
if (_PyMem_SetupAllocators(opt) < 0) { if (_PyMem_SetupAllocators(opt) < 0) {
fprintf(stderr, fprintf(stderr,
"Error in PYTHONMALLOC: unknown allocator \"%s\"!\n", opt); "Error in PYTHONMALLOC: unknown allocator \"%s\"!\n", opt);
exit(1); exit(1);
} }
#endif
_PyRandom_Init(); _PyRandom_Init();

View file

@ -29,6 +29,7 @@
#define uint unsigned int /* assuming >= 16 bits */ #define uint unsigned int /* assuming >= 16 bits */
/* Forward declaration */ /* Forward declaration */
#if IsModeDbg()
static void *_PyMem_DebugRawMalloc(void *, size_t); static void *_PyMem_DebugRawMalloc(void *, size_t);
static void *_PyMem_DebugRawCalloc(void *, size_t, size_t); static void *_PyMem_DebugRawCalloc(void *, size_t, size_t);
static void *_PyMem_DebugRawRealloc(void *, void *, size_t); static void *_PyMem_DebugRawRealloc(void *, void *, size_t);
@ -37,6 +38,7 @@ static void *_PyMem_DebugMalloc(void *, size_t);
static void *_PyMem_DebugCalloc(void *, size_t, size_t); static void *_PyMem_DebugCalloc(void *, size_t, size_t);
static void *_PyMem_DebugRealloc(void *, void *, size_t); static void *_PyMem_DebugRealloc(void *, void *, size_t);
static void _PyMem_DebugFree(void *, void *); static void _PyMem_DebugFree(void *, void *);
#endif
static void _PyObject_DebugDumpAddress(const void *); static void _PyObject_DebugDumpAddress(const void *);
static void _PyMem_DebugCheckAddress(char, const void *); static void _PyMem_DebugCheckAddress(char, const void *);
@ -82,6 +84,13 @@ static void* _PyObject_Malloc(void *ctx, size_t size);
static void* _PyObject_Calloc(void *ctx, size_t nelem, size_t elsize); static void* _PyObject_Calloc(void *ctx, size_t nelem, size_t elsize);
static void _PyObject_Free(void *ctx, void *p); static void _PyObject_Free(void *ctx, void *p);
static void* _PyObject_Realloc(void *ctx, void *ptr, size_t size); static void* _PyObject_Realloc(void *ctx, void *ptr, size_t size);
#else
/* in MODE=asan, no pymalloc, so use macro */
#define _PyObject_Malloc(ctx, size) _PyMem_RawMalloc((ctx), (size))
#define _PyObject_Calloc(ctx, nelem, elsize) _PyMem_RawCalloc((ctx), (nelem), (elsize))
#define _PyObject_Realloc(ctx, ptr, size) _PyMem_RawRealloc((ctx), (ptr), (size))
#define _PyObject_Free(ctx, p) _PyMem_RawFree((ctx), (p))
#endif #endif
static inline void * static inline void *
@ -207,6 +216,7 @@ _PyObject_ArenaFree(void *ctx, void *ptr, size_t size)
} }
#endif #endif
#if IsModeDbg()
#define PYRAW_FUNCS _PyMem_RawMalloc, _PyMem_RawCalloc, _PyMem_RawRealloc, _PyMem_RawFree #define PYRAW_FUNCS _PyMem_RawMalloc, _PyMem_RawCalloc, _PyMem_RawRealloc, _PyMem_RawFree
#ifdef WITH_PYMALLOC #ifdef WITH_PYMALLOC
# define PYOBJ_FUNCS _PyObject_Malloc, _PyObject_Calloc, _PyObject_Realloc, _PyObject_Free # define PYOBJ_FUNCS _PyObject_Malloc, _PyObject_Calloc, _PyObject_Realloc, _PyObject_Free
@ -428,6 +438,7 @@ PyObject_SetArenaAllocator(PyObjectArenaAllocator *allocator)
{ {
_PyObject_Arena = *allocator; _PyObject_Arena = *allocator;
} }
#endif
void * void *
PyMem_RawMalloc(size_t size) PyMem_RawMalloc(size_t size)
@ -440,7 +451,11 @@ PyMem_RawMalloc(size_t size)
*/ */
if (size > (size_t)PY_SSIZE_T_MAX) if (size > (size_t)PY_SSIZE_T_MAX)
return NULL; return NULL;
#if IsModeDbg()
return _PyMem_Raw.malloc(_PyMem_Raw.ctx, size); return _PyMem_Raw.malloc(_PyMem_Raw.ctx, size);
#else
return _PyMem_RawMalloc(NULL, size);
#endif
} }
void * void *
@ -449,7 +464,11 @@ PyMem_RawCalloc(size_t nelem, size_t elsize)
/* see PyMem_RawMalloc() */ /* see PyMem_RawMalloc() */
if (elsize != 0 && nelem > (size_t)PY_SSIZE_T_MAX / elsize) if (elsize != 0 && nelem > (size_t)PY_SSIZE_T_MAX / elsize)
return NULL; return NULL;
#if IsModeDbg()
return _PyMem_Raw.calloc(_PyMem_Raw.ctx, nelem, elsize); return _PyMem_Raw.calloc(_PyMem_Raw.ctx, nelem, elsize);
#else
return _PyMem_RawCalloc(NULL, nelem, elsize);
#endif
} }
void* void*
@ -458,12 +477,20 @@ PyMem_RawRealloc(void *ptr, size_t new_size)
/* see PyMem_RawMalloc() */ /* see PyMem_RawMalloc() */
if (new_size > (size_t)PY_SSIZE_T_MAX) if (new_size > (size_t)PY_SSIZE_T_MAX)
return NULL; return NULL;
#if IsModeDbg()
return _PyMem_Raw.realloc(_PyMem_Raw.ctx, ptr, new_size); return _PyMem_Raw.realloc(_PyMem_Raw.ctx, ptr, new_size);
#else
return _PyMem_RawRealloc(NULL, ptr, new_size);
#endif
} }
void PyMem_RawFree(void *ptr) void PyMem_RawFree(void *ptr)
{ {
#if IsModeDbg()
_PyMem_Raw.free(_PyMem_Raw.ctx, ptr); _PyMem_Raw.free(_PyMem_Raw.ctx, ptr);
#else
_PyMem_RawFree(NULL, ptr);
#endif
} }
void * void *
@ -472,7 +499,11 @@ PyMem_Malloc(size_t size)
/* see PyMem_RawMalloc() */ /* see PyMem_RawMalloc() */
if (size > (size_t)PY_SSIZE_T_MAX) if (size > (size_t)PY_SSIZE_T_MAX)
return NULL; return NULL;
#if IsModeDbg()
return _PyMem.malloc(_PyMem.ctx, size); return _PyMem.malloc(_PyMem.ctx, size);
#else
return _PyObject_Malloc(NULL, size);
#endif
} }
void * void *
@ -481,7 +512,11 @@ PyMem_Calloc(size_t nelem, size_t elsize)
/* see PyMem_RawMalloc() */ /* see PyMem_RawMalloc() */
if (elsize != 0 && nelem > (size_t)PY_SSIZE_T_MAX / elsize) if (elsize != 0 && nelem > (size_t)PY_SSIZE_T_MAX / elsize)
return NULL; return NULL;
#if IsModeDbg()
return _PyMem.calloc(_PyMem.ctx, nelem, elsize); return _PyMem.calloc(_PyMem.ctx, nelem, elsize);
#else
return _PyObject_Calloc(NULL, nelem, elsize);
#endif
} }
void * void *
@ -490,13 +525,21 @@ PyMem_Realloc(void *ptr, size_t new_size)
/* see PyMem_RawMalloc() */ /* see PyMem_RawMalloc() */
if (new_size > (size_t)PY_SSIZE_T_MAX) if (new_size > (size_t)PY_SSIZE_T_MAX)
return NULL; return NULL;
#if IsModeDbg()
return _PyMem.realloc(_PyMem.ctx, ptr, new_size); return _PyMem.realloc(_PyMem.ctx, ptr, new_size);
#else
return _PyObject_Realloc(NULL, ptr, new_size);
#endif
} }
void void
(PyMem_Free)(void *ptr) (PyMem_Free)(void *ptr)
{ {
#if IsModeDbg()
_PyMem.free(_PyMem.ctx, ptr); _PyMem.free(_PyMem.ctx, ptr);
#else
return _PyObject_Free(NULL, ptr);
#endif
} }
char * char *
@ -531,7 +574,11 @@ void *
/* see PyMem_RawMalloc() */ /* see PyMem_RawMalloc() */
if (size > (size_t)PY_SSIZE_T_MAX) if (size > (size_t)PY_SSIZE_T_MAX)
return NULL; return NULL;
#if IsModeDbg()
return _PyObject.malloc(_PyObject.ctx, size); return _PyObject.malloc(_PyObject.ctx, size);
#else
return _PyObject_Malloc(NULL, size);
#endif
} }
void * void *
@ -540,7 +587,11 @@ PyObject_Calloc(size_t nelem, size_t elsize)
/* see PyMem_RawMalloc() */ /* see PyMem_RawMalloc() */
if (elsize != 0 && nelem > (size_t)PY_SSIZE_T_MAX / elsize) if (elsize != 0 && nelem > (size_t)PY_SSIZE_T_MAX / elsize)
return NULL; return NULL;
#if IsModeDbg()
return _PyObject.calloc(_PyObject.ctx, nelem, elsize); return _PyObject.calloc(_PyObject.ctx, nelem, elsize);
#else
return _PyObject_Calloc(NULL, nelem, elsize);
#endif
} }
void * void *
@ -549,13 +600,21 @@ PyObject_Realloc(void *ptr, size_t new_size)
/* see PyMem_RawMalloc() */ /* see PyMem_RawMalloc() */
if (new_size > (size_t)PY_SSIZE_T_MAX) if (new_size > (size_t)PY_SSIZE_T_MAX)
return NULL; return NULL;
#if IsModeDbg()
return _PyObject.realloc(_PyObject.ctx, ptr, new_size); return _PyObject.realloc(_PyObject.ctx, ptr, new_size);
#else
return _PyObject_Realloc(NULL, ptr, new_size);
#endif
} }
void void
PyObject_Free(void *ptr) PyObject_Free(void *ptr)
{ {
#if IsModeDbg()
_PyObject.free(_PyObject.ctx, ptr); _PyObject.free(_PyObject.ctx, ptr);
#else
return _PyObject_Free(NULL, ptr);
#endif
} }
@ -1137,7 +1196,11 @@ new_arena(void)
arenaobj = unused_arena_objects; arenaobj = unused_arena_objects;
unused_arena_objects = arenaobj->nextarena; unused_arena_objects = arenaobj->nextarena;
assert(arenaobj->address == 0); assert(arenaobj->address == 0);
#if IsModeDbg()
address = _PyObject_Arena.alloc(_PyObject_Arena.ctx, ARENA_SIZE); address = _PyObject_Arena.alloc(_PyObject_Arena.ctx, ARENA_SIZE);
#else
address = _PyObject_ArenaMmap(NULL, ARENA_SIZE);
#endif
if (address == NULL) { if (address == NULL) {
/* The allocation failed: return NULL after putting the /* The allocation failed: return NULL after putting the
* arenaobj back. * arenaobj back.
@ -1615,8 +1678,13 @@ _PyObject_Free(void *ctx, void *p)
unused_arena_objects = ao; unused_arena_objects = ao;
/* Free the entire arena. */ /* Free the entire arena. */
#if IsModeDbg()
_PyObject_Arena.free(_PyObject_Arena.ctx, _PyObject_Arena.free(_PyObject_Arena.ctx,
(void *)ao->address, ARENA_SIZE); (void *)ao->address, ARENA_SIZE);
#else
_PyObject_ArenaMunmap(NULL,
(void *)ao->address, ARENA_SIZE);
#endif
ao->address = 0; /* mark unassociated */ ao->address = 0; /* mark unassociated */
--narenas_currently_allocated; --narenas_currently_allocated;
@ -1853,6 +1921,7 @@ write_size_t(void *p, size_t n)
WRITE64BE((char *)p, n); WRITE64BE((char *)p, n);
} }
#if IsModeDbg()
/* Let S = sizeof(size_t). The debug malloc asks for 4*S extra bytes and /* Let S = sizeof(size_t). The debug malloc asks for 4*S extra bytes and
fills them with useful stuff, here calling the underlying malloc's result p: fills them with useful stuff, here calling the underlying malloc's result p:
@ -1941,7 +2010,7 @@ _PyMem_DebugRawCalloc(void *ctx, size_t nelem, size_t elsize)
Python memory allocators which fills the memory with DEADBYTE (0xDB) when Python memory allocators which fills the memory with DEADBYTE (0xDB) when
memory is deallocated. */ memory is deallocated. */
int int
_PyMem_IsFreed(void *ptr, size_t size) (_PyMem_IsFreed)(void *ptr, size_t size)
{ {
unsigned char *bytes = ptr; unsigned char *bytes = ptr;
for (size_t i=0; i < size; i++) { for (size_t i=0; i < size; i++) {
@ -2222,11 +2291,12 @@ _PyObject_DebugDumpAddress(const void *p)
fputc('\n', stderr); fputc('\n', stderr);
fflush(stderr); fflush(stderr);
#ifdef USE_TRACEMALLOC #if IsModeDbg()
PYTHON_YOINK("_tracemalloc"); PYTHON_YOINK("_tracemalloc");
_PyMem_DumpTraceback(fileno(stderr), p); _PyMem_DumpTraceback(fileno(stderr), p);
#endif #endif
} }
#endif
static size_t static size_t
@ -2417,8 +2487,10 @@ _PyObject_DebugMallocStats(FILE *out)
quantization += p * ((POOL_SIZE - POOL_OVERHEAD) % size); quantization += p * ((POOL_SIZE - POOL_OVERHEAD) % size);
} }
fputc('\n', out); fputc('\n', out);
#if IsModeDbg()
if (_PyMem_DebugEnabled()) if (_PyMem_DebugEnabled())
(void)printone(out, "# times object malloc called", serialno); (void)printone(out, "# times object malloc called", serialno);
#endif
(void)printone(out, "# arenas allocated total", ntimes_arena_allocated); (void)printone(out, "# arenas allocated total", ntimes_arena_allocated);
(void)printone(out, "# arenas reclaimed", ntimes_arena_allocated - narenas); (void)printone(out, "# arenas reclaimed", ntimes_arena_allocated - narenas);
(void)printone(out, "# arenas highwater mark", narenas_highwater); (void)printone(out, "# arenas highwater mark", narenas_highwater);

View file

@ -121,7 +121,7 @@ Py_FinalizeEx(void)
_PyGC_CollectIfEnabled(); _PyGC_CollectIfEnabled();
#endif #endif
#ifdef MODE_DBG #if IsModeDbg()
/* Disable tracemalloc after all Python objects have been destroyed, /* Disable tracemalloc after all Python objects have been destroyed,
so it is possible to use tracemalloc in objects destructor. */ so it is possible to use tracemalloc in objects destructor. */
_PyTraceMalloc_Fini(); _PyTraceMalloc_Fini();
@ -219,11 +219,13 @@ Py_FinalizeEx(void)
_Py_PrintReferenceAddresses(stderr); _Py_PrintReferenceAddresses(stderr);
#endif /* Py_TRACE_REFS */ #endif /* Py_TRACE_REFS */
#ifdef WITH_PYMALLOC #ifdef WITH_PYMALLOC
#if IsModeDbg()
if (_PyMem_PymallocEnabled()) { if (_PyMem_PymallocEnabled()) {
char *opt = Py_GETENV("PYTHONMALLOCSTATS"); char *opt = Py_GETENV("PYTHONMALLOCSTATS");
if (opt != NULL && *opt != '\0') if (opt != NULL && *opt != '\0')
_PyObject_DebugMallocStats(stderr); _PyObject_DebugMallocStats(stderr);
} }
#endif
#endif #endif
_Py_CallLlExitFuncs(); _Py_CallLlExitFuncs();

View file

@ -274,7 +274,7 @@ _Py_InitializeEx_Private(int install_sigs, int install_importlib)
if (install_sigs) if (install_sigs)
_Py_InitSigs(); /* Signal handling stuff, including initintr() */ _Py_InitSigs(); /* Signal handling stuff, including initintr() */
#ifdef USE_TRACEMALLOC #if IsModeDbg()
if (_PyTraceMalloc_Init() < 0) if (_PyTraceMalloc_Init() < 0)
Py_FatalError("Py_Initialize: can't initialize tracemalloc"); Py_FatalError("Py_Initialize: can't initialize tracemalloc");
#endif #endif

View file

@ -91,7 +91,9 @@ PYTHON_PROVIDE("sys.float_info");
PYTHON_PROVIDE("sys.float_repr_style"); PYTHON_PROVIDE("sys.float_repr_style");
PYTHON_PROVIDE("sys.get_asyncgen_hooks"); PYTHON_PROVIDE("sys.get_asyncgen_hooks");
PYTHON_PROVIDE("sys.get_coroutine_wrapper"); PYTHON_PROVIDE("sys.get_coroutine_wrapper");
#if IsModeDbg()
PYTHON_PROVIDE("sys.getallocatedblocks"); PYTHON_PROVIDE("sys.getallocatedblocks");
#endif
PYTHON_PROVIDE("sys.getcheckinterval"); PYTHON_PROVIDE("sys.getcheckinterval");
PYTHON_PROVIDE("sys.getdefaultencoding"); PYTHON_PROVIDE("sys.getdefaultencoding");
PYTHON_PROVIDE("sys.getdlopenflags"); PYTHON_PROVIDE("sys.getdlopenflags");
@ -1262,6 +1264,7 @@ one higher than you might expect, because it includes the (temporary)\n\
reference as an argument to getrefcount()." reference as an argument to getrefcount()."
); );
#if IsModeDbg()
static PyObject * static PyObject *
sys_getallocatedblocks(PyObject *self) sys_getallocatedblocks(PyObject *self)
{ {
@ -1274,6 +1277,7 @@ PyDoc_STRVAR(getallocatedblocks_doc,
Return the number of memory blocks currently allocated, regardless of their\n\ Return the number of memory blocks currently allocated, regardless of their\n\
size." size."
); );
#endif
#ifdef COUNT_ALLOCS #ifdef COUNT_ALLOCS
static PyObject * static PyObject *
@ -1382,10 +1386,12 @@ static PyObject *
sys_debugmallocstats(PyObject *self, PyObject *args) sys_debugmallocstats(PyObject *self, PyObject *args)
{ {
#ifdef WITH_PYMALLOC #ifdef WITH_PYMALLOC
#if IsModeDbg()
if (_PyMem_PymallocEnabled()) { if (_PyMem_PymallocEnabled()) {
_PyObject_DebugMallocStats(stderr); _PyObject_DebugMallocStats(stderr);
fputc('\n', stderr); fputc('\n', stderr);
} }
#endif
#endif #endif
_PyObject_DebugTypeStats(stderr); _PyObject_DebugTypeStats(stderr);
@ -1455,8 +1461,10 @@ static PyMethodDef sys_methods[] = {
{"getdlopenflags", (PyCFunction)sys_getdlopenflags, METH_NOARGS, {"getdlopenflags", (PyCFunction)sys_getdlopenflags, METH_NOARGS,
getdlopenflags_doc}, getdlopenflags_doc},
#endif #endif
#if IsModeDbg()
{"getallocatedblocks", (PyCFunction)sys_getallocatedblocks, METH_NOARGS, {"getallocatedblocks", (PyCFunction)sys_getallocatedblocks, METH_NOARGS,
getallocatedblocks_doc}, getallocatedblocks_doc},
#endif
#ifdef COUNT_ALLOCS #ifdef COUNT_ALLOCS
{"getcounts", (PyCFunction)sys_getcounts, METH_NOARGS}, {"getcounts", (PyCFunction)sys_getcounts, METH_NOARGS},
#endif #endif

View file

@ -573,9 +573,8 @@
#define OPENSSL_NO_COMP 1 #define OPENSSL_NO_COMP 1
#define HAVE_LANGINFO_H 1 #define HAVE_LANGINFO_H 1
#ifdef MODE_DBG #if IsModeDbg()
#define Py_DEBUG 1 #define Py_DEBUG 1
#define USE_TRACEMALLOC 1
#endif #endif
/* #define FAST_LOOPS 1 /\* froot loops *\/ */ /* #define FAST_LOOPS 1 /\* froot loops *\/ */

View file

@ -280,8 +280,10 @@ RunPythonModule(int argc, char **argv)
linenoiseSetHintsCallback(TerminalHint); linenoiseSetHintsCallback(TerminalHint);
linenoiseSetFreeHintsCallback(free); linenoiseSetFreeHintsCallback(free);
#if IsModeDbg()
/* Force malloc() allocator to bootstrap Python */ /* Force malloc() allocator to bootstrap Python */
_PyMem_SetupAllocators("malloc"); _PyMem_SetupAllocators("malloc");
#endif
argv_copy = (wchar_t **)PyMem_RawMalloc(sizeof(wchar_t*) * (argc+1)); argv_copy = (wchar_t **)PyMem_RawMalloc(sizeof(wchar_t*) * (argc+1));
argv_copy2 = (wchar_t **)PyMem_RawMalloc(sizeof(wchar_t*) * (argc+1)); argv_copy2 = (wchar_t **)PyMem_RawMalloc(sizeof(wchar_t*) * (argc+1));
@ -324,9 +326,11 @@ RunPythonModule(int argc, char **argv)
res = Py_Main(argc, argv_copy); res = Py_Main(argc, argv_copy);
#if IsModeDbg()
/* Force again malloc() allocator to release memory blocks allocated /* Force again malloc() allocator to release memory blocks allocated
before Py_Main() */ before Py_Main() */
_PyMem_SetupAllocators("malloc"); _PyMem_SetupAllocators("malloc");
#endif
for (i = 0; i < argc; i++) { for (i = 0; i < argc; i++) {
PyMem_RawFree(argv_copy2[i]); PyMem_RawFree(argv_copy2[i]);