From 57f0eed3823d92453e060188dac98f80b15cd093 Mon Sep 17 00:00:00 2001 From: Gautham <41098605+ahgamut@users.noreply.github.com> Date: Sat, 2 Oct 2021 13:58:51 +0530 Subject: [PATCH] Fix Pyston speedups (#281) We remove (i.e. hide behind a debug ifdef) the recursion checking methods, and the memory hooks and memory allocator methods. ASAN mode has no PYMALLOC, so we need a macro. Fix build break with des.c stack allocation. --- third_party/mbedtls/mbedtls.mk | 4 + third_party/python/Include/abstract.h | 2 +- third_party/python/Include/ceval.h | 5 ++ third_party/python/Include/objimpl.h | 5 +- third_party/python/Include/pyerrors.h | 2 +- third_party/python/Include/pymem.h | 8 +- .../Lib/ctypes/test/test_as_parameter.py | 2 + third_party/python/Lib/test/list_tests.py | 3 + third_party/python/Lib/test/mapping_tests.py | 2 + third_party/python/Lib/test/pickletester.py | 3 +- third_party/python/Lib/test/test_capi.py | 5 ++ third_party/python/Lib/test/test_class.py | 2 + third_party/python/Lib/test/test_copy.py | 16 ++-- third_party/python/Lib/test/test_descr.py | 3 + third_party/python/Lib/test/test_dict.py | 8 +- third_party/python/Lib/test/test_dictviews.py | 2 + .../python/Lib/test/test_exceptions.py | 22 ++++-- third_party/python/Lib/test/test_fileio.py | 1 + third_party/python/Lib/test/test_functools.py | 18 +++-- third_party/python/Lib/test/test_io.py | 3 + .../python/Lib/test/test_isinstance.py | 3 + .../Lib/test/test_json/test_recursion.py | 5 ++ third_party/python/Lib/test/test_plistlib.py | 2 + third_party/python/Lib/test/test_repl.py | 2 + third_party/python/Lib/test/test_richcmp.py | 2 + third_party/python/Lib/test/test_runpy.py | 2 + third_party/python/Lib/test/test_sys.py | 5 ++ third_party/python/Lib/test/test_threading.py | 2 + third_party/python/Lib/test/test_traceback.py | 2 + .../python/Lib/test/test_tracemalloc.py | 9 ++- .../python/Lib/test/test_warnings/__init__.py | 2 + third_party/python/Lib/test/test_xml_etree.py | 2 + third_party/python/Lib/test/test_zlib.py | 2 + third_party/python/Modules/_testcapimodule.c | 8 ++ third_party/python/Modules/_tracemalloc.c | 60 +++++++++------ third_party/python/Modules/main.c | 2 + third_party/python/Objects/obmalloc.c | 76 ++++++++++++++++++- third_party/python/Python/finalize.c | 4 +- third_party/python/Python/pylifecycle.c | 2 +- third_party/python/Python/sysmodule.c | 8 ++ third_party/python/pyconfig.h | 3 +- third_party/python/repl.c | 4 + 42 files changed, 260 insertions(+), 63 deletions(-) diff --git a/third_party/mbedtls/mbedtls.mk b/third_party/mbedtls/mbedtls.mk index 030e78dd5..e78eab34a 100644 --- a/third_party/mbedtls/mbedtls.mk +++ b/third_party/mbedtls/mbedtls.mk @@ -58,6 +58,10 @@ o/$(MODE)/third_party/mbedtls/everest.o: \ OVERRIDE_CFLAGS += \ -O3 +o/$(MODE)/third_party/mbedtls/des.o: \ + OVERRIDE_CFLAGS += \ + -DSTACK_FRAME_UNLIMITED + o/$(MODE)/third_party/mbedtls/bigmul4.o \ o/$(MODE)/third_party/mbedtls/bigmul6.o: \ OVERRIDE_CFLAGS += \ diff --git a/third_party/python/Include/abstract.h b/third_party/python/Include/abstract.h index 63370f91c..bdca099b3 100644 --- a/third_party/python/Include/abstract.h +++ b/third_party/python/Include/abstract.h @@ -35,7 +35,7 @@ PyObject *_PyObject_FastCallKeywords(PyObject *func, PyObject **args, PyObject *_PyObject_Call_Prepend(PyObject *func, PyObject *obj, PyObject *args, PyObject *kwargs); -#ifdef USE_CHECKFUNCRESULT +#if IsModeDbg() PyObject *_Py_CheckFunctionResult(PyObject *func, PyObject *result, const char *where); #else diff --git a/third_party/python/Include/ceval.h b/third_party/python/Include/ceval.h index 097e1af77..34864036a 100644 --- a/third_party/python/Include/ceval.h +++ b/third_party/python/Include/ceval.h @@ -77,6 +77,7 @@ int Py_MakePendingCalls(void); void Py_SetRecursionLimit(int); int Py_GetRecursionLimit(void); +#if IsModeDbg() #define Py_EnterRecursiveCall(where) \ (_Py_MakeRecCheck(PyThreadState_GET()->recursion_depth) && \ _Py_CheckRecursiveCall(where)) @@ -84,6 +85,10 @@ int Py_GetRecursionLimit(void); do{ if(_Py_MakeEndRecCheck(PyThreadState_GET()->recursion_depth)) \ PyThreadState_GET()->overflowed = 0; \ } while(0) +#else +#define Py_EnterRecursiveCall(where) (0) +#define Py_LeaveRecursiveCall(where) ((void)0) +#endif int _Py_CheckRecursiveCall(const char *where); extern int _Py_CheckRecursionLimit; diff --git a/third_party/python/Include/objimpl.h b/third_party/python/Include/objimpl.h index 78bec2d99..ff84738d7 100644 --- a/third_party/python/Include/objimpl.h +++ b/third_party/python/Include/objimpl.h @@ -17,8 +17,10 @@ void * PyObject_Realloc(void *, size_t); void PyObject_Free(void *); #ifndef Py_LIMITED_API +#if IsModeDbg() /* This function returns the number of allocated memory blocks, regardless of size */ Py_ssize_t _Py_GetAllocatedBlocks(void); +#endif #endif /* !Py_LIMITED_API */ /* Macros */ @@ -117,6 +119,7 @@ PyVarObject * _PyObject_NewVar(PyTypeObject *, Py_ssize_t); constructor you would start directly with PyObject_Init/InitVar */ +#if IsModeDbg() #ifndef Py_LIMITED_API typedef struct { /* user context passed as the first argument to the 2 functions */ @@ -135,7 +138,7 @@ void PyObject_GetArenaAllocator(PyObjectArenaAllocator *allocator); /* Set the arena allocator. */ void PyObject_SetArenaAllocator(PyObjectArenaAllocator *allocator); #endif - +#endif /* * Garbage Collection Support diff --git a/third_party/python/Include/pyerrors.h b/third_party/python/Include/pyerrors.h index b153ebd95..fff039e3b 100644 --- a/third_party/python/Include/pyerrors.h +++ b/third_party/python/Include/pyerrors.h @@ -316,7 +316,7 @@ void PyErr_BadInternalCall(void); void _PyErr_BadInternalCall(const char *filename, int lineno); /* Mask the old API with a call to the new API for code compiled under Python 2.0: */ -#ifdef USE_BADINTERNALCALL +#if IsModeDbg() #define PyErr_BadInternalCall() _PyErr_BadInternalCall(__FILE__, __LINE__) #else #define PyErr_BadInternalCall() diff --git a/third_party/python/Include/pymem.h b/third_party/python/Include/pymem.h index 1c18e7344..4bd581657 100644 --- a/third_party/python/Include/pymem.h +++ b/third_party/python/Include/pymem.h @@ -64,13 +64,15 @@ PyObject* _PyTraceMalloc_GetTraceback( _PyTraceMalloc_domain_t domain, uintptr_t ptr); -#ifdef USE_TRACEMALLOC +int _PyMem_IsFreed(void *ptr, size_t size); + +#if !IsModeDbg() #define _PyTraceMalloc_Track(domain, ptr, size) (-2) #define _PyTraceMalloc_Untrack(domain, ptr) (-2) #define _PyTraceMalloc_GetTraceback(domain, ptr) (&_Py_NoneStruct) +#define _PyMem_IsFreed(ptr, size) (0) #endif -int _PyMem_IsFreed(void *ptr, size_t size); #endif /* !defined(Py_LIMITED_API) */ @@ -172,6 +174,7 @@ char * _PyMem_Strdup(const char *str); #define PyMem_Del PyMem_Free #define PyMem_DEL PyMem_FREE +#if IsModeDbg() #ifndef Py_LIMITED_API typedef enum { /* PyMem_RawMalloc(), PyMem_RawRealloc() and PyMem_RawFree() */ @@ -237,6 +240,7 @@ void PyMem_SetAllocator(PyMemAllocatorDomain domain, The function does nothing if Python is not compiled is debug mode. */ void PyMem_SetupDebugHooks(void); #endif +#endif COSMOPOLITAN_C_END_ #endif /* !Py_PYMEM_H */ diff --git a/third_party/python/Lib/ctypes/test/test_as_parameter.py b/third_party/python/Lib/ctypes/test/test_as_parameter.py index f9d27cb89..ba4b14756 100644 --- a/third_party/python/Lib/ctypes/test/test_as_parameter.py +++ b/third_party/python/Lib/ctypes/test/test_as_parameter.py @@ -2,6 +2,7 @@ import unittest from ctypes import * from ctypes.test import need_symbol import _ctypes_test +import cosmo dll = CDLL(_ctypes_test.__file__) @@ -190,6 +191,7 @@ class BasicWrapTestCase(unittest.TestCase): self.assertEqual((s8i.a, s8i.b, s8i.c, s8i.d, s8i.e, s8i.f, s8i.g, s8i.h), (9*2, 8*3, 7*4, 6*5, 5*6, 4*7, 3*8, 2*9)) + @unittest.skipUnless(cosmo.MODE == "dbg", "no recursion checking") def test_recursive_as_param(self): from ctypes import c_int diff --git a/third_party/python/Lib/test/list_tests.py b/third_party/python/Lib/test/list_tests.py index 05e771f94..eba6f71b7 100644 --- a/third_party/python/Lib/test/list_tests.py +++ b/third_party/python/Lib/test/list_tests.py @@ -4,6 +4,8 @@ Tests common to list and UserList.UserList import sys import os +import unittest +import cosmo from functools import cmp_to_key from test import support, seq_tests @@ -53,6 +55,7 @@ class CommonTest(seq_tests.CommonTest): self.assertEqual(str(a2), "[0, 1, 2, [...], 3]") self.assertEqual(repr(a2), "[0, 1, 2, [...], 3]") + @unittest.skipUnless(cosmo.MODE == "dbg", "disabled recursion checking") def test_repr_deep(self): a = self.type2test([]) for i in range(sys.getrecursionlimit() + 100): diff --git a/third_party/python/Lib/test/mapping_tests.py b/third_party/python/Lib/test/mapping_tests.py index 53f29f605..c5900182d 100644 --- a/third_party/python/Lib/test/mapping_tests.py +++ b/third_party/python/Lib/test/mapping_tests.py @@ -2,6 +2,7 @@ import unittest import collections import sys +import cosmo class BasicTestMappingProtocol(unittest.TestCase): @@ -620,6 +621,7 @@ class TestHashMappingProtocol(TestMappingProtocol): d = self._full_mapping({1: BadRepr()}) self.assertRaises(Exc, repr, d) + @unittest.skipUnless(cosmo.MODE == "dbg", "disabled recursion checking") def test_repr_deep(self): d = self._empty_mapping() for i in range(sys.getrecursionlimit() + 100): diff --git a/third_party/python/Lib/test/pickletester.py b/third_party/python/Lib/test/pickletester.py index d66c4456b..72a726ea7 100644 --- a/third_party/python/Lib/test/pickletester.py +++ b/third_party/python/Lib/test/pickletester.py @@ -1943,8 +1943,7 @@ class AbstractPickleTests(unittest.TestCase): self.assertEqual(y._reduce_called, 1) @no_tracing - @unittest.skipIf(cosmo.MODE in ("asan", "dbg"), - "extremely slow in asan mode") + @unittest.skipIf(True, "disabled recursion checking + slow in asan, dbg") def test_bad_getattr(self): # Issue #3514: crash when there is an infinite loop in __getattr__ x = BadGetattr() diff --git a/third_party/python/Lib/test/test_capi.py b/third_party/python/Lib/test/test_capi.py index 0c7adbd76..b2306f2bd 100644 --- a/third_party/python/Lib/test/test_capi.py +++ b/third_party/python/Lib/test/test_capi.py @@ -1,6 +1,7 @@ # Run the _testcapi module tests (tests for the Python/C API): by defn, # these are all functions _testcapi exports whose name begins with 'test_'. +import cosmo import os import pickle import random @@ -179,6 +180,7 @@ class CAPITest(unittest.TestCase): o @= m1 self.assertEqual(o, ("matmul", 42, m1)) + @unittest.skipUnless(cosmo.MODE == "dbg", "disabled recursion check") def test_return_null_without_error(self): # Issue #23571: A function must not return NULL without setting an # error @@ -207,6 +209,7 @@ class CAPITest(unittest.TestCase): 'return_null_without_error.* ' 'returned NULL without setting an error') + @unittest.skipUnless(cosmo.MODE == "dbg", "disabled recursion check") def test_return_result_with_error(self): # Issue #23571: A function must not return a result with an error set if Py_DEBUG: @@ -242,6 +245,7 @@ class CAPITest(unittest.TestCase): def test_buildvalue_N(self): _testcapi.test_buildvalue_N() + @unittest.skipUnless(cosmo.MODE == "dbg", "disabled memory hooks") def test_set_nomemory(self): code = """if 1: import _testcapi @@ -510,6 +514,7 @@ class Test_testcapi(unittest.TestCase): if name.startswith('test_') and not name.endswith('_code')) +@unittest.skipUnless(cosmo.MODE == "dbg", "disabled memory debugging") class PyMemDebugTests(unittest.TestCase): PYTHONMALLOC = 'debug' # '0x04c06e0' or '04C06E0' diff --git a/third_party/python/Lib/test/test_class.py b/third_party/python/Lib/test/test_class.py index 507c723e7..38e3bd056 100644 --- a/third_party/python/Lib/test/test_class.py +++ b/third_party/python/Lib/test/test_class.py @@ -1,6 +1,7 @@ "Test the functionality of Python classes implementing operators." import unittest +import cosmo testmeths = [ @@ -490,6 +491,7 @@ class ClassTests(unittest.TestCase): self.assertRaises(TypeError, hash, C2()) + @unittest.skipUnless(cosmo.MODE == "dbg", "disabled recursion checking") def testSFBug532646(self): # Test for SF bug 532646 diff --git a/third_party/python/Lib/test/test_copy.py b/third_party/python/Lib/test/test_copy.py index 45a692022..996b7c991 100644 --- a/third_party/python/Lib/test/test_copy.py +++ b/third_party/python/Lib/test/test_copy.py @@ -1,5 +1,6 @@ """Unit tests for the copy module.""" +import cosmo import copy import copyreg import weakref @@ -372,8 +373,9 @@ class TestCopy(unittest.TestCase): x = [] x.append(x) y = copy.deepcopy(x) - for op in comparisons: - self.assertRaises(RecursionError, op, y, x) + if cosmo.MODE == "dbg": # requires recursion checking + for op in comparisons: + self.assertRaises(RecursionError, op, y, x) self.assertIsNot(y, x) self.assertIs(y[0], y) self.assertEqual(len(y), 1) @@ -399,8 +401,9 @@ class TestCopy(unittest.TestCase): x = ([],) x[0].append(x) y = copy.deepcopy(x) - for op in comparisons: - self.assertRaises(RecursionError, op, y, x) + if cosmo.MODE == "dbg": # requires recursion checking + for op in comparisons: + self.assertRaises(RecursionError, op, y, x) self.assertIsNot(y, x) self.assertIsNot(y[0], x[0]) self.assertIs(y[0][0], y) @@ -418,8 +421,9 @@ class TestCopy(unittest.TestCase): y = copy.deepcopy(x) for op in order_comparisons: self.assertRaises(TypeError, op, y, x) - for op in equality_comparisons: - self.assertRaises(RecursionError, op, y, x) + if cosmo.MODE == "dbg": # requires recursion checking + for op in equality_comparisons: + self.assertRaises(RecursionError, op, y, x) self.assertIsNot(y, x) self.assertIs(y['foo'], y) self.assertEqual(len(y), 1) diff --git a/third_party/python/Lib/test/test_descr.py b/third_party/python/Lib/test/test_descr.py index 0d33e9179..f48fe4689 100644 --- a/third_party/python/Lib/test/test_descr.py +++ b/third_party/python/Lib/test/test_descr.py @@ -5,6 +5,7 @@ import itertools import math import pickle import sys +import cosmo import types import unittest import warnings @@ -3469,6 +3470,7 @@ order (MRO) for bases """ list.__init__(a, sequence=[0, 1, 2]) self.assertEqual(a, [0, 1, 2]) + @unittest.skipUnless(cosmo.MODE == "dbg", "disabled recursion checking") def test_recursive_call(self): # Testing recursive __call__() by setting to instance of class... class A(object): @@ -4494,6 +4496,7 @@ order (MRO) for bases """ with self.assertRaises(TypeError): str.__add__(fake_str, "abc") + @unittest.skipUnless(cosmo.MODE == "dbg", "disabled recursion checking") def test_repr_as_str(self): # Issue #11603: crash or infinite loop when rebinding __str__ as # __repr__. diff --git a/third_party/python/Lib/test/test_dict.py b/third_party/python/Lib/test/test_dict.py index 5dfc5e14a..7ed0e79ec 100644 --- a/third_party/python/Lib/test/test_dict.py +++ b/third_party/python/Lib/test/test_dict.py @@ -469,6 +469,7 @@ class DictTest(unittest.TestCase): d = {1: BadRepr()} self.assertRaises(Exc, repr, d) + @unittest.skipUnless(cosmo.MODE == "dbg", "disabled recursion checking") def test_repr_deep(self): d = {} for i in range(sys.getrecursionlimit() + 100): @@ -1221,11 +1222,12 @@ class CAPITest(unittest.TestCase): self.assertEqual(dict_getitem_knownhash(d, 'y', hash('y')), 2) self.assertEqual(dict_getitem_knownhash(d, 'z', hash('z')), 3) - # # TODO: Did this break? What did this do? + # # TODO: Did this break? What did this do? + # (likely related to disabling BadInternalCall in #264) # # not a dict # # find the APE compilation mode, run this test in dbg only # - # if cosmo.MODE == "dbg": - # self.assertRaises(SystemError, dict_getitem_knownhash, [], 1, hash(1)) + if cosmo.MODE == "dbg": + self.assertRaises(SystemError, dict_getitem_knownhash, [], 1, hash(1)) # key does not exist self.assertRaises(KeyError, dict_getitem_knownhash, {}, 1, hash(1)) diff --git a/third_party/python/Lib/test/test_dictviews.py b/third_party/python/Lib/test/test_dictviews.py index 080747632..73b1aa287 100644 --- a/third_party/python/Lib/test/test_dictviews.py +++ b/third_party/python/Lib/test/test_dictviews.py @@ -2,6 +2,7 @@ import collections import copy import pickle import sys +import cosmo import unittest class DictSetTest(unittest.TestCase): @@ -213,6 +214,7 @@ class DictSetTest(unittest.TestCase): # Again. self.assertIsInstance(r, str) + @unittest.skipUnless(cosmo.MODE == "dbg", "disabled recursion checking") def test_deeply_nested_repr(self): d = {} for i in range(sys.getrecursionlimit() + 100): diff --git a/third_party/python/Lib/test/test_exceptions.py b/third_party/python/Lib/test/test_exceptions.py index 700247372..45155e0a4 100644 --- a/third_party/python/Lib/test/test_exceptions.py +++ b/third_party/python/Lib/test/test_exceptions.py @@ -515,6 +515,7 @@ class ExceptionTests(unittest.TestCase): self.assertEqual(x.fancy_arg, 42) @no_tracing + @unittest.skipUnless(cosmo.MODE == "dbg", "disabled recursion checking") def testInfiniteRecursion(self): def f(): return f() @@ -928,15 +929,17 @@ class ExceptionTests(unittest.TestCase): else: self.fail("Should have raised KeyError") - def g(): - try: - return g() - except RecursionError: - return sys.exc_info() - e, v, tb = g() - self.assertTrue(isinstance(v, RecursionError), type(v)) - self.assertIn("maximum recursion depth exceeded", str(v)) + if cosmo.MODE == "dbg": + def g(): + try: + return g() + except RecursionError: + return sys.exc_info() + e, v, tb = g() + self.assertTrue(isinstance(v, RecursionError), type(v)) + self.assertIn("maximum recursion depth exceeded", str(v)) + @unittest.skipUnless(cosmo.MODE == "dbg", "disabled recursion checking") @cpython_only def test_recursion_normalizing_exception(self): # Issue #22898. @@ -1014,6 +1017,7 @@ class ExceptionTests(unittest.TestCase): b'while normalizing an exception', err) self.assertIn(b'Done.', out) + @unittest.skipUnless(cosmo.MODE == "dbg", "disabled recursion checking") @cpython_only def test_recursion_normalizing_with_no_memory(self): # Issue #30697. Test that in the abort that occurs when there is no @@ -1119,6 +1123,7 @@ class ExceptionTests(unittest.TestCase): self.assertEqual(wr(), None) @no_tracing + @unittest.skipUnless(cosmo.MODE == "dbg", "disabled recursion checking") def test_recursion_error_cleanup(self): # Same test as above, but with "recursion exceeded" errors class C: @@ -1208,6 +1213,7 @@ class ExceptionTests(unittest.TestCase): self.assertIn("test message", report) self.assertTrue(report.endswith("\n")) + @unittest.skipUnless(cosmo.MODE == "dbg", "disabled memory hooks") @cpython_only def test_memory_error_in_PyErr_PrintEx(self): code = """if 1: diff --git a/third_party/python/Lib/test/test_fileio.py b/third_party/python/Lib/test/test_fileio.py index b228d2535..2232d9393 100644 --- a/third_party/python/Lib/test/test_fileio.py +++ b/third_party/python/Lib/test/test_fileio.py @@ -177,6 +177,7 @@ class AutoFileTests: finally: os.close(fd) + @unittest.skipUnless(cosmo.MODE == "dbg", "disabled recursion checking") def testRecursiveRepr(self): # Issue #25455 with swap_attr(self.f, 'name', self.f): diff --git a/third_party/python/Lib/test/test_functools.py b/third_party/python/Lib/test/test_functools.py index 2f18451fc..6889b7922 100644 --- a/third_party/python/Lib/test/test_functools.py +++ b/third_party/python/Lib/test/test_functools.py @@ -217,6 +217,7 @@ class TestPartial: [f'{name}({capture!r}, {args_repr}, {kwargs_repr})' for kwargs_repr in kwargs_reprs]) + @unittest.skipUnless(cosmo.MODE == "dbg", "disabled recursion checking") def test_recursive_repr(self): if self.partial in (c_functools.partial, py_functools.partial): name = 'functools.partial' @@ -329,14 +330,15 @@ class TestPartial: def test_recursive_pickle(self): with self.AllowPickle(): - f = self.partial(capture) - f.__setstate__((f, (), {}, {})) - try: - for proto in range(pickle.HIGHEST_PROTOCOL + 1): - with self.assertRaises(RecursionError): - pickle.dumps(f, proto) - finally: - f.__setstate__((capture, (), {}, {})) + if cosmo.MODE == "dbg": + f = self.partial(capture) + f.__setstate__((f, (), {}, {})) + try: + for proto in range(pickle.HIGHEST_PROTOCOL + 1): + with self.assertRaises(RecursionError): + pickle.dumps(f, proto) + finally: + f.__setstate__((capture, (), {}, {})) f = self.partial(capture) f.__setstate__((capture, (f,), {}, {})) diff --git a/third_party/python/Lib/test/test_io.py b/third_party/python/Lib/test/test_io.py index 341e325b3..05622cb30 100644 --- a/third_party/python/Lib/test/test_io.py +++ b/third_party/python/Lib/test/test_io.py @@ -28,6 +28,7 @@ import pickle import random import signal import sys +import cosmo import time import unittest import warnings @@ -1099,6 +1100,7 @@ class CommonBufferedTests: raw.name = b"dummy" self.assertEqual(repr(b), "<%s name=b'dummy'>" % clsname) + @unittest.skipUnless(cosmo.MODE == "dbg", "disabled recursion checking") def test_recursive_repr(self): # Issue #25455 raw = self.MockRawIO() @@ -2540,6 +2542,7 @@ class TextIOWrapperTest(unittest.TestCase): t.buffer.detach() repr(t) # Should not raise an exception + @unittest.skipUnless(cosmo.MODE == "dbg", "disabled recursion checking") def test_recursive_repr(self): # Issue #25455 raw = self.BytesIO() diff --git a/third_party/python/Lib/test/test_isinstance.py b/third_party/python/Lib/test/test_isinstance.py index e63d59b34..09531fc0a 100644 --- a/third_party/python/Lib/test/test_isinstance.py +++ b/third_party/python/Lib/test/test_isinstance.py @@ -4,6 +4,7 @@ import unittest import sys +import cosmo @@ -257,11 +258,13 @@ class TestIsInstanceIsSubclass(unittest.TestCase): self.assertEqual(True, issubclass(int, (int, (float, int)))) self.assertEqual(True, issubclass(str, (str, (Child, NewChild, str)))) + @unittest.skipUnless(cosmo.MODE == "dbg", "disabled recursion checking") def test_subclass_recursion_limit(self): # make sure that issubclass raises RecursionError before the C stack is # blown self.assertRaises(RecursionError, blowstack, issubclass, str, str) + @unittest.skipUnless(cosmo.MODE == "dbg", "disabled recursion checking") def test_isinstance_recursion_limit(self): # make sure that issubclass raises RecursionError before the C stack is # blown diff --git a/third_party/python/Lib/test/test_json/test_recursion.py b/third_party/python/Lib/test/test_json/test_recursion.py index 877dc448b..d6ebfbac1 100644 --- a/third_party/python/Lib/test/test_json/test_recursion.py +++ b/third_party/python/Lib/test/test_json/test_recursion.py @@ -1,4 +1,6 @@ from test.test_json import PyTest, CTest +import unittest +import cosmo class JSONTestObject: @@ -65,6 +67,7 @@ class TestRecursion: self.fail("didn't raise ValueError on default recursion") + @unittest.skipUnless(cosmo.MODE == "dbg", "disabled recursion checking") def test_highly_nested_objects_decoding(self): # test that loading highly-nested objects doesn't segfault when C # accelerations are used. See #12017 @@ -75,6 +78,7 @@ class TestRecursion: with self.assertRaises(RecursionError): self.loads('[' * 100000 + '1' + ']' * 100000) + @unittest.skipUnless(cosmo.MODE == "dbg", "disabled recursion checking") def test_highly_nested_objects_encoding(self): # See #12051 l, d = [], {} @@ -85,6 +89,7 @@ class TestRecursion: with self.assertRaises(RecursionError): self.dumps(d) + @unittest.skipUnless(cosmo.MODE == "dbg", "disabled recursion checking") def test_endless_recursion(self): # See #12051 class EndlessJSONEncoder(self.json.JSONEncoder): diff --git a/third_party/python/Lib/test/test_plistlib.py b/third_party/python/Lib/test/test_plistlib.py index 8424470d1..eb92addcc 100644 --- a/third_party/python/Lib/test/test_plistlib.py +++ b/third_party/python/Lib/test/test_plistlib.py @@ -1,5 +1,6 @@ # Copyright (C) 2003-2013 Python Software Foundation +import cosmo import struct import unittest import plistlib @@ -813,6 +814,7 @@ class TestBinaryPlistlib(unittest.TestCase): b = plistlib.loads(plistlib.dumps(a, fmt=plistlib.FMT_BINARY)) self.assertIs(b['x'], b) + @unittest.skipUnless(cosmo.MODE == "dbg", "disabled recursion checking") def test_deep_nesting(self): for N in [300, 100000]: chunks = [b'\xa1' + (i + 1).to_bytes(4, 'big') for i in range(N)] diff --git a/third_party/python/Lib/test/test_repl.py b/third_party/python/Lib/test/test_repl.py index 9efd459a6..867061f7c 100644 --- a/third_party/python/Lib/test/test_repl.py +++ b/third_party/python/Lib/test/test_repl.py @@ -1,6 +1,7 @@ """Test the interactive interpreter.""" import sys +import cosmo import os import unittest import subprocess @@ -36,6 +37,7 @@ def spawn_repl(*args, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, **kw): class TestInteractiveInterpreter(unittest.TestCase): + @unittest.skipUnless(cosmo.MODE == "dbg", "disabled memory hooks") @cpython_only def test_no_memory(self): # Issue #30696: Fix the interactive interpreter looping endlessly when diff --git a/third_party/python/Lib/test/test_richcmp.py b/third_party/python/Lib/test/test_richcmp.py index 58729a9fe..36a7317fc 100644 --- a/third_party/python/Lib/test/test_richcmp.py +++ b/third_party/python/Lib/test/test_richcmp.py @@ -1,6 +1,7 @@ # Tests for rich comparisons import unittest +import cosmo from test import support import operator @@ -220,6 +221,7 @@ class MiscTest(unittest.TestCase): for func in (do, operator.not_): self.assertRaises(Exc, func, Bad()) + @unittest.skipUnless(cosmo.MODE == "dbg", "disabled recursion checking") @support.no_tracing def test_recursion(self): # Check that comparison for recursive objects fails gracefully diff --git a/third_party/python/Lib/test/test_runpy.py b/third_party/python/Lib/test/test_runpy.py index 02b4d6256..85baf8e4d 100644 --- a/third_party/python/Lib/test/test_runpy.py +++ b/third_party/python/Lib/test/test_runpy.py @@ -3,6 +3,7 @@ import unittest import os import os.path import sys +import cosmo import re import tempfile import importlib, importlib.machinery, importlib.util @@ -723,6 +724,7 @@ class RunPathTestCase(unittest.TestCase, CodeExecutionMixin): self._check_import_error(zip_name, msg) @no_tracing + @unittest.skipUnless(cosmo.MODE == "dbg", "disabled recursion checking") def test_main_recursion_error(self): with temp_dir() as script_dir, temp_dir() as dummy_dir: mod_name = '__main__' diff --git a/third_party/python/Lib/test/test_sys.py b/third_party/python/Lib/test/test_sys.py index b5b805900..f13d78b60 100644 --- a/third_party/python/Lib/test/test_sys.py +++ b/third_party/python/Lib/test/test_sys.py @@ -1,6 +1,7 @@ import unittest, test.support from test.support.script_helper import assert_python_ok, assert_python_failure import sys, io, os +import cosmo import struct import subprocess import textwrap @@ -190,6 +191,7 @@ class SysModuleTest(unittest.TestCase): finally: sys.setswitchinterval(orig) + @unittest.skipUnless(cosmo.MODE == "dbg", "disabled recursion checking") def test_recursionlimit(self): self.assertRaises(TypeError, sys.getrecursionlimit, 42) oldlimit = sys.getrecursionlimit() @@ -199,6 +201,7 @@ class SysModuleTest(unittest.TestCase): self.assertEqual(sys.getrecursionlimit(), 10000) sys.setrecursionlimit(oldlimit) + @unittest.skipUnless(cosmo.MODE == "dbg", "disabled recursion checking") def test_recursionlimit_recovery(self): if hasattr(sys, 'gettrace') and sys.gettrace(): self.skipTest('fatal error if run with a trace function') @@ -222,6 +225,7 @@ class SysModuleTest(unittest.TestCase): finally: sys.setrecursionlimit(oldlimit) + @unittest.skipUnless(cosmo.MODE == "dbg", "disabled recursion checking") @test.support.cpython_only def test_setrecursionlimit_recursion_depth(self): # Issue #25274: Setting a low recursion limit must be blocked if the @@ -257,6 +261,7 @@ class SysModuleTest(unittest.TestCase): finally: sys.setrecursionlimit(oldlimit) + @unittest.skipUnless(cosmo.MODE == "dbg", "disabled recursion checking") def test_recursionlimit_fatalerror(self): # A fatal error occurs if a second recursion limit is hit when recovering # from a first one. diff --git a/third_party/python/Lib/test/test_threading.py b/third_party/python/Lib/test/test_threading.py index 429e610f0..638100f1a 100644 --- a/third_party/python/Lib/test/test_threading.py +++ b/third_party/python/Lib/test/test_threading.py @@ -9,6 +9,7 @@ from test.support.script_helper import assert_python_ok, assert_python_failure import random import sys +import cosmo _thread = import_module('_thread') threading = import_module('threading') import time @@ -882,6 +883,7 @@ class ThreadingExceptionTests(BaseTestCase): lock = threading.Lock() self.assertRaises(RuntimeError, lock.release) + @unittest.skipUnless(cosmo.MODE == "dbg", "disabled recursion checking") @unittest.skipUnless(sys.platform == 'darwin' and test.support.python_is_optimized(), 'test macosx problem') def test_recursion_limit(self): diff --git a/third_party/python/Lib/test/test_traceback.py b/third_party/python/Lib/test/test_traceback.py index 6fb508b70..67e5d06cb 100644 --- a/third_party/python/Lib/test/test_traceback.py +++ b/third_party/python/Lib/test/test_traceback.py @@ -4,6 +4,7 @@ from collections import namedtuple from io import StringIO import linecache import sys +import cosmo import unittest import re from test import support @@ -300,6 +301,7 @@ class TracebackFormatTests(unittest.TestCase): ]) # issue 26823 - Shrink recursive tracebacks + @unittest.skipUnless(cosmo.MODE == "dbg", "disabled recursion checking") def _check_recursive_traceback_display(self, render_exc): # Always show full diffs when this test fails # Note that rearranging things may require adjusting diff --git a/third_party/python/Lib/test/test_tracemalloc.py b/third_party/python/Lib/test/test_tracemalloc.py index 6bfc58071..b251c17b0 100644 --- a/third_party/python/Lib/test/test_tracemalloc.py +++ b/third_party/python/Lib/test/test_tracemalloc.py @@ -1,7 +1,9 @@ import contextlib import os import sys -import tracemalloc +import cosmo +if cosmo.MODE == "dbg": + import tracemalloc import unittest from unittest.mock import patch from test.support.script_helper import (assert_python_ok, assert_python_failure, @@ -87,6 +89,7 @@ def traceback_filename(filename): return traceback_lineno(filename, 0) +@unittest.skipUnless(cosmo.MODE == "dbg", "requires APE debug build") class TestTracemallocEnabled(unittest.TestCase): def setUp(self): if tracemalloc.is_tracing(): @@ -297,6 +300,7 @@ class TestTracemallocEnabled(unittest.TestCase): self.assertEqual(exitcode, 0) +@unittest.skipUnless(cosmo.MODE == "dbg", "requires APE debug build") class TestSnapshot(unittest.TestCase): maxDiff = 4000 @@ -591,6 +595,7 @@ class TestSnapshot(unittest.TestCase): []) +@unittest.skipUnless(cosmo.MODE == "dbg", "requires APE debug build") class TestFilters(unittest.TestCase): maxDiff = 2048 @@ -802,6 +807,7 @@ class TestFilters(unittest.TestCase): self.assertFalse(f._match_traceback(unknown)) +@unittest.skipUnless(cosmo.MODE == "dbg", "requires APE debug build") class TestCommandLine(unittest.TestCase): def test_env_var_disabled_by_default(self): # not tracing by default @@ -874,6 +880,7 @@ class TestCommandLine(unittest.TestCase): assert_python_ok('-X', 'tracemalloc', '-c', code) +@unittest.skipUnless(cosmo.MODE == "dbg", "requires APE debug build") @unittest.skipIf(_testcapi is None, 'need _testcapi') class TestCAPI(unittest.TestCase): maxDiff = 80 * 20 diff --git a/third_party/python/Lib/test/test_warnings/__init__.py b/third_party/python/Lib/test/test_warnings/__init__.py index 3dc88f5e3..c0c5547ca 100644 --- a/third_party/python/Lib/test/test_warnings/__init__.py +++ b/third_party/python/Lib/test/test_warnings/__init__.py @@ -1,6 +1,7 @@ from contextlib import contextmanager import linecache import os +import cosmo from io import StringIO import re import sys @@ -922,6 +923,7 @@ class CWarningsDisplayTests(WarningsDisplayTests, unittest.TestCase): class PyWarningsDisplayTests(WarningsDisplayTests, unittest.TestCase): module = py_warnings + @unittest.skipUnless(cosmo.MODE == "dbg", "requires APE debug build") def test_tracemalloc(self): self.addCleanup(support.unlink, support.TESTFN) diff --git a/third_party/python/Lib/test/test_xml_etree.py b/third_party/python/Lib/test/test_xml_etree.py index 59c7c8f77..82e9a6c54 100644 --- a/third_party/python/Lib/test/test_xml_etree.py +++ b/third_party/python/Lib/test/test_xml_etree.py @@ -5,6 +5,7 @@ # For this purpose, the module-level "ET" symbol is temporarily # monkey-patched when running the "test_xml_etree_c" test suite. +import cosmo import copy import html import io @@ -1920,6 +1921,7 @@ class BadElementTest(ElementTestCase, unittest.TestCase): e.extend([ET.Element('bar')]) self.assertRaises(ValueError, e.remove, X('baz')) + @unittest.skipUnless(cosmo.MODE == "dbg", "disabled recursion checking") def test_recursive_repr(self): # Issue #25455 e = ET.Element('foo') diff --git a/third_party/python/Lib/test/test_zlib.py b/third_party/python/Lib/test/test_zlib.py index 2659c5f91..d124c7204 100644 --- a/third_party/python/Lib/test/test_zlib.py +++ b/third_party/python/Lib/test/test_zlib.py @@ -4,6 +4,7 @@ import binascii import pickle import random import sys +import cosmo import zlib from test.support import bigmemtest, _1G, _4G @@ -707,6 +708,7 @@ class CompressObjectTestCase(BaseCompressTestCase, unittest.TestCase): # Memory use of the following functions takes into account overallocation + @unittest.skipUnless(cosmo.MODE == "dbg", "disabled tracemalloc") @bigmemtest(size=_1G + 1024 * 1024, memuse=3) def test_big_compress_buffer(self, size): import tracemalloc diff --git a/third_party/python/Modules/_testcapimodule.c b/third_party/python/Modules/_testcapimodule.c index 2be74d116..a21510762 100644 --- a/third_party/python/Modules/_testcapimodule.c +++ b/third_party/python/Modules/_testcapimodule.c @@ -3488,6 +3488,7 @@ test_pymem_alloc0(PyObject *self) Py_RETURN_NONE; } +#if IsModeDbg() typedef struct { PyMemAllocatorEx alloc; @@ -3780,6 +3781,7 @@ remove_mem_hooks(PyObject *self) fm_remove_hooks(); Py_RETURN_NONE; } +#endif PyDoc_STRVAR(docstring_empty, "" @@ -4234,6 +4236,7 @@ test_PyTime_AsMicroseconds(PyObject *self, PyObject *args) return _PyTime_AsNanosecondsObject(ms); } +#if IsModeDbg() static PyObject* get_recursion_depth(PyObject *self, PyObject *args) { @@ -4242,6 +4245,7 @@ get_recursion_depth(PyObject *self, PyObject *args) /* subtract one to ignore the frame of the get_recursion_depth() call */ return PyLong_FromLong(tstate->recursion_depth - 1); } +#endif static PyObject* pymem_buffer_overflow(PyObject *self, PyObject *args) @@ -4656,6 +4660,7 @@ static PyMethodDef TestMethods[] = { {"create_cfunction", create_cfunction, METH_NOARGS}, {"test_pymem_alloc0", (PyCFunction)test_pymem_alloc0, METH_NOARGS}, +#if IsModeDbg() {"test_pymem_setrawallocators", (PyCFunction)test_pymem_setrawallocators, METH_NOARGS}, {"test_pymem_setallocators", @@ -4666,6 +4671,7 @@ static PyMethodDef TestMethods[] = { PyDoc_STR("set_nomemory(start:int, stop:int = 0)")}, {"remove_mem_hooks", (PyCFunction)remove_mem_hooks, METH_NOARGS, PyDoc_STR("Remove memory hooks.")}, +#endif {"no_docstring", (PyCFunction)test_with_docstring, METH_NOARGS}, {"docstring_empty", @@ -4723,6 +4729,7 @@ static PyMethodDef TestMethods[] = { #endif {"PyTime_AsMilliseconds", test_PyTime_AsMilliseconds, METH_VARARGS}, {"PyTime_AsMicroseconds", test_PyTime_AsMicroseconds, METH_VARARGS}, +#if IsModeDbg() {"get_recursion_depth", get_recursion_depth, METH_NOARGS}, {"pymem_buffer_overflow", pymem_buffer_overflow, METH_NOARGS}, {"pymem_api_misuse", pymem_api_misuse, METH_NOARGS}, @@ -4731,6 +4738,7 @@ static PyMethodDef TestMethods[] = { {"tracemalloc_track", tracemalloc_track, METH_VARARGS}, {"tracemalloc_untrack", tracemalloc_untrack, METH_VARARGS}, {"tracemalloc_get_traceback", tracemalloc_get_traceback, METH_VARARGS}, +#endif {"dict_get_version", dict_get_version, METH_VARARGS}, {"pyobject_fastcall", test_pyobject_fastcall, METH_VARARGS}, {"pyobject_fastcalldict", test_pyobject_fastcalldict, METH_VARARGS}, diff --git a/third_party/python/Modules/_tracemalloc.c b/third_party/python/Modules/_tracemalloc.c index e036d3fc2..0c0f04a92 100644 --- a/third_party/python/Modules/_tracemalloc.c +++ b/third_party/python/Modules/_tracemalloc.c @@ -41,6 +41,7 @@ PYTHON_PROVIDE("_tracemalloc.is_tracing"); PYTHON_PROVIDE("_tracemalloc.start"); PYTHON_PROVIDE("_tracemalloc.stop"); +#if IsModeDbg() /* Trace memory blocks allocated by PyMem_RawMalloc() */ #define TRACE_RAW_MALLOC @@ -1686,30 +1687,6 @@ static PyMethodDef module_methods[] = { PyDoc_STRVAR(module_doc, "Debug module to trace memory blocks allocated by Python."); -static struct PyModuleDef module_def = { - PyModuleDef_HEAD_INIT, - "_tracemalloc", - module_doc, - 0, /* non-negative size to be able to unload the module */ - module_methods, - NULL, -}; - -PyMODINIT_FUNC -PyInit__tracemalloc(void) -{ - PyObject *m; - m = PyModule_Create(&module_def); - if (m == NULL) - return NULL; - - if (tracemalloc_init() < 0) - return NULL; - - return m; -} - - static int parse_sys_xoptions(PyObject *value) { @@ -1862,6 +1839,41 @@ PyObject* return traceback_to_pyobject(traceback, NULL); } +#endif + +static struct PyModuleDef module_def = { + PyModuleDef_HEAD_INIT, + "_tracemalloc", +#ifdef USETRACEMALLOC + module_doc, +#else + NULL, +#endif + 0, /* non-negative size to be able to unload the module */ +#ifdef USETRACEMALLOC + module_methods, +#else + NULL, +#endif + NULL, +}; + + +PyMODINIT_FUNC +PyInit__tracemalloc(void) +{ + PyObject *m; + m = PyModule_Create(&module_def); + if (m == NULL) + return NULL; + +#ifdef USETRACEMALLOC + if (tracemalloc_init() < 0) + return NULL; +#endif + + return m; +} _Section(".rodata.pytab.1") const struct _inittab _PyImport_Inittab__tracemalloc = { "_tracemalloc", diff --git a/third_party/python/Modules/main.c b/third_party/python/Modules/main.c index 0559802f6..bfaf906d2 100644 --- a/third_party/python/Modules/main.c +++ b/third_party/python/Modules/main.c @@ -424,12 +424,14 @@ Py_Main(int argc, wchar_t **argv) } } +#if IsModeDbg() opt = Py_GETENV("PYTHONMALLOC"); if (_PyMem_SetupAllocators(opt) < 0) { fprintf(stderr, "Error in PYTHONMALLOC: unknown allocator \"%s\"!\n", opt); exit(1); } +#endif _PyRandom_Init(); diff --git a/third_party/python/Objects/obmalloc.c b/third_party/python/Objects/obmalloc.c index 147efbb7c..e1b0051a0 100644 --- a/third_party/python/Objects/obmalloc.c +++ b/third_party/python/Objects/obmalloc.c @@ -29,6 +29,7 @@ #define uint unsigned int /* assuming >= 16 bits */ /* Forward declaration */ +#if IsModeDbg() static void *_PyMem_DebugRawMalloc(void *, size_t); static void *_PyMem_DebugRawCalloc(void *, size_t, size_t); static void *_PyMem_DebugRawRealloc(void *, void *, size_t); @@ -37,6 +38,7 @@ static void *_PyMem_DebugMalloc(void *, size_t); static void *_PyMem_DebugCalloc(void *, size_t, size_t); static void *_PyMem_DebugRealloc(void *, void *, size_t); static void _PyMem_DebugFree(void *, void *); +#endif static void _PyObject_DebugDumpAddress(const void *); static void _PyMem_DebugCheckAddress(char, const void *); @@ -82,6 +84,13 @@ static void* _PyObject_Malloc(void *ctx, size_t size); static void* _PyObject_Calloc(void *ctx, size_t nelem, size_t elsize); static void _PyObject_Free(void *ctx, void *p); static void* _PyObject_Realloc(void *ctx, void *ptr, size_t size); + +#else +/* in MODE=asan, no pymalloc, so use macro */ +#define _PyObject_Malloc(ctx, size) _PyMem_RawMalloc((ctx), (size)) +#define _PyObject_Calloc(ctx, nelem, elsize) _PyMem_RawCalloc((ctx), (nelem), (elsize)) +#define _PyObject_Realloc(ctx, ptr, size) _PyMem_RawRealloc((ctx), (ptr), (size)) +#define _PyObject_Free(ctx, p) _PyMem_RawFree((ctx), (p)) #endif static inline void * @@ -207,6 +216,7 @@ _PyObject_ArenaFree(void *ctx, void *ptr, size_t size) } #endif +#if IsModeDbg() #define PYRAW_FUNCS _PyMem_RawMalloc, _PyMem_RawCalloc, _PyMem_RawRealloc, _PyMem_RawFree #ifdef WITH_PYMALLOC # define PYOBJ_FUNCS _PyObject_Malloc, _PyObject_Calloc, _PyObject_Realloc, _PyObject_Free @@ -428,6 +438,7 @@ PyObject_SetArenaAllocator(PyObjectArenaAllocator *allocator) { _PyObject_Arena = *allocator; } +#endif void * PyMem_RawMalloc(size_t size) @@ -440,7 +451,11 @@ PyMem_RawMalloc(size_t size) */ if (size > (size_t)PY_SSIZE_T_MAX) return NULL; +#if IsModeDbg() return _PyMem_Raw.malloc(_PyMem_Raw.ctx, size); +#else + return _PyMem_RawMalloc(NULL, size); +#endif } void * @@ -449,7 +464,11 @@ PyMem_RawCalloc(size_t nelem, size_t elsize) /* see PyMem_RawMalloc() */ if (elsize != 0 && nelem > (size_t)PY_SSIZE_T_MAX / elsize) return NULL; +#if IsModeDbg() return _PyMem_Raw.calloc(_PyMem_Raw.ctx, nelem, elsize); +#else + return _PyMem_RawCalloc(NULL, nelem, elsize); +#endif } void* @@ -458,12 +477,20 @@ PyMem_RawRealloc(void *ptr, size_t new_size) /* see PyMem_RawMalloc() */ if (new_size > (size_t)PY_SSIZE_T_MAX) return NULL; +#if IsModeDbg() return _PyMem_Raw.realloc(_PyMem_Raw.ctx, ptr, new_size); +#else + return _PyMem_RawRealloc(NULL, ptr, new_size); +#endif } void PyMem_RawFree(void *ptr) { +#if IsModeDbg() _PyMem_Raw.free(_PyMem_Raw.ctx, ptr); +#else + _PyMem_RawFree(NULL, ptr); +#endif } void * @@ -472,7 +499,11 @@ PyMem_Malloc(size_t size) /* see PyMem_RawMalloc() */ if (size > (size_t)PY_SSIZE_T_MAX) return NULL; +#if IsModeDbg() return _PyMem.malloc(_PyMem.ctx, size); +#else + return _PyObject_Malloc(NULL, size); +#endif } void * @@ -481,7 +512,11 @@ PyMem_Calloc(size_t nelem, size_t elsize) /* see PyMem_RawMalloc() */ if (elsize != 0 && nelem > (size_t)PY_SSIZE_T_MAX / elsize) return NULL; +#if IsModeDbg() return _PyMem.calloc(_PyMem.ctx, nelem, elsize); +#else + return _PyObject_Calloc(NULL, nelem, elsize); +#endif } void * @@ -490,13 +525,21 @@ PyMem_Realloc(void *ptr, size_t new_size) /* see PyMem_RawMalloc() */ if (new_size > (size_t)PY_SSIZE_T_MAX) return NULL; +#if IsModeDbg() return _PyMem.realloc(_PyMem.ctx, ptr, new_size); +#else + return _PyObject_Realloc(NULL, ptr, new_size); +#endif } void (PyMem_Free)(void *ptr) { +#if IsModeDbg() _PyMem.free(_PyMem.ctx, ptr); +#else + return _PyObject_Free(NULL, ptr); +#endif } char * @@ -531,7 +574,11 @@ void * /* see PyMem_RawMalloc() */ if (size > (size_t)PY_SSIZE_T_MAX) return NULL; +#if IsModeDbg() return _PyObject.malloc(_PyObject.ctx, size); +#else + return _PyObject_Malloc(NULL, size); +#endif } void * @@ -540,7 +587,11 @@ PyObject_Calloc(size_t nelem, size_t elsize) /* see PyMem_RawMalloc() */ if (elsize != 0 && nelem > (size_t)PY_SSIZE_T_MAX / elsize) return NULL; +#if IsModeDbg() return _PyObject.calloc(_PyObject.ctx, nelem, elsize); +#else + return _PyObject_Calloc(NULL, nelem, elsize); +#endif } void * @@ -549,13 +600,21 @@ PyObject_Realloc(void *ptr, size_t new_size) /* see PyMem_RawMalloc() */ if (new_size > (size_t)PY_SSIZE_T_MAX) return NULL; +#if IsModeDbg() return _PyObject.realloc(_PyObject.ctx, ptr, new_size); +#else + return _PyObject_Realloc(NULL, ptr, new_size); +#endif } void PyObject_Free(void *ptr) { +#if IsModeDbg() _PyObject.free(_PyObject.ctx, ptr); +#else + return _PyObject_Free(NULL, ptr); +#endif } @@ -1137,7 +1196,11 @@ new_arena(void) arenaobj = unused_arena_objects; unused_arena_objects = arenaobj->nextarena; assert(arenaobj->address == 0); +#if IsModeDbg() address = _PyObject_Arena.alloc(_PyObject_Arena.ctx, ARENA_SIZE); +#else + address = _PyObject_ArenaMmap(NULL, ARENA_SIZE); +#endif if (address == NULL) { /* The allocation failed: return NULL after putting the * arenaobj back. @@ -1615,8 +1678,13 @@ _PyObject_Free(void *ctx, void *p) unused_arena_objects = ao; /* Free the entire arena. */ +#if IsModeDbg() _PyObject_Arena.free(_PyObject_Arena.ctx, (void *)ao->address, ARENA_SIZE); +#else + _PyObject_ArenaMunmap(NULL, + (void *)ao->address, ARENA_SIZE); +#endif ao->address = 0; /* mark unassociated */ --narenas_currently_allocated; @@ -1853,6 +1921,7 @@ write_size_t(void *p, size_t n) WRITE64BE((char *)p, n); } +#if IsModeDbg() /* Let S = sizeof(size_t). The debug malloc asks for 4*S extra bytes and fills them with useful stuff, here calling the underlying malloc's result p: @@ -1941,7 +2010,7 @@ _PyMem_DebugRawCalloc(void *ctx, size_t nelem, size_t elsize) Python memory allocators which fills the memory with DEADBYTE (0xDB) when memory is deallocated. */ int -_PyMem_IsFreed(void *ptr, size_t size) +(_PyMem_IsFreed)(void *ptr, size_t size) { unsigned char *bytes = ptr; for (size_t i=0; i < size; i++) { @@ -2222,11 +2291,12 @@ _PyObject_DebugDumpAddress(const void *p) fputc('\n', stderr); fflush(stderr); -#ifdef USE_TRACEMALLOC +#if IsModeDbg() PYTHON_YOINK("_tracemalloc"); _PyMem_DumpTraceback(fileno(stderr), p); #endif } +#endif static size_t @@ -2417,8 +2487,10 @@ _PyObject_DebugMallocStats(FILE *out) quantization += p * ((POOL_SIZE - POOL_OVERHEAD) % size); } fputc('\n', out); +#if IsModeDbg() if (_PyMem_DebugEnabled()) (void)printone(out, "# times object malloc called", serialno); +#endif (void)printone(out, "# arenas allocated total", ntimes_arena_allocated); (void)printone(out, "# arenas reclaimed", ntimes_arena_allocated - narenas); (void)printone(out, "# arenas highwater mark", narenas_highwater); diff --git a/third_party/python/Python/finalize.c b/third_party/python/Python/finalize.c index cdbf5fec6..b68b910c0 100644 --- a/third_party/python/Python/finalize.c +++ b/third_party/python/Python/finalize.c @@ -121,7 +121,7 @@ Py_FinalizeEx(void) _PyGC_CollectIfEnabled(); #endif -#ifdef MODE_DBG +#if IsModeDbg() /* Disable tracemalloc after all Python objects have been destroyed, so it is possible to use tracemalloc in objects destructor. */ _PyTraceMalloc_Fini(); @@ -219,11 +219,13 @@ Py_FinalizeEx(void) _Py_PrintReferenceAddresses(stderr); #endif /* Py_TRACE_REFS */ #ifdef WITH_PYMALLOC +#if IsModeDbg() if (_PyMem_PymallocEnabled()) { char *opt = Py_GETENV("PYTHONMALLOCSTATS"); if (opt != NULL && *opt != '\0') _PyObject_DebugMallocStats(stderr); } +#endif #endif _Py_CallLlExitFuncs(); diff --git a/third_party/python/Python/pylifecycle.c b/third_party/python/Python/pylifecycle.c index 6ddc484dd..cdc0abdfa 100644 --- a/third_party/python/Python/pylifecycle.c +++ b/third_party/python/Python/pylifecycle.c @@ -274,7 +274,7 @@ _Py_InitializeEx_Private(int install_sigs, int install_importlib) if (install_sigs) _Py_InitSigs(); /* Signal handling stuff, including initintr() */ -#ifdef USE_TRACEMALLOC +#if IsModeDbg() if (_PyTraceMalloc_Init() < 0) Py_FatalError("Py_Initialize: can't initialize tracemalloc"); #endif diff --git a/third_party/python/Python/sysmodule.c b/third_party/python/Python/sysmodule.c index 0ed9d48c8..12852cac9 100644 --- a/third_party/python/Python/sysmodule.c +++ b/third_party/python/Python/sysmodule.c @@ -91,7 +91,9 @@ PYTHON_PROVIDE("sys.float_info"); PYTHON_PROVIDE("sys.float_repr_style"); PYTHON_PROVIDE("sys.get_asyncgen_hooks"); PYTHON_PROVIDE("sys.get_coroutine_wrapper"); +#if IsModeDbg() PYTHON_PROVIDE("sys.getallocatedblocks"); +#endif PYTHON_PROVIDE("sys.getcheckinterval"); PYTHON_PROVIDE("sys.getdefaultencoding"); PYTHON_PROVIDE("sys.getdlopenflags"); @@ -1262,6 +1264,7 @@ one higher than you might expect, because it includes the (temporary)\n\ reference as an argument to getrefcount()." ); +#if IsModeDbg() static PyObject * sys_getallocatedblocks(PyObject *self) { @@ -1274,6 +1277,7 @@ PyDoc_STRVAR(getallocatedblocks_doc, Return the number of memory blocks currently allocated, regardless of their\n\ size." ); +#endif #ifdef COUNT_ALLOCS static PyObject * @@ -1382,10 +1386,12 @@ static PyObject * sys_debugmallocstats(PyObject *self, PyObject *args) { #ifdef WITH_PYMALLOC +#if IsModeDbg() if (_PyMem_PymallocEnabled()) { _PyObject_DebugMallocStats(stderr); fputc('\n', stderr); } +#endif #endif _PyObject_DebugTypeStats(stderr); @@ -1455,8 +1461,10 @@ static PyMethodDef sys_methods[] = { {"getdlopenflags", (PyCFunction)sys_getdlopenflags, METH_NOARGS, getdlopenflags_doc}, #endif +#if IsModeDbg() {"getallocatedblocks", (PyCFunction)sys_getallocatedblocks, METH_NOARGS, getallocatedblocks_doc}, +#endif #ifdef COUNT_ALLOCS {"getcounts", (PyCFunction)sys_getcounts, METH_NOARGS}, #endif diff --git a/third_party/python/pyconfig.h b/third_party/python/pyconfig.h index 2cb277fa1..5bea67ecf 100644 --- a/third_party/python/pyconfig.h +++ b/third_party/python/pyconfig.h @@ -573,9 +573,8 @@ #define OPENSSL_NO_COMP 1 #define HAVE_LANGINFO_H 1 -#ifdef MODE_DBG +#if IsModeDbg() #define Py_DEBUG 1 -#define USE_TRACEMALLOC 1 #endif /* #define FAST_LOOPS 1 /\* froot loops *\/ */ diff --git a/third_party/python/repl.c b/third_party/python/repl.c index bc3dba248..48cdb9572 100644 --- a/third_party/python/repl.c +++ b/third_party/python/repl.c @@ -280,8 +280,10 @@ RunPythonModule(int argc, char **argv) linenoiseSetHintsCallback(TerminalHint); linenoiseSetFreeHintsCallback(free); +#if IsModeDbg() /* Force malloc() allocator to bootstrap Python */ _PyMem_SetupAllocators("malloc"); +#endif argv_copy = (wchar_t **)PyMem_RawMalloc(sizeof(wchar_t*) * (argc+1)); argv_copy2 = (wchar_t **)PyMem_RawMalloc(sizeof(wchar_t*) * (argc+1)); @@ -324,9 +326,11 @@ RunPythonModule(int argc, char **argv) res = Py_Main(argc, argv_copy); +#if IsModeDbg() /* Force again malloc() allocator to release memory blocks allocated before Py_Main() */ _PyMem_SetupAllocators("malloc"); +#endif for (i = 0; i < argc; i++) { PyMem_RawFree(argv_copy2[i]);