From 7521bf9e73d340eb36d76f3938eb29e07278ee69 Mon Sep 17 00:00:00 2001 From: Justine Tunney Date: Sat, 2 Oct 2021 10:31:24 -0700 Subject: [PATCH] Add stack overflow checking to Python --- third_party/python/Include/ceval.h | 32 ++++++++++++------- third_party/python/Lib/test/test_class.py | 4 +-- third_party/python/Lib/test/test_descr.py | 13 ++++---- third_party/python/Lib/test/test_dictviews.py | 7 +++- .../python/Lib/test/test_exceptions.py | 10 +++--- third_party/python/Lib/test/test_fileio.py | 8 +++-- third_party/python/Lib/test/test_functools.py | 1 - third_party/python/Lib/test/test_io.py | 2 -- .../Lib/test/test_json/test_recursion.py | 3 -- third_party/python/Lib/test/test_plistlib.py | 3 +- third_party/python/Lib/test/test_richcmp.py | 2 +- third_party/python/Lib/test/test_runpy.py | 1 - third_party/python/Lib/test/test_sys.py | 4 --- third_party/python/Lib/test/test_threading.py | 1 - third_party/python/Lib/test/test_traceback.py | 1 - third_party/python/Lib/test/test_xml_etree.py | 1 - 16 files changed, 47 insertions(+), 46 deletions(-) diff --git a/third_party/python/Include/ceval.h b/third_party/python/Include/ceval.h index 34864036a..4b69cf6a5 100644 --- a/third_party/python/Include/ceval.h +++ b/third_party/python/Include/ceval.h @@ -1,6 +1,8 @@ #ifndef Py_CEVAL_H #define Py_CEVAL_H +#include "libc/bits/likely.h" #include "third_party/python/Include/object.h" +#include "third_party/python/Include/pyerrors.h" #include "third_party/python/Include/pystate.h" #include "third_party/python/Include/pythonrun.h" COSMOPOLITAN_C_START_ @@ -77,18 +79,24 @@ int Py_MakePendingCalls(void); void Py_SetRecursionLimit(int); int Py_GetRecursionLimit(void); -#if IsModeDbg() -#define Py_EnterRecursiveCall(where) \ - (_Py_MakeRecCheck(PyThreadState_GET()->recursion_depth) && \ - _Py_CheckRecursiveCall(where)) -#define Py_LeaveRecursiveCall() \ - do{ if(_Py_MakeEndRecCheck(PyThreadState_GET()->recursion_depth)) \ - PyThreadState_GET()->overflowed = 0; \ - } while(0) -#else -#define Py_EnterRecursiveCall(where) (0) -#define Py_LeaveRecursiveCall(where) ((void)0) -#endif +forceinline int Py_EnterRecursiveCall(const char *where) { + const char *rsp, *bot; + extern char ape_stack_vaddr[] __attribute__((__weak__)); + if (!IsTiny()) { + rsp = __builtin_frame_address(0); + asm(".weak\tape_stack_vaddr\n\t" + "movabs\t$ape_stack_vaddr+12288,%0" : "=r"(bot)); + if (UNLIKELY(rsp < bot)) { + PyErr_Format(PyExc_MemoryError, "Stack overflow%s", where); + return -1; + } + } + return 0; +} + +forceinline void Py_LeaveRecursiveCall(void) { +} + int _Py_CheckRecursiveCall(const char *where); extern int _Py_CheckRecursionLimit; diff --git a/third_party/python/Lib/test/test_class.py b/third_party/python/Lib/test/test_class.py index 38e3bd056..b98b4d176 100644 --- a/third_party/python/Lib/test/test_class.py +++ b/third_party/python/Lib/test/test_class.py @@ -490,8 +490,6 @@ class ClassTests(unittest.TestCase): self.assertRaises(TypeError, hash, C2()) - - @unittest.skipUnless(cosmo.MODE == "dbg", "disabled recursion checking") def testSFBug532646(self): # Test for SF bug 532646 @@ -502,7 +500,7 @@ class ClassTests(unittest.TestCase): try: a() # This should not segfault - except RecursionError: + except (RecursionError, MemoryError): pass else: self.fail("Failed to raise RecursionError") diff --git a/third_party/python/Lib/test/test_descr.py b/third_party/python/Lib/test/test_descr.py index f48fe4689..c00afea85 100644 --- a/third_party/python/Lib/test/test_descr.py +++ b/third_party/python/Lib/test/test_descr.py @@ -3470,16 +3470,14 @@ order (MRO) for bases """ list.__init__(a, sequence=[0, 1, 2]) self.assertEqual(a, [0, 1, 2]) - @unittest.skipUnless(cosmo.MODE == "dbg", "disabled recursion checking") def test_recursive_call(self): # Testing recursive __call__() by setting to instance of class... class A(object): pass - A.__call__ = A() try: A()() - except RecursionError: + except (RecursionError, MemoryError): pass else: self.fail("Recursion limit should have been reached for __call__()") @@ -4496,7 +4494,6 @@ order (MRO) for bases """ with self.assertRaises(TypeError): str.__add__(fake_str, "abc") - @unittest.skipUnless(cosmo.MODE == "dbg", "disabled recursion checking") def test_repr_as_str(self): # Issue #11603: crash or infinite loop when rebinding __str__ as # __repr__. @@ -4504,8 +4501,12 @@ order (MRO) for bases """ pass Foo.__repr__ = Foo.__str__ foo = Foo() - self.assertRaises(RecursionError, str, foo) - self.assertRaises(RecursionError, repr, foo) + if cosmo.MODE == 'dbg': + self.assertRaises(RecursionError, str, foo) + self.assertRaises(RecursionError, repr, foo) + else: + self.assertRaises(MemoryError, str, foo) + self.assertRaises(MemoryError, repr, foo) def test_mixing_slot_wrappers(self): class X(dict): diff --git a/third_party/python/Lib/test/test_dictviews.py b/third_party/python/Lib/test/test_dictviews.py index 73b1aa287..cbb8e9df0 100644 --- a/third_party/python/Lib/test/test_dictviews.py +++ b/third_party/python/Lib/test/test_dictviews.py @@ -219,7 +219,12 @@ class DictSetTest(unittest.TestCase): d = {} for i in range(sys.getrecursionlimit() + 100): d = {42: d.values()} - self.assertRaises(RecursionError, repr, d) + try: + repr(d) + except (RecursionError, MemoryError): + pass + else: + assert False def test_copy(self): d = {1: 10, "a": "ABC"} diff --git a/third_party/python/Lib/test/test_exceptions.py b/third_party/python/Lib/test/test_exceptions.py index 45155e0a4..1838631ca 100644 --- a/third_party/python/Lib/test/test_exceptions.py +++ b/third_party/python/Lib/test/test_exceptions.py @@ -515,12 +515,11 @@ class ExceptionTests(unittest.TestCase): self.assertEqual(x.fancy_arg, 42) @no_tracing - @unittest.skipUnless(cosmo.MODE == "dbg", "disabled recursion checking") + @unittest.skipUnless(cosmo.MODE == "dbg", "TODO: disabled recursion checking") def testInfiniteRecursion(self): def f(): return f() self.assertRaises(RecursionError, f) - def g(): try: return g() @@ -939,8 +938,8 @@ class ExceptionTests(unittest.TestCase): self.assertTrue(isinstance(v, RecursionError), type(v)) self.assertIn("maximum recursion depth exceeded", str(v)) - @unittest.skipUnless(cosmo.MODE == "dbg", "disabled recursion checking") @cpython_only + @unittest.skipUnless(cosmo.MODE == "dbg", "TODO: disabled recursion checking") def test_recursion_normalizing_exception(self): # Issue #22898. # Test that a RecursionError is raised when tstate->recursion_depth is @@ -1017,8 +1016,8 @@ class ExceptionTests(unittest.TestCase): b'while normalizing an exception', err) self.assertIn(b'Done.', out) - @unittest.skipUnless(cosmo.MODE == "dbg", "disabled recursion checking") @cpython_only + @unittest.skipUnless(cosmo.MODE == "dbg", "TODO: disabled recursion checking") def test_recursion_normalizing_with_no_memory(self): # Issue #30697. Test that in the abort that occurs when there is no # memory left and the size of the Python frames stack is greater than @@ -1123,7 +1122,8 @@ class ExceptionTests(unittest.TestCase): self.assertEqual(wr(), None) @no_tracing - @unittest.skipUnless(cosmo.MODE == "dbg", "disabled recursion checking") + @unittest.skipIf(cosmo.MODE != 'dbg', + "WUT: Fatal Python error: Cannot recover from MemoryErrors while normalizing exceptions.") def test_recursion_error_cleanup(self): # Same test as above, but with "recursion exceeded" errors class C: diff --git a/third_party/python/Lib/test/test_fileio.py b/third_party/python/Lib/test/test_fileio.py index 2232d9393..2fcedfeb9 100644 --- a/third_party/python/Lib/test/test_fileio.py +++ b/third_party/python/Lib/test/test_fileio.py @@ -177,12 +177,16 @@ class AutoFileTests: finally: os.close(fd) - @unittest.skipUnless(cosmo.MODE == "dbg", "disabled recursion checking") + # @unittest.skipUnless(cosmo.MODE == "dbg", "disabled recursion checking") def testRecursiveRepr(self): # Issue #25455 with swap_attr(self.f, 'name', self.f): - with self.assertRaises(RuntimeError): + try: repr(self.f) # Should not crash + except (RuntimeError, MemoryError): + pass + else: + assert False def testErrors(self): f = self.f diff --git a/third_party/python/Lib/test/test_functools.py b/third_party/python/Lib/test/test_functools.py index 6889b7922..eef6c72f0 100644 --- a/third_party/python/Lib/test/test_functools.py +++ b/third_party/python/Lib/test/test_functools.py @@ -217,7 +217,6 @@ class TestPartial: [f'{name}({capture!r}, {args_repr}, {kwargs_repr})' for kwargs_repr in kwargs_reprs]) - @unittest.skipUnless(cosmo.MODE == "dbg", "disabled recursion checking") def test_recursive_repr(self): if self.partial in (c_functools.partial, py_functools.partial): name = 'functools.partial' diff --git a/third_party/python/Lib/test/test_io.py b/third_party/python/Lib/test/test_io.py index 05622cb30..829c67315 100644 --- a/third_party/python/Lib/test/test_io.py +++ b/third_party/python/Lib/test/test_io.py @@ -1100,7 +1100,6 @@ class CommonBufferedTests: raw.name = b"dummy" self.assertEqual(repr(b), "<%s name=b'dummy'>" % clsname) - @unittest.skipUnless(cosmo.MODE == "dbg", "disabled recursion checking") def test_recursive_repr(self): # Issue #25455 raw = self.MockRawIO() @@ -2542,7 +2541,6 @@ class TextIOWrapperTest(unittest.TestCase): t.buffer.detach() repr(t) # Should not raise an exception - @unittest.skipUnless(cosmo.MODE == "dbg", "disabled recursion checking") def test_recursive_repr(self): # Issue #25455 raw = self.BytesIO() diff --git a/third_party/python/Lib/test/test_json/test_recursion.py b/third_party/python/Lib/test/test_json/test_recursion.py index d6ebfbac1..03b851013 100644 --- a/third_party/python/Lib/test/test_json/test_recursion.py +++ b/third_party/python/Lib/test/test_json/test_recursion.py @@ -67,7 +67,6 @@ class TestRecursion: self.fail("didn't raise ValueError on default recursion") - @unittest.skipUnless(cosmo.MODE == "dbg", "disabled recursion checking") def test_highly_nested_objects_decoding(self): # test that loading highly-nested objects doesn't segfault when C # accelerations are used. See #12017 @@ -78,7 +77,6 @@ class TestRecursion: with self.assertRaises(RecursionError): self.loads('[' * 100000 + '1' + ']' * 100000) - @unittest.skipUnless(cosmo.MODE == "dbg", "disabled recursion checking") def test_highly_nested_objects_encoding(self): # See #12051 l, d = [], {} @@ -89,7 +87,6 @@ class TestRecursion: with self.assertRaises(RecursionError): self.dumps(d) - @unittest.skipUnless(cosmo.MODE == "dbg", "disabled recursion checking") def test_endless_recursion(self): # See #12051 class EndlessJSONEncoder(self.json.JSONEncoder): diff --git a/third_party/python/Lib/test/test_plistlib.py b/third_party/python/Lib/test/test_plistlib.py index eb92addcc..d3473727c 100644 --- a/third_party/python/Lib/test/test_plistlib.py +++ b/third_party/python/Lib/test/test_plistlib.py @@ -814,13 +814,12 @@ class TestBinaryPlistlib(unittest.TestCase): b = plistlib.loads(plistlib.dumps(a, fmt=plistlib.FMT_BINARY)) self.assertIs(b['x'], b) - @unittest.skipUnless(cosmo.MODE == "dbg", "disabled recursion checking") def test_deep_nesting(self): for N in [300, 100000]: chunks = [b'\xa1' + (i + 1).to_bytes(4, 'big') for i in range(N)] try: result = self.decode(*chunks, b'\x54seed', offset_size=4, ref_size=4) - except RecursionError: + except (RecursionError, MemoryError): pass else: for i in range(N): diff --git a/third_party/python/Lib/test/test_richcmp.py b/third_party/python/Lib/test/test_richcmp.py index 36a7317fc..2565086eb 100644 --- a/third_party/python/Lib/test/test_richcmp.py +++ b/third_party/python/Lib/test/test_richcmp.py @@ -221,7 +221,7 @@ class MiscTest(unittest.TestCase): for func in (do, operator.not_): self.assertRaises(Exc, func, Bad()) - @unittest.skipUnless(cosmo.MODE == "dbg", "disabled recursion checking") + @unittest.skipUnless(cosmo.MODE == "dbg", "TODO: disabled recursion checking") @support.no_tracing def test_recursion(self): # Check that comparison for recursive objects fails gracefully diff --git a/third_party/python/Lib/test/test_runpy.py b/third_party/python/Lib/test/test_runpy.py index 85baf8e4d..eebbc9e96 100644 --- a/third_party/python/Lib/test/test_runpy.py +++ b/third_party/python/Lib/test/test_runpy.py @@ -724,7 +724,6 @@ class RunPathTestCase(unittest.TestCase, CodeExecutionMixin): self._check_import_error(zip_name, msg) @no_tracing - @unittest.skipUnless(cosmo.MODE == "dbg", "disabled recursion checking") def test_main_recursion_error(self): with temp_dir() as script_dir, temp_dir() as dummy_dir: mod_name = '__main__' diff --git a/third_party/python/Lib/test/test_sys.py b/third_party/python/Lib/test/test_sys.py index f13d78b60..f1de4f923 100644 --- a/third_party/python/Lib/test/test_sys.py +++ b/third_party/python/Lib/test/test_sys.py @@ -191,7 +191,6 @@ class SysModuleTest(unittest.TestCase): finally: sys.setswitchinterval(orig) - @unittest.skipUnless(cosmo.MODE == "dbg", "disabled recursion checking") def test_recursionlimit(self): self.assertRaises(TypeError, sys.getrecursionlimit, 42) oldlimit = sys.getrecursionlimit() @@ -201,7 +200,6 @@ class SysModuleTest(unittest.TestCase): self.assertEqual(sys.getrecursionlimit(), 10000) sys.setrecursionlimit(oldlimit) - @unittest.skipUnless(cosmo.MODE == "dbg", "disabled recursion checking") def test_recursionlimit_recovery(self): if hasattr(sys, 'gettrace') and sys.gettrace(): self.skipTest('fatal error if run with a trace function') @@ -225,7 +223,6 @@ class SysModuleTest(unittest.TestCase): finally: sys.setrecursionlimit(oldlimit) - @unittest.skipUnless(cosmo.MODE == "dbg", "disabled recursion checking") @test.support.cpython_only def test_setrecursionlimit_recursion_depth(self): # Issue #25274: Setting a low recursion limit must be blocked if the @@ -261,7 +258,6 @@ class SysModuleTest(unittest.TestCase): finally: sys.setrecursionlimit(oldlimit) - @unittest.skipUnless(cosmo.MODE == "dbg", "disabled recursion checking") def test_recursionlimit_fatalerror(self): # A fatal error occurs if a second recursion limit is hit when recovering # from a first one. diff --git a/third_party/python/Lib/test/test_threading.py b/third_party/python/Lib/test/test_threading.py index 638100f1a..22f5ea720 100644 --- a/third_party/python/Lib/test/test_threading.py +++ b/third_party/python/Lib/test/test_threading.py @@ -883,7 +883,6 @@ class ThreadingExceptionTests(BaseTestCase): lock = threading.Lock() self.assertRaises(RuntimeError, lock.release) - @unittest.skipUnless(cosmo.MODE == "dbg", "disabled recursion checking") @unittest.skipUnless(sys.platform == 'darwin' and test.support.python_is_optimized(), 'test macosx problem') def test_recursion_limit(self): diff --git a/third_party/python/Lib/test/test_traceback.py b/third_party/python/Lib/test/test_traceback.py index 67e5d06cb..735ced111 100644 --- a/third_party/python/Lib/test/test_traceback.py +++ b/third_party/python/Lib/test/test_traceback.py @@ -301,7 +301,6 @@ class TracebackFormatTests(unittest.TestCase): ]) # issue 26823 - Shrink recursive tracebacks - @unittest.skipUnless(cosmo.MODE == "dbg", "disabled recursion checking") def _check_recursive_traceback_display(self, render_exc): # Always show full diffs when this test fails # Note that rearranging things may require adjusting diff --git a/third_party/python/Lib/test/test_xml_etree.py b/third_party/python/Lib/test/test_xml_etree.py index 82e9a6c54..715f5ecb5 100644 --- a/third_party/python/Lib/test/test_xml_etree.py +++ b/third_party/python/Lib/test/test_xml_etree.py @@ -1921,7 +1921,6 @@ class BadElementTest(ElementTestCase, unittest.TestCase): e.extend([ET.Element('bar')]) self.assertRaises(ValueError, e.remove, X('baz')) - @unittest.skipUnless(cosmo.MODE == "dbg", "disabled recursion checking") def test_recursive_repr(self): # Issue #25455 e = ET.Element('foo')