From 3a29dfc535fa03980bfa3d29edd212c64f3a057b Mon Sep 17 00:00:00 2001 From: Jake Moshenko Date: Mon, 23 Nov 2015 15:50:25 -0500 Subject: [PATCH] Reducing in a tree to avoid recursion depth limits --- data/model/storage.py | 26 +++++++++++++++++++++++++- 1 file changed, 25 insertions(+), 1 deletion(-) diff --git a/data/model/storage.py b/data/model/storage.py index 1b702f2e0..143cb5d1c 100644 --- a/data/model/storage.py +++ b/data/model/storage.py @@ -238,7 +238,31 @@ def lookup_repo_storages_by_content_checksum(repo, checksums): .select(SQL('*')) .from_(candidate_subq)) - return reduce(lambda l, r: l.union_all(r), queries) + return _reduce_as_tree(queries) + + +def _reduce_as_tree(queries_to_reduce): + """ This method will split a list of queries into halves recursively until we reach individual + queries, at which point it will start unioning the queries, or the already unioned subqueries. + This works around a bug in peewee SQL generation where reducing linearly generates a chain + of queries that will exceed the recursion depth limit when it has around 80 queries. + """ + mid = len(queries_to_reduce)/2 + left = queries_to_reduce[:mid] + right = queries_to_reduce[mid:] + + to_reduce_right = right[0] + if len(right) > 1: + to_reduce_right = _reduce_as_tree(right) + + if len(left) > 1: + to_reduce_left = _reduce_as_tree(left) + elif len(left) == 1: + to_reduce_left = left[0] + else: + return to_reduce_right + + return to_reduce_left.union_all(to_reduce_right) def get_storage_locations(uuid):