diff --git a/data/model/storage.py b/data/model/storage.py index 1b702f2e0..143cb5d1c 100644 --- a/data/model/storage.py +++ b/data/model/storage.py @@ -238,7 +238,31 @@ def lookup_repo_storages_by_content_checksum(repo, checksums): .select(SQL('*')) .from_(candidate_subq)) - return reduce(lambda l, r: l.union_all(r), queries) + return _reduce_as_tree(queries) + + +def _reduce_as_tree(queries_to_reduce): + """ This method will split a list of queries into halves recursively until we reach individual + queries, at which point it will start unioning the queries, or the already unioned subqueries. + This works around a bug in peewee SQL generation where reducing linearly generates a chain + of queries that will exceed the recursion depth limit when it has around 80 queries. + """ + mid = len(queries_to_reduce)/2 + left = queries_to_reduce[:mid] + right = queries_to_reduce[mid:] + + to_reduce_right = right[0] + if len(right) > 1: + to_reduce_right = _reduce_as_tree(right) + + if len(left) > 1: + to_reduce_left = _reduce_as_tree(left) + elif len(left) == 1: + to_reduce_left = left[0] + else: + return to_reduce_right + + return to_reduce_left.union_all(to_reduce_right) def get_storage_locations(uuid):