import logging import json from data.database import Image, ImageStorage, Repository, User, configure from data import model from app import app, storage as store logger = logging.getLogger(__name__) logging.basicConfig(level=logging.DEBUG) configure(app.config) # Turn off debug logging for boto logging.getLogger('boto').setLevel(logging.CRITICAL) query = (Image .select(Image, ImageStorage, Repository, User) .join(ImageStorage) .switch(Image) .join(Repository) .join(User) .where(ImageStorage.uploading == False)) bad_count = 0 good_count = 0 def resolve_or_create(repo, docker_image_id, new_ancestry): existing = model.get_repo_image(repo.namespace_user.username, repo.name, docker_image_id) if existing: logger.debug('Found existing image: %s, %s', existing.id, docker_image_id) return existing else: # we need to find some storage to link it to try: to_link = (ImageStorage .select() .join(Image) .where(Image.docker_image_id == docker_image_id) .get()) logger.debug('Linking to storage: %s' % to_link.uuid) created = Image.create(docker_image_id=docker_image_id, repository=repo, storage=to_link, ancestors=new_ancestry) logger.debug('Created image: %s' % created) return created except ImageStorage.DoesNotExist: msg = 'No image available anywhere for storage: %s in namespace: %s' logger.error(msg, docker_image_id, repo.namespace_user.username) raise RuntimeError() def all_ancestors_exist(ancestors): if not ancestors: return True found_count = len(list(Image .select() .where(Image.id << ancestors))) return found_count == len(ancestors) cant_fix = [] for img in query: try: with_locations = model.get_repo_image(img.repository.namespace_user.username, img.repository.name, img.docker_image_id) ancestry_storage = store.image_ancestry_path(img.storage.uuid) if store.exists(with_locations.storage.locations, ancestry_storage): full_ancestry = json.loads(store.get_content(with_locations.storage.locations, ancestry_storage))[1:] full_ancestry.reverse() ancestor_dbids = [int(anc_id) for anc_id in img.ancestors.split('/')[1:-1]] if len(full_ancestry) != len(ancestor_dbids) or not all_ancestors_exist(ancestor_dbids): logger.error('Image has incomplete ancestry: %s, %s, %s, %s', img.id, img.docker_image_id, full_ancestry, ancestor_dbids) fixed_ancestry = '/' for ancestor in full_ancestry: ancestor_img = resolve_or_create(img.repository, ancestor, fixed_ancestry) fixed_ancestry += str(ancestor_img.id) + '/' img.ancestors = fixed_ancestry img.save() bad_count += 1 else: good_count += 1 else: bad_count += 1 except RuntimeError: cant_fix.append(img) logger.debug('Bad: %s Good: %s Can\'t Fix: %s', bad_count, good_count, len(cant_fix)) for cant in cant_fix: logger.error('Unable to fix %s in repo %s/%s', cant.id, cant.repository.namespace_user.username, cant.repository.name)