Merge branch 'master' of https://bitbucket.org/yackob03/quay

tools/auditancestry.py

@@ -1,7 +1,7 @@
 import logging
 import json
 
-from data.database import Image, ImageStorage, Repository, configure
+from data.database import Image, ImageStorage, Repository, User, configure
 from data import model
 from app import app, storage as store
 
@@ -16,17 +16,18 @@ logging.getLogger('boto').setLevel(logging.CRITICAL)
 
 
 query = (Image
-  .select(Image, ImageStorage, Repository)
-  .join(ImageStorage)
-  .switch(Image)
-  .join(Repository)
-  .where(ImageStorage.uploading == False))
+         .select(Image, ImageStorage, Repository, User)
+         .join(ImageStorage)
+         .switch(Image)
+         .join(Repository)
+         .join(User)
+         .where(ImageStorage.uploading == False))
 
 bad_count = 0
 good_count = 0
 
 def resolve_or_create(repo, docker_image_id, new_ancestry):
-  existing = model.get_repo_image(repo.namespace, repo.name, docker_image_id)
+  existing = model.get_repo_image(repo.namespace_user.username, repo.name, docker_image_id)
   if existing:
     logger.debug('Found existing image: %s, %s', existing.id, docker_image_id)
     return existing
@@ -45,7 +46,7 @@ def resolve_or_create(repo, docker_image_id, new_ancestry):
       return created
     except ImageStorage.DoesNotExist:
       msg = 'No image available anywhere for storage: %s in namespace: %s'
-      logger.error(msg, docker_image_id, repo.namespace)
+      logger.error(msg, docker_image_id, repo.namespace_user.username)
       raise RuntimeError()
 
 
@@ -62,20 +63,19 @@ def all_ancestors_exist(ancestors):
 cant_fix = []
 for img in query:
   try:
-    with_locations = model.get_repo_image(img.repository.namespace, img.repository.name,
-                                          img.docker_image_id)
+    with_locations = model.get_repo_image(img.repository.namespace_user.username,
+                                          img.repository.name, img.docker_image_id)
     ancestry_storage = store.image_ancestry_path(img.storage.uuid)
     if store.exists(with_locations.storage.locations, ancestry_storage):
-      full_ancestry = json.loads(store.get_content(with_locations.storage.locations, ancestry_storage))[1:]
+      full_ancestry = json.loads(store.get_content(with_locations.storage.locations,
+                                                   ancestry_storage))[1:]
       full_ancestry.reverse()
 
-      ancestor_dbids = [int(anc_id) 
-                        for anc_id in img.ancestors.split('/')[1:-1]]
+      ancestor_dbids = [int(anc_id) for anc_id in img.ancestors.split('/')[1:-1]]
 
       if len(full_ancestry) != len(ancestor_dbids) or not all_ancestors_exist(ancestor_dbids):
-        logger.error('Image has incomplete ancestry: %s, %s, %s, %s' %
-                     (img.id, img.docker_image_id, full_ancestry,
-                      ancestor_dbids))
+        logger.error('Image has incomplete ancestry: %s, %s, %s, %s', img.id, img.docker_image_id,
+                     full_ancestry, ancestor_dbids)
 
         fixed_ancestry = '/'
         for ancestor in full_ancestry:
@@ -99,5 +99,5 @@ for img in query:
                len(cant_fix))
 
 for cant in cant_fix:
-  logger.error('Unable to fix %s in repo %s/%s', cant.id,
-               cant.repository.namespace, cant.repository.name)
+  logger.error('Unable to fix %s in repo %s/%s', cant.id, cant.repository.namespace_user.username,
+               cant.repository.name)

tools/orphans.py

@@ -1,22 +1,15 @@
-from data.database import Image
-from app import app, storage as store
+from data.database import Image, ImageStorage
+from peewee import JOIN_LEFT_OUTER, fn
+from app import app
 
-live_image_id_set = set()
+orphaned = (ImageStorage
+            .select()
+            .where(ImageStorage.uploading == False)
+            .join(Image, JOIN_LEFT_OUTER)
+            .group_by(ImageStorage)
+            .having(fn.Count(Image.id) == 0))
 
-for image in Image.select():
-  live_image_id_set.add(image.docker_image_id)
-
-storage_image_id_set = set()
-for customer in store.list_directory('images/'):
-  for repo in store.list_directory(customer):
-    for image in store.list_directory(repo):
-      storage_image_id_set.add(image.split('/')[-1])
-
-orphans = storage_image_id_set.difference(live_image_id_set)
-missing_image_data = live_image_id_set.difference(storage_image_id_set)
-
-for orphan in orphans:
-  print "Orphan: %s" % orphan
-
-for missing in missing_image_data:
-  print "Missing: %s" % missing
+counter = 0
+for orphan in orphaned:
+  counter += 1
+  print orphan.uuid

tools/relationships.py

@@ -9,7 +9,7 @@ with open('outfile.dot', 'w') as outfile:
   outfile.write('digraph relationships {\n')
 
   for repo in Repository.select():
-    ns = fix_ident(repo.namespace)
+    ns = fix_ident(repo.namespace_user.username)
     outfile.write('%s_%s -> %s\n' % (ns, fix_ident(repo.name), ns))
 
   teams_in_orgs = set()

tools/uncompressedsize.py

@@ -36,9 +36,13 @@ def backfill_sizes_from_json():
         uuid = image_storage.uuid
         with_locations = model.get_storage_by_uuid(uuid)
 
-        json_string = store.get_content(with_locations.locations, store.image_json_path(uuid))
-        json_data = json.loads(json_string)
-        size = json_data.get('Size', json_data.get('size', -1))
+        try:
+          json_string = store.get_content(with_locations.locations, store.image_json_path(uuid))
+          json_data = json.loads(json_string)
+          size = json_data.get('Size', json_data.get('size', -1))
+        except IOError:
+          logger.debug('Image storage with no json %s', uuid)
+          size = -1
 
         if size == -1:
           missing += 1