vbatts/quay
Archived
1
0
Fork 0
Code Issues Pull requests Releases Wiki Activity

Phase 2 of migrating repo namespaces to referencing user objects, backfilling the rows without a value for namespace_user, and changing all accesses to go through the namespace_user object. All tests are passing, manual testing still required.

Browse source
This commit is contained in:
Jake Moshenko 2014-09-24 18:01:35 -04:00
parent 6070c251ae
commit 03190efde3
19 changed files with 373 additions and 305 deletions
Download patch file Download diff file Expand all files Collapse all files

36
tools/auditancestry.py
View file

@ -1,7 +1,7 @@
import logging
import json
from data.database import Image, ImageStorage, Repository, configure
from data.database import Image, ImageStorage, Repository, User, configure
from data import model
from app import app, storage as store
@ -16,17 +16,18 @@ logging.getLogger('boto').setLevel(logging.CRITICAL)
query = (Image
.select(Image, ImageStorage, Repository)
.join(ImageStorage)
.switch(Image)
.join(Repository)
.where(ImageStorage.uploading == False))
.select(Image, ImageStorage, Repository, User)
.join(ImageStorage)
.switch(Image)
.join(Repository)
.join(User)
.where(ImageStorage.uploading == False))
bad_count = 0
good_count = 0
def resolve_or_create(repo, docker_image_id, new_ancestry):
existing = model.get_repo_image(repo.namespace, repo.name, docker_image_id)
existing = model.get_repo_image(repo.namespace_user.username, repo.name, docker_image_id)
if existing:
logger.debug('Found existing image: %s, %s', existing.id, docker_image_id)
return existing
@ -45,7 +46,7 @@ def resolve_or_create(repo, docker_image_id, new_ancestry):
return created
except ImageStorage.DoesNotExist:
msg = 'No image available anywhere for storage: %s in namespace: %s'
logger.error(msg, docker_image_id, repo.namespace)
logger.error(msg, docker_image_id, repo.namespace_user.username)
raise RuntimeError()
@ -62,20 +63,19 @@ def all_ancestors_exist(ancestors):
cant_fix = []
for img in query:
try:
with_locations = model.get_repo_image(img.repository.namespace, img.repository.name,
img.docker_image_id)
with_locations = model.get_repo_image(img.repository.namespace_user.username,
img.repository.name, img.docker_image_id)
ancestry_storage = store.image_ancestry_path(img.storage.uuid)
if store.exists(with_locations.storage.locations, ancestry_storage):
full_ancestry = json.loads(store.get_content(with_locations.storage.locations, ancestry_storage))[1:]
full_ancestry = json.loads(store.get_content(with_locations.storage.locations,
ancestry_storage))[1:]
full_ancestry.reverse()
ancestor_dbids = [int(anc_id)
for anc_id in img.ancestors.split('/')[1:-1]]
ancestor_dbids = [int(anc_id) for anc_id in img.ancestors.split('/')[1:-1]]
if len(full_ancestry) != len(ancestor_dbids) or not all_ancestors_exist(ancestor_dbids):
logger.error('Image has incomplete ancestry: %s, %s, %s, %s' %
(img.id, img.docker_image_id, full_ancestry,
ancestor_dbids))
logger.error('Image has incomplete ancestry: %s, %s, %s, %s', img.id, img.docker_image_id,
full_ancestry, ancestor_dbids)
fixed_ancestry = '/'
for ancestor in full_ancestry:
@ -99,5 +99,5 @@ for img in query:
len(cant_fix))
for cant in cant_fix:
logger.error('Unable to fix %s in repo %s/%s', cant.id,
cant.repository.namespace, cant.repository.name)
logger.error('Unable to fix %s in repo %s/%s', cant.id, cant.repository.namespace_user.username,
cant.repository.name)