Switch the checksums to use the registry computed value, remove all assumptions of namespaced paths for legacy storage, fix an upload race condition in the registry code.

This commit is contained in:
Jake Moshenko 2014-06-11 15:37:45 -04:00
parent 246a216f80
commit 78c5aec5b9
11 changed files with 112 additions and 264 deletions

View file

@ -5,9 +5,6 @@ from data.database import Image, ImageStorage, Repository
from data import model
from app import app, storage as store
import boto.s3.connection
import boto.s3.key
logger = logging.getLogger(__name__)
logging.basicConfig(level=logging.DEBUG)
@ -22,23 +19,9 @@ query = (Image
.switch(Image)
.join(Repository))
bad_count = 0
good_count = 0
s3_conn = boto.s3.connection.S3Connection(app.config['AWS_ACCESS_KEY'],
app.config['AWS_SECRET_KEY'])
s3_bucket = s3_conn.get_bucket('quay-registry')
PATHS = [
store.image_json_path,
store.image_checksum_path,
store.image_layer_path,
store.image_ancestry_path,
store.image_file_trie_path,
store.image_file_diffs_path,
]
def resolve_or_create(repo, docker_image_id, new_ancestry):
existing = model.get_repo_image(repo.namespace, repo.name, docker_image_id)
if existing:
@ -58,42 +41,9 @@ def resolve_or_create(repo, docker_image_id, new_ancestry):
logger.debug('Created image: %s' % created)
return created
except ImageStorage.DoesNotExist:
logger.warning('No storage for ancestor, tring to find it anywhere: %s',
docker_image_id)
try:
found = Image.get(docker_image_id=docker_image_id)
logger.debug('Found some legacy storage for docker_image_id: %s',
docker_image_id)
new_storage = ImageStorage.create(checksum=found.checksum,
created=found.created,
comment=found.comment,
command=found.command,
image_size=found.image_size)
logger.debug('Migrating data to new storage: %s' % new_storage.uuid)
for path in PATHS:
old_path = path(found.repository.namespace, found.repository.name,
docker_image_id, None)
new_path = path(None, None, None, new_storage.uuid)
logger.debug('Copying %s -> %s', old_path, new_path)
old_path_key = s3_bucket.get_key(old_path)
old_path_key.copy('quay-registry', new_path, encrypt_key=True,
validate_dst_bucket=False)
logger.debug('Creating new image from copied legacy storage: %s',
new_storage.uuid)
created = Image.create(docker_image_id=docker_image_id,
repository=repo,
storage=new_storage, ancestors=new_ancestry)
logger.debug('Created image: %s' % created)
return created
except Image.DoesNotExist:
msg = 'No image available anywhere for storage: %s in namespace: %s'
logger.error(msg, docker_image_id, repo.namespace)
raise RuntimeError()
msg = 'No image available anywhere for storage: %s in namespace: %s'
logger.error(msg, docker_image_id, repo.namespace)
raise RuntimeError()
def all_ancestors_exist(ancestors):
@ -109,11 +59,7 @@ def all_ancestors_exist(ancestors):
cant_fix = []
for img in query:
try:
uuid = img.storage.uuid
ancestry_storage = store.image_ancestry_path(img.repository.namespace,
img.repository.name,
img.docker_image_id,
uuid)
ancestry_storage = store.image_ancestry_path(img.storage.uuid)
if store.exists(ancestry_storage):
full_ancestry = json.loads(store.get_content(ancestry_storage))[1:]
full_ancestry.reverse()

View file

@ -1,45 +0,0 @@
from data.database import Image, RepositoryTag, Repository
from app import storage as store
tag_query = (RepositoryTag
.select(RepositoryTag, Image, Repository)
.join(Repository)
.switch(RepositoryTag)
.join(Image))
for tag in tag_query:
if tag.image.repository.id != tag.repository.id:
print('Repository tag pointing to external image: %s/%s:%s' %
(tag.repository.namespace, tag.repository.name, tag.name))
proper_image_layer_path = store.image_layer_path(tag.repository.namespace,
tag.repository.name,
tag.image.docker_image_id)
has_storage = False
if store.exists(proper_image_layer_path):
print('Storage already in place: %s' % proper_image_layer_path)
has_storage = True
else:
print('Storage missing: %s' % proper_image_layer_path)
has_db_entry = False
new_image = None
try:
new_image = Image.get(Image.docker_image_id == tag.image.docker_image_id,
Image.repository == tag.repository)
has_db_entry = True
print('DB image in place: %s invalid image id: %s' % (new_image.id,
tag.image.id))
except Image.DoesNotExist:
print('DB image missing: %s' % tag.image.docker_image_id)
if has_storage and has_db_entry:
print('Switching tag to proper image %s/%s/%s -> %s' %
(tag.repository.namespace, tag.repository.name, tag.name,
new_image.id))
tag.image = new_image
tag.save()
print('Done')