Switch the checksums to use the registry computed value, remove all assumptions of namespaced paths for legacy storage, fix an upload race condition in the registry code.
This commit is contained in:
Jake Moshenko
parent
246a216f80
commit
78c5aec5b9
11 changed files with 112 additions and 264 deletions
|
|
@ -5,9 +5,6 @@ from data.database import Image, ImageStorage, Repository
|
|||
from data import model
|
||||
from app import app, storage as store
|
||||
|
||||
import boto.s3.connection
|
||||
import boto.s3.key
|
||||
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
logging.basicConfig(level=logging.DEBUG)
|
||||
|
|
@ -22,23 +19,9 @@ query = (Image
|
|||
.switch(Image)
|
||||
.join(Repository))
|
||||
|
||||
|
||||
bad_count = 0
|
||||
good_count = 0
|
||||
|
||||
s3_conn = boto.s3.connection.S3Connection(app.config['AWS_ACCESS_KEY'],
|
||||
app.config['AWS_SECRET_KEY'])
|
||||
s3_bucket = s3_conn.get_bucket('quay-registry')
|
||||
|
||||
PATHS = [
|
||||
store.image_json_path,
|
||||
store.image_checksum_path,
|
||||
store.image_layer_path,
|
||||
store.image_ancestry_path,
|
||||
store.image_file_trie_path,
|
||||
store.image_file_diffs_path,
|
||||
]
|
||||
|
||||
def resolve_or_create(repo, docker_image_id, new_ancestry):
|
||||
existing = model.get_repo_image(repo.namespace, repo.name, docker_image_id)
|
||||
if existing:
|
||||
|
|
@ -58,42 +41,9 @@ def resolve_or_create(repo, docker_image_id, new_ancestry):
|
|||
logger.debug('Created image: %s' % created)
|
||||
return created
|
||||
except ImageStorage.DoesNotExist:
|
||||
logger.warning('No storage for ancestor, tring to find it anywhere: %s',
|
||||
docker_image_id)
|
||||
try:
|
||||
found = Image.get(docker_image_id=docker_image_id)
|
||||
logger.debug('Found some legacy storage for docker_image_id: %s',
|
||||
docker_image_id)
|
||||
new_storage = ImageStorage.create(checksum=found.checksum,
|
||||
created=found.created,
|
||||
comment=found.comment,
|
||||
command=found.command,
|
||||
image_size=found.image_size)
|
||||
|
||||
logger.debug('Migrating data to new storage: %s' % new_storage.uuid)
|
||||
|
||||
for path in PATHS:
|
||||
old_path = path(found.repository.namespace, found.repository.name,
|
||||
docker_image_id, None)
|
||||
new_path = path(None, None, None, new_storage.uuid)
|
||||
logger.debug('Copying %s -> %s', old_path, new_path)
|
||||
|
||||
old_path_key = s3_bucket.get_key(old_path)
|
||||
old_path_key.copy('quay-registry', new_path, encrypt_key=True,
|
||||
validate_dst_bucket=False)
|
||||
|
||||
logger.debug('Creating new image from copied legacy storage: %s',
|
||||
new_storage.uuid)
|
||||
created = Image.create(docker_image_id=docker_image_id,
|
||||
repository=repo,
|
||||
storage=new_storage, ancestors=new_ancestry)
|
||||
logger.debug('Created image: %s' % created)
|
||||
return created
|
||||
|
||||
except Image.DoesNotExist:
|
||||
msg = 'No image available anywhere for storage: %s in namespace: %s'
|
||||
logger.error(msg, docker_image_id, repo.namespace)
|
||||
raise RuntimeError()
|
||||
msg = 'No image available anywhere for storage: %s in namespace: %s'
|
||||
logger.error(msg, docker_image_id, repo.namespace)
|
||||
raise RuntimeError()
|
||||
|
||||
|
||||
def all_ancestors_exist(ancestors):
|
||||
|
|
@ -109,11 +59,7 @@ def all_ancestors_exist(ancestors):
|
|||
cant_fix = []
|
||||
for img in query:
|
||||
try:
|
||||
uuid = img.storage.uuid
|
||||
ancestry_storage = store.image_ancestry_path(img.repository.namespace,
|
||||
img.repository.name,
|
||||
img.docker_image_id,
|
||||
uuid)
|
||||
ancestry_storage = store.image_ancestry_path(img.storage.uuid)
|
||||
if store.exists(ancestry_storage):
|
||||
full_ancestry = json.loads(store.get_content(ancestry_storage))[1:]
|
||||
full_ancestry.reverse()
|
||||
|
|
|
|||
|
|
@ -1,45 +0,0 @@
|
|||
from data.database import Image, RepositoryTag, Repository
|
||||
|
||||
from app import storage as store
|
||||
|
||||
|
||||
tag_query = (RepositoryTag
|
||||
.select(RepositoryTag, Image, Repository)
|
||||
.join(Repository)
|
||||
.switch(RepositoryTag)
|
||||
.join(Image))
|
||||
|
||||
for tag in tag_query:
|
||||
if tag.image.repository.id != tag.repository.id:
|
||||
print('Repository tag pointing to external image: %s/%s:%s' %
|
||||
(tag.repository.namespace, tag.repository.name, tag.name))
|
||||
|
||||
proper_image_layer_path = store.image_layer_path(tag.repository.namespace,
|
||||
tag.repository.name,
|
||||
tag.image.docker_image_id)
|
||||
|
||||
has_storage = False
|
||||
if store.exists(proper_image_layer_path):
|
||||
print('Storage already in place: %s' % proper_image_layer_path)
|
||||
has_storage = True
|
||||
else:
|
||||
print('Storage missing: %s' % proper_image_layer_path)
|
||||
|
||||
has_db_entry = False
|
||||
new_image = None
|
||||
try:
|
||||
new_image = Image.get(Image.docker_image_id == tag.image.docker_image_id,
|
||||
Image.repository == tag.repository)
|
||||
has_db_entry = True
|
||||
print('DB image in place: %s invalid image id: %s' % (new_image.id,
|
||||
tag.image.id))
|
||||
except Image.DoesNotExist:
|
||||
print('DB image missing: %s' % tag.image.docker_image_id)
|
||||
|
||||
if has_storage and has_db_entry:
|
||||
print('Switching tag to proper image %s/%s/%s -> %s' %
|
||||
(tag.repository.namespace, tag.repository.name, tag.name,
|
||||
new_image.id))
|
||||
tag.image = new_image
|
||||
tag.save()
|
||||
print('Done')
|
||||
Reference in a new issue