vbatts/quay
Archived
1
0
Fork 0
Code Issues Pull requests Releases Wiki Activity

Switch the checksums to use the registry computed value, remove all assumptions of namespaced paths for legacy storage, fix an upload race condition in the registry code.

Browse source
This commit is contained in:
Jake Moshenko 2014-06-11 15:37:45 -04:00
parent 246a216f80
commit 78c5aec5b9
11 changed files with 112 additions and 264 deletions
Download patch file Download diff file Expand all files Collapse all files

62
tools/auditancestry.py
View file

@ -5,9 +5,6 @@ from data.database import Image, ImageStorage, Repository
from data import model
from app import app, storage as store
import boto.s3.connection
import boto.s3.key
logger = logging.getLogger(__name__)
logging.basicConfig(level=logging.DEBUG)
@ -22,23 +19,9 @@ query = (Image
.switch(Image)
.join(Repository))
bad_count = 0
good_count = 0
s3_conn = boto.s3.connection.S3Connection(app.config['AWS_ACCESS_KEY'],
app.config['AWS_SECRET_KEY'])
s3_bucket = s3_conn.get_bucket('quay-registry')
PATHS = [
store.image_json_path,
store.image_checksum_path,
store.image_layer_path,
store.image_ancestry_path,
store.image_file_trie_path,
store.image_file_diffs_path,
]
def resolve_or_create(repo, docker_image_id, new_ancestry):
existing = model.get_repo_image(repo.namespace, repo.name, docker_image_id)
if existing:
@ -58,42 +41,9 @@ def resolve_or_create(repo, docker_image_id, new_ancestry):
logger.debug('Created image: %s' % created)
return created
except ImageStorage.DoesNotExist:
logger.warning('No storage for ancestor, tring to find it anywhere: %s',
docker_image_id)
try:
found = Image.get(docker_image_id=docker_image_id)
logger.debug('Found some legacy storage for docker_image_id: %s',
docker_image_id)
new_storage = ImageStorage.create(checksum=found.checksum,
created=found.created,
comment=found.comment,
command=found.command,
image_size=found.image_size)
logger.debug('Migrating data to new storage: %s' % new_storage.uuid)
for path in PATHS:
old_path = path(found.repository.namespace, found.repository.name,
docker_image_id, None)
new_path = path(None, None, None, new_storage.uuid)
logger.debug('Copying %s -> %s', old_path, new_path)
old_path_key = s3_bucket.get_key(old_path)
old_path_key.copy('quay-registry', new_path, encrypt_key=True,
validate_dst_bucket=False)
logger.debug('Creating new image from copied legacy storage: %s',
new_storage.uuid)
created = Image.create(docker_image_id=docker_image_id,
repository=repo,
storage=new_storage, ancestors=new_ancestry)
logger.debug('Created image: %s' % created)
return created
except Image.DoesNotExist:
msg = 'No image available anywhere for storage: %s in namespace: %s'
logger.error(msg, docker_image_id, repo.namespace)
raise RuntimeError()
msg = 'No image available anywhere for storage: %s in namespace: %s'
logger.error(msg, docker_image_id, repo.namespace)
raise RuntimeError()
def all_ancestors_exist(ancestors):
@ -109,11 +59,7 @@ def all_ancestors_exist(ancestors):
cant_fix = []
for img in query:
try:
uuid = img.storage.uuid
ancestry_storage = store.image_ancestry_path(img.repository.namespace,
img.repository.name,
img.docker_image_id,
uuid)
ancestry_storage = store.image_ancestry_path(img.storage.uuid)
if store.exists(ancestry_storage):
full_ancestry = json.loads(store.get_content(ancestry_storage))[1:]
full_ancestry.reverse()