vbatts/quay
Archived
1
0
Fork 0
Code Issues Pull requests Releases Wiki Activity

Add automatic storage replication

Browse source 
Adds a worker to automatically replicate data between storages and update the database accordingly
This commit is contained in:
Joseph Schorr 2015-06-28 13:29:22 +03:00 • committed by Joseph Schorr
parent c693afca6a
commit 724b1607d7
18 changed files with 259 additions and 35 deletions
Download patch file Download diff file Expand all files Collapse all files

55
endpoints/v1/registry.py
View file

@ -1,12 +1,13 @@
import logging
import json
import features
from flask import make_response, request, session, Response, redirect, abort as flask_abort
from functools import wraps
from datetime import datetime
from time import time
from app import storage as store, image_diff_queue, app
from app import storage as store, image_diff_queue, image_replication_queue, app
from auth.auth import process_auth, extract_namespace_repo_from_session
from auth.auth_context import get_authenticated_user, get_grant_user_context
from digest import checksums
@ -55,6 +56,30 @@ def set_uploading_flag(repo_image, is_image_uploading):
repo_image.storage.save()
def _finish_image(namespace, repository, repo_image):
# Checksum is ok, we remove the marker
set_uploading_flag(repo_image, False)
image_id = repo_image.docker_image_id
# The layer is ready for download, send a job to the work queue to
# process it.
logger.debug('Adding layer to diff queue')
repo = model.repository.get_repository(namespace, repository)
image_diff_queue.put([repo.namespace_user.username, repository, image_id], json.dumps({
'namespace_user_id': repo.namespace_user.id,
'repository': repository,
'image_id': image_id,
}))
# Send a job to the work queue to replicate the image layer.
if features.STORAGE_REPLICATION:
image_replication_queue.put([repo_image.storage.uuid], json.dumps({
'namespace_user_id': repo.namespace_user.id,
'storage_id': repo_image.storage.uuid,
}))
def require_completion(f):
"""This make sure that the image push correctly finished."""
@wraps(f)
@ -260,18 +285,8 @@ def put_image_layer(namespace, repository, image_id):
abort(400, 'Checksum mismatch; ignoring the layer for image %(image_id)s',
issue='checksum-mismatch', image_id=image_id)
# Checksum is ok, we remove the marker
set_uploading_flag(repo_image, False)
# The layer is ready for download, send a job to the work queue to
# process it.
logger.debug('Adding layer to diff queue')
repo = model.repository.get_repository(namespace, repository)
image_diff_queue.put([repo.namespace_user.username, repository, image_id], json.dumps({
'namespace_user_id': repo.namespace_user.id,
'repository': repository,
'image_id': image_id,
}))
# Mark the image as uploaded.
_finish_image(namespace, repository, repo_image)
return make_response('true', 200)
@ -335,18 +350,8 @@ def put_image_checksum(namespace, repository, image_id):
abort(400, 'Checksum mismatch for image: %(image_id)s',
issue='checksum-mismatch', image_id=image_id)
# Checksum is ok, we remove the marker
set_uploading_flag(repo_image, False)
# The layer is ready for download, send a job to the work queue to
# process it.
logger.debug('Adding layer to diff queue')
repo = model.repository.get_repository(namespace, repository)
image_diff_queue.put([repo.namespace_user.username, repository, image_id], json.dumps({
'namespace_user_id': repo.namespace_user.id,
'repository': repository,
'image_id': image_id,
}))
# Mark the image as uploaded.
_finish_image(namespace, repository, repo_image)
return make_response('true', 200)