vbatts/quay
Archived
1
0
Fork 0
Code Issues Pull requests Releases Wiki Activity

Merge pull request #1502 from coreos-inc/image-replication

Browse source 
Enable storage replication for V2 and add backfill tool
This commit is contained in:
josephschorr 2016-06-02 15:02:53 -04:00
commit cad8746f9d
6 changed files with 93 additions and 12 deletions
Download patch file Download diff file Expand all files Collapse all files

11
endpoints/v1/registry.py
View file

@ -1,13 +1,12 @@
import logging
import json
import features
from flask import make_response, request, session, Response, redirect, abort as flask_abort
from functools import wraps
from datetime import datetime
from time import time
from app import storage as store, image_replication_queue, app
from app import storage as store, app
from auth.auth import process_auth, extract_namespace_repo_from_session
from auth.auth_context import get_authenticated_user
from auth.registry_jwt_auth import get_granted_username
@ -21,6 +20,7 @@ from util.registry import gzipstream
from util.registry.torrent import PieceHasher
from endpoints.v1 import v1_bp
from endpoints.decorators import anon_protect
from util.registry.replication import queue_storage_replication
logger = logging.getLogger(__name__)
@ -43,12 +43,7 @@ def _finish_image(namespace, repository, repo_image):
set_uploading_flag(repo_image, False)
# Send a job to the work queue to replicate the image layer.
if features.STORAGE_REPLICATION:
repo = model.repository.get_repository(namespace, repository)
image_replication_queue.put([repo_image.storage.uuid], json.dumps({
'namespace_user_id': repo.namespace_user.id,
'storage_id': repo_image.storage.uuid,
}))
queue_storage_replication(namespace, repo_image.storage)
def require_completion(f):

11
endpoints/v2/manifest.py
View file

@ -23,6 +23,7 @@ from endpoints.v2.errors import (BlobUnknown, ManifestInvalid, ManifestUnknown,
NameInvalid)
from endpoints.trackhelper import track_and_log
from endpoints.notificationhelper import spawn_notification
from util.registry.replication import queue_storage_replication
from digest import digest_tools
from data import model
from data.database import RepositoryTag
@ -441,7 +442,6 @@ def _write_manifest_itself(namespace_name, repo_name, manifest):
image = model.image.synthesize_v1_image(repo, blob_storage, working_docker_id,
v1_mdata.created, v1_mdata.comment, v1_mdata.command,
v1_metadata_json, parent_image)
images_map[v1_mdata.docker_id] = image
if not layers:
@ -453,6 +453,15 @@ def _write_manifest_itself(namespace_name, repo_name, manifest):
leaf_layer_id = images_map[layers[-1].v1_metadata.docker_id].docker_image_id
model.tag.store_tag_manifest(namespace_name, repo_name, tag_name, leaf_layer_id, manifest_digest,
manifest.bytes)
# Queue all blob manifests for replication.
# TODO(jschorr): Find a way to optimize this insertion.
if features.STORAGE_REPLICATION:
for mdata in layers:
digest_str = str(mdata.digest)
blob_storage = storage_map.get(digest_str)
queue_storage_replication(namespace_name, blob_storage)
return (repo, tag_name, manifest_digest)