Another huge batch of registry v2 changes
Add patch support and resumeable sha Implement all actual registry methods Add a simple database generation option
This commit is contained in:
Jake Moshenko
parent
5ba3521e67
commit
e1b3e9e6ae
29 changed files with 1095 additions and 430 deletions
|
|
@ -12,6 +12,7 @@ from auth.auth_context import get_authenticated_user, get_grant_user_context
|
|||
from digest import checksums
|
||||
from util.registry import changes
|
||||
from util.http import abort, exact_abort
|
||||
from util.registry.filelike import SocketReader
|
||||
from auth.permissions import (ReadRepositoryPermission,
|
||||
ModifyRepositoryPermission)
|
||||
from data import model, database
|
||||
|
|
@ -23,26 +24,6 @@ from endpoints.decorators import anon_protect
|
|||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class SocketReader(object):
|
||||
def __init__(self, fp):
|
||||
self._fp = fp
|
||||
self.handlers = []
|
||||
|
||||
def add_handler(self, handler):
|
||||
self.handlers.append(handler)
|
||||
|
||||
def read(self, n=-1):
|
||||
buf = self._fp.read(n)
|
||||
if not buf:
|
||||
return ''
|
||||
for handler in self.handlers:
|
||||
handler(buf)
|
||||
return buf
|
||||
|
||||
def tell(self):
|
||||
raise IOError('Stream is not seekable.')
|
||||
|
||||
|
||||
def image_is_uploading(repo_image):
|
||||
if repo_image is None:
|
||||
return False
|
||||
|
|
@ -145,9 +126,11 @@ def get_image_layer(namespace, repository, image_id, headers):
|
|||
abort(404, 'Image %(image_id)s not found', issue='unknown-image',
|
||||
image_id=image_id)
|
||||
|
||||
logger.debug('Looking up the layer path')
|
||||
try:
|
||||
path = store.image_layer_path(repo_image.storage.uuid)
|
||||
path = store.blob_path(repo_image.storage.checksum)
|
||||
if not repo_image.storage.cas_path:
|
||||
path = store.v1_image_layer_path(repo_image.storage.uuid)
|
||||
logger.info('Serving legacy v1 image from path: %s', path)
|
||||
|
||||
logger.debug('Looking up the direct download URL')
|
||||
direct_download_url = store.get_direct_download_url(repo_image.storage.locations, path)
|
||||
|
|
@ -186,14 +169,15 @@ def put_image_layer(namespace, repository, image_id):
|
|||
try:
|
||||
logger.debug('Retrieving image data')
|
||||
uuid = repo_image.storage.uuid
|
||||
json_data = store.get_content(repo_image.storage.locations, store.image_json_path(uuid))
|
||||
json_data = (repo_image.v1_json_metadata or
|
||||
store.get_content(repo_image.storage.locations, store.image_json_path(uuid)))
|
||||
except (IOError, AttributeError):
|
||||
logger.exception('Exception when retrieving image data')
|
||||
abort(404, 'Image %(image_id)s not found', issue='unknown-image',
|
||||
image_id=image_id)
|
||||
|
||||
logger.debug('Retrieving image path info')
|
||||
layer_path = store.image_layer_path(uuid)
|
||||
layer_path = store.v1_image_layer_path(uuid)
|
||||
logger.info('Storing layer at v1 path: %s', layer_path)
|
||||
|
||||
if (store.exists(repo_image.storage.locations, layer_path) and not
|
||||
image_is_uploading(repo_image)):
|
||||
|
|
@ -315,7 +299,8 @@ def put_image_checksum(namespace, repository, image_id):
|
|||
uuid = repo_image.storage.uuid
|
||||
|
||||
logger.debug('Looking up repo layer data')
|
||||
if not store.exists(repo_image.storage.locations, store.image_json_path(uuid)):
|
||||
if (repo_image.v1_json_metadata is None and
|
||||
not store.exists(repo_image.storage.locations, store.image_json_path(uuid))):
|
||||
abort(404, 'Image not found: %(image_id)s', issue='unknown-image', image_id=image_id)
|
||||
|
||||
logger.debug('Marking image path')
|
||||
|
|
@ -369,13 +354,17 @@ def get_image_json(namespace, repository, image_id, headers):
|
|||
logger.debug('Looking up repo layer data')
|
||||
try:
|
||||
uuid = repo_image.storage.uuid
|
||||
data = store.get_content(repo_image.storage.locations, store.image_json_path(uuid))
|
||||
data = (repo_image.v1_json_metadata or
|
||||
store.get_content(repo_image.storage.locations, store.image_json_path(uuid)))
|
||||
except (IOError, AttributeError):
|
||||
flask_abort(404)
|
||||
|
||||
logger.debug('Looking up repo layer size')
|
||||
size = repo_image.storage.image_size
|
||||
headers['X-Docker-Size'] = str(size)
|
||||
|
||||
headers['Content-Type'] = 'application/json'
|
||||
if size is not None:
|
||||
headers['X-Docker-Size'] = str(size)
|
||||
|
||||
response = make_response(data, 200)
|
||||
response.headers.extend(headers)
|
||||
|
|
@ -394,37 +383,18 @@ def get_image_ancestry(namespace, repository, image_id, headers):
|
|||
if not permission.can() and not model.repository.repository_is_public(namespace, repository):
|
||||
abort(403)
|
||||
|
||||
logger.debug('Looking up repo image')
|
||||
repo_image = model.image.get_repo_image_extended(namespace, repository, image_id)
|
||||
image = model.image.get_image_by_id(namespace, repository, image_id)
|
||||
parents = model.image.get_parent_images(namespace, repository, image)
|
||||
|
||||
logger.debug('Looking up image data')
|
||||
try:
|
||||
uuid = repo_image.storage.uuid
|
||||
data = store.get_content(repo_image.storage.locations, store.image_ancestry_path(uuid))
|
||||
except (IOError, AttributeError):
|
||||
abort(404, 'Image %(image_id)s not found', issue='unknown-image',
|
||||
image_id=image_id)
|
||||
ancestry_docker_ids = [image.docker_image_id]
|
||||
ancestry_docker_ids.extend([parent.docker_image_id for parent in reversed(parents)])
|
||||
|
||||
logger.debug('Converting to <-> from JSON')
|
||||
response = make_response(json.dumps(json.loads(data)), 200)
|
||||
# We can not use jsonify here because we are returning a list not an object
|
||||
response = make_response(json.dumps(ancestry_docker_ids), 200)
|
||||
response.headers.extend(headers)
|
||||
|
||||
logger.debug('Done')
|
||||
return response
|
||||
|
||||
|
||||
def generate_ancestry(image_id, uuid, locations, parent_id=None, parent_uuid=None,
|
||||
parent_locations=None):
|
||||
if not parent_id:
|
||||
store.put_content(locations, store.image_ancestry_path(uuid), json.dumps([image_id]))
|
||||
return
|
||||
|
||||
data = store.get_content(parent_locations, store.image_ancestry_path(parent_uuid))
|
||||
data = json.loads(data)
|
||||
data.insert(0, image_id)
|
||||
store.put_content(locations, store.image_ancestry_path(uuid), json.dumps(data))
|
||||
|
||||
|
||||
def store_checksum(image_storage, checksum):
|
||||
checksum_parts = checksum.split(':')
|
||||
if len(checksum_parts) != 2:
|
||||
|
|
@ -447,7 +417,8 @@ def put_image_json(namespace, repository, image_id):
|
|||
|
||||
logger.debug('Parsing image JSON')
|
||||
try:
|
||||
data = json.loads(request.data.decode('utf8'))
|
||||
v1_metadata = request.data
|
||||
data = json.loads(v1_metadata.decode('utf8'))
|
||||
except ValueError:
|
||||
pass
|
||||
|
||||
|
|
@ -479,63 +450,38 @@ def put_image_json(namespace, repository, image_id):
|
|||
model.tag.create_temporary_hidden_tag(repo, repo_image,
|
||||
app.config['PUSH_TEMP_TAG_EXPIRATION_SEC'])
|
||||
|
||||
uuid = repo_image.storage.uuid
|
||||
|
||||
if image_id != data['id']:
|
||||
abort(400, 'JSON data contains invalid id for image: %(image_id)s',
|
||||
issue='invalid-request', image_id=image_id)
|
||||
|
||||
parent_id = data.get('parent')
|
||||
parent_id = data.get('parent', None)
|
||||
|
||||
parent_image = None
|
||||
if parent_id:
|
||||
logger.debug('Looking up parent image')
|
||||
parent_image = model.image.get_repo_image_extended(namespace, repository, parent_id)
|
||||
|
||||
parent_uuid = parent_image and parent_image.storage.uuid
|
||||
parent_locations = parent_image and parent_image.storage.locations
|
||||
if not parent_image or parent_image.storage.uploading:
|
||||
abort(400, 'Image %(image_id)s depends on non existing parent image %(parent_id)s',
|
||||
issue='invalid-request', image_id=image_id, parent_id=parent_id)
|
||||
|
||||
if parent_id:
|
||||
logger.debug('Looking up parent image data')
|
||||
|
||||
if (parent_id and not
|
||||
store.exists(parent_locations, store.image_json_path(parent_uuid))):
|
||||
abort(400, 'Image %(image_id)s depends on non existing parent image %(parent_id)s',
|
||||
issue='invalid-request', image_id=image_id, parent_id=parent_id)
|
||||
|
||||
logger.debug('Looking up image storage paths')
|
||||
json_path = store.image_json_path(uuid)
|
||||
|
||||
logger.debug('Checking if image already exists')
|
||||
if (store.exists(repo_image.storage.locations, json_path) and not
|
||||
image_is_uploading(repo_image)):
|
||||
json_path = store.image_json_path(repo_image.storage.uuid)
|
||||
if (not image_is_uploading(repo_image) and
|
||||
(repo_image.v1_json_metadata is not None or
|
||||
store.exists(repo_image.storage.locations, json_path))):
|
||||
exact_abort(409, 'Image already exists')
|
||||
|
||||
set_uploading_flag(repo_image, True)
|
||||
|
||||
# If we reach that point, it means that this is a new image or a retry
|
||||
# on a failed push
|
||||
# save the metadata
|
||||
# on a failed push, save the metadata
|
||||
command_list = data.get('container_config', {}).get('Cmd', None)
|
||||
command = json.dumps(command_list) if command_list else None
|
||||
|
||||
logger.debug('Setting image metadata')
|
||||
model.image.set_image_metadata(image_id, namespace, repository, data.get('created'),
|
||||
data.get('comment'), command, parent_image)
|
||||
data.get('comment'), command, v1_metadata, parent_image)
|
||||
|
||||
logger.debug('Putting json path')
|
||||
store.put_content(repo_image.storage.locations, json_path, request.data)
|
||||
|
||||
logger.debug('Generating image ancestry')
|
||||
|
||||
try:
|
||||
generate_ancestry(image_id, uuid, repo_image.storage.locations, parent_id, parent_uuid,
|
||||
parent_locations)
|
||||
except IOError as ioe:
|
||||
logger.debug('Error when generating ancestry: %s', ioe.message)
|
||||
abort(404)
|
||||
|
||||
logger.debug('Done')
|
||||
return make_response('true', 200)
|
||||
|
||||
|
||||
|
|
@ -572,7 +518,11 @@ def process_image_changes(namespace, repository, image_id):
|
|||
parent_trie.frombytes(parent_trie_bytes)
|
||||
|
||||
# Read in the file entries from the layer tar file
|
||||
layer_path = store.image_layer_path(uuid)
|
||||
layer_path = store.blob_path(repo_image.storage.checksum)
|
||||
if not repo_image.storage.cas_path:
|
||||
logger.info('Processing diffs for newly stored v1 image at %s', layer_path)
|
||||
layer_path = store.v1_image_layer_path(uuid)
|
||||
|
||||
with store.stream_read_file(image.storage.locations, layer_path) as layer_tar_stream:
|
||||
removed_files = set()
|
||||
layer_files = changes.files_and_dirs_from_tar(layer_tar_stream,
|
||||
|
|
|
|||
Reference in a new issue