2015-07-06 19:00:07 +00:00
|
|
|
import logging
|
2015-08-12 20:39:32 +00:00
|
|
|
import re
|
2015-07-06 19:00:07 +00:00
|
|
|
|
2016-08-09 16:28:00 +00:00
|
|
|
from flask import url_for, request, redirect, Response, abort as flask_abort
|
2015-07-06 19:00:07 +00:00
|
|
|
|
2016-01-11 21:43:20 +00:00
|
|
|
import resumablehashlib
|
|
|
|
|
2015-07-15 21:25:41 +00:00
|
|
|
from app import storage, app
|
2015-12-09 20:07:37 +00:00
|
|
|
from auth.registry_jwt_auth import process_registry_jwt_auth
|
2016-08-02 00:48:34 +00:00
|
|
|
from data import database
|
2016-09-23 21:50:09 +00:00
|
|
|
from data.interfaces.v2 import pre_oci_model as model
|
2015-07-06 19:00:07 +00:00
|
|
|
from digest import digest_tools
|
2016-03-09 21:20:28 +00:00
|
|
|
from endpoints.common import parse_repository_name
|
2015-07-06 19:00:07 +00:00
|
|
|
from endpoints.v2 import v2_bp, require_repo_read, require_repo_write, get_input_stream
|
2015-12-07 17:30:28 +00:00
|
|
|
from endpoints.v2.errors import (BlobUnknown, BlobUploadInvalid, BlobUploadUnknown, Unsupported,
|
|
|
|
NameUnknown)
|
2015-07-06 19:00:07 +00:00
|
|
|
from endpoints.decorators import anon_protect
|
2015-08-12 20:39:32 +00:00
|
|
|
from util.cache import cache_control
|
2015-09-02 21:31:44 +00:00
|
|
|
from util.registry.filelike import wrap_with_handler, StreamSlice
|
2015-11-30 19:25:01 +00:00
|
|
|
from util.registry.gzipstream import calculate_size_handler
|
2015-12-30 22:19:19 +00:00
|
|
|
from util.registry.torrent import PieceHasher
|
2015-07-06 19:00:07 +00:00
|
|
|
|
|
|
|
|
|
|
|
logger = logging.getLogger(__name__)
|
|
|
|
|
|
|
|
|
2016-01-21 20:40:51 +00:00
|
|
|
BASE_BLOB_ROUTE = '/<repopath:repository>/blobs/<regex("{0}"):digest>'
|
2015-07-16 19:49:06 +00:00
|
|
|
BLOB_DIGEST_ROUTE = BASE_BLOB_ROUTE.format(digest_tools.DIGEST_PATTERN)
|
2015-08-12 20:39:32 +00:00
|
|
|
RANGE_HEADER_REGEX = re.compile(r'^bytes=([0-9]+)-([0-9]+)$')
|
2015-11-30 20:45:45 +00:00
|
|
|
BLOB_CONTENT_TYPE = 'application/octet-stream'
|
2015-08-12 20:39:32 +00:00
|
|
|
|
|
|
|
|
|
|
|
class _InvalidRangeHeader(Exception):
|
|
|
|
pass
|
2015-07-16 19:49:06 +00:00
|
|
|
|
|
|
|
|
2015-08-12 20:39:32 +00:00
|
|
|
@v2_bp.route(BLOB_DIGEST_ROUTE, methods=['HEAD'])
|
2016-03-09 21:20:28 +00:00
|
|
|
@parse_repository_name()
|
2016-03-09 23:09:20 +00:00
|
|
|
@process_registry_jwt_auth(scopes=['pull'])
|
2015-08-12 20:39:32 +00:00
|
|
|
@require_repo_read
|
|
|
|
@anon_protect
|
|
|
|
@cache_control(max_age=31436000)
|
2016-03-09 21:20:28 +00:00
|
|
|
def check_blob_exists(namespace_name, repo_name, digest):
|
2016-08-02 00:48:34 +00:00
|
|
|
# Find the blob.
|
2016-08-30 19:05:15 +00:00
|
|
|
blob = model.get_blob_by_digest(namespace_name, repo_name, digest)
|
2016-08-02 00:48:34 +00:00
|
|
|
if blob is None:
|
|
|
|
raise BlobUnknown()
|
2015-08-12 20:39:32 +00:00
|
|
|
|
2016-08-02 00:48:34 +00:00
|
|
|
# Build the response headers.
|
|
|
|
headers = {
|
|
|
|
'Docker-Content-Digest': digest,
|
|
|
|
'Content-Length': blob.size,
|
|
|
|
'Content-Type': BLOB_CONTENT_TYPE,
|
|
|
|
}
|
|
|
|
|
2016-08-09 20:06:12 +00:00
|
|
|
# If our storage supports range requests, let the client know.
|
2016-08-02 00:48:34 +00:00
|
|
|
if storage.get_supports_resumable_downloads(blob.locations):
|
|
|
|
headers['Accept-Ranges'] = 'bytes'
|
|
|
|
|
2016-08-09 20:06:12 +00:00
|
|
|
# Write the response to the client.
|
2016-08-09 16:28:00 +00:00
|
|
|
return Response(headers=headers)
|
2015-07-06 19:00:07 +00:00
|
|
|
|
|
|
|
|
2015-07-16 19:49:06 +00:00
|
|
|
@v2_bp.route(BLOB_DIGEST_ROUTE, methods=['GET'])
|
2016-03-09 21:20:28 +00:00
|
|
|
@parse_repository_name()
|
2016-03-09 23:09:20 +00:00
|
|
|
@process_registry_jwt_auth(scopes=['pull'])
|
2015-07-06 19:00:07 +00:00
|
|
|
@require_repo_read
|
|
|
|
@anon_protect
|
2015-08-12 20:39:32 +00:00
|
|
|
@cache_control(max_age=31536000)
|
2016-03-09 21:20:28 +00:00
|
|
|
def download_blob(namespace_name, repo_name, digest):
|
2016-08-02 00:48:34 +00:00
|
|
|
# Find the blob.
|
2016-08-30 19:05:15 +00:00
|
|
|
blob = model.get_blob_by_digest(namespace_name, repo_name, digest)
|
2016-08-02 00:48:34 +00:00
|
|
|
if blob is None:
|
|
|
|
raise BlobUnknown()
|
2015-08-12 20:39:32 +00:00
|
|
|
|
2016-08-02 00:48:34 +00:00
|
|
|
# Build the response headers.
|
|
|
|
headers = {'Docker-Content-Digest': digest}
|
|
|
|
|
2016-08-09 20:06:12 +00:00
|
|
|
# If our storage supports range requests, let the client know.
|
2016-08-02 00:48:34 +00:00
|
|
|
if storage.get_supports_resumable_downloads(blob.locations):
|
|
|
|
headers['Accept-Ranges'] = 'bytes'
|
|
|
|
|
|
|
|
# Find the storage path for the blob.
|
2016-08-30 19:05:15 +00:00
|
|
|
path = model.get_blob_path(blob)
|
2015-08-12 20:39:32 +00:00
|
|
|
|
2016-08-02 00:48:34 +00:00
|
|
|
# Short-circuit by redirecting if the storage supports it.
|
|
|
|
logger.debug('Looking up the direct download URL for path: %s', path)
|
|
|
|
direct_download_url = storage.get_direct_download_url(blob.locations, path)
|
2015-08-12 20:39:32 +00:00
|
|
|
if direct_download_url:
|
|
|
|
logger.debug('Returning direct download URL')
|
|
|
|
resp = redirect(direct_download_url)
|
|
|
|
resp.headers.extend(headers)
|
|
|
|
return resp
|
|
|
|
|
2016-08-02 00:48:34 +00:00
|
|
|
# Close the database connection before we stream the download.
|
|
|
|
logger.debug('Closing database connection before streaming layer data')
|
2016-08-03 19:18:51 +00:00
|
|
|
with database.CloseForLongOperation(app.config):
|
2016-08-09 20:06:12 +00:00
|
|
|
# Stream the response to the client.
|
2016-08-03 19:18:51 +00:00
|
|
|
return Response(
|
|
|
|
storage.stream_read(blob.locations, path),
|
|
|
|
headers=headers.update({
|
|
|
|
'Content-Length': blob.size,
|
|
|
|
'Content-Type': BLOB_CONTENT_TYPE,
|
|
|
|
}),
|
|
|
|
)
|
|
|
|
|
2015-07-06 19:00:07 +00:00
|
|
|
|
|
|
|
|
2016-01-21 20:40:51 +00:00
|
|
|
@v2_bp.route('/<repopath:repository>/blobs/uploads/', methods=['POST'])
|
2016-03-09 21:20:28 +00:00
|
|
|
@parse_repository_name()
|
2016-03-09 23:09:20 +00:00
|
|
|
@process_registry_jwt_auth(scopes=['pull', 'push'])
|
2015-07-06 19:00:07 +00:00
|
|
|
@require_repo_write
|
|
|
|
@anon_protect
|
2016-03-09 21:20:28 +00:00
|
|
|
def start_blob_upload(namespace_name, repo_name):
|
2016-08-02 00:48:34 +00:00
|
|
|
# Begin the blob upload process in the database and storage.
|
2015-08-12 20:39:32 +00:00
|
|
|
location_name = storage.preferred_locations[0]
|
2015-08-26 21:08:42 +00:00
|
|
|
new_upload_uuid, upload_metadata = storage.initiate_chunked_upload(location_name)
|
2016-08-30 19:05:15 +00:00
|
|
|
repository_exists = model.create_blob_upload(namespace_name, repo_name, new_upload_uuid,
|
|
|
|
location_name, upload_metadata)
|
2016-08-02 00:48:34 +00:00
|
|
|
if not repository_exists:
|
2015-12-07 17:30:28 +00:00
|
|
|
raise NameUnknown()
|
2015-08-12 20:39:32 +00:00
|
|
|
|
|
|
|
digest = request.args.get('digest', None)
|
|
|
|
if digest is None:
|
2016-08-02 00:48:34 +00:00
|
|
|
# Short-circuit because the user will send the blob data in another request.
|
2016-08-09 16:28:00 +00:00
|
|
|
return Response(
|
|
|
|
status=202,
|
|
|
|
headers={
|
|
|
|
'Docker-Upload-UUID': new_upload_uuid,
|
|
|
|
'Range': _render_range(0),
|
|
|
|
'Location': url_for('v2.upload_chunk', repository='%s/%s' % (namespace_name, repo_name),
|
|
|
|
upload_uuid=new_upload_uuid)
|
|
|
|
},
|
|
|
|
)
|
2015-08-12 20:39:32 +00:00
|
|
|
|
2016-08-02 00:48:34 +00:00
|
|
|
# The user plans to send us the entire body right now.
|
|
|
|
# Find the upload.
|
2016-08-30 19:05:15 +00:00
|
|
|
blob_upload = model.blob_upload_by_uuid(namespace_name, repo_name, new_upload_uuid)
|
2016-08-02 00:48:34 +00:00
|
|
|
if blob_upload is None:
|
|
|
|
raise BlobUploadUnknown()
|
|
|
|
|
|
|
|
# Upload the chunk to storage while calculating some metadata and updating
|
|
|
|
# the upload state.
|
2016-08-09 20:02:49 +00:00
|
|
|
updated_blob_upload = _upload_chunk(blob_upload, request.headers.get('range'))
|
2016-08-02 00:48:34 +00:00
|
|
|
if updated_blob_upload is None:
|
2016-08-16 19:23:00 +00:00
|
|
|
_abort_range_not_satisfiable(blob_upload.byte_count, new_upload_uuid)
|
2016-07-20 21:53:43 +00:00
|
|
|
|
2016-08-02 00:48:34 +00:00
|
|
|
# Save the upload state to the database.
|
2016-08-30 19:05:15 +00:00
|
|
|
model.update_blob_upload(updated_blob_upload)
|
2016-08-02 00:48:34 +00:00
|
|
|
|
|
|
|
# Finalize the upload process in the database and storage.
|
|
|
|
_finish_upload(namespace_name, repo_name, updated_blob_upload, digest)
|
|
|
|
|
2016-08-09 20:06:12 +00:00
|
|
|
# Write the response to the client.
|
2016-08-09 16:28:00 +00:00
|
|
|
return Response(
|
|
|
|
status=201,
|
|
|
|
headers={
|
|
|
|
'Docker-Content-Digest': digest,
|
|
|
|
'Location': url_for('v2.download_blob', repository='%s/%s' % (namespace_name, repo_name),
|
|
|
|
digest=digest),
|
|
|
|
},
|
|
|
|
)
|
2015-08-12 20:39:32 +00:00
|
|
|
|
|
|
|
|
2016-01-21 20:40:51 +00:00
|
|
|
@v2_bp.route('/<repopath:repository>/blobs/uploads/<upload_uuid>', methods=['GET'])
|
2016-03-09 21:20:28 +00:00
|
|
|
@parse_repository_name()
|
2016-03-09 23:09:20 +00:00
|
|
|
@process_registry_jwt_auth(scopes=['pull'])
|
2015-08-12 20:39:32 +00:00
|
|
|
@require_repo_write
|
|
|
|
@anon_protect
|
2016-03-09 21:20:28 +00:00
|
|
|
def fetch_existing_upload(namespace_name, repo_name, upload_uuid):
|
2016-08-30 19:05:15 +00:00
|
|
|
blob_upload = model.blob_upload_by_uuid(namespace_name, repo_name, upload_uuid)
|
2016-08-02 00:48:34 +00:00
|
|
|
if blob_upload is None:
|
|
|
|
raise BlobUploadUnknown()
|
|
|
|
|
2016-08-09 16:28:00 +00:00
|
|
|
return Response(
|
|
|
|
status=204,
|
|
|
|
headers={
|
|
|
|
'Docker-Upload-UUID': upload_uuid,
|
2016-08-09 20:06:12 +00:00
|
|
|
'Range': _render_range(blob_upload.byte_count+1), # byte ranges are exclusive
|
2016-08-09 16:28:00 +00:00
|
|
|
},
|
|
|
|
)
|
2016-08-02 00:48:34 +00:00
|
|
|
|
|
|
|
|
|
|
|
@v2_bp.route('/<repopath:repository>/blobs/uploads/<upload_uuid>', methods=['PATCH'])
|
|
|
|
@parse_repository_name()
|
|
|
|
@process_registry_jwt_auth(scopes=['pull', 'push'])
|
|
|
|
@require_repo_write
|
|
|
|
@anon_protect
|
|
|
|
def upload_chunk(namespace_name, repo_name, upload_uuid):
|
|
|
|
# Find the upload.
|
2016-08-30 19:05:15 +00:00
|
|
|
blob_upload = model.blob_upload_by_uuid(namespace_name, repo_name, upload_uuid)
|
2016-08-02 00:48:34 +00:00
|
|
|
if blob_upload is None:
|
2015-08-12 20:39:32 +00:00
|
|
|
raise BlobUploadUnknown()
|
|
|
|
|
2016-08-02 00:48:34 +00:00
|
|
|
# Upload the chunk to storage while calculating some metadata and updating
|
|
|
|
# the upload state.
|
2016-08-09 20:02:49 +00:00
|
|
|
updated_blob_upload = _upload_chunk(blob_upload, request.headers.get('range'))
|
2016-08-02 00:48:34 +00:00
|
|
|
if updated_blob_upload is None:
|
2016-08-16 19:23:00 +00:00
|
|
|
_abort_range_not_satisfiable(blob_upload.byte_count, upload_uuid)
|
2016-08-02 00:48:34 +00:00
|
|
|
|
|
|
|
# Save the upload state to the database.
|
2016-08-30 19:05:15 +00:00
|
|
|
model.update_blob_upload(updated_blob_upload)
|
2016-08-02 00:48:34 +00:00
|
|
|
|
2016-08-09 20:06:12 +00:00
|
|
|
# Write the response to the client.
|
2016-08-09 16:28:00 +00:00
|
|
|
return Response(
|
|
|
|
status=204,
|
|
|
|
headers={
|
|
|
|
'Location': _current_request_path(),
|
|
|
|
'Range': _render_range(updated_blob_upload.byte_count, with_bytes_prefix=False),
|
|
|
|
'Docker-Upload-UUID': upload_uuid,
|
|
|
|
},
|
|
|
|
)
|
2015-08-12 20:39:32 +00:00
|
|
|
|
|
|
|
|
2016-08-02 00:48:34 +00:00
|
|
|
@v2_bp.route('/<repopath:repository>/blobs/uploads/<upload_uuid>', methods=['PUT'])
|
|
|
|
@parse_repository_name()
|
|
|
|
@process_registry_jwt_auth(scopes=['pull', 'push'])
|
|
|
|
@require_repo_write
|
|
|
|
@anon_protect
|
|
|
|
def monolithic_upload_or_last_chunk(namespace_name, repo_name, upload_uuid):
|
|
|
|
# Ensure the digest is present before proceeding.
|
|
|
|
digest = request.args.get('digest', None)
|
|
|
|
if digest is None:
|
2016-08-02 22:45:30 +00:00
|
|
|
raise BlobUploadInvalid(detail={'reason': 'Missing digest arg on monolithic upload'})
|
2016-08-02 00:48:34 +00:00
|
|
|
|
|
|
|
# Find the upload.
|
2016-08-30 19:05:15 +00:00
|
|
|
blob_upload = model.blob_upload_by_uuid(namespace_name, repo_name, upload_uuid)
|
2016-08-02 00:48:34 +00:00
|
|
|
if blob_upload is None:
|
|
|
|
raise BlobUploadUnknown()
|
|
|
|
|
|
|
|
# Upload the chunk to storage while calculating some metadata and updating
|
|
|
|
# the upload state.
|
2016-08-09 20:02:49 +00:00
|
|
|
updated_blob_upload = _upload_chunk(blob_upload, request.headers.get('range'))
|
2016-08-02 00:48:34 +00:00
|
|
|
if updated_blob_upload is None:
|
2016-08-16 19:23:00 +00:00
|
|
|
_abort_range_not_satisfiable(blob_upload.byte_count, upload_uuid)
|
2016-08-02 00:48:34 +00:00
|
|
|
|
|
|
|
# Finalize the upload process in the database and storage.
|
|
|
|
_finish_upload(namespace_name, repo_name, updated_blob_upload, digest)
|
|
|
|
|
2016-08-09 20:06:12 +00:00
|
|
|
# Write the response to the client.
|
2016-08-09 16:28:00 +00:00
|
|
|
return Response(
|
|
|
|
status=201,
|
|
|
|
headers={
|
|
|
|
'Docker-Content-Digest': digest,
|
|
|
|
'Location': url_for('v2.download_blob', repository='%s/%s' % (namespace_name, repo_name),
|
|
|
|
digest=digest),
|
|
|
|
}
|
|
|
|
)
|
2016-08-02 00:48:34 +00:00
|
|
|
|
|
|
|
|
|
|
|
@v2_bp.route('/<repopath:repository>/blobs/uploads/<upload_uuid>', methods=['DELETE'])
|
|
|
|
@parse_repository_name()
|
|
|
|
@process_registry_jwt_auth(scopes=['pull', 'push'])
|
|
|
|
@require_repo_write
|
|
|
|
@anon_protect
|
|
|
|
def cancel_upload(namespace_name, repo_name, upload_uuid):
|
2016-08-30 19:05:15 +00:00
|
|
|
blob_upload = model.blob_upload_by_uuid(namespace_name, repo_name, upload_uuid)
|
2016-08-16 19:23:00 +00:00
|
|
|
if blob_upload is None:
|
2016-08-02 00:48:34 +00:00
|
|
|
raise BlobUploadUnknown()
|
|
|
|
|
|
|
|
# We delete the record for the upload first, since if the partial upload in
|
2016-08-16 19:23:00 +00:00
|
|
|
# storage fails to delete, it doesn't break anything.
|
2016-08-30 19:05:15 +00:00
|
|
|
model.delete_blob_upload(namespace_name, repo_name, upload_uuid)
|
2016-08-16 19:23:00 +00:00
|
|
|
storage.cancel_chunked_upload({blob_upload.location_name}, blob_upload.uuid,
|
|
|
|
blob_upload.storage_metadata)
|
2016-08-02 00:48:34 +00:00
|
|
|
|
2016-08-09 16:28:00 +00:00
|
|
|
return Response(status=204)
|
2016-08-02 00:48:34 +00:00
|
|
|
|
|
|
|
|
|
|
|
@v2_bp.route('/<repopath:repository>/blobs/<digest>', methods=['DELETE'])
|
|
|
|
@parse_repository_name()
|
|
|
|
@process_registry_jwt_auth(scopes=['pull', 'push'])
|
|
|
|
@require_repo_write
|
|
|
|
@anon_protect
|
|
|
|
def delete_digest(namespace_name, repo_name, upload_uuid):
|
|
|
|
# We do not support deleting arbitrary digests, as they break repo images.
|
|
|
|
raise Unsupported()
|
|
|
|
|
|
|
|
|
|
|
|
def _render_range(num_uploaded_bytes, with_bytes_prefix=True):
|
2016-08-02 22:45:30 +00:00
|
|
|
"""
|
|
|
|
Returns a string formatted to be used in the Range header.
|
|
|
|
"""
|
2016-08-02 00:48:34 +00:00
|
|
|
return '{0}0-{1}'.format('bytes=' if with_bytes_prefix else '', num_uploaded_bytes - 1)
|
|
|
|
|
|
|
|
|
2015-08-12 20:39:32 +00:00
|
|
|
def _current_request_path():
|
|
|
|
return '{0}{1}'.format(request.script_root, request.path)
|
|
|
|
|
|
|
|
|
2016-08-02 00:48:34 +00:00
|
|
|
def _abort_range_not_satisfiable(valid_end, upload_uuid):
|
|
|
|
"""
|
|
|
|
Writes a failure response for scenarios where the registry cannot function
|
|
|
|
with the provided range.
|
|
|
|
|
|
|
|
TODO(jzelinskie): Unify this with the V2RegistryException class.
|
|
|
|
"""
|
2016-08-09 16:28:00 +00:00
|
|
|
flask_abort(Response(status=416, headers={'Location': _current_request_path(),
|
|
|
|
'Range': '0-{0}'.format(valid_end),
|
|
|
|
'Docker-Upload-UUID': upload_uuid}))
|
2015-08-12 20:39:32 +00:00
|
|
|
|
|
|
|
|
2015-09-25 15:51:50 +00:00
|
|
|
def _parse_range_header(range_header_text):
|
2016-08-02 00:48:34 +00:00
|
|
|
"""
|
|
|
|
Parses the range header.
|
|
|
|
|
|
|
|
Returns a tuple of the start offset and the length.
|
|
|
|
If the parse fails, raises _InvalidRangeHeader.
|
2015-08-12 20:39:32 +00:00
|
|
|
"""
|
|
|
|
found = RANGE_HEADER_REGEX.match(range_header_text)
|
|
|
|
if found is None:
|
|
|
|
raise _InvalidRangeHeader()
|
|
|
|
|
|
|
|
start = int(found.group(1))
|
|
|
|
length = int(found.group(2)) - start
|
|
|
|
|
2015-09-25 15:51:50 +00:00
|
|
|
if length <= 0:
|
2015-08-12 20:39:32 +00:00
|
|
|
raise _InvalidRangeHeader()
|
|
|
|
|
|
|
|
return (start, length)
|
|
|
|
|
|
|
|
|
2016-08-09 20:02:49 +00:00
|
|
|
def _start_offset_and_length(range_header):
|
2016-08-02 00:48:34 +00:00
|
|
|
"""
|
|
|
|
Returns a tuple of the start offset and the length.
|
|
|
|
If the range header doesn't exist, defaults to (0, -1).
|
|
|
|
If parsing fails, returns (None, None).
|
2015-08-12 20:39:32 +00:00
|
|
|
"""
|
|
|
|
start_offset, length = 0, -1
|
|
|
|
if range_header is not None:
|
|
|
|
try:
|
2015-09-25 15:51:50 +00:00
|
|
|
start_offset, length = _parse_range_header(range_header)
|
2015-08-12 20:39:32 +00:00
|
|
|
except _InvalidRangeHeader:
|
2016-08-02 00:48:34 +00:00
|
|
|
return None, None
|
2016-08-02 22:45:30 +00:00
|
|
|
|
2016-08-02 00:48:34 +00:00
|
|
|
return start_offset, length
|
2015-08-12 20:39:32 +00:00
|
|
|
|
2015-09-02 21:31:44 +00:00
|
|
|
|
2016-08-09 20:02:49 +00:00
|
|
|
def _upload_chunk(blob_upload, range_header):
|
2016-08-02 00:48:34 +00:00
|
|
|
"""
|
|
|
|
Calculates metadata while uploading a chunk to storage.
|
|
|
|
|
|
|
|
Returns a BlobUpload object or None if there was a failure.
|
|
|
|
"""
|
2016-08-09 20:02:49 +00:00
|
|
|
# Get the offset and length of the current chunk.
|
|
|
|
start_offset, length = _start_offset_and_length(range_header)
|
2016-08-16 19:23:00 +00:00
|
|
|
if blob_upload is None or None in {start_offset, length}:
|
2016-08-09 20:02:49 +00:00
|
|
|
logger.error('Invalid arguments provided to _upload_chunk')
|
2016-08-02 00:48:34 +00:00
|
|
|
return None
|
2016-08-02 22:45:30 +00:00
|
|
|
|
2016-08-02 00:48:34 +00:00
|
|
|
if start_offset > 0 and start_offset > blob_upload.byte_count:
|
2016-08-09 19:11:35 +00:00
|
|
|
logger.error('start_offset provided to _upload_chunk greater than blob.upload.byte_count')
|
2016-08-02 00:48:34 +00:00
|
|
|
return None
|
|
|
|
|
|
|
|
location_set = {blob_upload.location_name}
|
2015-09-25 15:51:50 +00:00
|
|
|
|
2016-07-20 21:53:43 +00:00
|
|
|
upload_error = None
|
2015-12-07 19:07:10 +00:00
|
|
|
with database.CloseForLongOperation(app.config):
|
|
|
|
input_fp = get_input_stream(request)
|
2015-09-02 21:31:44 +00:00
|
|
|
|
2016-08-02 00:48:34 +00:00
|
|
|
if start_offset > 0 and start_offset < blob_upload.byte_count:
|
2015-12-07 19:07:10 +00:00
|
|
|
# Skip the bytes which were received on a previous push, which are already stored and
|
|
|
|
# included in the sha calculation
|
2016-08-02 00:48:34 +00:00
|
|
|
overlap_size = blob_upload.byte_count - start_offset
|
2015-12-10 18:14:11 +00:00
|
|
|
input_fp = StreamSlice(input_fp, overlap_size)
|
|
|
|
|
|
|
|
# Update our upload bounds to reflect the skipped portion of the overlap
|
2016-08-02 00:48:34 +00:00
|
|
|
start_offset = blob_upload.byte_count
|
2015-12-10 18:14:11 +00:00
|
|
|
length = max(length - overlap_size, 0)
|
|
|
|
|
|
|
|
# We use this to escape early in case we have already processed all of the bytes the user
|
|
|
|
# wants to upload
|
|
|
|
if length == 0:
|
2016-08-02 00:48:34 +00:00
|
|
|
return blob_upload
|
2015-08-12 20:39:32 +00:00
|
|
|
|
2016-08-02 00:48:34 +00:00
|
|
|
input_fp = wrap_with_handler(input_fp, blob_upload.sha_state.update)
|
2015-12-07 19:07:10 +00:00
|
|
|
|
2016-01-12 22:32:55 +00:00
|
|
|
# Add a hasher for calculating SHA1s for torrents if this is the first chunk and/or we have
|
|
|
|
# already calculated hash data for the previous chunk(s).
|
2015-12-30 22:19:19 +00:00
|
|
|
piece_hasher = None
|
2016-08-02 00:48:34 +00:00
|
|
|
if blob_upload.chunk_count == 0 or blob_upload.piece_sha_state:
|
|
|
|
initial_sha1_value = blob_upload.piece_sha_state or resumablehashlib.sha1()
|
|
|
|
initial_sha1_pieces_value = blob_upload.piece_hashes or ''
|
2016-01-12 22:32:55 +00:00
|
|
|
|
2016-01-22 20:52:28 +00:00
|
|
|
piece_hasher = PieceHasher(app.config['BITTORRENT_PIECE_SIZE'], start_offset,
|
2016-08-02 00:48:34 +00:00
|
|
|
initial_sha1_pieces_value, initial_sha1_value)
|
2016-01-12 22:32:55 +00:00
|
|
|
|
2015-12-30 22:19:19 +00:00
|
|
|
input_fp = wrap_with_handler(input_fp, piece_hasher.update)
|
|
|
|
|
2015-11-30 19:25:01 +00:00
|
|
|
# If this is the first chunk and we're starting at the 0 offset, add a handler to gunzip the
|
|
|
|
# stream so we can determine the uncompressed size. We'll throw out this data if another chunk
|
2016-08-09 20:06:12 +00:00
|
|
|
# comes in, but in the common case the docker client only sends one chunk.
|
2015-11-30 19:25:01 +00:00
|
|
|
size_info = None
|
2016-08-02 00:48:34 +00:00
|
|
|
if start_offset == 0 and blob_upload.chunk_count == 0:
|
2015-11-30 19:25:01 +00:00
|
|
|
size_info, fn = calculate_size_handler()
|
|
|
|
input_fp = wrap_with_handler(input_fp, fn)
|
|
|
|
|
2016-08-16 19:23:00 +00:00
|
|
|
length_written, new_metadata, upload_error = storage.stream_upload_chunk(
|
2016-08-09 19:11:35 +00:00
|
|
|
location_set,
|
|
|
|
blob_upload.uuid,
|
|
|
|
start_offset,
|
|
|
|
length,
|
|
|
|
input_fp,
|
|
|
|
blob_upload.storage_metadata,
|
|
|
|
content_type=BLOB_CONTENT_TYPE,
|
|
|
|
)
|
2016-08-16 19:23:00 +00:00
|
|
|
|
|
|
|
if upload_error is not None:
|
|
|
|
logger.error('storage.stream_upload_chunk returned error %s', upload_error)
|
2016-08-02 00:48:34 +00:00
|
|
|
return None
|
2015-08-12 20:39:32 +00:00
|
|
|
|
2015-11-30 19:25:01 +00:00
|
|
|
# If we determined an uncompressed size and this is the first chunk, add it to the blob.
|
|
|
|
# Otherwise, we clear the size from the blob as it was uploaded in multiple chunks.
|
2016-08-02 00:48:34 +00:00
|
|
|
if size_info is not None and blob_upload.chunk_count == 0 and size_info.is_valid:
|
|
|
|
blob_upload.uncompressed_byte_count = size_info.uncompressed_size
|
2015-11-30 19:25:01 +00:00
|
|
|
elif length_written > 0:
|
|
|
|
# Otherwise, if we wrote some bytes and the above conditions were not met, then we don't
|
|
|
|
# know the uncompressed size.
|
2016-08-02 00:48:34 +00:00
|
|
|
blob_upload.uncompressed_byte_count = None
|
2015-11-30 19:25:01 +00:00
|
|
|
|
2015-12-30 22:19:19 +00:00
|
|
|
if piece_hasher is not None:
|
2016-08-02 00:48:34 +00:00
|
|
|
blob_upload.piece_hashes = piece_hasher.piece_hashes
|
|
|
|
blob_upload.piece_sha_state = piece_hasher.hash_fragment
|
2016-08-16 19:23:00 +00:00
|
|
|
|
2016-08-02 00:48:34 +00:00
|
|
|
blob_upload.storage_metadata = new_metadata
|
|
|
|
blob_upload.byte_count += length_written
|
|
|
|
blob_upload.chunk_count += 1
|
|
|
|
return blob_upload
|
2015-12-30 22:19:19 +00:00
|
|
|
|
2015-08-12 20:39:32 +00:00
|
|
|
|
2016-08-02 00:48:34 +00:00
|
|
|
def _validate_digest(blob_upload, expected_digest):
|
|
|
|
"""
|
|
|
|
Verifies that the digest's SHA matches that of the uploaded data.
|
|
|
|
"""
|
|
|
|
computed_digest = digest_tools.sha256_digest_from_hashlib(blob_upload.sha_state)
|
2015-08-12 20:39:32 +00:00
|
|
|
if not digest_tools.digests_equal(computed_digest, expected_digest):
|
2016-08-02 16:15:30 +00:00
|
|
|
logger.error('Digest mismatch for upload %s: Expected digest %s, found digest %s',
|
2016-08-02 22:45:30 +00:00
|
|
|
blob_upload.uuid, expected_digest, computed_digest)
|
2016-08-02 16:15:30 +00:00
|
|
|
raise BlobUploadInvalid(detail={'reason': 'Digest mismatch on uploaded blob'})
|
2015-08-12 20:39:32 +00:00
|
|
|
|
2016-08-02 00:48:34 +00:00
|
|
|
|
|
|
|
def _finalize_blob_storage(blob_upload, expected_digest):
|
|
|
|
"""
|
|
|
|
When an upload is successful, this ends the uploading process from the
|
|
|
|
storage's perspective.
|
|
|
|
|
|
|
|
Returns True if the blob already existed.
|
|
|
|
"""
|
2015-08-12 20:39:32 +00:00
|
|
|
final_blob_location = digest_tools.content_path(expected_digest)
|
2015-09-02 21:31:44 +00:00
|
|
|
|
2015-12-03 16:19:39 +00:00
|
|
|
# Move the storage into place, or if this was a re-upload, cancel it
|
|
|
|
with database.CloseForLongOperation(app.config):
|
2016-08-02 00:48:34 +00:00
|
|
|
already_existed = storage.exists({blob_upload.location_name}, final_blob_location)
|
|
|
|
if already_existed:
|
|
|
|
# It already existed, clean up our upload which served as proof that the
|
|
|
|
# uploader had the blob.
|
|
|
|
storage.cancel_chunked_upload({blob_upload.location_name}, blob_upload.uuid,
|
|
|
|
blob_upload.storage_metadata)
|
2015-12-03 16:19:39 +00:00
|
|
|
|
|
|
|
else:
|
|
|
|
# We were the first ones to upload this image (at least to this location)
|
|
|
|
# Let's copy it into place
|
2016-08-02 00:48:34 +00:00
|
|
|
storage.complete_chunked_upload({blob_upload.location_name}, blob_upload.uuid,
|
|
|
|
final_blob_location, blob_upload.storage_metadata)
|
|
|
|
return already_existed
|
2015-07-06 19:00:07 +00:00
|
|
|
|
|
|
|
|
2016-08-02 00:48:34 +00:00
|
|
|
def _finalize_blob_database(namespace_name, repo_name, blob_upload, digest, already_existed):
|
|
|
|
"""
|
|
|
|
When an upload is successful, this ends the uploading process from the
|
|
|
|
database's perspective.
|
|
|
|
"""
|
|
|
|
# Create the blob and temporarily tag it.
|
2016-08-30 19:05:15 +00:00
|
|
|
blob_storage = model.create_blob_and_temp_tag(
|
2016-08-02 00:48:34 +00:00
|
|
|
namespace_name,
|
|
|
|
repo_name,
|
|
|
|
digest,
|
2016-08-16 19:23:00 +00:00
|
|
|
blob_upload,
|
2016-08-02 00:48:34 +00:00
|
|
|
app.config['PUSH_TEMP_TAG_EXPIRATION_SEC'],
|
|
|
|
)
|
|
|
|
|
|
|
|
# If it doesn't already exist, create the BitTorrent pieces for the blob.
|
|
|
|
if blob_upload.piece_sha_state is not None and not already_existed:
|
|
|
|
piece_bytes = blob_upload.piece_hashes + blob_upload.piece_sha_state.digest()
|
2016-08-30 19:05:15 +00:00
|
|
|
model.save_bittorrent_pieces(blob_storage, app.config['BITTORRENT_PIECE_SIZE'], piece_bytes)
|
2016-08-02 00:48:34 +00:00
|
|
|
|
|
|
|
# Delete the blob upload.
|
2016-08-30 19:05:15 +00:00
|
|
|
model.delete_blob_upload(namespace_name, repo_name, blob_upload.uuid)
|
2016-08-02 00:48:34 +00:00
|
|
|
|
|
|
|
|
|
|
|
def _finish_upload(namespace_name, repo_name, blob_upload, digest):
|
|
|
|
"""
|
|
|
|
When an upload is successful, this ends the uploading process.
|
|
|
|
"""
|
|
|
|
_validate_digest(blob_upload, digest)
|
|
|
|
_finalize_blob_database(
|
|
|
|
namespace_name,
|
|
|
|
repo_name,
|
|
|
|
blob_upload,
|
|
|
|
digest,
|
|
|
|
_finalize_blob_storage(blob_upload, digest),
|
|
|
|
)
|