vbatts/quay
Archived
1
0
Fork 0
Code Issues Pull requests Releases Wiki Activity

Fix torrent hash generation to work in mixed stacks

Browse source
This commit is contained in:
Jake Moshenko 2016-01-11 16:43:20 -05:00
parent 8bc8e2acc0
commit 8ab6c8a22d
3 changed files with 12 additions and 7 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

4
data/database.py
View file

@ -809,8 +809,8 @@ class BlobUpload(BaseModel):
chunk_count = IntegerField(default=0) chunk_count = IntegerField(default=0)
uncompressed_byte_count = IntegerField(null=True) uncompressed_byte_count = IntegerField(null=True)
created = DateTimeField(default=datetime.now, index=True) created = DateTimeField(default=datetime.now, index=True)
piece_sha_state = ResumableSHA1Field(null=True, default=resumablehashlib.sha1) piece_sha_state = ResumableSHA1Field(null=True)
piece_hashes = Base64BinaryField(null=True, default='') piece_hashes = Base64BinaryField(null=True)
class Meta: class Meta:
database = db database = db

7
data/fields.py
View file

@ -10,6 +10,9 @@ class _ResumableSHAField(TextField):
raise NotImplementedError raise NotImplementedError
def db_value(self, value): def db_value(self, value):
if value is None:
return None
sha_state = value.state() sha_state = value.state()
# One of the fields is a byte string, let's base64 encode it to make sure # One of the fields is a byte string, let's base64 encode it to make sure
@ -19,14 +22,14 @@ class _ResumableSHAField(TextField):
return json.dumps(sha_state) return json.dumps(sha_state)
def python_value(self, value): def python_value(self, value):
to_resume = self._create_sha()
if value is None: if value is None:
return to_resume return None
sha_state = json.loads(value) sha_state = json.loads(value)
# We need to base64 decode the data bytestring. # We need to base64 decode the data bytestring.
sha_state[3] = base64.b64decode(sha_state[3]) sha_state[3] = base64.b64decode(sha_state[3])
to_resume = self._create_sha()
to_resume.set_state(sha_state) to_resume.set_state(sha_state)
return to_resume return to_resume

8
endpoints/v2/blob.py
View file

@ -3,6 +3,8 @@ import re
from flask import make_response, url_for, request, redirect, Response, abort as flask_abort from flask import make_response, url_for, request, redirect, Response, abort as flask_abort
import resumablehashlib
from app import storage, app from app import storage, app
from auth.registry_jwt_auth import process_registry_jwt_auth from auth.registry_jwt_auth import process_registry_jwt_auth
from data import model, database from data import model, database
@ -224,9 +226,9 @@ def _upload_chunk(namespace, repo_name, upload_uuid):
piece_hasher = None piece_hasher = None
# TODO remove this when all in-progress blob uploads reliably contain piece hashes # TODO remove this when all in-progress blob uploads reliably contain piece hashes
if start_offset == 0 or found.piece_sha_state is not None: if start_offset == 0:
piece_hasher = PieceHasher(app.config['TORRENT_PIECE_SIZE'], start_offset, found.piece_hashes, piece_hasher = PieceHasher(app.config['TORRENT_PIECE_SIZE'], start_offset, '',
found.piece_sha_state) resumablehashlib.sha1())
input_fp = wrap_with_handler(input_fp, piece_hasher.update) input_fp = wrap_with_handler(input_fp, piece_hasher.update)
# If this is the first chunk and we're starting at the 0 offset, add a handler to gunzip the # If this is the first chunk and we're starting at the 0 offset, add a handler to gunzip the