vbatts/quay
Archived
1
0
Fork 0
Code Issues Pull requests Releases Wiki Activity

Address torrent feature review comments.

Browse source
This commit is contained in:
Jake Moshenko 2016-01-08 16:38:02 -05:00
parent 8a924aae4a
commit 1ae101c917
7 changed files with 31 additions and 29 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
config.py
View file

@ -275,7 +275,7 @@ class DefaultConfig(object):
}
# Torrent management flags
FEATURE_BITTORRENT = True
FEATURE_BITTORRENT = False
TORRENT_PIECE_SIZE = 512 * 1024
TORRENT_ANNOUNCE_URL = 'https://localhost:6881/announce'
TORRENT_NAMING_SALT = '3ae93fef-c30a-427e-9ba0-eea0fd710419'

2
data/database.py
View file

@ -810,7 +810,7 @@ class BlobUpload(BaseModel):
uncompressed_byte_count = IntegerField(null=True)
created = DateTimeField(default=datetime.now, index=True)
piece_sha_state = ResumableSHA1Field(null=True, default=resumablehashlib.sha1)
piece_hashes = Base64BinaryField(default='')
piece_hashes = Base64BinaryField(null=True, default='')
class Meta:
database = db

2
data/migrations/versions/23ca04d0bc8e_add_the_torrentinfo_table_and_torrent_.py
View file

@ -27,7 +27,7 @@ def upgrade(tables):
)
op.create_index('torrentinfo_storage_id', 'torrentinfo', ['storage_id'], unique=False)
op.create_index('torrentinfo_storage_id_piece_length', 'torrentinfo', ['storage_id', 'piece_length'], unique=True)
op.add_column(u'blobupload', sa.Column('piece_hashes', UTF8LongText(), nullable=False))
op.add_column(u'blobupload', sa.Column('piece_hashes', UTF8LongText(), nullable=True))
op.add_column(u'blobupload', sa.Column('piece_sha_state', UTF8LongText(), nullable=True))
### end Alembic commands ###

21
data/model/image.py
View file

@ -9,7 +9,7 @@ from data.model import (DataModelException, db_transaction, _basequery, storage,
InvalidImageException, config)
from data.database import (Image, Repository, ImageStoragePlacement, Namespace, ImageStorage,
ImageStorageLocation, RepositoryPermission, DerivedStorageForImage,
ImageStorageTransformation, db_random_func, db_for_update)
ImageStorageTransformation, db_random_func)
logger = logging.getLogger(__name__)
@ -299,17 +299,16 @@ def set_image_metadata(docker_image_id, namespace_name, repository_name, created
""" Sets metadata that is specific to how a binary piece of storage fits into the layer tree.
"""
with db_transaction():
query = (Image
.select(Image, ImageStorage)
.join(Repository)
.join(Namespace, on=(Repository.namespace_user == Namespace.id))
.switch(Image)
.join(ImageStorage)
.where(Repository.name == repository_name, Namespace.username == namespace_name,
Image.docker_image_id == docker_image_id))
try:
fetched = db_for_update(query).get()
fetched = (Image
.select(Image, ImageStorage)
.join(Repository)
.join(Namespace, on=(Repository.namespace_user == Namespace.id))
.switch(Image)
.join(ImageStorage)
.where(Repository.name == repository_name, Namespace.username == namespace_name,
Image.docker_image_id == docker_image_id)
.get())
except Image.DoesNotExist:
raise DataModelException('No image with specified id and repository')

25
data/model/storage.py
View file

@ -6,8 +6,7 @@ from data.model import (config, db_transaction, InvalidImageException, TorrentIn
DataModelException, _basequery)
from data.database import (ImageStorage, Image, ImageStoragePlacement, ImageStorageLocation,
ImageStorageTransformation, ImageStorageSignature,
ImageStorageSignatureKind, Repository, Namespace, TorrentInfo,
db_for_update)
ImageStorageSignatureKind, Repository, Namespace, TorrentInfo)
logger = logging.getLogger(__name__)
@ -213,17 +212,16 @@ def set_image_storage_metadata(docker_image_id, namespace_name, repository_name,
if image_size is None:
raise DataModelException('Empty image size field')
query = (Image
.select(Image, ImageStorage)
.join(Repository)
.join(Namespace, on=(Repository.namespace_user == Namespace.id))
.switch(Image)
.join(ImageStorage)
.where(Repository.name == repository_name, Namespace.username == namespace_name,
Image.docker_image_id == docker_image_id))
try:
image = db_for_update(query).get()
image = (Image
.select(Image, ImageStorage)
.join(Repository)
.join(Namespace, on=(Repository.namespace_user == Namespace.id))
.switch(Image)
.join(ImageStorage)
.where(Repository.name == repository_name, Namespace.username == namespace_name,
Image.docker_image_id == docker_image_id)
.get())
except ImageStorage.DoesNotExist:
raise InvalidImageException('No image with specified id and repository')
@ -257,8 +255,7 @@ def get_torrent_info(blob):
try:
return (TorrentInfo
.select()
.join(ImageStorage)
.where(blob.id == ImageStorage.id)
.where(blob == TorrentInfo.storage)
.get())
except TorrentInfo.DoesNotExist:
raise TorrentInfoDoesNotExist

4
endpoints/verbs.py
View file

@ -309,6 +309,7 @@ def get_tag_torrent(namespace, repo_name, digest):
user = get_authenticated_user()
if user is None and not public_repo:
# We can not generate a private torrent cluster without a user uuid (e.g. token auth)
abort(403)
try:
@ -320,7 +321,7 @@ def get_tag_torrent(namespace, repo_name, digest):
webseed = storage.get_direct_download_url(blob.locations, path)
if webseed is None:
# We cannot support webseeds for storages that cannot provide direct downloads.
abort(501)
abort(make_response('Storage engine does not support seeding.', 501))
try:
torrent_info = model.storage.get_torrent_info(blob)
@ -331,6 +332,7 @@ def get_tag_torrent(namespace, repo_name, digest):
name = public_torrent_filename(blob.uuid)
else:
name = per_user_torrent_filename(user.uuid, blob.uuid)
torrent_file = make_torrent(name, webseed, blob.image_size,
torrent_info.piece_length, torrent_info.pieces)

4
util/registry/torrent.py
View file

@ -62,6 +62,10 @@ def per_user_torrent_filename(user_uuid, blob_uuid):
class PieceHasher(object):
""" Utility for computing torrent piece hashes as the data flows through the update
method of this class. Users should get the final value by calling final_piece_hashes
since new chunks are allocated lazily.
"""
def __init__(self, piece_size, starting_offset=0, starting_piece_hash_bytes='',
hash_fragment_to_resume=None):
if not isinstance(starting_offset, (int, long)):