vbatts/quay
Archived
1
0
Fork 0
Code Issues Pull requests Releases Wiki Activity

Merge pull request #1161 from jzelinskie/torrenthmac

Browse source 
misc torrent changes
This commit is contained in:
Jimmy Zelinskie 2016-01-22 23:02:44 -05:00
commit 85ae1a2a0a
7 changed files with 14 additions and 14 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

6
config.py
View file

@ -286,6 +286,6 @@ class DefaultConfig(object):
# Torrent management flags # Torrent management flags
FEATURE_BITTORRENT = False FEATURE_BITTORRENT = False
TORRENT_PIECE_SIZE = 512 * 1024 BITTORRENT_PIECE_SIZE = 512 * 1024
TORRENT_ANNOUNCE_URL = 'https://localhost:6881/announce' BITTORRENT_ANNOUNCE_URL = 'https://localhost:6881/announce'
TORRENT_NAMING_SALT = '3ae93fef-c30a-427e-9ba0-eea0fd710419' BITTORRENT_FILENAME_PEPPER = '3ae93fef-c30a-427e-9ba0-eea0fd710419'

2
data/migrations/versions/956a0833223_add_backfill_for_torrent_shas_and_.py
View file

@ -16,7 +16,7 @@ from util.migrate.backfill_content_checksums_and_torrent_pieces import backfill_
def upgrade(tables): def upgrade(tables):
### commands auto generated by Alembic - please adjust! ### ### commands auto generated by Alembic - please adjust! ###
backfill_content_checksums_and_torrent_pieces(app.config['TORRENT_PIECE_SIZE']) backfill_content_checksums_and_torrent_pieces(app.config['BITTORRENT_PIECE_SIZE'])
### end Alembic commands ### ### end Alembic commands ###

4
endpoints/v1/registry.py
View file

@ -216,7 +216,7 @@ def put_image_layer(namespace, repository, image_id):
sr.add_handler(size_hndlr) sr.add_handler(size_hndlr)
# Add a handler to hash the chunks of the upload for torrenting # Add a handler to hash the chunks of the upload for torrenting
piece_hasher = PieceHasher(app.config['TORRENT_PIECE_SIZE']) piece_hasher = PieceHasher(app.config['BITTORRENT_PIECE_SIZE'])
sr.add_handler(piece_hasher.update) sr.add_handler(piece_hasher.update)
# Add a handler which computes the checksum. # Add a handler which computes the checksum.
@ -240,7 +240,7 @@ def put_image_layer(namespace, repository, image_id):
size_info.compressed_size, size_info.compressed_size,
size_info.uncompressed_size) size_info.uncompressed_size)
pieces_bytes = piece_hasher.final_piece_hashes() pieces_bytes = piece_hasher.final_piece_hashes()
model.storage.save_torrent_info(updated_storage, app.config['TORRENT_PIECE_SIZE'], pieces_bytes) model.storage.save_torrent_info(updated_storage, app.config['BITTORRENT_PIECE_SIZE'], pieces_bytes)
# Append the computed checksum. # Append the computed checksum.
csums = [] csums = []

4
endpoints/v2/blob.py
View file

@ -238,7 +238,7 @@ def _upload_chunk(namespace, repo_name, upload_uuid):
initial_sha1_value = found.piece_sha_state or resumablehashlib.sha1() initial_sha1_value = found.piece_sha_state or resumablehashlib.sha1()
initial_sha1_pieces_value = found.piece_hashes or '' initial_sha1_pieces_value = found.piece_hashes or ''
piece_hasher = PieceHasher(app.config['TORRENT_PIECE_SIZE'], start_offset, piece_hasher = PieceHasher(app.config['BITTORRENT_PIECE_SIZE'], start_offset,
initial_sha1_pieces_value, initial_sha1_pieces_value,
initial_sha1_value) initial_sha1_value)
@ -311,7 +311,7 @@ def _finish_upload(namespace, repo_name, upload_obj, expected_digest):
if upload_obj.piece_sha_state is not None and not already_exists: if upload_obj.piece_sha_state is not None and not already_exists:
piece_bytes = upload_obj.piece_hashes + upload_obj.piece_sha_state.digest() piece_bytes = upload_obj.piece_hashes + upload_obj.piece_sha_state.digest()
model.storage.save_torrent_info(blob_storage, app.config['TORRENT_PIECE_SIZE'], piece_bytes) model.storage.save_torrent_info(blob_storage, app.config['BITTORRENT_PIECE_SIZE'], piece_bytes)
# Delete the upload tracking row. # Delete the upload tracking row.
upload_obj.delete_instance() upload_obj.delete_instance()

4
endpoints/verbs.py
View file

@ -282,11 +282,11 @@ def _repo_verb(namespace, repository, tag, verb, formatter, sign=False, checker=
# Close any existing DB connection once the process has exited. # Close any existing DB connection once the process has exited.
database.close_db_filter(None) database.close_db_filter(None)
hasher = PieceHasher(app.config['TORRENT_PIECE_SIZE']) hasher = PieceHasher(app.config['BITTORRENT_PIECE_SIZE'])
def _store_metadata_and_cleanup(): def _store_metadata_and_cleanup():
with database.UseThenDisconnect(app.config): with database.UseThenDisconnect(app.config):
model.storage.save_torrent_info(derived, app.config['TORRENT_PIECE_SIZE'], model.storage.save_torrent_info(derived, app.config['BITTORRENT_PIECE_SIZE'],
hasher.final_piece_hashes()) hasher.final_piece_hashes())
derived.image_size = hasher.hashed_bytes derived.image_size = hasher.hashed_bytes
derived.save() derived.save()

2
util/migrate/backfill_content_checksums_and_torrent_pieces.py
View file

@ -142,4 +142,4 @@ if __name__ == '__main__':
#logging.getLogger('peewee').setLevel(logging.WARNING) #logging.getLogger('peewee').setLevel(logging.WARNING)
logging.getLogger('boto').setLevel(logging.WARNING) logging.getLogger('boto').setLevel(logging.WARNING)
logging.getLogger('data.database').setLevel(logging.WARNING) logging.getLogger('data.database').setLevel(logging.WARNING)
backfill_content_checksums_and_torrent_pieces(app.config['TORRENT_PIECE_SIZE']) backfill_content_checksums_and_torrent_pieces(app.config['BITTORRENT_PIECE_SIZE'])

6
util/registry/torrent.py
View file

@ -11,8 +11,8 @@ from cachetools import lru_cache
from app import app from app import app
ANNOUNCE_URL = app.config.get('TORRENT_ANNOUNCE_URL') ANNOUNCE_URL = app.config.get('BITTORRENT_ANNOUNCE_URL')
NAMING_SALT = app.config.get('TORRENT_NAMING_SALT') FILENAME_PEPPER = app.config.get('BITTORRENT_FILENAME_PEPPER')
REGISTRY_TITLE = app.config.get('REGISTRY_TITLE') REGISTRY_TITLE = app.config.get('REGISTRY_TITLE')
JWT_ISSUER = app.config.get('JWT_AUTH_TOKEN_ISSUER') JWT_ISSUER = app.config.get('JWT_AUTH_TOKEN_ISSUER')
@ -58,7 +58,7 @@ def public_torrent_filename(blob_uuid):
return hashlib.sha256(blob_uuid).hexdigest() return hashlib.sha256(blob_uuid).hexdigest()
def per_user_torrent_filename(user_uuid, blob_uuid): def per_user_torrent_filename(user_uuid, blob_uuid):
return hashlib.sha256(blob_uuid + user_uuid + NAMING_SALT).hexdigest() return hashlib.sha256(FILENAME_PEPPER + "||" + blob_uuid + "||" + user_uuid).hexdigest()
class PieceHasher(object): class PieceHasher(object):