Merge pull request #1161 from jzelinskie/torrenthmac

misc torrent changes

config.py

@@ -286,6 +286,6 @@ class DefaultConfig(object):
 
   # Torrent management flags
   FEATURE_BITTORRENT = False
-  TORRENT_PIECE_SIZE = 512 * 1024
-  TORRENT_ANNOUNCE_URL = 'https://localhost:6881/announce'
-  TORRENT_NAMING_SALT = '3ae93fef-c30a-427e-9ba0-eea0fd710419'
+  BITTORRENT_PIECE_SIZE = 512 * 1024
+  BITTORRENT_ANNOUNCE_URL = 'https://localhost:6881/announce'
+  BITTORRENT_FILENAME_PEPPER = '3ae93fef-c30a-427e-9ba0-eea0fd710419'

data/migrations/versions/956a0833223_add_backfill_for_torrent_shas_and_.py

@@ -16,7 +16,7 @@ from util.migrate.backfill_content_checksums_and_torrent_pieces import backfill_
 
 def upgrade(tables):
     ### commands auto generated by Alembic - please adjust! ###
-    backfill_content_checksums_and_torrent_pieces(app.config['TORRENT_PIECE_SIZE'])
+    backfill_content_checksums_and_torrent_pieces(app.config['BITTORRENT_PIECE_SIZE'])
     ### end Alembic commands ###
 
 

endpoints/v1/registry.py

@@ -216,7 +216,7 @@ def put_image_layer(namespace, repository, image_id):
   sr.add_handler(size_hndlr)
 
   # Add a handler to hash the chunks of the upload for torrenting
-  piece_hasher = PieceHasher(app.config['TORRENT_PIECE_SIZE'])
+  piece_hasher = PieceHasher(app.config['BITTORRENT_PIECE_SIZE'])
   sr.add_handler(piece_hasher.update)
 
   # Add a handler which computes the checksum.
@@ -240,7 +240,7 @@ def put_image_layer(namespace, repository, image_id):
                                                              size_info.compressed_size,
                                                              size_info.uncompressed_size)
   pieces_bytes = piece_hasher.final_piece_hashes()
-  model.storage.save_torrent_info(updated_storage, app.config['TORRENT_PIECE_SIZE'], pieces_bytes)
+  model.storage.save_torrent_info(updated_storage, app.config['BITTORRENT_PIECE_SIZE'], pieces_bytes)
 
   # Append the computed checksum.
   csums = []

endpoints/v2/blob.py

@@ -238,7 +238,7 @@ def _upload_chunk(namespace, repo_name, upload_uuid):
       initial_sha1_value = found.piece_sha_state or resumablehashlib.sha1()
       initial_sha1_pieces_value = found.piece_hashes or ''
 
-      piece_hasher = PieceHasher(app.config['TORRENT_PIECE_SIZE'], start_offset,
+      piece_hasher = PieceHasher(app.config['BITTORRENT_PIECE_SIZE'], start_offset,
                                  initial_sha1_pieces_value,
                                  initial_sha1_value)
 
@@ -311,7 +311,7 @@ def _finish_upload(namespace, repo_name, upload_obj, expected_digest):
 
   if upload_obj.piece_sha_state is not None and not already_exists:
     piece_bytes = upload_obj.piece_hashes + upload_obj.piece_sha_state.digest()
-    model.storage.save_torrent_info(blob_storage, app.config['TORRENT_PIECE_SIZE'], piece_bytes)
+    model.storage.save_torrent_info(blob_storage, app.config['BITTORRENT_PIECE_SIZE'], piece_bytes)
 
   # Delete the upload tracking row.
   upload_obj.delete_instance()

endpoints/verbs.py

@@ -282,11 +282,11 @@ def _repo_verb(namespace, repository, tag, verb, formatter, sign=False, checker=
     # Close any existing DB connection once the process has exited.
     database.close_db_filter(None)
 
-  hasher = PieceHasher(app.config['TORRENT_PIECE_SIZE'])
+  hasher = PieceHasher(app.config['BITTORRENT_PIECE_SIZE'])
 
   def _store_metadata_and_cleanup():
     with database.UseThenDisconnect(app.config):
-      model.storage.save_torrent_info(derived, app.config['TORRENT_PIECE_SIZE'],
+      model.storage.save_torrent_info(derived, app.config['BITTORRENT_PIECE_SIZE'],
                                       hasher.final_piece_hashes())
       derived.image_size = hasher.hashed_bytes
       derived.save()

util/migrate/backfill_content_checksums_and_torrent_pieces.py

@@ -142,4 +142,4 @@ if __name__ == '__main__':
   #logging.getLogger('peewee').setLevel(logging.WARNING)
   logging.getLogger('boto').setLevel(logging.WARNING)
   logging.getLogger('data.database').setLevel(logging.WARNING)
-  backfill_content_checksums_and_torrent_pieces(app.config['TORRENT_PIECE_SIZE'])
+  backfill_content_checksums_and_torrent_pieces(app.config['BITTORRENT_PIECE_SIZE'])

util/registry/torrent.py

@@ -11,8 +11,8 @@ from cachetools import lru_cache
 from app import app
 
 
-ANNOUNCE_URL = app.config.get('TORRENT_ANNOUNCE_URL')
-NAMING_SALT = app.config.get('TORRENT_NAMING_SALT')
+ANNOUNCE_URL = app.config.get('BITTORRENT_ANNOUNCE_URL')
+FILENAME_PEPPER = app.config.get('BITTORRENT_FILENAME_PEPPER')
 REGISTRY_TITLE = app.config.get('REGISTRY_TITLE')
 JWT_ISSUER = app.config.get('JWT_AUTH_TOKEN_ISSUER')
 
@@ -58,7 +58,7 @@ def public_torrent_filename(blob_uuid):
   return hashlib.sha256(blob_uuid).hexdigest()
 
 def per_user_torrent_filename(user_uuid, blob_uuid):
-  return hashlib.sha256(blob_uuid + user_uuid + NAMING_SALT).hexdigest()
+  return hashlib.sha256(FILENAME_PEPPER + "||" + blob_uuid + "||" + user_uuid).hexdigest()
 
 
 class PieceHasher(object):