Merge pull request #3117 from quay/joseph.schorr/QUAY-969/extract-torrent

Extract app from torrent handling code

endpoints/api/suconfig.py

@@ -7,7 +7,8 @@ import subprocess
 
 from flask import abort
 
-from app import app, config_provider, superusers, OVERRIDE_CONFIG_DIRECTORY, ip_resolver
+from app import (app, config_provider, superusers, OVERRIDE_CONFIG_DIRECTORY, ip_resolver,
+                 instance_keys)
 from auth.permissions import SuperUserPermission
 from auth.auth_context import get_authenticated_user
 from data.database import configure
@@ -15,12 +16,12 @@ from data.runmigration import run_alembic_migration
 from data.users import get_federated_service_name, get_users_handler
 from endpoints.api.suconfig_models_pre_oci import pre_oci_model as model
 from endpoints.api import (ApiResource, nickname, resource, internal_only, show_if,
-                           require_fresh_login, request, validate_json_request, verify_not_prod,
+                           require_fresh_login, request, validate_json_request, verify_not_prod)
-                           InvalidRequest)
 from endpoints.common import common_login
 from util.config.configutil import add_enterprise_config_defaults
 from util.config.database import sync_database_with_config
-from util.config.validator import validate_service_for_config, is_valid_config_upload_filename, ValidatorContext
+from util.config.validator import (validate_service_for_config, is_valid_config_upload_filename,
+                                   ValidatorContext)
 
 import features
 
@@ -405,7 +406,9 @@ class SuperUserConfigValidate(ApiResource):
     # this is also safe since this method does not access any information not given in the request.
     if not config_provider.config_exists() or SuperUserPermission().can():
       config = request.get_json()['config']
-      validator_context = ValidatorContext.from_app(app, config, request.get_json().get('password', ''),
+      validator_context = ValidatorContext.from_app(app, config,
+                                                    request.get_json().get('password', ''),
+                                                    instance_keys=instance_keys,
                                                     ip_resolver=ip_resolver,
                                                     config_provider=config_provider)
 

endpoints/verbs/__init__.py

@@ -5,7 +5,7 @@ from flask import redirect, Blueprint, abort, send_file, make_response, request
 
 import features
 
-from app import app, signer, storage, metric_queue, config_provider, ip_resolver
+from app import app, signer, storage, metric_queue, config_provider, ip_resolver, instance_keys
 from auth.auth_context import get_authenticated_user
 from auth.decorators import process_auth
 from auth.permissions import ReadRepositoryPermission
@@ -22,8 +22,8 @@ from util.registry.filelike import wrap_with_handler
 from util.registry.queuefile import QueueFile
 from util.registry.queueprocess import QueueProcess
 from util.registry.tarlayerformat import TarLayerFormatterReporter
-from util.registry.torrent import (
+from util.registry.torrent import (make_torrent, per_user_torrent_filename, public_torrent_filename,
-  make_torrent, per_user_torrent_filename, public_torrent_filename, PieceHasher)
+                                   PieceHasher, TorrentConfiguration)
 
 logger = logging.getLogger(__name__)
 
@@ -137,6 +137,9 @@ def _torrent_for_blob(blob, is_public):
     # We cannot support webseeds for storages that cannot provide direct downloads.
     exact_abort(501, 'Storage engine does not support seeding.')
 
+  # Load the config for building torrents.
+  torrent_config = TorrentConfiguration.from_app_config(instance_keys, app.config)
+
   # Build the filename for the torrent.
   if is_public:
     name = public_torrent_filename(blob.uuid)
@@ -145,10 +148,10 @@ def _torrent_for_blob(blob, is_public):
     if not user:
       abort(403)
 
-    name = per_user_torrent_filename(user.uuid, blob.uuid)
+    name = per_user_torrent_filename(torrent_config, user.uuid, blob.uuid)
 
   # Return the torrent file.
-  torrent_file = make_torrent(name, webseed, blob.size, torrent_info.piece_length,
+  torrent_file = make_torrent(torrent_config, name, webseed, blob.size, torrent_info.piece_length,
                               torrent_info.pieces)
 
   headers = {

util/config/validator.py

@@ -102,7 +102,7 @@ class ValidatorContext(object):
   def __init__(self, config, user_password=None, http_client=None, context=None,
                url_scheme_and_hostname=None, jwt_auth_max=None, registry_title=None,
                ip_resolver=None, feature_sec_scanner=False, is_testing=False,
-               uri_creator=None, config_provider=None):
+               uri_creator=None, config_provider=None, instance_keys=None):
     self.config = config
     self.user = get_authenticated_user()
     self.user_password = user_password
@@ -116,14 +116,17 @@ class ValidatorContext(object):
     self.is_testing = is_testing
     self.uri_creator = uri_creator
     self.config_provider = config_provider
+    self.instance_keys = instance_keys
 
   @classmethod
-  def from_app(cls, app, config, user_password, ip_resolver, client=None, config_provider=None):
+  def from_app(cls, app, config, user_password, ip_resolver, instance_keys, client=None,
+               config_provider=None):
     """
     Creates a ValidatorContext from an app config, with a given config to validate
     :param app: the Flask app to pull configuration information from
     :param config: the config to validate
     :param user_password: request password
+    :param instance_keys: The instance keys handler
     :param ip_resolver: an App
     :param client:
     :param config_provider:
@@ -139,11 +142,8 @@ class ValidatorContext(object):
                app.config.get('JWT_AUTH_MAX_FRESH_S', 300),
                app.config['REGISTRY_TITLE'],
                ip_resolver,
+               instance_keys,
                app.config.get('FEATURE_SECURITY_SCANNER', False),
                app.config.get('TESTING', False),
                get_blob_download_uri_getter(app.test_request_context('/'), url_scheme_and_hostname),
                config_provider)
-
-
-
-

util/config/validators/test/test_validate_torrent.py

@@ -1,8 +1,9 @@
 import pytest
 
+from config import build_requests_session
 from httmock import urlmatch, HTTMock
 
-from config import build_requests_session
+from app import instance_keys
 from util.config.validator import ValidatorContext
 from util.config.validators import ConfigValidationException
 from util.config.validators.validate_torrent import BittorrentValidator
@@ -25,13 +26,13 @@ def test_validate_torrent(unvalidated_config, expected, app):
     validator = BittorrentValidator()
     if expected is not None:
       with pytest.raises(expected):
-        config = ValidatorContext(unvalidated_config)
+        config = ValidatorContext(unvalidated_config, instance_keys=instance_keys)
         config.http_client = build_requests_session()
 
         validator.validate(config)
       assert not announcer_hit[0]
     else:
-      config = ValidatorContext(unvalidated_config)
+      config = ValidatorContext(unvalidated_config, instance_keys=instance_keys)
       config.http_client = build_requests_session()
 
       validator.validate(config)

util/config/validators/validate_torrent.py

@@ -3,9 +3,7 @@ import logging
 from hashlib import sha1
 
 from util.config.validators import BaseValidator, ConfigValidationException
-# Temporarily removed because registry.torrent imports from app, add encoded_jwt back once extracted
+from util.registry.torrent import jwt_from_infohash, TorrentConfiguration
-# TODO(jschorr): extract app from following package and re-enable jwt_from_infohash in validator
-# from util.registry.torrent import jwt_from_infohash
 
 logger = logging.getLogger(__name__)
 
@@ -33,8 +31,10 @@ class BittorrentValidator(BaseValidator):
       'port': 80,
     }
 
-    # encoded_jwt = jwt_from_infohash(params['info_hash'])
+    torrent_config = TorrentConfiguration.for_testing(validator_context.instance_keys, announce_url,
-    # params['jwt'] = encoded_jwt
+                                                      validator_context.registry_title)
+    encoded_jwt = jwt_from_infohash(torrent_config, params['info_hash'])
+    params['jwt'] = encoded_jwt
 
     resp = client.get(announce_url, timeout=5, params=params)
     logger.debug('Got tracker response: %s: %s', resp.status_code, resp.text)

util/registry/test/test_torrent.py

@@ -1,4 +0,0 @@
-from util.registry.torrent import make_torrent
-
-def test_make_torrent_unicode_url():
-  make_torrent('foo', unicode('bar'), 10, 20, 'hello world')

util/registry/torrent.py

@@ -2,49 +2,52 @@ import hashlib
 import time
 
 from binascii import hexlify
-from cachetools import lru_cache
 
 import bencode
 import jwt
 import resumablehashlib
 
-from app import app, instance_keys
+
+class TorrentConfiguration(object):
+  def __init__(self, instance_keys, announce_url, filename_pepper, registry_title):
+    self.instance_keys = instance_keys
+    self.announce_url = announce_url
+    self.filename_pepper = filename_pepper
+    self.registry_title = registry_title
+
+  @classmethod
+  def for_testing(cls, instance_keys, announce_url, registry_title):
+    return TorrentConfiguration(instance_keys, announce_url, 'somepepper', registry_title)
+
+  @classmethod
+  def from_app_config(cls, instance_keys, config):
+    return TorrentConfiguration(instance_keys, config['BITTORRENT_ANNOUNCE_URL'],
+                                config['BITTORRENT_FILENAME_PEPPER'], config['REGISTRY_TITLE'])
 
 
-ANNOUNCE_URL = app.config['BITTORRENT_ANNOUNCE_URL']
+def _jwt_from_infodict(torrent_config, infodict):
-FILENAME_PEPPER = app.config['BITTORRENT_FILENAME_PEPPER']
-REGISTRY_TITLE = app.config['REGISTRY_TITLE']
-
-
-@lru_cache(maxsize=1)
-def _load_private_key(private_key_file_path):
-  with open(private_key_file_path) as private_key_file:
-    return private_key_file.read()
-
-
-def jwt_from_infodict(infodict):
   """ Returns an encoded JWT for the given BitTorrent info dict, signed by the local instance's
       private key.
   """
   digest = hashlib.sha1()
   digest.update(bencode.bencode(infodict))
-  return jwt_from_infohash(digest.digest())
+  return jwt_from_infohash(torrent_config, digest.digest())
 
 
-def jwt_from_infohash(infohash_digest):
+def jwt_from_infohash(torrent_config, infohash_digest):
   """ Returns an encoded JWT for the given BitTorrent infohash, signed by the local instance's
       private key.
   """
   token_data = {
-    'iss': instance_keys.service_name,
+    'iss': torrent_config.instance_keys.service_name,
-    'aud': ANNOUNCE_URL,
+    'aud': torrent_config.announce_url,
     'infohash': hexlify(infohash_digest),
   }
-  return jwt.encode(token_data, instance_keys.local_private_key, algorithm='RS256',
+  return jwt.encode(token_data, torrent_config.instance_keys.local_private_key, algorithm='RS256',
-                    headers={'kid': instance_keys.local_key_id})
+                    headers={'kid': torrent_config.instance_keys.local_key_id})
 
 
-def make_torrent(name, webseed, length, piece_length, pieces):
+def make_torrent(torrent_config, name, webseed, length, piece_length, pieces):
   info_dict = {
     'name': name,
     'length': length,
@@ -53,22 +56,26 @@ def make_torrent(name, webseed, length, piece_length, pieces):
     'private': 1,
   }
 
+  info_jwt = _jwt_from_infodict(torrent_config, info_dict)
   return bencode.bencode({
-    'announce': ANNOUNCE_URL + "?jwt=" + jwt_from_infodict(info_dict),
+    'announce': torrent_config.announce_url + "?jwt=" + info_jwt,
     'url-list': str(webseed),
     'encoding': 'UTF-8',
-    'created by': REGISTRY_TITLE,
+    'created by': torrent_config.registry_title,
     'creation date': int(time.time()),
     'info': info_dict,
   })
 
 
 def public_torrent_filename(blob_uuid):
+  """ Returns the filename for the given blob UUID in a public image. """
   return hashlib.sha256(blob_uuid).hexdigest()
 
 
-def per_user_torrent_filename(user_uuid, blob_uuid):
+def per_user_torrent_filename(torrent_config, user_uuid, blob_uuid):
-  return hashlib.sha256(FILENAME_PEPPER + "||" + blob_uuid + "||" + user_uuid).hexdigest()
+  """ Returns the filename for the given blob UUID for a private image. """
+  joined = torrent_config.filename_pepper + "||" + blob_uuid + "||" + user_uuid
+  return hashlib.sha256(joined).hexdigest()
 
 
 class PieceHasher(object):