From 0fdefd78e9208cad3c5061c952938913414537a2 Mon Sep 17 00:00:00 2001 From: Joseph Schorr Date: Thu, 14 Jun 2018 17:29:39 -0400 Subject: [PATCH] Extract app from torrent handling code Fixes https://jira.coreos.com/browse/QUAY-969 --- endpoints/api/suconfig.py | 13 +++-- endpoints/verbs/__init__.py | 13 +++-- util/config/validator.py | 12 ++-- .../validators/test/test_validate_torrent.py | 7 ++- util/config/validators/validate_torrent.py | 10 ++-- util/registry/test/test_torrent.py | 4 -- util/registry/torrent.py | 57 +++++++++++-------- 7 files changed, 63 insertions(+), 53 deletions(-) delete mode 100644 util/registry/test/test_torrent.py diff --git a/endpoints/api/suconfig.py b/endpoints/api/suconfig.py index 11984d5ab..2ed481219 100644 --- a/endpoints/api/suconfig.py +++ b/endpoints/api/suconfig.py @@ -7,7 +7,8 @@ import subprocess from flask import abort -from app import app, config_provider, superusers, OVERRIDE_CONFIG_DIRECTORY, ip_resolver +from app import (app, config_provider, superusers, OVERRIDE_CONFIG_DIRECTORY, ip_resolver, + instance_keys) from auth.permissions import SuperUserPermission from auth.auth_context import get_authenticated_user from data.database import configure @@ -15,12 +16,12 @@ from data.runmigration import run_alembic_migration from data.users import get_federated_service_name, get_users_handler from endpoints.api.suconfig_models_pre_oci import pre_oci_model as model from endpoints.api import (ApiResource, nickname, resource, internal_only, show_if, - require_fresh_login, request, validate_json_request, verify_not_prod, - InvalidRequest) + require_fresh_login, request, validate_json_request, verify_not_prod) from endpoints.common import common_login from util.config.configutil import add_enterprise_config_defaults from util.config.database import sync_database_with_config -from util.config.validator import validate_service_for_config, is_valid_config_upload_filename, ValidatorContext +from util.config.validator import (validate_service_for_config, is_valid_config_upload_filename, + ValidatorContext) import features @@ -405,7 +406,9 @@ class SuperUserConfigValidate(ApiResource): # this is also safe since this method does not access any information not given in the request. if not config_provider.config_exists() or SuperUserPermission().can(): config = request.get_json()['config'] - validator_context = ValidatorContext.from_app(app, config, request.get_json().get('password', ''), + validator_context = ValidatorContext.from_app(app, config, + request.get_json().get('password', ''), + instance_keys=instance_keys, ip_resolver=ip_resolver, config_provider=config_provider) diff --git a/endpoints/verbs/__init__.py b/endpoints/verbs/__init__.py index bf3823fe9..2a0a190ed 100644 --- a/endpoints/verbs/__init__.py +++ b/endpoints/verbs/__init__.py @@ -5,7 +5,7 @@ from flask import redirect, Blueprint, abort, send_file, make_response, request import features -from app import app, signer, storage, metric_queue, config_provider, ip_resolver +from app import app, signer, storage, metric_queue, config_provider, ip_resolver, instance_keys from auth.auth_context import get_authenticated_user from auth.decorators import process_auth from auth.permissions import ReadRepositoryPermission @@ -22,8 +22,8 @@ from util.registry.filelike import wrap_with_handler from util.registry.queuefile import QueueFile from util.registry.queueprocess import QueueProcess from util.registry.tarlayerformat import TarLayerFormatterReporter -from util.registry.torrent import ( - make_torrent, per_user_torrent_filename, public_torrent_filename, PieceHasher) +from util.registry.torrent import (make_torrent, per_user_torrent_filename, public_torrent_filename, + PieceHasher, TorrentConfiguration) logger = logging.getLogger(__name__) @@ -137,6 +137,9 @@ def _torrent_for_blob(blob, is_public): # We cannot support webseeds for storages that cannot provide direct downloads. exact_abort(501, 'Storage engine does not support seeding.') + # Load the config for building torrents. + torrent_config = TorrentConfiguration.from_app_config(instance_keys, app.config) + # Build the filename for the torrent. if is_public: name = public_torrent_filename(blob.uuid) @@ -145,10 +148,10 @@ def _torrent_for_blob(blob, is_public): if not user: abort(403) - name = per_user_torrent_filename(user.uuid, blob.uuid) + name = per_user_torrent_filename(torrent_config, user.uuid, blob.uuid) # Return the torrent file. - torrent_file = make_torrent(name, webseed, blob.size, torrent_info.piece_length, + torrent_file = make_torrent(torrent_config, name, webseed, blob.size, torrent_info.piece_length, torrent_info.pieces) headers = { diff --git a/util/config/validator.py b/util/config/validator.py index 6b0640c1a..f8f296a58 100644 --- a/util/config/validator.py +++ b/util/config/validator.py @@ -102,7 +102,7 @@ class ValidatorContext(object): def __init__(self, config, user_password=None, http_client=None, context=None, url_scheme_and_hostname=None, jwt_auth_max=None, registry_title=None, ip_resolver=None, feature_sec_scanner=False, is_testing=False, - uri_creator=None, config_provider=None): + uri_creator=None, config_provider=None, instance_keys=None): self.config = config self.user = get_authenticated_user() self.user_password = user_password @@ -116,14 +116,17 @@ class ValidatorContext(object): self.is_testing = is_testing self.uri_creator = uri_creator self.config_provider = config_provider + self.instance_keys = instance_keys @classmethod - def from_app(cls, app, config, user_password, ip_resolver, client=None, config_provider=None): + def from_app(cls, app, config, user_password, ip_resolver, instance_keys, client=None, + config_provider=None): """ Creates a ValidatorContext from an app config, with a given config to validate :param app: the Flask app to pull configuration information from :param config: the config to validate :param user_password: request password + :param instance_keys: The instance keys handler :param ip_resolver: an App :param client: :param config_provider: @@ -139,11 +142,8 @@ class ValidatorContext(object): app.config.get('JWT_AUTH_MAX_FRESH_S', 300), app.config['REGISTRY_TITLE'], ip_resolver, + instance_keys, app.config.get('FEATURE_SECURITY_SCANNER', False), app.config.get('TESTING', False), get_blob_download_uri_getter(app.test_request_context('/'), url_scheme_and_hostname), config_provider) - - - - diff --git a/util/config/validators/test/test_validate_torrent.py b/util/config/validators/test/test_validate_torrent.py index 1ad3664b0..f87304a05 100644 --- a/util/config/validators/test/test_validate_torrent.py +++ b/util/config/validators/test/test_validate_torrent.py @@ -1,8 +1,9 @@ import pytest +from config import build_requests_session from httmock import urlmatch, HTTMock -from config import build_requests_session +from app import instance_keys from util.config.validator import ValidatorContext from util.config.validators import ConfigValidationException from util.config.validators.validate_torrent import BittorrentValidator @@ -25,13 +26,13 @@ def test_validate_torrent(unvalidated_config, expected, app): validator = BittorrentValidator() if expected is not None: with pytest.raises(expected): - config = ValidatorContext(unvalidated_config) + config = ValidatorContext(unvalidated_config, instance_keys=instance_keys) config.http_client = build_requests_session() validator.validate(config) assert not announcer_hit[0] else: - config = ValidatorContext(unvalidated_config) + config = ValidatorContext(unvalidated_config, instance_keys=instance_keys) config.http_client = build_requests_session() validator.validate(config) diff --git a/util/config/validators/validate_torrent.py b/util/config/validators/validate_torrent.py index 567285f0b..b4dcae7ce 100644 --- a/util/config/validators/validate_torrent.py +++ b/util/config/validators/validate_torrent.py @@ -3,9 +3,7 @@ import logging from hashlib import sha1 from util.config.validators import BaseValidator, ConfigValidationException -# Temporarily removed because registry.torrent imports from app, add encoded_jwt back once extracted -# TODO(jschorr): extract app from following package and re-enable jwt_from_infohash in validator -# from util.registry.torrent import jwt_from_infohash +from util.registry.torrent import jwt_from_infohash, TorrentConfiguration logger = logging.getLogger(__name__) @@ -33,8 +31,10 @@ class BittorrentValidator(BaseValidator): 'port': 80, } - # encoded_jwt = jwt_from_infohash(params['info_hash']) - # params['jwt'] = encoded_jwt + torrent_config = TorrentConfiguration.for_testing(validator_context.instance_keys, announce_url, + validator_context.registry_title) + encoded_jwt = jwt_from_infohash(torrent_config, params['info_hash']) + params['jwt'] = encoded_jwt resp = client.get(announce_url, timeout=5, params=params) logger.debug('Got tracker response: %s: %s', resp.status_code, resp.text) diff --git a/util/registry/test/test_torrent.py b/util/registry/test/test_torrent.py deleted file mode 100644 index 4b909bd1e..000000000 --- a/util/registry/test/test_torrent.py +++ /dev/null @@ -1,4 +0,0 @@ -from util.registry.torrent import make_torrent - -def test_make_torrent_unicode_url(): - make_torrent('foo', unicode('bar'), 10, 20, 'hello world') diff --git a/util/registry/torrent.py b/util/registry/torrent.py index 3d3452dad..9abfd53db 100644 --- a/util/registry/torrent.py +++ b/util/registry/torrent.py @@ -2,49 +2,52 @@ import hashlib import time from binascii import hexlify -from cachetools import lru_cache import bencode import jwt import resumablehashlib -from app import app, instance_keys + +class TorrentConfiguration(object): + def __init__(self, instance_keys, announce_url, filename_pepper, registry_title): + self.instance_keys = instance_keys + self.announce_url = announce_url + self.filename_pepper = filename_pepper + self.registry_title = registry_title + + @classmethod + def for_testing(cls, instance_keys, announce_url, registry_title): + return TorrentConfiguration(instance_keys, announce_url, 'somepepper', registry_title) + + @classmethod + def from_app_config(cls, instance_keys, config): + return TorrentConfiguration(instance_keys, config['BITTORRENT_ANNOUNCE_URL'], + config['BITTORRENT_FILENAME_PEPPER'], config['REGISTRY_TITLE']) -ANNOUNCE_URL = app.config['BITTORRENT_ANNOUNCE_URL'] -FILENAME_PEPPER = app.config['BITTORRENT_FILENAME_PEPPER'] -REGISTRY_TITLE = app.config['REGISTRY_TITLE'] - - -@lru_cache(maxsize=1) -def _load_private_key(private_key_file_path): - with open(private_key_file_path) as private_key_file: - return private_key_file.read() - - -def jwt_from_infodict(infodict): +def _jwt_from_infodict(torrent_config, infodict): """ Returns an encoded JWT for the given BitTorrent info dict, signed by the local instance's private key. """ digest = hashlib.sha1() digest.update(bencode.bencode(infodict)) - return jwt_from_infohash(digest.digest()) + return jwt_from_infohash(torrent_config, digest.digest()) -def jwt_from_infohash(infohash_digest): +def jwt_from_infohash(torrent_config, infohash_digest): """ Returns an encoded JWT for the given BitTorrent infohash, signed by the local instance's private key. """ token_data = { - 'iss': instance_keys.service_name, - 'aud': ANNOUNCE_URL, + 'iss': torrent_config.instance_keys.service_name, + 'aud': torrent_config.announce_url, 'infohash': hexlify(infohash_digest), } - return jwt.encode(token_data, instance_keys.local_private_key, algorithm='RS256', - headers={'kid': instance_keys.local_key_id}) + return jwt.encode(token_data, torrent_config.instance_keys.local_private_key, algorithm='RS256', + headers={'kid': torrent_config.instance_keys.local_key_id}) -def make_torrent(name, webseed, length, piece_length, pieces): +def make_torrent(torrent_config, name, webseed, length, piece_length, pieces): info_dict = { 'name': name, 'length': length, @@ -53,22 +56,26 @@ def make_torrent(name, webseed, length, piece_length, pieces): 'private': 1, } + info_jwt = _jwt_from_infodict(torrent_config, info_dict) return bencode.bencode({ - 'announce': ANNOUNCE_URL + "?jwt=" + jwt_from_infodict(info_dict), + 'announce': torrent_config.announce_url + "?jwt=" + info_jwt, 'url-list': str(webseed), 'encoding': 'UTF-8', - 'created by': REGISTRY_TITLE, + 'created by': torrent_config.registry_title, 'creation date': int(time.time()), 'info': info_dict, }) def public_torrent_filename(blob_uuid): + """ Returns the filename for the given blob UUID in a public image. """ return hashlib.sha256(blob_uuid).hexdigest() -def per_user_torrent_filename(user_uuid, blob_uuid): - return hashlib.sha256(FILENAME_PEPPER + "||" + blob_uuid + "||" + user_uuid).hexdigest() +def per_user_torrent_filename(torrent_config, user_uuid, blob_uuid): + """ Returns the filename for the given blob UUID for a private image. """ + joined = torrent_config.filename_pepper + "||" + blob_uuid + "||" + user_uuid + return hashlib.sha256(joined).hexdigest() class PieceHasher(object):