yapf

endpoints/verbs/__init__.py

@@ -22,16 +22,14 @@ from util.http import exact_abort
 from util.registry.filelike import wrap_with_handler
 from util.registry.queuefile import QueueFile
 from util.registry.queueprocess import QueueProcess
-from util.registry.torrent import (make_torrent, per_user_torrent_filename, public_torrent_filename,
+from util.registry.torrent import (
-                                   PieceHasher)
+  make_torrent, per_user_torrent_filename, public_torrent_filename, PieceHasher)
-
 
 logger = logging.getLogger(__name__)
 
 verbs = Blueprint('verbs', __name__)
 license_validator.enforce_license_before_request(verbs)
 
-
 LAYER_MIMETYPE = 'binary/octet-stream'
 
 
@@ -60,7 +58,8 @@ def _open_stream(formatter, repo_image, tag, derived_image_id, handlers):
       logger.debug('Returning image layer %s: %s', current_image.image_id, current_image_path)
       yield current_image_stream
 
-  stream = formatter.build_stream(repo_image, tag, derived_image_id, get_next_image, get_next_layer)
+  stream = formatter.build_stream(repo_image, tag, derived_image_id, get_next_image,
+                                  get_next_layer)
 
   for handler_fn in handlers:
     stream = wrap_with_handler(stream, handler_fn)
@@ -89,6 +88,7 @@ def _write_derived_image_to_storage(verb, derived_image, queue_file):
   """ Read from the generated stream and write it back to the storage engine. This method runs in a
       separate process.
   """
+
   def handle_exception(ex):
     logger.debug('Exception when building %s derived image %s: %s', verb, derived_image.ref, ex)
 
@@ -139,7 +139,8 @@ def _torrent_for_blob(blob, is_public):
   torrent_file = make_torrent(name, webseed, blob.size, torrent_info.piece_length,
                               torrent_info.pieces)
 
-  headers = {'Content-Type': 'application/x-bittorrent',
+  headers = {
+    'Content-Type': 'application/x-bittorrent',
     'Content-Disposition': 'attachment; filename={0}.torrent'.format(name)}
 
   return make_response(torrent_file, 200, headers)
@@ -158,8 +159,7 @@ def _torrent_repo_verb(repo_image, tag, verb, **kwargs):
     abort(406)
 
   # Return the torrent.
-  repo = model.get_repository(repo_image.repository.namespace_name,
+  repo = model.get_repository(repo_image.repository.namespace_name, repo_image.repository.name)
-                              repo_image.repository.name)
   repo_is_public = repo is not None and repo.is_public
   torrent = _torrent_for_blob(derived_image.blob, repo_is_public)
 
@@ -229,15 +229,14 @@ def _repo_verb(namespace, repository, tag, verb, formatter, sign=False, checker=
   metric_queue.repository_pull.Inc(labelvalues=[namespace, repository, verb, True])
 
   # Lookup/create the derived image for the verb and repo image.
-  derived_image = model.lookup_or_create_derived_image(repo_image, verb,
+  derived_image = model.lookup_or_create_derived_image(
-                                                       storage.preferred_locations[0],
+    repo_image, verb, storage.preferred_locations[0], varying_metadata={'tag': tag})
-                                                       varying_metadata={'tag': tag})
   if not derived_image.blob.uploading:
     logger.debug('Derived %s image %s exists in storage', verb, derived_image.ref)
     derived_layer_path = model.get_blob_path(derived_image.blob)
     is_head_request = request.method == 'HEAD'
-    download_url = storage.get_direct_download_url(derived_image.blob.locations, derived_layer_path,
+    download_url = storage.get_direct_download_url(derived_image.blob.locations,
-                                                   head=is_head_request)
+                                                   derived_layer_path, head=is_head_request)
     if download_url:
       logger.debug('Redirecting to download URL for derived %s image %s', verb, derived_image.ref)
       return redirect(download_url)
@@ -246,7 +245,8 @@ def _repo_verb(namespace, repository, tag, verb, formatter, sign=False, checker=
     database.close_db_filter(None)
 
     logger.debug('Sending cached derived %s image %s', verb, derived_image.ref)
-    return send_file(storage.stream_read_file(derived_image.blob.locations, derived_layer_path),
+    return send_file(
+      storage.stream_read_file(derived_image.blob.locations, derived_layer_path),
       mimetype=LAYER_MIMETYPE)
 
   logger.debug('Building and returning derived %s image %s', verb, derived_image.ref)
@@ -270,9 +270,12 @@ def _repo_verb(namespace, repository, tag, verb, formatter, sign=False, checker=
   # and send the results to the client and storage.
   handlers = [hasher.update]
   args = (formatter, repo_image, tag, derived_image_id, handlers)
-  queue_process = QueueProcess(_open_stream,
+  queue_process = QueueProcess(
-                               8 * 1024, 10 * 1024 * 1024, # 8K/10M chunk/max
+    _open_stream,
-                               args, finished=_store_metadata_and_cleanup)
+    8 * 1024,
+    10 * 1024 * 1024,  # 8K/10M chunk/max
+    args,
+    finished=_store_metadata_and_cleanup)
 
   client_queue_file = QueueFile(queue_process.create_queue(), 'client')
   storage_queue_file = QueueFile(queue_process.create_queue(), 'storage')
@@ -336,11 +339,13 @@ def get_aci_signature(server, namespace, repository, tag, os, arch):
 
 @route_show_if(features.ACI_CONVERSION)
 @anon_protect
-@verbs.route('/aci/<server>/<namespace>/<repository>/<tag>/aci/<os>/<arch>/', methods=['GET', 'HEAD'])
+@verbs.route('/aci/<server>/<namespace>/<repository>/<tag>/aci/<os>/<arch>/', methods=[
+  'GET', 'HEAD'])
 @process_auth
 def get_aci_image(server, namespace, repository, tag, os, arch):
-  return _repo_verb(namespace, repository, tag, 'aci', AppCImageFormatter(),
+  return _repo_verb(namespace, repository, tag, 'aci',
-                    sign=True, checker=os_arch_checker(os, arch), os=os, arch=arch)
+                    AppCImageFormatter(), sign=True, checker=os_arch_checker(os, arch), os=os,
+                    arch=arch)
 
 
 @anon_protect

endpoints/verbs/models_interface.py

@@ -3,8 +3,10 @@ from collections import namedtuple
 
 from six import add_metaclass
 
-class Repository(namedtuple('Repository', ['id', 'name', 'namespace_name', 'description',
+
-                                           'is_public', 'kind'])):
+class Repository(
+    namedtuple('Repository', ['id', 'name', 'namespace_name', 'description', 'is_public',
+                              'kind'])):
   """
   Repository represents a namespaced collection of tags.
   :type id: int
@@ -21,22 +23,27 @@ class DerivedImage(namedtuple('DerivedImage', ['ref', 'blob', 'internal_source_i
   DerivedImage represents a user-facing alias for an image which was derived from another image.
   """
 
+
 class RepositoryReference(namedtuple('RepositoryReference', ['id', 'name', 'namespace_name'])):
   """
   RepositoryReference represents a reference to a Repository, without its full metadata.
   """
 
-class ImageWithBlob(namedtuple('Image', ['image_id', 'blob', 'compat_metadata', 'repository',
+
-                                         'internal_db_id', 'v1_metadata'])):
+class ImageWithBlob(
+    namedtuple('Image', [
+      'image_id', 'blob', 'compat_metadata', 'repository', 'internal_db_id', 'v1_metadata'])):
   """
   ImageWithBlob represents a user-facing alias for referencing an image, along with its blob.
   """
 
+
 class Blob(namedtuple('Blob', ['uuid', 'size', 'uncompressed_size', 'uploading', 'locations'])):
   """
   Blob represents an opaque binary blob saved to the storage system.
   """
 
+
 class TorrentInfo(namedtuple('TorrentInfo', ['piece_length', 'pieces'])):
   """
   TorrentInfo represents the torrent piece information associated with a blob.
@@ -49,6 +56,7 @@ class VerbsDataInterface(object):
   Interface that represents all data store interactions required by the registry's custom HTTP
   verbs.
   """
+
   @abstractmethod
   def get_repository(self, namespace_name, repo_name):
     """

endpoints/verbs/models_pre_oci.py

@@ -10,8 +10,8 @@ from endpoints.verbs.models_interface import (
   Repository,
   RepositoryReference,
   TorrentInfo,
-  VerbsDataInterface,
+  VerbsDataInterface,)
-)
+
 
 class PreOCIModel(VerbsDataInterface):
   """
@@ -27,13 +27,11 @@ class PreOCIModel(VerbsDataInterface):
     return _repository_for_repo(repo)
 
   def get_manifest_layers_with_blobs(self, repo_image):
-    repo_image_record = model.image.get_image_by_id(repo_image.repository.namespace_name,
+    repo_image_record = model.image.get_image_by_id(
-                                                    repo_image.repository.name,
+      repo_image.repository.namespace_name, repo_image.repository.name, repo_image.image_id)
-                                                    repo_image.image_id)
 
-    parents = model.image.get_parent_images_with_placements(repo_image.repository.namespace_name,
+    parents = model.image.get_parent_images_with_placements(
-                                                            repo_image.repository.name,
+      repo_image.repository.namespace_name, repo_image.repository.name, repo_image_record)
-                                                            repo_image_record)
 
     yield repo_image
 
@@ -51,8 +49,7 @@ class PreOCIModel(VerbsDataInterface):
         compat_metadata=metadata,
         v1_metadata=_docker_v1_metadata(repo_image.repository.namespace_name,
                                         repo_image.repository.name, parent),
-        internal_db_id=parent.id,
+        internal_db_id=parent.id,)
-      )
 
   def get_derived_image_signature(self, derived_image, signer_name):
     storage = model.storage.get_storage_by_uuid(derived_image.blob.uuid)
@@ -100,8 +97,7 @@ class PreOCIModel(VerbsDataInterface):
 
     return TorrentInfo(
       pieces=torrent_info.pieces,
-      piece_length=torrent_info.piece_length,
+      piece_length=torrent_info.piece_length,)
-    )
 
   def set_torrent_info(self, blob, piece_length, pieces):
     blob_record = model.storage.get_storage_by_uuid(blob.uuid)
@@ -138,12 +134,10 @@ class PreOCIModel(VerbsDataInterface):
       repository=RepositoryReference(
         namespace_name=namespace_name,
         name=repo_name,
-        id=found.repository_id,
+        id=found.repository_id,),
-      ),
       compat_metadata=metadata,
       v1_metadata=_docker_v1_metadata(namespace_name, repo_name, found),
-      internal_db_id=found.id,
+      internal_db_id=found.id,)
-    )
 
 
 pre_oci_model = PreOCIModel()
@@ -168,8 +162,7 @@ def _docker_v1_metadata(namespace_name, repo_name, repo_image):
 
     # Note: These are not needed in verbs and are expensive to load, so we just skip them.
     content_checksum=None,
-    parent_image_id=None,
+    parent_image_id=None,)
-  )
 
 
 def _derived_image(blob_record, repo_image):
@@ -179,8 +172,7 @@ def _derived_image(blob_record, repo_image):
   return DerivedImage(
     ref=repo_image.internal_db_id,
     blob=_blob(blob_record),
-    internal_source_image_db_id=repo_image.internal_db_id,
+    internal_source_image_db_id=repo_image.internal_db_id,)
-  )
 
 
 def _blob(blob_record):
@@ -197,8 +189,8 @@ def _blob(blob_record):
     size=blob_record.image_size,
     uncompressed_size=blob_record.uncompressed_size,
     uploading=blob_record.uploading,
-    locations=locations,
+    locations=locations,)
-  )
+
 
 def _repository_for_repo(repo):
   """ Returns a Repository object representing the Pre-OCI data model repo instance given. """
@@ -208,5 +200,4 @@ def _repository_for_repo(repo):
     namespace_name=repo.namespace_user.username,
     description=repo.description,
     is_public=model.repository.is_repository_public(repo),
-    kind=model.repository.get_repo_kind_name(repo),
+    kind=model.repository.get_repo_kind_name(repo),)
-  )

endpoints/verbs/test/test_security.py

@@ -18,17 +18,18 @@ ACI_ARGS = {
   'server': 'someserver',
   'tag': 'fake',
   'os': 'linux',
-  'arch': 'x64',
+  'arch': 'x64',}
-}
+
 
 @pytest.mark.parametrize('user', [
   (0, None),
   (1, NO_ACCESS_USER),
   (2, READ_ACCESS_USER),
   (3, CREATOR_ACCESS_USER),
-  (4, ADMIN_ACCESS_USER),
+  (4, ADMIN_ACCESS_USER),])
-])
+@pytest.mark.parametrize(
-@pytest.mark.parametrize('endpoint,method,repository,single_repo_path,params,expected_statuses', [
+  'endpoint,method,repository,single_repo_path,params,expected_statuses',
+  [
     ('get_aci_signature', 'GET', PUBLIC_REPO, False, ACI_ARGS, (404, 404, 404, 404, 404)),
     ('get_aci_signature', 'GET', PRIVATE_REPO, False, ACI_ARGS, (403, 403, 404, 403, 404)),
     ('get_aci_signature', 'GET', ORG_REPO, False, ACI_ARGS, (403, 403, 404, 403, 404)),
@@ -44,14 +45,18 @@ ACI_ARGS = {
     ('get_squashed_tag', 'GET', PUBLIC_REPO, False, dict(tag='fake'), (404, 404, 404, 404, 404)),
     ('get_squashed_tag', 'GET', PRIVATE_REPO, False, dict(tag='fake'), (403, 403, 404, 403, 404)),
     ('get_squashed_tag', 'GET', ORG_REPO, False, dict(tag='fake'), (403, 403, 404, 403, 404)),
-  ('get_squashed_tag', 'GET', ANOTHER_ORG_REPO, False, dict(tag='fake'), (403, 403, 403, 403, 404)),
+    ('get_squashed_tag', 'GET', ANOTHER_ORG_REPO, False, dict(tag='fake'), (403, 403, 403, 403,
+                                                                            404)),
 
     # get_tag_torrent
-  ('get_tag_torrent', 'GET', PUBLIC_REPO, True, dict(digest='sha256:1234'), (404, 404, 404, 404, 404)),
+    ('get_tag_torrent', 'GET', PUBLIC_REPO, True, dict(digest='sha256:1234'), (404, 404, 404, 404,
-  ('get_tag_torrent', 'GET', PRIVATE_REPO, True, dict(digest='sha256:1234'), (403, 403, 404, 403, 404)),
+                                                                               404)),
-  ('get_tag_torrent', 'GET', ORG_REPO, True, dict(digest='sha256:1234'), (403, 403, 404, 403, 404)),
+    ('get_tag_torrent', 'GET', PRIVATE_REPO, True, dict(digest='sha256:1234'), (403, 403, 404, 403,
-  ('get_tag_torrent', 'GET', ANOTHER_ORG_REPO, True, dict(digest='sha256:1234'), (403, 403, 403, 403, 404)),
+                                                                                404)),
-])
+    ('get_tag_torrent', 'GET', ORG_REPO, True, dict(digest='sha256:1234'), (403, 403, 404, 403,
+                                                                            404)),
+    ('get_tag_torrent', 'GET', ANOTHER_ORG_REPO, True, dict(digest='sha256:1234'), (403, 403, 403,
+                                                                                    403, 404)),])
 def test_verbs_security(user, endpoint, method, repository, single_repo_path, params,
                         expected_statuses, app, client):
   headers = {}