Merge branch 'master' into delete-setup-page

.travis.yml

@@ -48,6 +48,9 @@ jobs:
     - stage: test
       script: scripts/ci registry_old
 
+    - stage: test
+      script: scripts/ci certs_test
+
     - stage: database
       script: scripts/ci mysql
 

Makefile

@@ -60,6 +60,9 @@ registry-test-old:
 	--timeout=3600 --verbose --show-count -x \
 	./test/registry_tests.py
 
+certs-test:
+	./test/test_certs_install.sh
+
 full-db-test: ensure-test-db
 	TEST=true PYTHONPATH=. alembic upgrade head
 	TEST=true PYTHONPATH=. SKIP_DB_SCHEMA=true py.test --timeout=7200 \

buildman/component/buildcomponent.py

@@ -17,9 +17,10 @@ from buildman.jobutil.buildstatus import StatusHandler
 from buildman.jobutil.workererror import WorkerError
 
 from app import app
-from data import model
 from data.database import BUILD_PHASE, UseThenDisconnect
 from data.model import InvalidRepositoryBuildException
+from data.registry_model import registry_model
+from data.registry_model.datatypes import RepositoryReference
 from util import slash_join
 
 HEARTBEAT_DELTA = datetime.timedelta(seconds=60)
@@ -29,6 +30,9 @@ INITIAL_TIMEOUT = 25
 
 SUPPORTED_WORKER_VERSIONS = ['0.3']
 
+# Label which marks a manifest with its source build ID.
+INTERNAL_LABEL_BUILD_UUID = 'quay.build.uuid'
+
 logger = logging.getLogger(__name__)
 
 class ComponentStatus(object):
@@ -357,18 +361,17 @@ class BuildComponent(BaseComponent):
 
       # Label the pushed manifests with the build metadata.
       manifest_digests = kwargs.get('digests') or []
-      for digest in manifest_digests:
+      repository = registry_model.lookup_repository(self._current_job.namespace,
-        with UseThenDisconnect(app.config):
+                                                    self._current_job.repo_name)
-          try:
+      if repository is not None:
-            manifest = model.tag.load_manifest_by_digest(self._current_job.namespace,
+        for digest in manifest_digests:
-                                                         self._current_job.repo_name, digest)
+          with UseThenDisconnect(app.config):
-            model.label.create_manifest_label(manifest, model.label.INTERNAL_LABEL_BUILD_UUID,
+            manifest = registry_model.lookup_manifest_by_digest(repository, digest)
-                                              build_id, 'internal', 'text/plain')
+            if manifest is None:
-          except model.InvalidManifestException:
+              continue
-            logger.debug('Could not find built manifest with digest %s under repo %s/%s for build %s',
+            
-                         digest, self._current_job.namespace, self._current_job.repo_name,
+            registry_model.create_manifest_label(manifest, INTERNAL_LABEL_BUILD_UUID,
-                         build_id)
+                                                 build_id, 'internal', 'text/plain')
-            continue
 
       # Send the notification that the build has completed successfully.
       self._current_job.send_notification('build_success',

buildman/jobutil/buildjob.py

@@ -8,7 +8,6 @@ from data import model
 from data.registry_model import registry_model
 from data.registry_model.datatypes import RepositoryReference
 from data.database import UseThenDisconnect
-from util.imagetree import ImageTree
 from util.morecollections import AttrDict
 
 logger = logging.getLogger(__name__)

conf/init/certs_install.sh

@@ -1,9 +1,9 @@
 #! /bin/bash
 set -e
 QUAYPATH=${QUAYPATH:-"."}
-QUAYCONF=${QUAYCONF:-"$QUAYPATH/conf/stack"}
+QUAYCONF=${QUAYCONF:-"$QUAYPATH/conf"}
 QUAYCONFIG=${QUAYCONFIG:-"$QUAYCONF/stack"}
-CERTDIR=${QUAYCONFIG/extra_ca_certs}
+CERTDIR=${CERTDIR:-"$QUAYCONFIG/extra_ca_certs"}
 
 # If we're running under kube, the previous script (02_get_kube_certs.sh) will put the certs in a different location
 if [[ "$KUBERNETES_SERVICE_HOST" != "" ]];then
@@ -37,7 +37,7 @@ if [ -f $CERTDIR ]; then
 fi
 
 # Add extra trusted certificates (prefixed)
-for f in $(find $CERTDIR/ -maxdepth 1 -type f -name "extra_ca*")
+for f in $(find $QUAYCONFIG/ -maxdepth 1 -type f -name "extra_ca*")
 do
  echo "Installing extra cert $f"
  cp "$f" /usr/local/share/ca-certificates/

data/model/image.py

@@ -357,7 +357,11 @@ def set_image_metadata(docker_image_id, namespace_name, repository_name, created
 
 def get_image(repo, docker_image_id):
   try:
-    return Image.get(Image.docker_image_id == docker_image_id, Image.repository == repo)
+    return (Image
+            .select(Image, ImageStorage)
+            .join(ImageStorage)
+            .where(Image.docker_image_id == docker_image_id, Image.repository == repo)
+            .get())
   except Image.DoesNotExist:
     return None
 

data/model/label.py

@@ -11,9 +11,6 @@ from util.validation import is_json
 
 logger = logging.getLogger(__name__)
 
-# Label which marks a manifest with its source build ID.
-INTERNAL_LABEL_BUILD_UUID = 'quay.build.uuid'
-
 
 @lru_cache(maxsize=1)
 def get_label_source_types():

data/model/tag.py

@@ -208,8 +208,9 @@ def list_active_repo_tags(repo):
       and (if present), their manifest.
   """
   query = _tag_alive(RepositoryTag
-                     .select(RepositoryTag, Image, TagManifest.digest)
+                     .select(RepositoryTag, Image, ImageStorage, TagManifest.digest)
                      .join(Image)
+                     .join(ImageStorage)
                      .where(RepositoryTag.repository == repo, RepositoryTag.hidden == False)
                      .switch(RepositoryTag)
                      .join(TagManifest, JOIN.LEFT_OUTER))
@@ -470,8 +471,9 @@ def get_tag_image(namespace_name, repository_name, tag_name, include_storage=Fal
 
 def list_repository_tag_history(repo_obj, page=1, size=100, specific_tag=None):
   query = (RepositoryTag
-           .select(RepositoryTag, Image)
+           .select(RepositoryTag, Image, ImageStorage)
            .join(Image)
+           .join(ImageStorage)
            .switch(RepositoryTag)
            .where(RepositoryTag.repository == repo_obj)
            .where(RepositoryTag.hidden == False)
@@ -515,7 +517,7 @@ def restore_tag_to_manifest(repo_obj, tag_name, manifest_digest):
 
     # Change the tag manifest to point to the updated image.
     docker_image_id = tag_manifest.tag.image.docker_image_id
-    updated_tag = create_or_update_tag_for_repo(repo_obj.id, tag_name, docker_image_id,
+    updated_tag = create_or_update_tag_for_repo(repo_obj, tag_name, docker_image_id,
                                                 reversion=True)
     tag_manifest.tag = updated_tag
     tag_manifest.save()
@@ -544,8 +546,7 @@ def restore_tag_to_image(repo_obj, tag_name, docker_image_id):
     except DataModelException:
       existing_image = None
 
-    create_or_update_tag(repo_obj.namespace_user.username, repo_obj.name, tag_name,
+    create_or_update_tag_for_repo(repo_obj, tag_name, docker_image_id, reversion=True)
-                         docker_image_id, reversion=True)
     return existing_image
 
 
@@ -589,6 +590,16 @@ def get_active_tag(namespace, repo_name, tag_name):
                     .where(RepositoryTag.name == tag_name, Repository.name == repo_name,
                            Namespace.username == namespace)).get()
 
+def get_active_tag_for_repo(repo, tag_name):
+  try:
+    return _tag_alive(RepositoryTag
+                      .select(RepositoryTag, Image, ImageStorage)
+                      .join(Image)
+                      .join(ImageStorage)
+                      .where(RepositoryTag.name == tag_name,
+                             RepositoryTag.repository == repo)).get()
+  except RepositoryTag.DoesNotExist:
+    return None
 
 def get_possibly_expired_tag(namespace, repo_name, tag_name):
   return (RepositoryTag
@@ -641,6 +652,13 @@ def populate_manifest(repository, manifest, legacy_image, storage_ids):
     return manifest_row
 
 
+def get_tag_manifest(tag):
+  try:
+    return TagManifest.get(tag=tag)
+  except TagManifest.DoesNotExist:
+    return None
+
+
 def load_tag_manifest(namespace, repo_name, tag_name):
   try:
     return (_load_repo_manifests(namespace, repo_name)

data/registry_model/datatype.py

@@ -0,0 +1,49 @@
+# pylint: disable=protected-access
+
+from functools import wraps, total_ordering
+
+def datatype(name, static_fields):
+  """ Defines a base class for a datatype that will represent a row from the database,
+      in an abstracted form.
+  """
+  @total_ordering
+  class DataType(object):
+    __name__ = name
+
+    def __init__(self, **kwargs):
+      self._db_id = kwargs.pop('db_id', None)
+      self._inputs = kwargs.pop('inputs', None)
+      self._fields = kwargs
+
+      for name in static_fields:
+        assert name in self._fields, 'Missing field %s' % name
+
+    def __eq__(self, other):
+      return self._db_id == other._db_id
+
+    def __lt__(self, other):
+      return self._db_id < other._db_id
+
+    def __getattr__(self, name):
+      if name in static_fields:
+        return self._fields[name]
+
+      raise AttributeError('Unknown field `%s`' % name)
+
+  return DataType
+
+
+def requiresinput(input_name):
+  """ Marks a property on the data type as requiring an input to be invoked. """
+  def inner(func):
+    @wraps(func)
+    def wrapper(self, *args, **kwargs):
+      if self._inputs.get(input_name) is None:
+        raise Exception('Cannot invoke function with missing input `%s`' % input_name)
+
+      kwargs[input_name] = self._inputs[input_name]
+      result = func(self, *args, **kwargs)
+      return result
+
+    return wrapper
+  return inner

data/registry_model/datatypes.py

@@ -1,20 +1,120 @@
-from collections import namedtuple
+from enum import Enum, unique
 
-class RepositoryReference(object):
+from data.registry_model.datatype import datatype, requiresinput
+
+class RepositoryReference(datatype('Repository', [])):
   """ RepositoryReference is a reference to a repository, passed to registry interface methods. """
-  def __init__(self, repo_id):
-    self.repo_id = repo_id
-
   @classmethod
   def for_repo_obj(cls, repo_obj):
-    return RepositoryReference(repo_obj.id)
+    if repo_obj is None:
+      return None
+
+    return RepositoryReference(db_id=repo_obj.id)
 
 
-class Tag(namedtuple('Tag', ['id', 'name'])):
+class Label(datatype('Label', ['key', 'value', 'uuid', 'source_type_name', 'media_type_name'])):
+  """ Label represents a label on a manifest. """
+  @classmethod
+  def for_label(cls, label):
+    if label is None:
+      return None
+
+    return Label(db_id=label.id, key=label.key, value=label.value,
+                 uuid=label.uuid, media_type_name=label.media_type.name,
+                 source_type_name=label.source_type.name)
+
+
+class Tag(datatype('Tag', ['name', 'reversion', 'manifest_digest', 'lifetime_start_ts',
+                           'lifetime_end_ts'])):
   """ Tag represents a tag in a repository, which points to a manifest or image. """
   @classmethod
-  def for_repository_tag(cls, repository_tag):
+  def for_repository_tag(cls, repository_tag, manifest_digest=None, legacy_image=None):
     if repository_tag is None:
       return None
 
-    return Tag(id=repository_tag.id, name=repository_tag.name)
+    return Tag(db_id=repository_tag.id,
+               name=repository_tag.name,
+               reversion=repository_tag.reversion,
+               lifetime_start_ts=repository_tag.lifetime_start_ts,
+               lifetime_end_ts=repository_tag.lifetime_end_ts,
+               manifest_digest=manifest_digest,
+               inputs=dict(legacy_image=legacy_image))
+
+  @property
+  @requiresinput('legacy_image')
+  def legacy_image(self, legacy_image):
+    """ Returns the legacy Docker V1-style image for this tag. Note that this
+        will be None for tags whose manifests point to other manifests instead of images.
+    """
+    return legacy_image
+
+
+class Manifest(datatype('Manifest', ['digest', 'manifest_bytes'])):
+  """ Manifest represents a manifest in a repository. """
+  @classmethod
+  def for_tag_manifest(cls, tag_manifest, legacy_image=None):
+    if tag_manifest is None:
+      return None
+
+    return Manifest(db_id=tag_manifest.id, digest=tag_manifest.digest,
+                    manifest_bytes=tag_manifest.json_data,
+                    inputs=dict(legacy_image=legacy_image))
+
+  @property
+  @requiresinput('legacy_image')
+  def legacy_image(self, legacy_image):
+    """ Returns the legacy Docker V1-style image for this manifest. Note that this
+        will be None for manifests that point to other manifests instead of images.
+    """
+    return legacy_image
+
+
+class LegacyImage(datatype('LegacyImage', ['docker_image_id', 'created', 'comment', 'command',
+                                           'image_size', 'aggregate_size', 'uploading'])):
+  """ LegacyImage represents a Docker V1-style image found in a repository. """
+  @classmethod
+  def for_image(cls, image, images_map=None, tags_map=None):
+    if image is None:
+      return None
+
+    return LegacyImage(db_id=image.id,
+                       inputs=dict(images_map=images_map, tags_map=tags_map,
+                                   ancestor_id_list=image.ancestor_id_list()),
+                       docker_image_id=image.docker_image_id,
+                       created=image.created,
+                       comment=image.comment,
+                       command=image.command,
+                       image_size=image.storage.image_size,
+                       aggregate_size=image.aggregate_size,
+                       uploading=image.storage.uploading)
+
+  @property
+  @requiresinput('images_map')
+  @requiresinput('ancestor_id_list')
+  def parents(self, images_map, ancestor_id_list):
+    """ Returns the parent images for this image. Raises an exception if the parents have
+        not been loaded before this property is invoked.
+    """
+    return [LegacyImage.for_image(images_map[ancestor_id], images_map=images_map)
+            for ancestor_id in reversed(ancestor_id_list)
+            if images_map.get(ancestor_id)]
+
+  @property
+  @requiresinput('tags_map')
+  def tags(self, tags_map):
+    """ Returns the tags pointing to this image. Raises an exception if the tags have
+        not been loaded before this property is invoked.
+    """
+    tags = tags_map.get(self._db_id)
+    if not tags:
+      return []
+
+    return [Tag.for_repository_tag(tag) for tag in tags]
+
+
+@unique
+class SecurityScanStatus(Enum):
+  """ Security scan status enum """
+  SCANNED = 'scanned'
+  FAILED = 'failed'
+  QUEUED = 'queued'

data/registry_model/interface.py

@@ -19,3 +19,106 @@ class RegistryDataInterface(object):
     """ Returns the most recently pushed alive tag in the repository, if any. If none, returns
         None.
     """
+
+  @abstractmethod
+  def lookup_repository(self, namespace_name, repo_name, kind_filter=None):
+    """ Looks up and returns a reference to the repository with the given namespace and name,
+        or None if none. """
+
+  @abstractmethod
+  def get_manifest_for_tag(self, tag):
+    """ Returns the manifest associated with the given tag. """
+
+  @abstractmethod
+  def lookup_manifest_by_digest(self, repository_ref, manifest_digest, allow_dead=False):
+    """ Looks up the manifest with the given digest under the given repository and returns it
+        or None if none. """
+
+  @abstractmethod
+  def get_legacy_images(self, repository_ref):
+    """
+    Returns an iterator of all the LegacyImage's defined in the matching repository.
+    """
+
+  @abstractmethod
+  def get_legacy_image(self, repository_ref, docker_image_id, include_parents=False):
+    """
+    Returns the matching LegacyImages under the matching repository, if any. If none,
+    returns None.
+    """
+
+  @abstractmethod
+  def create_manifest_label(self, manifest, key, value, source_type_name, media_type_name=None):
+    """ Creates a label on the manifest with the given key and value.
+
+        Can raise InvalidLabelKeyException or InvalidMediaTypeException depending
+        on the validation errors.
+    """
+
+  @abstractmethod
+  def list_manifest_labels(self, manifest, key_prefix=None):
+    """ Returns all labels found on the manifest. If specified, the key_prefix will filter the
+        labels returned to those keys that start with the given prefix.
+    """
+
+  @abstractmethod
+  def get_manifest_label(self, manifest, label_uuid):
+    """ Returns the label with the specified UUID on the manifest or None if none. """
+
+  @abstractmethod
+  def delete_manifest_label(self, manifest, label_uuid):
+    """ Delete the label with the specified UUID on the manifest. Returns the label deleted
+        or None if none.
+    """
+
+  @abstractmethod
+  def list_repository_tags(self, repository_ref, include_legacy_images=False):
+    """
+    Returns a list of all the active tags in the repository. Note that this can be a *heavy*
+    operation on repositories with a lot of tags, and should be avoided for more targetted
+    operations wherever possible.
+    """
+
+  @abstractmethod
+  def list_repository_tag_history(self, repository_ref, page=1, size=100, specific_tag_name=None):
+    """
+    Returns the history of all tags in the repository (unless filtered). This includes tags that
+    have been made in-active due to newer versions of those tags coming into service.
+    """
+
+  @abstractmethod
+  def get_repo_tag(self, repository_ref, tag_name, include_legacy_image=False):
+    """
+    Returns the latest, *active* tag found in the repository, with the matching name
+    or None if none.
+    """
+
+  @abstractmethod
+  def retarget_tag(self, repository_ref, tag_name, manifest_or_legacy_image,
+                   is_reversion=False):
+    """
+    Creates, updates or moves a tag to a new entry in history, pointing to the manifest or
+    legacy image specified. If is_reversion is set to True, this operation is considered a
+    reversion over a previous tag move operation. Returns the updated Tag or None on error.
+    """
+
+  @abstractmethod
+  def delete_tag(self, repository_ref, tag_name):
+    """
+    Deletes the latest, *active* tag with the given name in the repository.
+    """
+
+  @abstractmethod
+  def change_repository_tag_expiration(self, tag, expiration_date):
+    """ Sets the expiration date of the tag under the matching repository to that given. If the
+        expiration date is None, then the tag will not expire. Returns a tuple of the previous
+        expiration timestamp in seconds (if any), and whether the operation succeeded.
+    """
+
+  @abstractmethod
+  def get_legacy_images_owned_by_tag(self, tag):
+    """ Returns all legacy images *solely owned and used* by the given tag. """
+
+  @abstractmethod
+  def get_security_status(self, manifest_or_legacy_image):
+    """ Returns the security status for the given manifest or legacy image or None if none. """

data/registry_model/registry_pre_oci_model.py

@@ -1,6 +1,12 @@
+# pylint: disable=protected-access
+
+from collections import defaultdict
+
+from data import database
 from data import model
 from data.registry_model.interface import RegistryDataInterface
-from data.registry_model.datatypes import Tag
+from data.registry_model.datatypes import (Tag, RepositoryReference, Manifest, LegacyImage, Label,
+                                           SecurityScanStatus)
 
 
 class PreOCIModel(RegistryDataInterface):
@@ -13,15 +19,275 @@ class PreOCIModel(RegistryDataInterface):
     """ Finds an alive tag in the repository matching one of the given tag names and returns it
         or None if none.
     """
-    found_tag = model.tag.find_matching_tag(repository_ref.repo_id, tag_names)
+    found_tag = model.tag.find_matching_tag(repository_ref._db_id, tag_names)
     return Tag.for_repository_tag(found_tag)
 
   def get_most_recent_tag(self, repository_ref):
     """ Returns the most recently pushed alive tag in the repository, if any. If none, returns
         None.
     """
-    found_tag = model.tag.get_most_recent_tag(repository_ref.repo_id)
+    found_tag = model.tag.get_most_recent_tag(repository_ref._db_id)
     return Tag.for_repository_tag(found_tag)
 
+  def lookup_repository(self, namespace_name, repo_name, kind_filter=None):
+    """ Looks up and returns a reference to the repository with the given namespace and name,
+        or None if none. """
+    repo = model.repository.get_repository(namespace_name, repo_name, kind_filter=kind_filter)
+    return RepositoryReference.for_repo_obj(repo)
+
+  def get_manifest_for_tag(self, tag):
+    """ Returns the manifest associated with the given tag. """
+    try:
+      tag_manifest = database.TagManifest.get(tag_id=tag._db_id)
+    except database.TagManifest.DoesNotExist:
+      return
+
+    return Manifest.for_tag_manifest(tag_manifest)
+
+  def lookup_manifest_by_digest(self, repository_ref, manifest_digest, allow_dead=False,
+                                include_legacy_image=False):
+    """ Looks up the manifest with the given digest under the given repository and returns it
+        or None if none. """
+    repo = model.repository.lookup_repository(repository_ref._db_id)
+    if repo is None:
+      return None
+
+    try:
+      tag_manifest = model.tag.load_manifest_by_digest(repo.namespace_user.username,
+                                                       repo.name,
+                                                       manifest_digest,
+                                                       allow_dead=allow_dead)
+    except model.tag.InvalidManifestException:
+      return None
+
+    legacy_image = None
+    if include_legacy_image:
+      legacy_image = self.get_legacy_image(repository_ref, tag_manifest.tag.image.docker_image_id,
+                                           include_parents=True)
+
+    return Manifest.for_tag_manifest(tag_manifest, legacy_image)
+
+  def get_legacy_images(self, repository_ref):
+    """
+    Returns an iterator of all the LegacyImage's defined in the matching repository.
+    """
+    repo = model.repository.lookup_repository(repository_ref._db_id)
+    if repo is None:
+      return None
+
+    all_images = model.image.get_repository_images_without_placements(repo)
+    all_images_map = {image.id: image for image in all_images}
+
+    all_tags = model.tag.list_repository_tags(repo.namespace_user.username, repo.name)
+    tags_by_image_id = defaultdict(list)
+    for tag in all_tags:
+      tags_by_image_id[tag.image_id].append(tag)
+
+    return [LegacyImage.for_image(image, images_map=all_images_map, tags_map=tags_by_image_id)
+            for image in all_images]
+
+  def get_legacy_image(self, repository_ref, docker_image_id, include_parents=False):
+    """
+    Returns the matching LegacyImages under the matching repository, if any. If none,
+    returns None.
+    """
+    repo = model.repository.lookup_repository(repository_ref._db_id)
+    if repo is None:
+      return None
+
+    image = model.image.get_image(repository_ref._db_id, docker_image_id)
+    if image is None:
+      return None
+
+    parent_images_map = None
+    if include_parents:
+      parent_images = model.image.get_parent_images(repo.namespace_user.username, repo.name, image)
+      parent_images_map = {image.id: image for image in parent_images}
+
+    return LegacyImage.for_image(image, images_map=parent_images_map)
+
+  def create_manifest_label(self, manifest, key, value, source_type_name, media_type_name=None):
+    """ Creates a label on the manifest with the given key and value. """
+    try:
+      tag_manifest = database.TagManifest.get(id=manifest._db_id)
+    except database.TagManifest.DoesNotExist:
+      return None
+
+    label = model.label.create_manifest_label(tag_manifest, key, value, source_type_name,
+                                              media_type_name)
+    return Label.for_label(label)
+
+  def list_manifest_labels(self, manifest, key_prefix=None):
+    """ Returns all labels found on the manifest. If specified, the key_prefix will filter the
+        labels returned to those keys that start with the given prefix.
+    """
+    labels = model.label.list_manifest_labels(manifest._db_id, prefix_filter=key_prefix)
+    return [Label.for_label(l) for l in labels]
+
+  def get_manifest_label(self, manifest, label_uuid):
+    """ Returns the label with the specified UUID on the manifest or None if none. """
+    return Label.for_label(model.label.get_manifest_label(label_uuid, manifest._db_id))
+
+  def delete_manifest_label(self, manifest, label_uuid):
+    """ Delete the label with the specified UUID on the manifest. Returns the label deleted
+        or None if none.
+    """
+    return Label.for_label(model.label.delete_manifest_label(label_uuid, manifest._db_id))
+
+  def list_repository_tags(self, repository_ref, include_legacy_images=False):
+    """
+    Returns a list of all the active tags in the repository. Note that this can be a *heavy*
+    operation on repositories with a lot of tags, and should be avoided for more targetted
+    operations wherever possible.
+    """
+    # NOTE: include_legacy_images isn't used here because `list_active_repo_tags` includes the
+    # information already, so we might as well just use it. However, the new model classes will
+    # *not* include it by default, so we make it a parameter now.
+    tags = model.tag.list_active_repo_tags(repository_ref._db_id)
+    return [Tag.for_repository_tag(tag,
+                                   legacy_image=LegacyImage.for_image(tag.image),
+                                   manifest_digest=(tag.tagmanifest.digest
+                                                    if hasattr(tag, 'tagmanifest')
+                                                    else None))
+            for tag in tags]
+
+  def list_repository_tag_history(self, repository_ref, page=1, size=100, specific_tag_name=None):
+    """
+    Returns the history of all tags in the repository (unless filtered). This includes tags that
+    have been made in-active due to newer versions of those tags coming into service.
+    """
+    tags, manifest_map, has_more = model.tag.list_repository_tag_history(repository_ref._db_id,
+                                                                         page, size,
+                                                                         specific_tag_name)
+    return [Tag.for_repository_tag(tag, manifest_map.get(tag.id),
+                                   legacy_image=LegacyImage.for_image(tag.image))
+            for tag in tags], has_more
+
+  def get_repo_tag(self, repository_ref, tag_name, include_legacy_image=False):
+    """
+    Returns the latest, *active* tag found in the repository, with the matching name
+    or None if none.
+    """
+    tag = model.tag.get_active_tag_for_repo(repository_ref._db_id, tag_name)
+    if tag is None:
+      return None
+
+    legacy_image = LegacyImage.for_image(tag.image) if include_legacy_image else None
+    tag_manifest = model.tag.get_tag_manifest(tag)
+    manifest_digest = tag_manifest.digest if tag_manifest else None
+    return Tag.for_repository_tag(tag, legacy_image=legacy_image, manifest_digest=manifest_digest)
+
+  def retarget_tag(self, repository_ref, tag_name, manifest_or_legacy_image,
+                   is_reversion=False):
+    """
+    Creates, updates or moves a tag to a new entry in history, pointing to the manifest or
+    legacy image specified. If is_reversion is set to True, this operation is considered a
+    reversion over a previous tag move operation. Returns the updated Tag or None on error.
+    """
+    # TODO: unify this.
+    if not is_reversion:
+      if isinstance(manifest_or_legacy_image, Manifest):
+        raise NotImplementedError('Not yet implemented')
+      else:
+        model.tag.create_or_update_tag_for_repo(repository_ref._db_id, tag_name,
+                                                manifest_or_legacy_image.docker_image_id)
+    else:
+      if isinstance(manifest_or_legacy_image, Manifest):
+        image = model.tag.restore_tag_to_manifest(repository_ref._db_id, tag_name,
+                                                  manifest_or_legacy_image.digest)
+        if image is None:
+          return None
+      else:
+        image = model.tag.restore_tag_to_image(repository_ref._db_id, tag_name,
+                                               manifest_or_legacy_image.docker_image_id)
+        if image is None:
+          return None
+
+    return self.get_repo_tag(repository_ref, tag_name, include_legacy_image=True)
+
+  def delete_tag(self, repository_ref, tag_name):
+    """
+    Deletes the latest, *active* tag with the given name in the repository.
+    """
+    repo = model.repository.lookup_repository(repository_ref._db_id)
+    if repo is None:
+      return None
+
+    deleted_tag = model.tag.delete_tag(repo.namespace_user.username, repo.name, tag_name)
+    return Tag.for_repository_tag(deleted_tag)
+
+  def change_repository_tag_expiration(self, tag, expiration_date):
+    """ Sets the expiration date of the tag under the matching repository to that given. If the
+        expiration date is None, then the tag will not expire. Returns a tuple of the previous
+        expiration timestamp in seconds (if any), and whether the operation succeeded.
+    """
+    try:
+      tag_obj = database.RepositoryTag.get(id=tag._db_id)
+    except database.RepositoryTag.DoesNotExist:
+      return (None, False)
+
+    return model.tag.change_tag_expiration(tag_obj, expiration_date)
+
+  def get_legacy_images_owned_by_tag(self, tag):
+    """ Returns all legacy images *solely owned and used* by the given tag. """
+    try:
+      tag_obj = database.RepositoryTag.get(id=tag._db_id)
+    except database.RepositoryTag.DoesNotExist:
+      return None
+
+    # Collect the IDs of all images that the tag uses.
+    tag_image_ids = set()
+    tag_image_ids.add(tag_obj.image.id)
+    tag_image_ids.update(tag_obj.image.ancestor_id_list())
+
+    # Remove any images shared by other tags.
+    for current_tag in model.tag.list_active_repo_tags(tag_obj.repository_id):
+      if current_tag == tag_obj:
+        continue
+
+      tag_image_ids.discard(current_tag.image.id)
+      tag_image_ids = tag_image_ids.difference(current_tag.image.ancestor_id_list())
+      if not tag_image_ids:
+        return []
+
+    if not tag_image_ids:
+      return []
+
+    # Load the images we need to return.
+    images = database.Image.select().where(database.Image.id << list(tag_image_ids))
+    all_image_ids = set()
+    for image in images:
+      all_image_ids.add(image.id)
+      all_image_ids.update(image.ancestor_id_list())
+
+    # Build a map of all the images and their parents.
+    images_map = {}
+    all_images = database.Image.select().where(database.Image.id << list(all_image_ids))
+    for image in all_images:
+      images_map[image.id] = image
+
+    return [LegacyImage.for_image(image, images_map=images_map) for image in images]
+
+  def get_security_status(self, manifest_or_legacy_image):
+    """ Returns the security status for the given manifest or legacy image or None if none. """
+    image = None
+
+    if isinstance(manifest_or_legacy_image, Manifest):
+      try:
+        tag_manifest = database.TagManifest.get(id=manifest_or_legacy_image._db_id)
+        image = tag_manifest.tag.image
+      except database.TagManifest.DoesNotExist:
+        return None
+    else:
+      try:
+        image = database.Image.get(id=manifest_or_legacy_image._db_id)
+      except database.Image.DoesNotExist:
+        return None
+
+    if image.security_indexed_engine is not None and image.security_indexed_engine >= 0:
+      return SecurityScanStatus.SCANNED if image.security_indexed else SecurityScanStatus.FAILED
+
+    return SecurityScanStatus.QUEUED
+
 
 pre_oci_model = PreOCIModel()

data/registry_model/test/test_pre_oci_model.py

@@ -1,8 +1,13 @@
+from datetime import datetime, timedelta
+
 import pytest
 
+from playhouse.test_utils import assert_query_count
+
 from data import model
 from data.registry_model.registry_pre_oci_model import PreOCIModel
 from data.registry_model.datatypes import RepositoryReference
+
 from test.fixtures import *
 
 @pytest.fixture()
@@ -28,7 +33,7 @@ def test_find_matching_tag(names, expected, pre_oci_model):
 
 
 @pytest.mark.parametrize('repo_namespace, repo_name, expected', [
-  ('devtable', 'simple', {'latest'}),
+  ('devtable', 'simple', {'latest', 'prod'}),
   ('buynlarge', 'orgrepo', {'latest', 'prod'}),
 ])
 def test_get_most_recent_tag(repo_namespace, repo_name, expected, pre_oci_model):
@@ -39,3 +44,274 @@ def test_get_most_recent_tag(repo_namespace, repo_name, expected, pre_oci_model)
     assert found is None
   else:
     assert found.name in expected
+
+
+@pytest.mark.parametrize('repo_namespace, repo_name, expected', [
+  ('devtable', 'simple', True),
+  ('buynlarge', 'orgrepo', True),
+  ('buynlarge', 'unknownrepo', False),
+])
+def test_lookup_repository(repo_namespace, repo_name, expected, pre_oci_model):
+  repo_ref = pre_oci_model.lookup_repository(repo_namespace, repo_name)
+  if expected:
+    assert repo_ref
+  else:
+    assert repo_ref is None
+
+
+@pytest.mark.parametrize('repo_namespace, repo_name', [
+  ('devtable', 'simple'),
+  ('buynlarge', 'orgrepo'),
+])
+def test_lookup_manifests(repo_namespace, repo_name, pre_oci_model):
+  repo = model.repository.get_repository(repo_namespace, repo_name)
+  repository_ref = RepositoryReference.for_repo_obj(repo)
+  found_tag = pre_oci_model.find_matching_tag(repository_ref, ['latest'])
+  found_manifest = pre_oci_model.get_manifest_for_tag(found_tag)
+  found = pre_oci_model.lookup_manifest_by_digest(repository_ref, found_manifest.digest,
+                                                  include_legacy_image=True)
+  assert found._db_id == found_manifest._db_id
+  assert found.digest == found_manifest.digest
+  assert found.legacy_image
+
+
+def test_lookup_unknown_manifest(pre_oci_model):
+  repo = model.repository.get_repository('devtable', 'simple')
+  repository_ref = RepositoryReference.for_repo_obj(repo)
+  found = pre_oci_model.lookup_manifest_by_digest(repository_ref, 'sha256:deadbeef')
+  assert found is None
+
+
+@pytest.mark.parametrize('repo_namespace, repo_name', [
+  ('devtable', 'simple'),
+  ('devtable', 'complex'),
+  ('devtable', 'history'),
+  ('buynlarge', 'orgrepo'),
+])
+def test_legacy_images(repo_namespace, repo_name, pre_oci_model):
+  repository_ref = pre_oci_model.lookup_repository(repo_namespace, repo_name)
+  legacy_images = pre_oci_model.get_legacy_images(repository_ref)
+  assert len(legacy_images)
+
+  found_tags = set()
+  for image in legacy_images:
+    found_image = pre_oci_model.get_legacy_image(repository_ref, image.docker_image_id,
+                                                 include_parents=True)
+
+    with assert_query_count(4 if found_image.parents else 3):
+      found_image = pre_oci_model.get_legacy_image(repository_ref, image.docker_image_id,
+                                                   include_parents=True)
+      assert found_image.docker_image_id == image.docker_image_id
+      assert found_image.parents == image.parents
+
+    # Check that the tags list can be retrieved.
+    assert image.tags is not None
+    found_tags.update({tag.name for tag in image.tags})
+
+    # Check against the actual DB row.
+    model_image = model.image.get_image(repository_ref._db_id, found_image.docker_image_id)
+    assert model_image.id == found_image._db_id
+    assert ([pid for pid in reversed(model_image.ancestor_id_list())] ==
+            [p._db_id for p in found_image.parents])
+
+    # Try without parents and ensure it raises an exception.
+    found_image = pre_oci_model.get_legacy_image(repository_ref, image.docker_image_id,
+                                                 include_parents=False)
+    with pytest.raises(Exception):
+      assert not found_image.parents
+
+  assert found_tags
+
+  unknown = pre_oci_model.get_legacy_image(repository_ref, 'unknown', include_parents=True)
+  assert unknown is None
+
+
+def test_manifest_labels(pre_oci_model):
+  repo = model.repository.get_repository('devtable', 'simple')
+  repository_ref = RepositoryReference.for_repo_obj(repo)
+  found_tag = pre_oci_model.find_matching_tag(repository_ref, ['latest'])
+  found_manifest = pre_oci_model.get_manifest_for_tag(found_tag)
+
+  # Create a new label.
+  created = pre_oci_model.create_manifest_label(found_manifest, 'foo', 'bar', 'api')
+  assert created.key == 'foo'
+  assert created.value == 'bar'
+  assert created.source_type_name == 'api'
+  assert created.media_type_name == 'text/plain'
+
+  # Ensure we can look it up.
+  assert pre_oci_model.get_manifest_label(found_manifest, created.uuid) == created
+
+  # Ensure it is in our list of labels.
+  assert created in pre_oci_model.list_manifest_labels(found_manifest)
+  assert created in pre_oci_model.list_manifest_labels(found_manifest, key_prefix='fo')
+
+  # Ensure it is *not* in our filtered list.
+  assert created not in pre_oci_model.list_manifest_labels(found_manifest, key_prefix='ba')
+
+  # Delete the label and ensure it is gone.
+  assert pre_oci_model.delete_manifest_label(found_manifest, created.uuid)
+  assert pre_oci_model.get_manifest_label(found_manifest, created.uuid) is None
+  assert created not in pre_oci_model.list_manifest_labels(found_manifest)
+
+
+@pytest.mark.parametrize('repo_namespace, repo_name', [
+  ('devtable', 'simple'),
+  ('devtable', 'complex'),
+  ('devtable', 'history'),
+  ('buynlarge', 'orgrepo'),
+])
+def test_repository_tags(repo_namespace, repo_name, pre_oci_model):
+  repository_ref = pre_oci_model.lookup_repository(repo_namespace, repo_name)
+
+  with assert_query_count(1):
+    tags = pre_oci_model.list_repository_tags(repository_ref, include_legacy_images=True)
+    assert len(tags)
+
+  for tag in tags:
+    with assert_query_count(2):
+      found_tag = pre_oci_model.get_repo_tag(repository_ref, tag.name, include_legacy_image=True)
+      assert found_tag == tag
+
+    if found_tag.legacy_image is None:
+      continue
+
+    with assert_query_count(2):
+      found_image = pre_oci_model.get_legacy_image(repository_ref,
+                                                   found_tag.legacy_image.docker_image_id)
+      assert found_image == found_tag.legacy_image
+
+
+def test_repository_tag_history(pre_oci_model):
+  repository_ref = pre_oci_model.lookup_repository('devtable', 'history')
+
+  with assert_query_count(2):
+    history, has_more = pre_oci_model.list_repository_tag_history(repository_ref)
+    assert not has_more
+    assert len(history) == 2
+
+
+@pytest.mark.parametrize('repo_namespace, repo_name', [
+  ('devtable', 'simple'),
+  ('devtable', 'complex'),
+  ('devtable', 'history'),
+  ('buynlarge', 'orgrepo'),
+])
+def test_delete_tags(repo_namespace, repo_name, pre_oci_model):
+  repository_ref = pre_oci_model.lookup_repository(repo_namespace, repo_name)
+  tags = pre_oci_model.list_repository_tags(repository_ref)
+  assert len(tags)
+
+  # Save history before the deletions.
+  previous_history, _ = pre_oci_model.list_repository_tag_history(repository_ref, size=1000)
+  assert len(previous_history) >= len(tags)
+
+  # Delete every tag in the repository.
+  for tag in tags:
+    assert pre_oci_model.delete_tag(repository_ref, tag.name)
+
+    # Make sure the tag is no longer found.
+    with assert_query_count(1):
+      found_tag = pre_oci_model.get_repo_tag(repository_ref, tag.name, include_legacy_image=True)
+      assert found_tag is None
+
+  # Ensure all tags have been deleted.
+  tags = pre_oci_model.list_repository_tags(repository_ref)
+  assert not len(tags)
+
+  # Ensure that the tags all live in history.
+  history, _ = pre_oci_model.list_repository_tag_history(repository_ref, size=1000)
+  assert len(history) == len(previous_history)
+
+
+@pytest.mark.parametrize('use_manifest', [
+  True,
+  False,
+])
+def test_retarget_tag_history(use_manifest, pre_oci_model):
+  repository_ref = pre_oci_model.lookup_repository('devtable', 'history')
+  history, _ = pre_oci_model.list_repository_tag_history(repository_ref)
+
+  if use_manifest:
+    manifest_or_legacy_image = pre_oci_model.lookup_manifest_by_digest(repository_ref,
+                                                                       history[1].manifest_digest,
+                                                                       allow_dead=True)
+  else:
+    manifest_or_legacy_image = history[1].legacy_image
+
+  # Retarget the tag.
+  assert manifest_or_legacy_image
+  updated_tag = pre_oci_model.retarget_tag(repository_ref, 'latest', manifest_or_legacy_image,
+                                           is_reversion=True)
+
+  # Ensure the tag has changed targets.
+  if use_manifest:
+    assert updated_tag.manifest_digest == manifest_or_legacy_image.digest
+  else:
+    assert updated_tag.legacy_image == manifest_or_legacy_image
+
+  # Ensure history has been updated.
+  new_history, _ = pre_oci_model.list_repository_tag_history(repository_ref)
+  assert len(new_history) == len(history) + 1
+
+
+def test_retarget_tag(pre_oci_model):
+  repository_ref = pre_oci_model.lookup_repository('devtable', 'complex')
+  history, _ = pre_oci_model.list_repository_tag_history(repository_ref)
+
+  prod_tag = pre_oci_model.get_repo_tag(repository_ref, 'prod', include_legacy_image=True)
+
+  # Retarget the tag.
+  updated_tag = pre_oci_model.retarget_tag(repository_ref, 'latest', prod_tag.legacy_image)
+
+  # Ensure the tag has changed targets.
+  assert updated_tag.legacy_image == prod_tag.legacy_image
+
+  # Ensure history has been updated.
+  new_history, _ = pre_oci_model.list_repository_tag_history(repository_ref)
+  assert len(new_history) == len(history) + 1
+
+
+def test_change_repository_tag_expiration(pre_oci_model):
+  repository_ref = pre_oci_model.lookup_repository('devtable', 'simple')
+  tag = pre_oci_model.get_repo_tag(repository_ref, 'latest')
+  assert tag.lifetime_end_ts is None
+
+  new_datetime = datetime.utcnow() + timedelta(days=2)
+  previous, okay = pre_oci_model.change_repository_tag_expiration(tag, new_datetime)
+
+  assert okay
+  assert previous is None
+
+  tag = pre_oci_model.get_repo_tag(repository_ref, 'latest')
+  assert tag.lifetime_end_ts is not None
+
+
+@pytest.mark.parametrize('repo_namespace, repo_name, expected_non_empty', [
+  ('devtable', 'simple', []),
+  ('devtable', 'complex', ['prod', 'v2.0']),
+  ('devtable', 'history', ['latest']),
+  ('buynlarge', 'orgrepo', []),
+  ('devtable', 'gargantuan', ['v2.0', 'v3.0', 'v4.0', 'v5.0', 'v6.0']),
+])
+def test_get_legacy_images_owned_by_tag(repo_namespace, repo_name, expected_non_empty,
+                                        pre_oci_model):
+  repository_ref = pre_oci_model.lookup_repository(repo_namespace, repo_name)
+  tags = pre_oci_model.list_repository_tags(repository_ref)
+  assert len(tags)
+
+  non_empty = set()
+  for tag in tags:
+    if pre_oci_model.get_legacy_images_owned_by_tag(tag):
+      non_empty.add(tag.name)
+
+  assert non_empty == set(expected_non_empty)
+
+
+def test_get_security_status(pre_oci_model):
+  repository_ref = pre_oci_model.lookup_repository('devtable', 'simple')
+  tags = pre_oci_model.list_repository_tags(repository_ref, include_legacy_images=True)
+  assert len(tags)
+
+  for tag in tags:
+    assert pre_oci_model.get_security_status(tag.legacy_image)

endpoints/api/image.py

@@ -1,11 +1,35 @@
 """ List and lookup repository images. """
+import json
 
+from data.registry_model import registry_model
 from endpoints.api import (resource, nickname, require_repo_read, RepositoryParamResource,
-                           path_param, disallow_for_app_repositories)
+                           path_param, disallow_for_app_repositories, format_date)
-from endpoints.api.image_models_pre_oci import pre_oci_model as model
 from endpoints.exception import NotFound
 
 
+def image_dict(image, with_history=False, with_tags=False):
+  image_data = {
+    'id': image.docker_image_id,
+    'created': format_date(image.created),
+    'comment': image.comment,
+    'command': json.loads(image.command) if image.command else None,
+    'size': image.image_size,
+    'uploading': image.uploading,
+    'sort_index': len(image.parents),
+  }
+
+  if with_tags:
+    image_data['tags'] = [tag.name for tag in image.tags]
+
+  if with_history:
+    image_data['history'] = [image_dict(parent) for parent in image.parents]
+
+  # Calculate the ancestors string, with the DBID's replaced with the docker IDs.
+  parent_docker_ids = [parent_image.docker_image_id for parent_image in image.parents]
+  image_data['ancestors'] = '/{0}/'.format('/'.join(parent_docker_ids))
+  return image_data
+
+
 @resource('/v1/repository/<apirepopath:repository>/image/')
 @path_param('repository', 'The full path of the repository. e.g. namespace/name')
 class RepositoryImageList(RepositoryParamResource):
@@ -16,11 +40,12 @@ class RepositoryImageList(RepositoryParamResource):
   @disallow_for_app_repositories
   def get(self, namespace, repository):
     """ List the images for the specified repository. """
-    images = model.get_repository_images(namespace, repository)
+    repo_ref = registry_model.lookup_repository(namespace, repository)
-    if images is None:
+    if repo_ref is None:
       raise NotFound()
 
-    return {'images': [image.to_dict() for image in images]}
+    images = registry_model.get_legacy_images(repo_ref)
+    return {'images': [image_dict(image, with_tags=True) for image in images]}
 
 
 @resource('/v1/repository/<apirepopath:repository>/image/<image_id>')
@@ -34,8 +59,12 @@ class RepositoryImage(RepositoryParamResource):
   @disallow_for_app_repositories
   def get(self, namespace, repository, image_id):
     """ Get the information available for the specified image. """
-    image = model.get_repository_image(namespace, repository, image_id)
+    repo_ref = registry_model.lookup_repository(namespace, repository)
+    if repo_ref is None:
+      raise NotFound()
+
+    image = registry_model.get_legacy_image(repo_ref, image_id, include_parents=True)
     if image is None:
       raise NotFound()
 
-    return image.to_dict()
+    return image_dict(image, with_history=True)

endpoints/api/image_models_interface.py

@@ -1,77 +0,0 @@
-import json
-
-from endpoints.api import format_date
-
-from abc import ABCMeta, abstractmethod
-from collections import namedtuple
-from six import add_metaclass
-
-class Image(namedtuple('Image', ['docker_image_id', 'created', 'comment', 'command', 'image_size',
-                                 'uploading', 'parents'])):
-  """
-  Image represents an image.
-  :type name: string
-  """
-
-  def to_dict(self):
-    image_data = {
-      'id': self.docker_image_id,
-      'created': format_date(self.created),
-      'comment': self.comment,
-      'command': json.loads(self.command) if self.command else None,
-      'size': self.image_size,
-      'uploading': self.uploading,
-      'sort_index': len(self.parents),
-    }
-
-    # Calculate the ancestors string, with the DBID's replaced with the docker IDs.
-    parent_docker_ids = [parent_image.docker_image_id for parent_image in self.parents]
-    image_data['ancestors'] = '/{0}/'.format('/'.join(parent_docker_ids))
-
-    return image_data
-
-class ImageWithTags(namedtuple('ImageWithTags', ['image', 'tag_names'])):
-  """
-  ImageWithTags represents an image, along with the tags that point to it.
-  :type image: Image
-  :type tag_names: list of string
-  """
-
-  def to_dict(self):
-    image_dict = self.image.to_dict()
-    image_dict['tags'] = self.tag_names
-    return image_dict
-
-
-class ImageWithHistory(namedtuple('ImageWithHistory', ['image'])):
-  """
-  ImageWithHistory represents an image, along with its full parent image dictionaries.
-  :type image: Image
-  :type history: list of Image parents (name is old and must be kept for compat)
-  """
-
-  def to_dict(self):
-    image_dict = self.image.to_dict()
-    image_dict['history'] = [parent_image.to_dict() for parent_image in self.image.parents]
-    return image_dict
-
-
-@add_metaclass(ABCMeta)
-class ImageInterface(object):
-  """
-  Interface that represents all data store interactions required by the image API endpoint.
-  """
-
-  @abstractmethod
-  def get_repository_images(self, namespace_name, repo_name):
-    """
-    Returns an iterator of all the ImageWithTag's defined in the matching repository. If the
-    repository doesn't exist, returns None.
-    """
-
-  @abstractmethod
-  def get_repository_image(self, namespace_name, repo_name, docker_image_id):
-    """
-    Returns the matching ImageWithHistory under the matching repository, if any. If none,
-    returns None.
-    """

endpoints/api/image_models_pre_oci.py

@@ -1,56 +0,0 @@
-from collections import defaultdict
-from data import model
-from endpoints.api.image_models_interface import (ImageInterface, ImageWithHistory, ImageWithTags,
-                                                  Image)
-
-def _image(namespace_name, repo_name, image, all_images, include_parents=True):
-  parent_image_tuples = []
-  if include_parents:
-    parent_images = [all_images[ancestor_id] for ancestor_id in image.ancestor_id_list()
-                     if all_images.get(ancestor_id)]
-    parent_image_tuples = [_image(namespace_name, repo_name, parent_image, all_images, False)
-                           for parent_image in parent_images]
-
-  return Image(image.docker_image_id, image.created, image.comment, image.command,
-               image.storage.image_size, image.storage.uploading, parent_image_tuples)
-
-def _tag_names(image, tags_by_docker_id):
-  return [tag.name for tag in tags_by_docker_id.get(image.docker_image_id, [])]
-
-
-class PreOCIModel(ImageInterface):
-  """
-  PreOCIModel implements the data model for the Image API using a database schema
-  before it was changed to support the OCI specification.
-  """
-  def get_repository_images(self, namespace_name, repo_name):
-    repo = model.repository.get_repository(namespace_name, repo_name)
-    if not repo:
-      return None
-
-    all_images = model.image.get_repository_images_without_placements(repo)
-    all_images_map = {image.id: image for image in all_images}
-
-    all_tags = list(model.tag.list_repository_tags(namespace_name, repo_name))
-
-    tags_by_docker_id = defaultdict(list)
-    for tag in all_tags:
-      tags_by_docker_id[tag.image.docker_image_id].append(tag)
-
-    def _build_image(image):
-      image_itself = _image(namespace_name, repo_name, image, all_images_map)
-      return ImageWithTags(image_itself, tag_names=_tag_names(image, tags_by_docker_id))
-
-    return [_build_image(image) for image in all_images]
-
-  def get_repository_image(self, namespace_name, repo_name, docker_image_id):
-    image = model.image.get_repo_image_and_storage(namespace_name, repo_name, docker_image_id)
-    if not image:
-      return None
-
-    parent_images = model.image.get_parent_images(namespace_name, repo_name, image)
-    all_images_map = {image.id: image for image in parent_images}
-    return ImageWithHistory(_image(namespace_name, repo_name, image, all_images_map))
-
-
-pre_oci_model = PreOCIModel()

endpoints/api/manifest.py

@@ -1,23 +1,43 @@
 """ Manage the manifests of a repository. """
-import json
+from flask import request
 
 from app import label_validator
-from flask import request
+from data.model import InvalidLabelKeyException, InvalidMediaTypeException
+from data.registry_model import registry_model
+from digest import digest_tools
 from endpoints.api import (resource, nickname, require_repo_read, require_repo_write,
                            RepositoryParamResource, log_action, validate_json_request,
                            path_param, parse_args, query_param, abort, api,
                            disallow_for_app_repositories)
+from endpoints.api.image import image_dict
 from endpoints.exception import NotFound
-from manifest_models_pre_oci import pre_oci_model as model
-from data.model import InvalidLabelKeyException, InvalidMediaTypeException
-
-from digest import digest_tools
 from util.validation import VALID_LABEL_KEY_REGEX
 
+
 BASE_MANIFEST_ROUTE = '/v1/repository/<apirepopath:repository>/manifest/<regex("{0}"):manifestref>'
 MANIFEST_DIGEST_ROUTE = BASE_MANIFEST_ROUTE.format(digest_tools.DIGEST_PATTERN)
 ALLOWED_LABEL_MEDIA_TYPES = ['text/plain', 'application/json']
 
+def _label_dict(label):
+  return {
+    'id': label.uuid,
+    'key': label.key,
+    'value': label.value,
+    'source_type': label.source_type_name,
+    'media_type': label.media_type_name,
+  }
+
+def _manifest_dict(manifest):
+  image = None
+  if manifest.legacy_image is not None:
+    image = image_dict(manifest.legacy_image, with_history=True)
+
+  return {
+    'digest': manifest.digest,
+    'manifest_data': manifest.manifest_bytes,
+    'image': image,
+  }
+
 
 @resource(MANIFEST_DIGEST_ROUTE)
 @path_param('repository', 'The full path of the repository. e.g. namespace/name')
@@ -28,11 +48,16 @@ class RepositoryManifest(RepositoryParamResource):
   @nickname('getRepoManifest')
   @disallow_for_app_repositories
   def get(self, namespace_name, repository_name, manifestref):
-    manifest = model.get_repository_manifest(namespace_name, repository_name, manifestref)
+    repo_ref = registry_model.lookup_repository(namespace_name, repository_name)
+    if repo_ref is None:
+      raise NotFound()
+
+    manifest = registry_model.lookup_manifest_by_digest(repo_ref, manifestref,
+                                                        include_legacy_image=True)
     if manifest is None:
       raise NotFound()
 
-    return manifest.to_dict()
+    return _manifest_dict(manifest)
 
 
 @resource(MANIFEST_DIGEST_ROUTE + '/labels')
@@ -74,11 +99,20 @@ class RepositoryManifestLabels(RepositoryParamResource):
   @query_param('filter', 'If specified, only labels matching the given prefix will be returned',
                type=str, default=None)
   def get(self, namespace_name, repository_name, manifestref, parsed_args):
-    labels = model.get_manifest_labels(namespace_name, repository_name, manifestref, filter=parsed_args['filter'])
+    repo_ref = registry_model.lookup_repository(namespace_name, repository_name)
+    if repo_ref is None:
+      raise NotFound()
+
+    manifest = registry_model.lookup_manifest_by_digest(repo_ref, manifestref)
+    if manifest is None:
+      raise NotFound()
+
+    labels = registry_model.list_manifest_labels(manifest, parsed_args['filter'])
     if labels is None:
       raise NotFound()
+
     return {
-      'labels': [label.to_dict() for label in labels]
+      'labels': [_label_dict(label) for label in labels]
     }
 
   @require_repo_write
@@ -93,24 +127,32 @@ class RepositoryManifestLabels(RepositoryParamResource):
     if label_validator.has_reserved_prefix(label_data['key']):
       abort(400, message='Label has a reserved prefix')
 
+    repo_ref = registry_model.lookup_repository(namespace_name, repository_name)
+    if repo_ref is None:
+      raise NotFound()
+
+    manifest = registry_model.lookup_manifest_by_digest(repo_ref, manifestref)
+    if manifest is None:
+      raise NotFound()
+
     label = None
     try:
-      label = model.create_manifest_label(namespace_name, 
+      label = registry_model.create_manifest_label(manifest,
-                                          repository_name, 
+                                                   label_data['key'],
-                                          manifestref, 
+                                                   label_data['value'],
-                                          label_data['key'], 
+                                                   'api',
-                                          label_data['value'], 
+                                                   label_data['media_type'])
-                                          'api', 
-                                          label_data['media_type'])
     except InvalidLabelKeyException:
-      abort(400, message='Label is of an invalid format or missing please use %s format for labels'.format(
+      message = ('Label is of an invalid format or missing please ' +
-            VALID_LABEL_KEY_REGEX))
+                 'use %s format for labels' % VALID_LABEL_KEY_REGEX)
+      abort(400, message=message)
     except InvalidMediaTypeException:
-      abort(400, message='Media type is invalid please use a valid media type of text/plain or application/json')
+      message = 'Media type is invalid please use a valid media type: text/plain, application/json'
+      abort(400, message=message)
 
     if label is None:
       raise NotFound()
-    
+
     metadata = {
       'id': label.uuid,
       'key': label.key,
@@ -123,7 +165,7 @@ class RepositoryManifestLabels(RepositoryParamResource):
 
     log_action('manifest_label_add', namespace_name, metadata, repo_name=repository_name)
 
-    resp = {'label': label.to_dict()}
+    resp = {'label': _label_dict(label)}
     repo_string = '%s/%s' % (namespace_name, repository_name)
     headers = {
       'Location': api.url_for(ManageRepositoryManifestLabel, repository=repo_string,
@@ -143,11 +185,19 @@ class ManageRepositoryManifestLabel(RepositoryParamResource):
   @disallow_for_app_repositories
   def get(self, namespace_name, repository_name, manifestref, labelid):
     """ Retrieves the label with the specific ID under the manifest. """
-    label = model.get_manifest_label(namespace_name, repository_name, manifestref, labelid)
+    repo_ref = registry_model.lookup_repository(namespace_name, repository_name)
+    if repo_ref is None:
+      raise NotFound()
+
+    manifest = registry_model.lookup_manifest_by_digest(repo_ref, manifestref)
+    if manifest is None:
+      raise NotFound()
+
+    label = registry_model.get_manifest_label(manifest, labelid)
     if label is None:
       raise NotFound()
 
-    return label.to_dict()
+    return _label_dict(label)
 
 
   @require_repo_write
@@ -155,7 +205,15 @@ class ManageRepositoryManifestLabel(RepositoryParamResource):
   @disallow_for_app_repositories
   def delete(self, namespace_name, repository_name, manifestref, labelid):
     """ Deletes an existing label from a manifest. """
-    deleted = model.delete_manifest_label(namespace_name, repository_name, manifestref, labelid)
+    repo_ref = registry_model.lookup_repository(namespace_name, repository_name)
+    if repo_ref is None:
+      raise NotFound()
+
+    manifest = registry_model.lookup_manifest_by_digest(repo_ref, manifestref)
+    if manifest is None:
+      raise NotFound()
+
+    deleted = registry_model.delete_manifest_label(manifest, labelid)
     if deleted is None:
       raise NotFound()
 
@@ -170,4 +228,3 @@ class ManageRepositoryManifestLabel(RepositoryParamResource):
 
     log_action('manifest_label_delete', namespace_name, metadata, repo_name=repository_name)
     return '', 204
-

endpoints/api/manifest_models_interface.py

@@ -1,117 +0,0 @@
-from abc import ABCMeta, abstractmethod
-from collections import namedtuple
-
-from six import add_metaclass
-
-
-class ManifestLabel(
-  namedtuple('ManifestLabel', [
-    'uuid',
-    'key',
-    'value',
-    'source_type_name',
-    'media_type_name',
-    ])):
-  """
-  ManifestLabel represents a label on a manifest
-  :type uuid: string
-  :type key: string 
-  :type value: string 
-  :type source_type_name: string  
-  :type media_type_name: string 
-  """ 
-  def to_dict(self):
-    return {
-      'id': self.uuid,
-      'key': self.key,
-      'value': self.value,
-      'source_type': self.source_type_name,
-      'media_type': self.media_type_name,
-    }
-
-  
-class ManifestAndImage(
-  namedtuple('ManifestAndImage', [
-    'digest',
-    'manifest_data',
-    'image',
-    ])):
-  def to_dict(self):
-    return {
-      'digest': self.digest,
-      'manifest_data': self.manifest_data,
-      'image': self.image.to_dict(),
-    }
-
-  
-@add_metaclass(ABCMeta)
-class ManifestLabelInterface(object):
-  """
-  Data interface that the manifest labels API uses
-  """
-  
-  @abstractmethod
-  def get_manifest_labels(self, namespace_name, repository_name, manifestref, filter=None):
-    """
-    
-    Args:
-      namespace_name: string
-      repository_name: string
-      manifestref: string
-      filter: string
-
-    Returns:
-      list(ManifestLabel) or None
-
-    """   
-
-  @abstractmethod
-  def create_manifest_label(self, namespace_name, repository_name, manifestref, key, value, source_type_name, media_type_name):
-    """
-    
-    Args:
-      namespace_name: string
-      repository_name: string
-      manifestref: string
-      key: string
-      value: string
-      source_type_name: string 
-      media_type_name: string
-
-    Returns:
-      ManifestLabel or None 
-    """
-  
-  @abstractmethod
-  def get_manifest_label(self, namespace_name, repository_name, manifestref, label_uuid):
-    """
-    
-    Args:
-      namespace_name: string
-      repository_name: string
-      manifestref: string
-      label_uuid: string
-
-    Returns:
-      ManifestLabel or None
-    """
-  
-  @abstractmethod
-  def delete_manifest_label(self, namespace_name, repository_name, manifestref, label_uuid):
-    """
-    
-    Args:
-      namespace_name: string
-      repository_name: string
-      manifestref: string
-      label_uuid: string
-
-    Returns:
-      ManifestLabel or None
-    """
-
-  @abstractmethod
-  def get_repository_manifest(self, namespace_name, repository_name, digest):
-    """
-    Returns the manifest and image for the manifest with the specified digest, if any.
-    """

endpoints/api/manifest_models_pre_oci.py

@@ -1,68 +0,0 @@
-import json
-
-from manifest_models_interface import ManifestLabel, ManifestLabelInterface, ManifestAndImage
-from data import model
-from image_models_pre_oci import pre_oci_model as image_models
-
-
-class ManifestLabelPreOCI(ManifestLabelInterface):
-  def get_manifest_labels(self, namespace_name, repository_name, manifestref, filter=None):
-    try:
-      tag_manifest = model.tag.load_manifest_by_digest(namespace_name, repository_name, manifestref)
-    except model.DataModelException:
-      return None
-
-    labels = model.label.list_manifest_labels(tag_manifest, prefix_filter=filter)
-    return [self._label(l) for l in labels]
-
-  def create_manifest_label(self, namespace_name, repository_name, manifestref, key, value, source_type_name, media_type_name):
-    try:
-      tag_manifest = model.tag.load_manifest_by_digest(namespace_name, repository_name, manifestref)
-    except model.DataModelException:
-      return None
-
-    return self._label(model.label.create_manifest_label(tag_manifest, key, value, source_type_name, media_type_name))
-  
-  def get_manifest_label(self, namespace_name, repository_name, manifestref, label_uuid):
-    try:
-      tag_manifest = model.tag.load_manifest_by_digest(namespace_name, repository_name, manifestref)
-    except model.DataModelException:
-      return None
-
-    return self._label(model.label.get_manifest_label(label_uuid, tag_manifest))
-
-  def delete_manifest_label(self, namespace_name, repository_name, manifestref, label_uuid):
-    try:
-      tag_manifest = model.tag.load_manifest_by_digest(namespace_name, repository_name, manifestref)
-    except model.DataModelException:
-      return None
-
-    return self._label(model.label.delete_manifest_label(label_uuid, tag_manifest))
-  
-  def get_repository_manifest(self, namespace_name, repository_name, digest):
-    try:
-      tag_manifest = model.tag.load_manifest_by_digest(namespace_name, repository_name, digest,
-                                                       allow_dead=True)
-    except model.DataModelException:
-      return None
-
-    # TODO: remove this dependency on image once we've moved to the new data model.
-    image = image_models.get_repository_image(namespace_name, repository_name,
-                                              tag_manifest.tag.image.docker_image_id)
-
-    manifest_data = json.loads(tag_manifest.json_data)
-    return ManifestAndImage(digest=digest, manifest_data=manifest_data, image=image)
-
-
-  def _label(self, label_obj):
-    if not label_obj:
-      return None
-    return ManifestLabel(
-      uuid=label_obj.uuid,
-      key=label_obj.key,
-      value=label_obj.value,
-      source_type_name=label_obj.source_type.name,
-      media_type_name=label_obj.media_type.name,
-    )
-
-pre_oci_model = ManifestLabelPreOCI()

endpoints/api/repository_models_pre_oci.py

@@ -5,6 +5,8 @@ from datetime import datetime, timedelta
 from auth.permissions import ReadRepositoryPermission
 from data import model
 from data.appr_model import channel as channel_model, release as release_model
+from data.registry_model import registry_model
+from data.registry_model.datatypes import RepositoryReference
 from endpoints.appr.models_cnr import model as appr_model
 from endpoints.api.repository_models_interface import RepositoryDataInterface, RepositoryBaseElement, Repository, \
   ApplicationRepository, ImageRepositoryRepository, Tag, Channel, Release, Count
@@ -154,13 +156,16 @@ class PreOCIModel(RepositoryDataInterface):
           for release in releases
         ])
 
-    tags = model.tag.list_active_repo_tags(repo)
+    repo_ref = RepositoryReference.for_repo_obj(repo)
+    tags = registry_model.list_repository_tags(repo_ref, include_legacy_images=True)
+
     start_date = datetime.now() - timedelta(days=MAX_DAYS_IN_3_MONTHS)
     counts = model.log.get_repository_action_counts(repo, start_date)
 
     return ImageRepositoryRepository(base, [
-      Tag(tag.name, tag.image.docker_image_id, tag.image.aggregate_size, tag.lifetime_start_ts,
+      Tag(tag.name, tag.legacy_image.docker_image_id, tag.legacy_image.aggregate_size,
-          tag.tagmanifest.digest if hasattr(tag, 'tagmanifest') else None,
+          tag.lifetime_start_ts,
+          tag.manifest_digest,
           tag.lifetime_end_ts) for tag in tags
     ], [Count(count.date, count.count) for count in counts], repo.badge_token, repo.trust_enabled)
 

endpoints/api/secscan.py

@@ -4,7 +4,8 @@ import logging
 import features
 
 from app import secscan_api
-from data import model
+from data.registry_model import registry_model
+from data.registry_model.datatypes import SecurityScanStatus
 from endpoints.api import (require_repo_read, path_param,
                            RepositoryParamResource, resource, nickname, show_if, parse_args,
                            query_param, truthy_bool, disallow_for_app_repositories)
@@ -15,37 +16,24 @@ from util.secscan.api import APIRequestFailure
 
 logger = logging.getLogger(__name__)
 
-
+def _security_info(manifest_or_legacy_image, include_vulnerabilities=True):
-class SCAN_STATUS(object):
-  """ Security scan status enum """
-  SCANNED = 'scanned'
-  FAILED = 'failed'
-  QUEUED = 'queued'
-
-
-def _get_status(repo_image):
-  """ Returns the SCAN_STATUS value for the given image. """
-  if repo_image.security_indexed_engine is not None and repo_image.security_indexed_engine >= 0:
-    return SCAN_STATUS.SCANNED if repo_image.security_indexed else SCAN_STATUS.FAILED
-
-  return SCAN_STATUS.QUEUED
-
-def _security_status_for_image(namespace, repository, repo_image, include_vulnerabilities=True):
   """ Returns a dict representing the result of a call to the security status API for the given
-      image.
+      manifest or image.
   """
-  if not repo_image.security_indexed:
+  status = registry_model.get_security_status(manifest_or_legacy_image)
-    logger.debug('Image %s under repository %s/%s not security indexed',
+  if status is None:
-                 repo_image.docker_image_id, namespace, repository)
+    raise NotFound()
+
+  if status != SecurityScanStatus.SCANNED:
     return {
-      'status': _get_status(repo_image),
+      'status': status.value,
     }
 
   try:
     if include_vulnerabilities:
-      data = secscan_api.get_layer_data(repo_image, include_vulnerabilities=True)
+      data = secscan_api.get_layer_data(manifest_or_legacy_image, include_vulnerabilities=True)
     else:
-      data = secscan_api.get_layer_data(repo_image, include_features=True)
+      data = secscan_api.get_layer_data(manifest_or_legacy_image, include_features=True)
   except APIRequestFailure as arf:
     raise DownstreamIssue(arf.message)
 
@@ -53,7 +41,7 @@ def _security_status_for_image(namespace, repository, repo_image, include_vulner
     raise NotFound()
 
   return {
-    'status': _get_status(repo_image),
+    'status': status.value,
     'data': data,
   }
 
@@ -73,12 +61,16 @@ class RepositoryImageSecurity(RepositoryParamResource):
                default=False)
   def get(self, namespace, repository, imageid, parsed_args):
     """ Fetches the features and vulnerabilities (if any) for a repository image. """
-    repo_image = model.image.get_repo_image(namespace, repository, imageid)
+    repo_ref = registry_model.lookup_repository(namespace, repository)
-    if repo_image is None:
+    if repo_ref is None:
       raise NotFound()
 
-    return _security_status_for_image(namespace, repository, repo_image,
+    legacy_image = registry_model.get_legacy_image(repo_ref, imageid)
-                                      parsed_args.vulnerabilities)
+    if legacy_image is None:
+      raise NotFound()
+
+    return _security_info(legacy_image, parsed_args.vulnerabilities)
+
 
 @resource(MANIFEST_DIGEST_ROUTE + '/security')
 @show_if(features.SECURITY_SCANNER)
@@ -94,12 +86,12 @@ class RepositoryManifestSecurity(RepositoryParamResource):
   @query_param('vulnerabilities', 'Include vulnerabilities informations', type=truthy_bool,
                default=False)
   def get(self, namespace, repository, manifestref, parsed_args):
-    try:
+    repo_ref = registry_model.lookup_repository(namespace, repository)
-      tag_manifest = model.tag.load_manifest_by_digest(namespace, repository, manifestref)
+    if repo_ref is None:
-    except model.DataModelException:
       raise NotFound()
 
-    repo_image = tag_manifest.tag.image
+    manifest = registry_model.lookup_manifest_by_digest(repo_ref, manifestref, allow_dead=True)
+    if manifest is None:
+      raise NotFound()
 
-    return _security_status_for_image(namespace, repository, repo_image,
+    return _security_info(manifest, parsed_args.vulnerabilities)
-                                      parsed_args.vulnerabilities)

endpoints/api/tag.py

@@ -1,20 +1,40 @@
 """ Manage the tags of a repository. """
 
-from datetime import datetime, timedelta
+from datetime import datetime
 from flask import request, abort
 
 from auth.auth_context import get_authenticated_user
-from data.model import DataModelException
+from data.registry_model import registry_model
 from endpoints.api import (resource, nickname, require_repo_read, require_repo_write,
                            RepositoryParamResource, log_action, validate_json_request, path_param,
                            parse_args, query_param, truthy_bool, disallow_for_app_repositories)
-from endpoints.api.tag_models_interface import Repository
+from endpoints.api.image import image_dict
-from endpoints.api.tag_models_pre_oci import pre_oci_model as model
 from endpoints.exception import NotFound, InvalidRequest
 from endpoints.v2.manifest import _generate_and_store_manifest
 from util.names import TAG_ERROR, TAG_REGEX
 
 
+def _tag_dict(tag):
+  tag_info = {
+    'name': tag.name,
+    'reversion': tag.reversion,
+  }
+
+  if tag.lifetime_start_ts > 0:
+    tag_info['start_ts'] = tag.lifetime_start_ts
+
+  if tag.lifetime_end_ts > 0:
+    tag_info['end_ts'] = tag.lifetime_end_ts
+
+  if tag.manifest_digest:
+    tag_info['manifest_digest'] = tag.manifest_digest
+
+  if tag.legacy_image:
+    tag_info['docker_image_id'] = tag.legacy_image.docker_image_id
+
+  return tag_info
+
+
 @resource('/v1/repository/<apirepopath:repository>/tag/')
 @path_param('repository', 'The full path of the repository. e.g. namespace/name')
 class ListRepositoryTags(RepositoryParamResource):
@@ -33,17 +53,17 @@ class ListRepositoryTags(RepositoryParamResource):
     page = max(1, parsed_args.get('page', 1))
     limit = min(100, max(1, parsed_args.get('limit', 50)))
 
-    tag_history = model.list_repository_tag_history(namespace_name=namespace,
+    repo_ref = registry_model.lookup_repository(namespace, repository)
-                                                    repository_name=repository, page=page,
+    if repo_ref is None:
-                                                    size=limit, specific_tag=specific_tag)
-
-    if not tag_history:
       raise NotFound()
 
+    history, has_more = registry_model.list_repository_tag_history(repo_ref, page=page,
+                                                                   size=limit,
+                                                                   specific_tag_name=specific_tag)
     return {
-      'tags': [tag.to_dict() for tag in tag_history.tags],
+      'tags': [_tag_dict(tag) for tag in history],
       'page': page,
-      'has_additional': tag_history.more,
+      'has_additional': has_more,
     }
 
 
@@ -75,59 +95,67 @@ class RepositoryTag(RepositoryParamResource):
   @validate_json_request('ChangeTag')
   def put(self, namespace, repository, tag):
     """ Change which image a tag points to or create a new tag."""
-
     if not TAG_REGEX.match(tag):
       abort(400, TAG_ERROR)
 
-    repo = model.get_repo(namespace, repository)
+    repo_ref = registry_model.lookup_repository(namespace, repository)
-    if not repo:
+    if repo_ref is None:
       raise NotFound()
 
     if 'expiration' in request.get_json():
+      tag_ref = registry_model.get_repo_tag(repo_ref, tag)
+      if tag_ref is None:
+        raise NotFound()
+
       expiration = request.get_json().get('expiration')
       expiration_date = None
       if expiration is not None:
-       try:
+        try:
-         expiration_date = datetime.utcfromtimestamp(float(expiration))
+          expiration_date = datetime.utcfromtimestamp(float(expiration))
-       except ValueError:
+        except ValueError:
-         abort(400)
+          abort(400)
 
-       if expiration_date <= datetime.now():
+        if expiration_date <= datetime.now():
-         abort(400)
+          abort(400)
 
-      existing_end_ts, ok = model.change_repository_tag_expiration(namespace, repository, tag,
+      existing_end_ts, ok = registry_model.change_repository_tag_expiration(tag_ref,
-                                                                   expiration_date)
+                                                                            expiration_date)
       if ok:
-       if not (existing_end_ts is None and expiration_date is None):
+        if not (existing_end_ts is None and expiration_date is None):
-         log_action('change_tag_expiration', namespace, {
+          log_action('change_tag_expiration', namespace, {
-           'username': get_authenticated_user().username,
+            'username': get_authenticated_user().username,
-           'repo': repository,
+            'repo': repository,
-           'tag': tag,
+            'tag': tag,
-           'namespace': namespace,
+            'namespace': namespace,
-           'expiration_date': expiration_date,
+            'expiration_date': expiration_date,
-           'old_expiration_date': existing_end_ts
+            'old_expiration_date': existing_end_ts
-         }, repo_name=repository)
+          }, repo_name=repository)
       else:
         raise InvalidRequest('Could not update tag expiration; Tag has probably changed')
 
     if 'image' in request.get_json():
+      existing_tag = registry_model.get_repo_tag(repo_ref, tag, include_legacy_image=True)
+
       image_id = request.get_json()['image']
-      image = model.get_repository_image(namespace, repository, image_id)
+      image = registry_model.get_legacy_image(repo_ref, image_id)
       if image is None:
         raise NotFound()
 
-      original_image_id = model.get_repo_tag_image(repo, tag)
+      if not registry_model.retarget_tag(repo_ref, tag, image):
-      model.create_or_update_tag(namespace, repository, tag, image_id)
+        raise InvalidRequest('Could not move tag')
 
       username = get_authenticated_user().username
-      log_action('move_tag' if original_image_id else 'create_tag', namespace, {
+
+      log_action('move_tag' if existing_tag else 'create_tag', namespace, {
         'username': username,
         'repo': repository,
         'tag': tag,
         'namespace': namespace,
         'image': image_id,
-        'original_image': original_image_id
+        'original_image': existing_tag.legacy_image.docker_image_id if existing_tag else None,
       }, repo_name=repository)
+
+      # TODO(jschorr): Move this into the retarget_tag call
       _generate_and_store_manifest(namespace, repository, tag)
 
     return 'Updated', 201
@@ -137,7 +165,11 @@ class RepositoryTag(RepositoryParamResource):
   @nickname('deleteFullTag')
   def delete(self, namespace, repository, tag):
     """ Delete the specified repository tag. """
-    model.delete_tag(namespace, repository, tag)
+    repo_ref = registry_model.lookup_repository(namespace, repository)
+    if repo_ref is None:
+      raise NotFound()
+
+    registry_model.delete_tag(repo_ref, tag)
 
     username = get_authenticated_user().username
     log_action('delete_tag', namespace,
@@ -163,37 +195,28 @@ class RepositoryTagImages(RepositoryParamResource):
                type=truthy_bool, default=False)
   def get(self, namespace, repository, tag, parsed_args):
     """ List the images for the specified repository tag. """
-    try:
+    repo_ref = registry_model.lookup_repository(namespace, repository)
-      tag_image = model.get_repo_tag_image(
+    if repo_ref is None:
-        Repository(namespace_name=namespace, repository_name=repository), tag)
-    except DataModelException:
       raise NotFound()
 
-    if tag_image is None:
+    tag_ref = registry_model.get_repo_tag(repo_ref, tag, include_legacy_image=True)
+    if tag_ref is None:
       raise NotFound()
 
-    # Find all the parent images for the tag.
+    image_id = tag_ref.legacy_image.docker_image_id
-    parent_images = model.get_parent_images(namespace, repository, tag_image.docker_image_id)
-    all_images = [tag_image] + list(parent_images)
-    image_map = {image.docker_image_id: image for image in all_images}
-    skip_set = set()
 
-    # Filter the images returned to those not found in the ancestry of any of the other tags in
+    all_images = None
-    # the repository.
     if parsed_args['owned']:
-      all_tags = model.list_repository_tags(namespace, repository)
+      all_images = registry_model.get_legacy_images_owned_by_tag(tag_ref)
-      for current_tag in all_tags:
+    else:
-        if current_tag.name == tag:
+      image_with_parents = registry_model.get_legacy_image(repo_ref, image_id, include_parents=True)
-          continue
+      if image_with_parents is None:
+        raise NotFound()
 
-        skip_set.add(current_tag.image.ancestor_id)
+      all_images = [image_with_parents] + image_with_parents.parents
-        skip_set = skip_set | set(current_tag.image.ancestor_id_list)
 
     return {
-      'images': [
+      'images': [image_dict(image) for image in all_images],
-        image.to_dict(image_map) for image in all_images
-        if not parsed_args['owned'] or (image.ancestor_id not in skip_set)
-      ]
     }
 
 
@@ -226,6 +249,9 @@ class RestoreTag(RepositoryParamResource):
   @validate_json_request('RestoreTag')
   def post(self, namespace, repository, tag):
     """ Restores a repository tag back to a previous image in the repository. """
+    repo_ref = registry_model.lookup_repository(namespace, repository)
+    if repo_ref is None:
+      raise NotFound()
 
     # Restore the tag back to the previous image.
     image_id = request.get_json()['image']
@@ -239,19 +265,26 @@ class RestoreTag(RepositoryParamResource):
       'tag': tag,
       'image': image_id,
     }
-    repo = Repository(namespace, repository)
-    if manifest_digest is not None:
-      existing_image = model.restore_tag_to_manifest(repo, tag, manifest_digest)
-    else:
-      existing_image = model.restore_tag_to_image(repo, tag, image_id)
-      _generate_and_store_manifest(namespace, repository, tag)
 
-    if existing_image is not None:
+    manifest_or_legacy_image = None
-      log_data['original_image'] = existing_image.docker_image_id
+    if manifest_digest is not None:
+      manifest_or_legacy_image = registry_model.lookup_manifest_by_digest(repo_ref, manifest_digest,
+                                                                          allow_dead=True)
+    else:
+      manifest_or_legacy_image = registry_model.get_legacy_image(repo_ref, image_id)
+
+    if manifest_or_legacy_image is None:
+      raise NotFound()
+
+    if not registry_model.retarget_tag(repo_ref, tag, manifest_or_legacy_image, is_reversion=True):
+      raise InvalidRequest('Could not restore tag')
+
+    if manifest_digest is None:
+      # TODO(jschorr): Move this into the retarget_tag call
+      _generate_and_store_manifest(namespace, repository, tag)
 
     log_action('revert_tag', namespace, log_data, repo_name=repository)
 
     return {
       'image_id': image_id,
-      'original_image_id': existing_image.docker_image_id if existing_image else None,
     }

endpoints/api/tag_models_interface.py

@@ -1,187 +0,0 @@
-import json
-from abc import ABCMeta, abstractmethod
-from collections import namedtuple
-
-from six import add_metaclass
-
-from endpoints.api import format_date
-
-
-class Tag(
-  namedtuple('Tag', [
-    'name', 'image', 'reversion', 'lifetime_start_ts', 'lifetime_end_ts', 'manifest_list',
-    'docker_image_id'
-  ])):
-  """
-  Tag represents a name to an image.
-  :type name: string
-  :type image: Image
-  :type reversion: boolean
-  :type lifetime_start_ts: int
-  :type lifetime_end_ts: int
-  :type manifest_list: [manifest_digest]
-  :type docker_image_id: string
-  """
-
-  def to_dict(self):
-    tag_info = {
-      'name': self.name,
-      'docker_image_id': self.docker_image_id,
-      'reversion': self.reversion,
-    }
-
-    if self.lifetime_start_ts > 0:
-      tag_info['start_ts'] = self.lifetime_start_ts
-
-    if self.lifetime_end_ts > 0:
-      tag_info['end_ts'] = self.lifetime_end_ts
-
-    if self.manifest_list:
-      tag_info['manifest_digest'] = self.manifest_list
-
-    return tag_info
-
-
-class RepositoryTagHistory(namedtuple('RepositoryTagHistory', ['tags', 'more'])):
-  """
-  Tag represents a name to an image.
-  :type tags: [Tag]
-  :type more: boolean
-  """
-
-
-class Repository(namedtuple('Repository', ['namespace_name', 'repository_name'])):
-  """
-  Repository a single quay repository
-  :type namespace_name: string
-  :type repository_name: string
-  """
-
-
-class Image(
-  namedtuple('Image', [
-    'docker_image_id', 'created', 'comment', 'command', 'storage_image_size',
-    'storage_uploading', 'ancestor_id_list', 'ancestor_id'
-  ])):
-  """
-  Image
-  :type docker_image_id: string
-  :type created: datetime
-  :type comment: string
-  :type command: string
-  :type storage_image_size: int
-  :type storage_uploading: boolean
-  :type ancestor_id_list: [int]
-  :type ancestor_id: int
-  """
-
-  def to_dict(self, image_map, include_ancestors=True):
-    command = self.command
-
-    def docker_id(aid):
-      if aid not in image_map:
-        return ''
-
-      return image_map[aid].docker_image_id
-
-    image_data = {
-      'id': self.docker_image_id,
-      'created': format_date(self.created),
-      'comment': self.comment,
-      'command': json.loads(command) if command else None,
-      'size': self.storage_image_size,
-      'uploading': self.storage_uploading,
-      'sort_index': len(self.ancestor_id_list),
-    }
-
-    if include_ancestors:
-      # Calculate the ancestors string, with the DBID's replaced with the docker IDs.
-      ancestors = [docker_id(a) for a in self.ancestor_id_list]
-      image_data['ancestors'] = '/{0}/'.format('/'.join(ancestors))
-
-    return image_data
-
-
-@add_metaclass(ABCMeta)
-class TagDataInterface(object):
-  """
-  Interface that represents all data store interactions required by a Tag.
-  """
-
-  @abstractmethod
-  def list_repository_tag_history(self, namespace_name, repository_name, page=1, size=100,
-                                  specific_tag=None):
-    """
-    Returns a RepositoryTagHistory with a list of historic tags and whether there are more tags then returned.
-    """
-
-  @abstractmethod
-  def get_repo(self, namespace_name, repository_name):
-    """
-    Returns a repository associated with the given namespace and repository name.
-    """
-
-  @abstractmethod
-  def get_repo_tag_image(self, repository, tag_name):
-    """
-    Returns an image associated with the repository and tag_name
-    """
-
-  @abstractmethod
-  def create_or_update_tag(self, namespace_name, repository_name, tag_name, docker_image_id):
-    """
-    Returns the repository tag if it is created.
-    """
-
-  @abstractmethod
-  def delete_tag(self, namespace_name, repository_name, tag_name):
-    """
-    Returns the tag for the given namespace and repository if it was created
-    """
-
-  @abstractmethod
-  def get_parent_images(self, namespace, repository, tag_name):
-    """
-    Returns a list of the parent images for the namespace, repository and tag specified.
-    """
-
-  @abstractmethod
-  def list_repository_tags(self, namespace_name, repository_name):
-    """
-    Returns a list of all tags associated with namespace_nam and repository_name
-    """
-
-  @abstractmethod
-  def get_repository(self, namespace_name, repository_name):
-    """
-    Returns the repository associated with the namespace_name and repository_name
-    """
-
-  @abstractmethod
-  def get_repository_image(self, namespace_name, repository_name, docker_image_id):
-    """
-    Returns the repository image associated with the namespace_name, repository_name, and docker
-    image ID.
-    """
-
-  @abstractmethod
-  def restore_tag_to_manifest(self, repository_name, tag_name, manifest_digest):
-    """
-    Returns the existing repo tag image if it exists or else returns None.
-    Side effects include adding the tag with associated name to the manifest_digest in the named repo.
-    """
-
-  @abstractmethod
-  def restore_tag_to_image(self, repository_name, tag_name, image_id):
-    """
-    Returns the existing repo tag image if it exists or else returns None
-    Side effects include adding the tag with associated name to the image with the associated id in the named repo.
-    """
-
-  @abstractmethod
-  def change_repository_tag_expiration(self, namespace_name, repository_name, tag_name,
-                                       expiration_date):
-    """ Sets the expiration date of the tag under the matching repository to that given. If the
-        expiration date is None, then the tag will not expire. Returns a tuple of the previous
-        expiration timestamp in seconds (if any), and whether the operation succeeded.
-    """

endpoints/api/tag_models_pre_oci.py

@@ -1,133 +0,0 @@
-from data import model
-from data.model import DataModelException, InvalidImageException
-from endpoints.api.tag_models_interface import TagDataInterface, Tag, RepositoryTagHistory, Repository, Image
-
-
-class PreOCIModel(TagDataInterface):
-  """
-  PreOCIModel implements the data model for the Tags using a database schema
-  before it was changed to support the OCI specification.
-  """
-
-  def list_repository_tag_history(self, namespace_name, repository_name, page=1, size=100,
-                                  specific_tag=None):
-    repository = model.repository.get_repository(namespace_name, repository_name)
-    if repository is None:
-      return None
-
-    tags, manifest_map, more = model.tag.list_repository_tag_history(repository, page, size,
-                                                                     specific_tag)
-    repository_tag_history = []
-    for tag in tags:
-      manifest_list = None
-      if tag.id in manifest_map:
-        manifest_list = manifest_map[tag.id]
-
-      repository_tag_history.append(convert_tag(tag, manifest_list))
-
-    return RepositoryTagHistory(tags=repository_tag_history, more=more)
-
-  def get_repo(self, namespace_name, repository_name):
-    repo = model.repository.get_repository(namespace_name, repository_name)
-    if repo is None:
-      return None
-
-    return Repository(repo.namespace_user, repo.name)
-
-  def get_repo_tag_image(self, repository, tag_name):
-    repo = model.repository.get_repository(str(repository.namespace_name), str(repository.repository_name))
-    if repo is None:
-      return None
-
-    try:
-      image = model.tag.get_repo_tag_image(repo, tag_name)
-    except DataModelException:
-      return None
-
-    return convert_image(image)
-
-  def create_or_update_tag(self, namespace_name, repository_name, tag_name, docker_image_id):
-    return model.tag.create_or_update_tag(namespace_name, repository_name, tag_name,
-                                          docker_image_id)
-
-  def delete_tag(self, namespace_name, repository_name, tag_name):
-    return model.tag.delete_tag(namespace_name, repository_name, tag_name)
-
-  def get_parent_images(self, namespace_name, repository_name, docker_image_id):
-    try:
-      image = model.image.get_image_by_id(namespace_name, repository_name, docker_image_id)
-    except InvalidImageException:
-      return []
-
-    parent_tags = model.image.get_parent_images(namespace_name, repository_name, image)
-    return_tags = []
-    for image in parent_tags:
-      return_tags.append(convert_image(image))
-
-    return return_tags
-
-  def list_repository_tags(self, namespace_name, repository_name):
-    tags = model.tag.list_repository_tags(namespace_name, repository_name)
-    new_tags = []
-    for tag in tags:
-      new_tags.append(convert_tag(tag))
-    return new_tags
-
-  def get_repository_image(self, namespace_name, repository_name, docker_image_id):
-    image = model.image.get_repo_image(namespace_name, repository_name, docker_image_id)
-    if image is None:
-      return None
-
-    return convert_image(image)
-
-  def get_repository(self, namespace_name, repository_name):
-    repo = model.repository.get_repository(namespace_name, repository_name)
-    if repo is None:
-      return None
-
-    return Repository(namespace_name=namespace_name, repository_name=repository_name)
-
-  def restore_tag_to_manifest(self, repository, tag_name, manifest_digest):
-    repo = model.repository.get_repository(repository.namespace_name, repository.repository_name)
-    if repo is None:
-      return None
-
-    image = model.tag.restore_tag_to_manifest(repo, tag_name, manifest_digest)
-    if image is None:
-      return None
-
-    return convert_image(image)
-
-  def restore_tag_to_image(self, repository, tag_name, image_id):
-    repo = model.repository.get_repository(repository.namespace_name, repository.repository_name)
-    if repo is None:
-      return None
-
-    image = model.tag.restore_tag_to_image(repo, tag_name, image_id)
-    if image is None:
-      return None
-
-    return convert_image(image)
-
-  def change_repository_tag_expiration(self, namespace_name, repository_name, tag_name,
-                                       expiration_date):
-    return model.tag.change_repository_tag_expiration(namespace_name, repository_name, tag_name,
-                                                      expiration_date)
-
-
-def convert_image(database_image):
-  return Image(docker_image_id=database_image.docker_image_id, created=database_image.created,
-               comment=database_image.comment, command=database_image.command,
-               storage_image_size=database_image.storage.image_size,
-               storage_uploading=database_image.storage.uploading,
-               ancestor_id_list=database_image.ancestor_id_list(),
-               ancestor_id=database_image.id)
-
-
-def convert_tag(tag, manifest_list=None):
-  return Tag(name=tag.name, image=convert_image(tag.image), reversion=tag.reversion,
-             lifetime_start_ts=tag.lifetime_start_ts, lifetime_end_ts=tag.lifetime_end_ts,
-             manifest_list=manifest_list, docker_image_id=tag.image.docker_image_id)
-
-
-pre_oci_model = PreOCIModel()

endpoints/api/test/test_manifest.py

@@ -1,22 +1,24 @@
-import pytest
+from data.registry_model import registry_model
-
-from data import model
 from endpoints.api.manifest import RepositoryManifest
 from endpoints.api.test.shared import conduct_api_call
 from endpoints.test.shared import client_with_identity
+
 from test.fixtures import *
 
 def test_repository_manifest(client):
   with client_with_identity('devtable', client) as cl:
-    tags = model.tag.list_repository_tags('devtable', 'simple')
+    repo_ref = registry_model.lookup_repository('devtable', 'simple')
-    digests = model.tag.get_tag_manifest_digests(tags)
+    tags = registry_model.list_repository_tags(repo_ref)
     for tag in tags:
-      manifest = digests[tag.id]
+      manifest_digest = tag.manifest_digest
+      if manifest_digest is None:
+        continue
+
       params = {
         'repository': 'devtable/simple',
-        'manifestref': manifest,
+        'manifestref': manifest_digest,
       }
       result = conduct_api_call(cl, RepositoryManifest, 'GET', params, None, 200).json
-      assert result['digest'] == manifest
+      assert result['digest'] == manifest_digest
       assert result['manifest_data']
       assert result['image']

endpoints/api/test/test_security.py

@@ -788,11 +788,11 @@ SECURITY_TESTS = [
   (RestoreTag, 'POST', {'tag': 'HP8R', 'repository': 'public/publicrepo'}, {u'image': 'WXNG'}, 'freshuser', 403),
   (RestoreTag, 'POST', {'tag': 'HP8R', 'repository': 'public/publicrepo'}, {u'image': 'WXNG'}, 'reader', 403),
   (RestoreTag, 'POST', {'tag': 'HP8R', 'repository': 'devtable/shared'}, {u'image': 'WXNG'}, None, 401),
-  (RestoreTag, 'POST', {'tag': 'HP8R', 'repository': 'devtable/shared'}, {u'image': 'WXNG'}, 'devtable', 400),
+  (RestoreTag, 'POST', {'tag': 'HP8R', 'repository': 'devtable/shared'}, {u'image': 'WXNG'}, 'devtable', 404),
   (RestoreTag, 'POST', {'tag': 'HP8R', 'repository': 'devtable/shared'}, {u'image': 'WXNG'}, 'freshuser', 403),
   (RestoreTag, 'POST', {'tag': 'HP8R', 'repository': 'devtable/shared'}, {u'image': 'WXNG'}, 'reader', 403),
   (RestoreTag, 'POST', {'tag': 'HP8R', 'repository': 'buynlarge/orgrepo'}, {u'image': 'WXNG'}, None, 401),
-  (RestoreTag, 'POST', {'tag': 'HP8R', 'repository': 'buynlarge/orgrepo'}, {u'image': 'WXNG'}, 'devtable', 400),
+  (RestoreTag, 'POST', {'tag': 'HP8R', 'repository': 'buynlarge/orgrepo'}, {u'image': 'WXNG'}, 'devtable', 404),
   (RestoreTag, 'POST', {'tag': 'HP8R', 'repository': 'buynlarge/orgrepo'}, {u'image': 'WXNG'}, 'freshuser', 403),
   (RestoreTag, 'POST', {'tag': 'HP8R', 'repository': 'buynlarge/orgrepo'}, {u'image': 'WXNG'}, 'reader', 403),
 

endpoints/api/test/test_tag.py

@@ -1,129 +1,13 @@
-import json
-
 import pytest
 
-from mock import patch, Mock, MagicMock, call
+from data.registry_model import registry_model
 
-from data.model import DataModelException
-from endpoints.api.tag_models_interface import RepositoryTagHistory, Tag
 from endpoints.api.test.shared import conduct_api_call
 from endpoints.test.shared import client_with_identity
 from endpoints.api.tag import RepositoryTag, RestoreTag, ListRepositoryTags, RepositoryTagImages
 
-from features import FeatureNameValue
-
 from test.fixtures import *
 
-
-@pytest.fixture()
-def get_repo_image():
-  def mock_callable(namespace, repository, image_id):
-    mock = Mock(namespace_user='devtable')
-    mock.name = 'simple'
-    img = Mock(repository=mock, docker_image_id=12) if image_id == 'image1' else None
-    return img
-
-  with patch('endpoints.api.tag_models_pre_oci.model.image.get_repo_image',
-             side_effect=mock_callable) as mk:
-    yield mk
-
-
-@pytest.fixture()
-def get_repository():
-  with patch('endpoints.api.tag_models_pre_oci.model.image.get_repo_image',
-             return_value='mock_repo') as mk:
-    yield mk
-
-
-@pytest.fixture()
-def get_repo_tag_image():
-  def mock_get_repo_tag_image(repository, tag):
-    storage_mock = Mock(image_size=1234, uploading='uploading')
-
-    def fake_ancestor_id_list():
-      return []
-
-    if tag == 'existing-tag':
-      return Mock(docker_image_id='mock_docker_image_id', created=12345, comment='comment',
-                  command='command', storage=storage_mock, ancestors=[],
-                  ancestor_id_list=fake_ancestor_id_list)
-    else:
-      raise DataModelException('Unable to find image for tag.')
-
-  with patch('endpoints.api.tag_models_pre_oci.model.tag.get_repo_tag_image',
-             side_effect=mock_get_repo_tag_image):
-    yield
-
-
-@pytest.fixture()
-def restore_tag_to_manifest():
-  def mock_restore_tag_to_manifest(repository, tag, manifest_digest):
-    tag_img = Mock(docker_image_id='mock_docker_image_id') if tag == 'existing-tag' else None
-    return tag_img
-
-  with patch('endpoints.api.tag_models_pre_oci.model.tag.restore_tag_to_manifest',
-             side_effect=mock_restore_tag_to_manifest):
-    yield
-
-
-@pytest.fixture()
-def restore_tag_to_image():
-  def mock_restore_tag_to_image(repository, tag, image_id):
-    tag_img = Mock(docker_image_id='mock_docker_image_id') if tag == 'existing-tag' else None
-    return tag_img
-
-  with patch('endpoints.api.tag_models_pre_oci.model.tag.restore_tag_to_image',
-             side_effect=mock_restore_tag_to_image):
-    yield
-
-
-@pytest.fixture()
-def create_or_update_tag():
-  with patch('endpoints.api.tag_models_pre_oci.model.tag.create_or_update_tag') as mk:
-    yield mk
-
-
-@pytest.fixture()
-def generate_manifest():
-  def mock_callable(namespace, repository, tag):
-    if tag == 'generatemanifestfail':
-      raise Exception('test_failure')
-
-  with patch('endpoints.api.tag._generate_and_store_manifest', side_effect=mock_callable) as mk:
-    yield mk
-
-
-@pytest.fixture()
-def authd_client(client):
-  with client_with_identity('devtable', client) as cl:
-    yield cl
-
-
-@pytest.fixture()
-def list_repository_tag_history():
-  def list_repository_tag_history(namespace_name, repository_name, page, size, specific_tag):
-    return RepositoryTagHistory(tags=[
-      Tag(name='First Tag', image='image', reversion=False, lifetime_start_ts=0, lifetime_end_ts=0,
-          manifest_list=[], docker_image_id='first docker image id'),
-      Tag(name='Second Tag', image='second image', reversion=True, lifetime_start_ts=10,
-          lifetime_end_ts=100, manifest_list=[], docker_image_id='second docker image id')
-    ], more=False)
-
-  with patch('endpoints.api.tag.model.list_repository_tag_history',
-             side_effect=list_repository_tag_history):
-    yield
-
-
-@pytest.fixture()
-def find_no_repo_tag_history():
-  def list_repository_tag_history(namespace_name, repository_name, page, size, specific_tag):
-    return None
-
-  with patch('endpoints.api.tag.model.list_repository_tag_history',
-             side_effect=list_repository_tag_history):
-    yield
-
-
 @pytest.mark.parametrize('expiration_time, expected_status', [
   (None, 201),
   ('aksdjhasd', 400),
@@ -161,126 +45,50 @@ def test_change_tag_expiration(client, app):
     assert tag.lifetime_end_ts == updated_expiration
 
 
-@pytest.mark.parametrize('test_image,test_tag,expected_status', [
+@pytest.mark.parametrize('image_exists,test_tag,expected_status', [
-  ('image1', '-INVALID-TAG-NAME', 400),
+  (True, '-INVALID-TAG-NAME', 400),
-  ('image1', '.INVALID-TAG-NAME', 400),
+  (True, '.INVALID-TAG-NAME', 400),
-  ('image1',
+  (True,
    'INVALID-TAG_NAME-BECAUSE-THIS-IS-WAY-WAY-TOO-LOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOONG',
    400),
-  ('nonexistantimage', 'newtag', 404),
+  (False, 'newtag', 404),
-  ('image1', 'generatemanifestfail', None),
+  (True, 'generatemanifestfail', None),
-  ('image1', 'existing-tag', 201),
+  (True, 'latest', 201),
-  ('image1', 'newtag', 201),
+  (True, 'newtag', 201),
 ])
-def test_move_tag(test_image, test_tag, expected_status, get_repo_image, get_repo_tag_image,
+def test_move_tag(image_exists, test_tag, expected_status, client, app):
-                  create_or_update_tag, generate_manifest, authd_client):
+  with client_with_identity('devtable', client) as cl:
-  params = {'repository': 'devtable/simple', 'tag': test_tag}
+    test_image = 'unknown'
-  request_body = {'image': test_image}
+    if image_exists:
-  if expected_status is None:
+      repo_ref = registry_model.lookup_repository('devtable', 'simple')
-    with pytest.raises(Exception):
+      tag_ref = registry_model.get_repo_tag(repo_ref, 'latest', include_legacy_image=True)
-      conduct_api_call(authd_client, RepositoryTag, 'put', params, request_body, expected_status)
+      assert tag_ref
-  else:
+
-    conduct_api_call(authd_client, RepositoryTag, 'put', params, request_body, expected_status)
+      test_image = tag_ref.legacy_image.docker_image_id
+
+    params = {'repository': 'devtable/simple', 'tag': test_tag}
+    request_body = {'image': test_image}
+    if expected_status is None:
+      with pytest.raises(Exception):
+        conduct_api_call(cl, RepositoryTag, 'put', params, request_body, expected_status)
+    else:
+      conduct_api_call(cl, RepositoryTag, 'put', params, request_body, expected_status)
 
 
-@pytest.mark.parametrize(
+@pytest.mark.parametrize('repo_namespace, repo_name', [
-  'namespace, repository, specific_tag, page, limit, expected_response_code, expected', [
+  ('devtable', 'simple'),
-    ('devtable', 'simple', None, 1, 10, 200, {
+  ('devtable', 'history'),
-      'has_additional': False
+  ('devtable', 'complex'),
-    }),
+  ('buynlarge', 'orgrepo'),
-    ('devtable', 'simple', None, 1, 10, 200, {
-      'page': 1
-    }),
-    ('devtable', 'simple', None, 1, 10, 200, {
-      'tags': [{
-        'docker_image_id': 'first docker image id',
-        'name': 'First Tag',
-        'reversion': False
-      }, {
-        'docker_image_id': 'second docker image id',
-        'end_ts': 100,
-        'name': 'Second Tag',
-        'reversion': True,
-        'start_ts': 10
-      }]
-    }),
-  ])
-def test_list_repository_tags_view_is_correct(namespace, repository, specific_tag, page, limit,
-                                              list_repository_tag_history, expected_response_code,
-                                              expected, authd_client):
-  params = {
-    'repository': namespace + '/' + repository,
-    'specificTag': specific_tag,
-    'page': page,
-    'limit': limit
-  }
-  response = conduct_api_call(authd_client, ListRepositoryTags, 'get', params,
-                              expected_code=expected_response_code)
-  compare_list_history_tags_response(expected, response.json)
-
-
-def compare_list_history_tags_response(expected, actual):
-  if 'has_additional' in expected:
-    assert expected['has_additional'] == actual['has_additional']
-
-  if 'page' in expected:
-    assert expected['page'] == actual['page']
-
-  if 'tags' in expected:
-    assert expected['tags'] == actual['tags']
-
-
-def test_no_repo_tag_history(find_no_repo_tag_history, authd_client):
-  params = {'repository': 'devtable/simple', 'specificTag': None, 'page': 1, 'limit': 10}
-  conduct_api_call(authd_client, ListRepositoryTags, 'get', params, expected_code=404)
-
-
-@pytest.mark.parametrize(
-  'specific_tag, page, limit, expected_specific_tag, expected_page, expected_limit', [
-    (None, None, None, None, 1, 50),
-    ('specific_tag', 12, 13, 'specific_tag', 12, 13),
-    ('specific_tag', -1, 101, 'specific_tag', 1, 100),
-    ('specific_tag', 0, 0, 'specific_tag', 1, 1),
-  ])
-def test_repo_tag_history_param_parse(specific_tag, page, limit, expected_specific_tag,
-                                      expected_page, expected_limit, authd_client):
-  mock = MagicMock()
-  mock.return_value = RepositoryTagHistory(tags=[], more=False)
-
-  with patch('endpoints.api.tag.model.list_repository_tag_history', side_effect=mock):
-    params = {
-      'repository': 'devtable/simple',
-      'specificTag': specific_tag,
-      'page': page,
-      'limit': limit
-    }
-    conduct_api_call(authd_client, ListRepositoryTags, 'get', params)
-
-  assert mock.call_args == call(namespace_name='devtable', repository_name='simple',
-                                page=expected_page, size=expected_limit,
-                                specific_tag=expected_specific_tag)
-
-
-@pytest.mark.parametrize('test_manifest,test_tag,manifest_generated,expected_status', [
-  (None, 'newtag', True, 200),
-  (None, 'generatemanifestfail', True, None),
-  ('manifest1', 'newtag', False, 200),
 ])
-def test_restore_tag(test_manifest, test_tag, manifest_generated, expected_status, get_repository,
+def test_list_repo_tags(repo_namespace, repo_name, client, app):
-                     restore_tag_to_manifest, restore_tag_to_image, generate_manifest,
+  params = {'repository': repo_namespace + '/' + repo_name}
-                     authd_client):
+  with client_with_identity('devtable', client) as cl:
-  params = {'repository': 'devtable/simple', 'tag': test_tag}
+    tags = conduct_api_call(cl, ListRepositoryTags, 'get', params).json['tags']
-  request_body = {'image': 'image1'}
+    repo_ref = registry_model.lookup_repository(repo_namespace, repo_name)
-  if test_manifest is not None:
+    history, _ = registry_model.list_repository_tag_history(repo_ref)
-    request_body['manifest_digest'] = test_manifest
+    assert len(tags) == len(history)
-  if expected_status is None:
-    with pytest.raises(Exception):
-      conduct_api_call(authd_client, RestoreTag, 'post', params, request_body, expected_status)
-  else:
-    conduct_api_call(authd_client, RestoreTag, 'post', params, request_body, expected_status)
 
-  if manifest_generated:
-    generate_manifest.assert_called_with('devtable', 'simple', test_tag)
 
 @pytest.mark.parametrize('repository, tag, owned, expect_images', [
   ('devtable/simple', 'prod', False, True),
@@ -291,7 +99,8 @@ def test_restore_tag(test_manifest, test_tag, manifest_generated, expected_statu
   ('devtable/complex', 'prod', False, True),
   ('devtable/complex', 'prod', True, True),
 ])
-def test_list_tag_images(repository, tag, owned, expect_images, authd_client):
+def test_list_tag_images(repository, tag, owned, expect_images, client, app):
-  params = {'repository': repository, 'tag': tag, 'owned': owned}
+  with client_with_identity('devtable', client) as cl:
-  result = conduct_api_call(authd_client, RepositoryTagImages, 'get', params, None, 200).json
+    params = {'repository': repository, 'tag': tag, 'owned': owned}
-  assert bool(result['images']) == expect_images
+    result = conduct_api_call(cl, RepositoryTagImages, 'get', params, None, 200).json
+    assert bool(result['images']) == expect_images

endpoints/api/test/test_tag_models_pre_oci.py

@@ -1,215 +0,0 @@
-import pytest
-
-from data.model import DataModelException, InvalidImageException
-from endpoints.api.tag_models_interface import RepositoryTagHistory, Tag, Repository
-from mock import Mock, call
-
-from data import model
-from endpoints.api.tag_models_pre_oci import pre_oci_model
-from util.morecollections import AttrDict
-
-EMPTY_REPOSITORY = 'empty_repository'
-EMPTY_NAMESPACE = 'empty_namespace'
-BAD_REPOSITORY_NAME = 'bad_repository_name'
-BAD_NAMESPACE_NAME = 'bad_namespace_name'
-
-
-@pytest.fixture
-def get_monkeypatch(monkeypatch):
-  return monkeypatch
-
-
-def mock_out_get_repository(monkeypatch, namespace_name, repository_name):
-  def return_none(namespace_name, repository_name):
-    return None
-
-  def return_repository(namespace_name, repository_name):
-    return 'repository'
-
-  if namespace_name == BAD_NAMESPACE_NAME or repository_name == BAD_REPOSITORY_NAME:
-    return_function = return_none
-  else:
-    return_function = return_repository
-
-  monkeypatch.setattr(model.repository, 'get_repository', return_function)
-
-
-def get_repo_mock(monkeypatch, return_value):
-  def return_return_value(namespace_name, repository_name):
-    return return_value
-
-  monkeypatch.setattr(model.repository, 'get_repository', return_return_value)
-
-
-def test_get_repo_not_exists(get_monkeypatch):
-  namespace_name = 'namespace_name'
-  repository_name = 'repository_name'
-  get_repo_mock(get_monkeypatch, None)
-  repo = pre_oci_model.get_repo(namespace_name, repository_name)
-
-  assert repo is None
-
-
-def test_get_repo_exists(get_monkeypatch):
-  namespace_name = 'namespace_name'
-  repository_name = 'repository_name'
-  mock = Mock()
-  mock.namespace_user = namespace_name
-  mock.name = repository_name
-  mock.repository = mock
-  get_repo_mock(get_monkeypatch, mock)
-
-  repo = pre_oci_model.get_repo(namespace_name, repository_name)
-
-  assert repo is not None
-  assert repo.repository_name == repository_name
-  assert repo.namespace_name == namespace_name
-
-
-def get_repository_mock(monkeypatch, return_value):
-  def return_return_value(namespace_name, repository_name, kind_filter=None):
-    return return_value
-
-  monkeypatch.setattr(model.repository, 'get_repository', return_return_value)
-
-
-def get_repo_tag_image_mock(monkeypatch, return_value):
-  def return_return_value(repo, tag_name, include_storage=False):
-    return return_value
-
-  monkeypatch.setattr(model.tag, 'get_repo_tag_image', return_return_value)
-
-
-def test_get_repo_tag_image_with_repo_and_repo_tag(get_monkeypatch):
-  mock_storage = Mock()
-  mock_image = Mock()
-  mock_image.docker_image_id = 'some docker image id'
-  mock_image.created = 1235
-  mock_image.comment = 'some comment'
-  mock_image.command = 'some command'
-  mock_image.storage = mock_storage
-  mock_image.ancestors = []
-
-  get_repository_mock(get_monkeypatch, mock_image)
-  get_repo_tag_image_mock(get_monkeypatch, mock_image)
-
-  image = pre_oci_model.get_repo_tag_image(
-    Repository('namespace_name', 'repository_name'), 'tag_name')
-
-  assert image is not None
-  assert image.docker_image_id == 'some docker image id'
-
-
-def test_get_repo_tag_image_without_repo(get_monkeypatch):
-  get_repository_mock(get_monkeypatch, None)
-
-  image = pre_oci_model.get_repo_tag_image(
-    Repository('namespace_name', 'repository_name'), 'tag_name')
-
-  assert image is None
-
-
-def test_get_repo_tag_image_without_repo_tag_image(get_monkeypatch):
-  mock = Mock()
-  mock.docker_image_id = 'some docker image id'
-  get_repository_mock(get_monkeypatch, mock)
-
-  def raise_exception(repo, tag_name, include_storage=False):
-    raise DataModelException()
-
-  get_monkeypatch.setattr(model.tag, 'get_repo_tag_image', raise_exception)
-
-  image = pre_oci_model.get_repo_tag_image(
-    Repository('namespace_name', 'repository_name'), 'tag_name')
-
-  assert image is None
-
-
-def test_create_or_update_tag(get_monkeypatch):
-  mock = Mock()
-  get_monkeypatch.setattr(model.tag, 'create_or_update_tag', mock)
-
-  pre_oci_model.create_or_update_tag('namespace_name', 'repository_name', 'tag_name',
-                                     'docker_image_id')
-
-  assert mock.call_count == 1
-  assert mock.call_args == call('namespace_name', 'repository_name', 'tag_name', 'docker_image_id')
-
-
-def test_delete_tag(get_monkeypatch):
-  mock = Mock()
-  get_monkeypatch.setattr(model.tag, 'delete_tag', mock)
-
-  pre_oci_model.delete_tag('namespace_name', 'repository_name', 'tag_name')
-
-  assert mock.call_count == 1
-  assert mock.call_args == call('namespace_name', 'repository_name', 'tag_name')
-
-
-def test_get_parent_images_with_exception(get_monkeypatch):
-  mock = Mock(side_effect=InvalidImageException)
-
-  get_monkeypatch.setattr(model.image, 'get_image_by_id', mock)
-
-  images = pre_oci_model.get_parent_images('namespace_name', 'repository_name', 'tag_name')
-  assert images == []
-
-
-def test_get_parent_images_empty_parent_images(get_monkeypatch):
-  get_image_by_id_mock = Mock()
-  get_monkeypatch.setattr(model.image, 'get_image_by_id', get_image_by_id_mock)
-
-  get_parent_images_mock = Mock(return_value=[])
-  get_monkeypatch.setattr(model.image, 'get_parent_images', get_parent_images_mock)
-
-  images = pre_oci_model.get_parent_images('namespace_name', 'repository_name', 'tag_name')
-  assert images == []
-
-
-def test_list_repository_tags(get_monkeypatch):
-  mock = Mock(return_value=[])
-
-  get_monkeypatch.setattr(model.tag, 'list_repository_tags', mock)
-
-  pre_oci_model.list_repository_tags('namespace_name', 'repository_name')
-
-  mock.assert_called_once_with('namespace_name', 'repository_name')
-
-
-def test_get_repository(get_monkeypatch):
-  mock = Mock()
-
-  get_monkeypatch.setattr(model.repository, 'get_repository', mock)
-
-  pre_oci_model.get_repository('namespace_name', 'repository_name')
-
-  mock.assert_called_once_with('namespace_name', 'repository_name')
-
-
-def test_tag_to_manifest(get_monkeypatch):
-  repo_mock = Mock()
-  restore_tag_mock = Mock(return_value=None)
-  get_repository_mock = Mock(return_value=repo_mock)
-
-  get_monkeypatch.setattr(model.tag, 'restore_tag_to_manifest', restore_tag_mock)
-  get_monkeypatch.setattr(model.repository, 'get_repository', get_repository_mock)
-
-  pre_oci_model.restore_tag_to_manifest(
-    Repository('namespace', 'repository'), 'tag_name', 'manifest_digest')
-
-  get_repository_mock.assert_called_once_with('namespace', 'repository')
-  restore_tag_mock.assert_called_once_with(repo_mock, 'tag_name', 'manifest_digest')
-
-
-def test__tag_to_image(get_monkeypatch):
-  repo_mock = Mock()
-  restore_tag_mock = Mock(return_value=None)
-  get_repository_mock = Mock(return_value=repo_mock)
-
-  get_monkeypatch.setattr(model.tag, 'restore_tag_to_image', restore_tag_mock)
-  get_monkeypatch.setattr(model.repository, 'get_repository', get_repository_mock)
-
-  pre_oci_model.restore_tag_to_image(Repository('namespace', 'repository'), 'tag_name', 'image_id')
-
-  get_repository_mock.assert_called_once_with('namespace', 'repository')
-  restore_tag_mock.assert_called_once_with(repo_mock, 'tag_name', 'image_id')

scripts/ci

@@ -69,6 +69,10 @@ registry_old() {
     load_image && quay_run make registry-test-old
 }
 
+certs_test() {
+    load_image && quay_run make certs-test
+}
+
 
 mysql_ping() {
     mysqladmin --connect-timeout=2 --wait=60 --host=127.0.0.1 \
@@ -146,6 +150,10 @@ case "$1" in
         registry_old
         ;;
 
+    certs_test)
+        certs_test
+        ;;
+
     mysql)
         mysql
         ;;

test/test_api_usage.py

@@ -25,6 +25,7 @@ from app import app, config_provider, all_queues, dockerfile_build_queue, notifi
 from buildtrigger.basehandler import BuildTriggerHandler
 from initdb import setup_database_for_testing, finished_database_for_testing
 from data import database, model, appr_model
+from data.registry_model import registry_model
 from data.appr_model.models import NEW_MODELS
 from data.database import RepositoryActionCount, Repository as RepositoryTable
 from test.helpers import assert_action_logged
@@ -2142,8 +2143,9 @@ class TestDeleteRepository(ApiTestCase):
     self.getResponse(Repository, params=dict(repository=self.COMPLEX_REPO))
 
     # Make sure the repository has some images and tags.
-    self.assertTrue(len(list(model.image.get_repository_images(ADMIN_ACCESS_USER, 'complex'))) > 0)
+    repo_ref = registry_model.lookup_repository(ADMIN_ACCESS_USER, 'complex')
-    self.assertTrue(len(list(model.tag.list_repository_tags(ADMIN_ACCESS_USER, 'complex'))) > 0)
+    self.assertTrue(len(list(registry_model.get_legacy_images(repo_ref))) > 0)
+    self.assertTrue(len(list(registry_model.list_repository_tags(repo_ref))) > 0)
 
     # Add some data for the repository, in addition to is already existing images and tags.
     repository = model.repository.get_repository(ADMIN_ACCESS_USER, 'complex')
@@ -2190,16 +2192,17 @@ class TestDeleteRepository(ApiTestCase):
     RepositoryActionCount.create(
       repository=repository, date=datetime.datetime.now() - datetime.timedelta(days=5), count=6)
 
+    repo_ref = registry_model.lookup_repository(ADMIN_ACCESS_USER, 'complex')
+    tag = registry_model.get_repo_tag(repo_ref, 'prod')
+    manifest = registry_model.get_manifest_for_tag(tag)
+
     # Create some labels.
-    pre_delete_label_count = database.Label.select().count()
+    registry_model.create_manifest_label(manifest, 'foo', 'bar', 'manifest')
+    registry_model.create_manifest_label(manifest, 'foo', 'baz', 'manifest')
+    registry_model.create_manifest_label(manifest, 'something', '{}', 'api',
+                                         media_type_name='application/json')
 
-    tag_manifest = model.tag.load_tag_manifest(ADMIN_ACCESS_USER, 'complex', 'prod')
+    registry_model.create_manifest_label(manifest, 'something', '{"some": "json"}', 'manifest')
-    model.label.create_manifest_label(tag_manifest, 'foo', 'bar', 'manifest')
-    model.label.create_manifest_label(tag_manifest, 'foo', 'baz', 'manifest')
-    model.label.create_manifest_label(tag_manifest, 'something', '{}', 'api',
-                                      media_type_name='application/json')
-
-    model.label.create_manifest_label(tag_manifest, 'something', '{"some": "json"}', 'manifest')
 
     # Delete the repository.
     with check_transitive_modifications():
@@ -2208,10 +2211,6 @@ class TestDeleteRepository(ApiTestCase):
     # Verify the repo was deleted.
     self.getResponse(Repository, params=dict(repository=self.COMPLEX_REPO), expected_code=404)
 
-    # Verify the labels are gone.
-    post_delete_label_count = database.Label.select().count()
-    self.assertEquals(post_delete_label_count, pre_delete_label_count)
-
 
 class TestGetRepository(ApiTestCase):
   PUBLIC_REPO = PUBLIC_USER + '/publicrepo'
@@ -2732,14 +2731,14 @@ class TestRestoreTag(ApiTestCase):
 
     self.postResponse(RestoreTag,
                       params=dict(repository=ADMIN_ACCESS_USER + '/history', tag='invalidtag'),
-                      data=dict(image='invalid_image'), expected_code=400)
+                      data=dict(image='invalid_image'), expected_code=404)
 
   def test_restoretag_invalidimage(self):
     self.login(ADMIN_ACCESS_USER)
 
     self.postResponse(RestoreTag,
                       params=dict(repository=ADMIN_ACCESS_USER + '/history', tag='latest'),
-                      data=dict(image='invalid_image'), expected_code=400)
+                      data=dict(image='invalid_image'), expected_code=404)
 
   def test_restoretag_invalidmanifest(self):
     self.login(ADMIN_ACCESS_USER)
@@ -2902,23 +2901,19 @@ class TestListAndDeleteTag(ApiTestCase):
   def test_listtagpagination(self):
     self.login(ADMIN_ACCESS_USER)
 
-    latest_image = model.tag.get_tag_image(ADMIN_ACCESS_USER, "complex", "prod")
+    repo_ref = registry_model.lookup_repository(ADMIN_ACCESS_USER, "simple")
+    latest_tag = registry_model.get_repo_tag(repo_ref, 'latest', include_legacy_image=True)
 
-    # Create 10 tags in an empty repo.
+    # Create 8 tags in the simple repo.
-    user = model.user.get_user_or_org(ADMIN_ACCESS_USER)
+    remaining_tags = {'latest', 'prod'}
-    repo = model.repository.create_repository(ADMIN_ACCESS_USER, "empty", user)
+    for i in xrange(1, 9):
-
-    image = model.image.find_create_or_link_image(latest_image.docker_image_id, repo,
-                                                  ADMIN_ACCESS_USER, {}, ['local_us'])
-    remaining_tags = set()
-    for i in xrange(1, 11):
       tag_name = "tag" + str(i)
       remaining_tags.add(tag_name)
-      model.tag.create_or_update_tag(ADMIN_ACCESS_USER, "empty", tag_name, image.docker_image_id)
+      registry_model.retarget_tag(repo_ref, tag_name, latest_tag.legacy_image)
 
     # Make sure we can iterate over all of them.
     json = self.getJsonResponse(ListRepositoryTags, params=dict(
-      repository=ADMIN_ACCESS_USER + '/empty', page=1, limit=5))
+      repository=ADMIN_ACCESS_USER + '/simple', page=1, limit=5))
     self.assertEquals(1, json['page'])
     self.assertEquals(5, len(json['tags']))
     self.assertTrue(json['has_additional'])
@@ -2928,7 +2923,7 @@ class TestListAndDeleteTag(ApiTestCase):
     self.assertEquals(5, len(remaining_tags))
 
     json = self.getJsonResponse(ListRepositoryTags, params=dict(
-      repository=ADMIN_ACCESS_USER + '/empty', page=2, limit=5))
+      repository=ADMIN_ACCESS_USER + '/simple', page=2, limit=5))
 
     self.assertEquals(2, json['page'])
     self.assertEquals(5, len(json['tags']))
@@ -2939,7 +2934,7 @@ class TestListAndDeleteTag(ApiTestCase):
     self.assertEquals(0, len(remaining_tags))
 
     json = self.getJsonResponse(ListRepositoryTags, params=dict(
-      repository=ADMIN_ACCESS_USER + '/empty', page=3, limit=5))
+      repository=ADMIN_ACCESS_USER + '/simple', page=3, limit=5))
 
     self.assertEquals(3, json['page'])
     self.assertEquals(0, len(json['tags']))
@@ -3984,80 +3979,6 @@ class TestSuperUserLogs(ApiTestCase):
     assert len(json['logs']) > 0
 
 
-
-
-class TestRepositoryImageSecurity(ApiTestCase):
-  def test_get_vulnerabilities(self):
-    self.login(ADMIN_ACCESS_USER)
-
-    tag = model.tag.get_active_tag(ADMIN_ACCESS_USER, 'simple', 'latest')
-    layer = model.tag.get_tag_image(ADMIN_ACCESS_USER, 'simple', 'latest')
-
-    tag_manifest = database.TagManifest.get(tag=tag)
-
-    # Grab the security info for the tag. It should be queued.
-    manifest_response = self.getJsonResponse(RepositoryManifestSecurity, params=dict(
-      repository=ADMIN_ACCESS_USER + '/simple', manifestref=tag_manifest.digest,
-      vulnerabilities='true'))
-
-    image_response = self.getJsonResponse(
-      RepositoryImageSecurity, params=dict(repository=ADMIN_ACCESS_USER + '/simple',
-                                           imageid=layer.docker_image_id, vulnerabilities='true'))
-
-    self.assertEquals(manifest_response, image_response)
-    self.assertEquals('queued', image_response['status'])
-
-    # Mark the layer as indexed.
-    layer.security_indexed = True
-    layer.security_indexed_engine = app.config['SECURITY_SCANNER_ENGINE_VERSION_TARGET']
-    layer.save()
-
-    # Grab the security info again.
-    with fake_security_scanner() as security_scanner:
-      security_scanner.add_layer(security_scanner.layer_id(layer))
-
-      manifest_response = self.getJsonResponse(RepositoryManifestSecurity, params=dict(
-        repository=ADMIN_ACCESS_USER + '/simple', manifestref=tag_manifest.digest,
-        vulnerabilities='true'))
-
-      image_response = self.getJsonResponse(RepositoryImageSecurity, params=dict(
-        repository=ADMIN_ACCESS_USER + '/simple', imageid=layer.docker_image_id,
-        vulnerabilities='true'))
-
-      self.assertEquals(manifest_response, image_response)
-      self.assertEquals('scanned', image_response['status'])
-      self.assertEquals(1, image_response['data']['Layer']['IndexedByVersion'])
-
-  def test_get_vulnerabilities_read_failover(self):
-    self.login(ADMIN_ACCESS_USER)
-
-    # Get a layer and mark it as indexed.
-    layer = model.tag.get_tag_image(ADMIN_ACCESS_USER, 'simple', 'latest')
-    layer.security_indexed = True
-    layer.security_indexed_engine = app.config['SECURITY_SCANNER_ENGINE_VERSION_TARGET']
-    layer.save()
-
-    with fake_security_scanner(hostname='failoverscanner') as security_scanner:
-      # Query the wrong security scanner URL without failover.
-      self.getResponse(RepositoryImageSecurity, params=dict(
-        repository=ADMIN_ACCESS_USER + '/simple', imageid=layer.docker_image_id,
-        vulnerabilities='true'), expected_code=520)
-
-      # Set the failover URL in the global config.
-      with AppConfigChange({
-          'SECURITY_SCANNER_READONLY_FAILOVER_ENDPOINTS': ['https://failoverscanner']
-      }):
-        # Configure the API to return 200 for this layer.
-        layer_id = security_scanner.layer_id(layer)
-        security_scanner.set_ok_layer_id(layer_id)
-
-        # Call the API and succeed on failover.
-        self.getResponse(RepositoryImageSecurity, params=dict(
-          repository=ADMIN_ACCESS_USER + '/simple', imageid=layer.docker_image_id,
-          vulnerabilities='true'), expected_code=200)
-
-
-
 class TestSuperUserTakeOwnership(ApiTestCase):
   def test_take_ownership_superuser(self):
     self.login(ADMIN_ACCESS_USER)
@@ -4242,46 +4163,46 @@ class TestRepositoryManifestLabels(ApiTestCase):
   def test_basic_labels(self):
     self.login(ADMIN_ACCESS_USER)
 
-    # Find the manifest digest for the prod tag in the complex repo.
+    repo_ref = registry_model.lookup_repository(ADMIN_ACCESS_USER, 'complex')
-    tag_manifest = model.tag.load_tag_manifest(ADMIN_ACCESS_USER, 'complex', 'prod')
+    tag = registry_model.get_repo_tag(repo_ref, 'prod')
     repository = ADMIN_ACCESS_USER + '/complex'
 
     # Check the existing labels on the complex repo, which should be empty
     json = self.getJsonResponse(
       RepositoryManifestLabels,
-      params=dict(repository=repository, manifestref=tag_manifest.digest))
+      params=dict(repository=repository, manifestref=tag.manifest_digest))
 
     self.assertEquals(0, len(json['labels']))
 
     self.postJsonResponse(RepositoryManifestLabels, params=dict(repository=repository,
-                                                                manifestref=tag_manifest.digest),
+                                                                manifestref=tag.manifest_digest),
                           data=dict(key='bad_label', value='world',
                                     media_type='text/plain'), expected_code=400)
 
     self.postJsonResponse(RepositoryManifestLabels, params=dict(repository=repository,
-                                                                manifestref=tag_manifest.digest),
+                                                                manifestref=tag.manifest_digest),
                           data=dict(key='hello', value='world',
                                     media_type='bad_media_type'), expected_code=400)
 
     # Add some labels to the manifest.
     with assert_action_logged('manifest_label_add'):
       label1 = self.postJsonResponse(RepositoryManifestLabels, params=dict(
-        repository=repository, manifestref=tag_manifest.digest), data=dict(
+        repository=repository, manifestref=tag.manifest_digest), data=dict(
           key='hello', value='world', media_type='text/plain'), expected_code=201)
 
     with assert_action_logged('manifest_label_add'):
       label2 = self.postJsonResponse(RepositoryManifestLabels, params=dict(
-        repository=repository, manifestref=tag_manifest.digest), data=dict(
+        repository=repository, manifestref=tag.manifest_digest), data=dict(
           key='hi', value='there', media_type='text/plain'), expected_code=201)
 
     with assert_action_logged('manifest_label_add'):
       label3 = self.postJsonResponse(RepositoryManifestLabels, params=dict(
-        repository=repository, manifestref=tag_manifest.digest), data=dict(
+        repository=repository, manifestref=tag.manifest_digest), data=dict(
           key='hello', value='someone', media_type='application/json'), expected_code=201)
 
     # Ensure we have *3* labels
     json = self.getJsonResponse(RepositoryManifestLabels, params=dict(
-      repository=repository, manifestref=tag_manifest.digest))
+      repository=repository, manifestref=tag.manifest_digest))
 
     self.assertEquals(3, len(json['labels']))
 
@@ -4296,73 +4217,75 @@ class TestRepositoryManifestLabels(ApiTestCase):
     # Ensure we can retrieve each of the labels.
     for label in json['labels']:
       label_json = self.getJsonResponse(ManageRepositoryManifestLabel, params=dict(
-        repository=repository, manifestref=tag_manifest.digest, labelid=label['id']))
+        repository=repository, manifestref=tag.manifest_digest, labelid=label['id']))
       self.assertEquals(label['id'], label_json['id'])
 
     # Delete a label.
     with assert_action_logged('manifest_label_delete'):
       self.deleteEmptyResponse(ManageRepositoryManifestLabel, params=dict(
-        repository=repository, manifestref=tag_manifest.digest, labelid=label1['label']['id']))
+        repository=repository, manifestref=tag.manifest_digest, labelid=label1['label']['id']))
 
     # Ensure the label is gone.
     json = self.getJsonResponse(RepositoryManifestLabels, params=dict(
-      repository=repository, manifestref=tag_manifest.digest))
+      repository=repository, manifestref=tag.manifest_digest))
 
     self.assertEquals(2, len(json['labels']))
 
     # Check filtering.
     json = self.getJsonResponse(RepositoryManifestLabels, params=dict(
-      repository=repository, manifestref=tag_manifest.digest, filter='hello'))
+      repository=repository, manifestref=tag.manifest_digest, filter='hello'))
 
     self.assertEquals(1, len(json['labels']))
 
   def test_prefixed_labels(self):
     self.login(ADMIN_ACCESS_USER)
 
-    # Find the manifest digest for the prod tag in the complex repo.
+    repo_ref = registry_model.lookup_repository(ADMIN_ACCESS_USER, 'complex')
-    tag_manifest = model.tag.load_tag_manifest(ADMIN_ACCESS_USER, 'complex', 'prod')
+    tag = registry_model.get_repo_tag(repo_ref, 'prod')
     repository = ADMIN_ACCESS_USER + '/complex'
 
     self.postJsonResponse(RepositoryManifestLabels, params=dict(repository=repository,
-                                                                manifestref=tag_manifest.digest),
+                                                                manifestref=tag.manifest_digest),
                           data=dict(key='com.dockers.whatever', value='pants',
                                     media_type='text/plain'), expected_code=201)
 
     self.postJsonResponse(RepositoryManifestLabels, params=dict(repository=repository,
-                                                                manifestref=tag_manifest.digest),
+                                                                manifestref=tag.manifest_digest),
                           data=dict(key='my.cool.prefix.for.my.label', value='value',
                                     media_type='text/plain'), expected_code=201)
 
   def test_add_invalid_media_type(self):
     self.login(ADMIN_ACCESS_USER)
 
-    tag_manifest = model.tag.load_tag_manifest(ADMIN_ACCESS_USER, 'complex', 'prod')
+    repo_ref = registry_model.lookup_repository(ADMIN_ACCESS_USER, 'complex')
+    tag = registry_model.get_repo_tag(repo_ref, 'prod')
     repository = ADMIN_ACCESS_USER + '/complex'
 
     self.postResponse(RepositoryManifestLabels, params=dict(repository=repository,
-                                                            manifestref=tag_manifest.digest),
+                                                            manifestref=tag.manifest_digest),
                       data=dict(key='hello', value='world', media_type='some/invalid'),
                       expected_code=400)
 
   def test_add_invalid_key(self):
     self.login(ADMIN_ACCESS_USER)
 
-    tag_manifest = model.tag.load_tag_manifest(ADMIN_ACCESS_USER, 'complex', 'prod')
+    repo_ref = registry_model.lookup_repository(ADMIN_ACCESS_USER, 'complex')
+    tag = registry_model.get_repo_tag(repo_ref, 'prod')
     repository = ADMIN_ACCESS_USER + '/complex'
 
     # Try to add an empty label key.
     self.postResponse(RepositoryManifestLabels, params=dict(repository=repository,
-                                                            manifestref=tag_manifest.digest),
+                                                            manifestref=tag.manifest_digest),
                       data=dict(key='', value='world'), expected_code=400)
 
     # Try to add an invalid label key.
     self.postResponse(RepositoryManifestLabels, params=dict(repository=repository,
-                                                            manifestref=tag_manifest.digest),
+                                                            manifestref=tag.manifest_digest),
                       data=dict(key='invalid___key', value='world'), expected_code=400)
 
     # Try to add a label key in a reserved namespace.
     self.postResponse(RepositoryManifestLabels, params=dict(repository=repository,
-                                                            manifestref=tag_manifest.digest),
+                                                            manifestref=tag.manifest_digest),
                       data=dict(key='io.docker.whatever', value='world'), expected_code=400)
 
 

test/test_certs_install.sh

@@ -0,0 +1,66 @@
+#!/usr/bin/env bash
+
+set -e
+
+echo "> Starting certs install test"
+
+# Set up all locations needed for the test
+QUAYPATH=${QUAYPATH:-"."}
+SCRIPT_LOCATION=${SCRIPT_LOCATION:-"/quay-registry/conf/init"}
+
+# Parameters: (quay config dir, certifcate dir, number of certs expected).
+function call_script_and_check_num_certs {
+    QUAYCONFIG=$1 CERTDIR=$2 ${SCRIPT_LOCATION}/certs_install.sh
+    if [ $? -ne 0 ]; then
+        echo "Failed to install $3 certs"
+        exit 1;
+    fi
+
+    certs_found=$(ls /usr/local/share/ca-certificates | wc -l)
+    if [ ${certs_found} -ne "$3" ]; then
+        echo "Expected there to be $3 in ca-certificates, found $certs_found"
+        exit 1
+    fi
+}
+
+# Create a dummy cert we can test to install
+echo '{"CN":"CA","key":{"algo":"rsa","size":2048}}' | cfssl gencert -initca - | cfssljson -bare test
+
+# Create temp dirs we can test with
+WORK_DIR=`mktemp -d`
+CERTS_WORKDIR=`mktemp -d`
+
+# deletes the temp directory
+function cleanup {
+  rm -rf "$WORK_DIR"
+  rm -rf "$CERTS_WORKDIR"
+  rm test.pem
+  rm test-key.pem
+}
+
+# register the cleanup function to be called on the EXIT signal
+trap cleanup EXIT
+
+# Test calling with empty directory to not fail
+call_script_and_check_num_certs ${WORK_DIR} ${CERTS_WORKDIR} 0
+if [ "$?" -ne 0 ]; then
+    echo "Failed to install certs with no files in the directory"
+    exit 1
+fi
+
+# Move an ldap cert into the temp directory and test that installation
+cp test.pem ${WORK_DIR}/ldap.crt
+call_script_and_check_num_certs ${WORK_DIR} ${CERTS_WORKDIR} 1
+
+# Move 1 cert to extra cert dir and test
+cp test.pem ${CERTS_WORKDIR}/cert1.crt
+call_script_and_check_num_certs ${WORK_DIR} ${CERTS_WORKDIR} 2
+
+
+# Move another cert to extra cer dir and test all three exist
+cp test.pem ${CERTS_WORKDIR}/cert2.crt
+call_script_and_check_num_certs ${WORK_DIR} ${CERTS_WORKDIR} 3
+
+
+echo "> Certs install script test succeeded"
+exit 0

util/imagetree.py

@@ -1,99 +0,0 @@
-from collections import defaultdict
-
-
-class ImageTreeNode(object):
-  """ A node in the image tree. """
-  def __init__(self, image, child_map):
-    self.image = image
-    self.parent = None
-    self.tags = []
-
-    self._child_map = child_map
-
-  @property
-  def children(self):
-    return self._child_map[self.image.id]
-
-  def add_tag(self, tag):
-    self.tags.append(tag)
-
-
-class ImageTree(object):
-  """ In-memory tree for easy traversal and lookup of images in a repository. """
-
-  def __init__(self, all_images, all_tags, base_filter=None):
-    self._image_map = {}
-    self._child_map = defaultdict(list)
-
-    self._build(all_images, all_tags, base_filter)
-
-  def _build(self, all_images, all_tags, base_filter=None):
-    # Build nodes for each of the images.
-    for image in all_images:
-      ancestors = image.ancestor_id_list()
-
-      # Filter any unneeded images.
-      if base_filter is not None:
-        if image.id != base_filter and not base_filter in ancestors:
-          continue
-
-      # Create the node for the image.
-      image_node = ImageTreeNode(image, self._child_map)
-      self._image_map[image.id] = image_node
-
-      # Add the node to the child map for its parent image (if any).
-      parent_image_id = image.parent_id
-      if parent_image_id is not None:
-        self._child_map[parent_image_id].append(image_node)
-
-    # Build the tag map.
-    for tag in all_tags:
-      image_node = self._image_map.get(tag.image.id)
-      if not image_node:
-        continue
-
-      image_node.add_tag(tag.name)
-
-  def find_longest_path(self, image_id, checker):
-    """ Returns a list of images representing the longest path that matches the given
-        checker function, starting from the given image_id *exclusive*.
-    """
-    start_node = self._image_map.get(image_id)
-    if not start_node:
-      return []
-
-    return self._find_longest_path(start_node, checker, -1)[1:]
-
-  def _find_longest_path(self, image_node, checker, index):
-    found_path = []
-
-    for child_node in image_node.children:
-      if not checker(index + 1, child_node.image):
-        continue
-
-      found = self._find_longest_path(child_node, checker, index + 1)
-      if found and len(found) > len(found_path):
-        found_path = found
-
-    return [image_node.image] + found_path
-
-  def tag_containing_image(self, image):
-    """ Returns the name of the closest tag containing the given image. """
-    if not image:
-      return None
-
-    # Check the current image for a tag.
-    image_node = self._image_map.get(image.id)
-    if image_node is None:
-      return None
-
-    if image_node.tags:
-      return image_node.tags[0]
-
-    # Check any deriving images for a tag.
-    for child_node in image_node.children:
-      found = self.tag_containing_image(child_node.image)
-      if found is not None:
-        return found
-
-    return None

util/secscan/api.py

@@ -7,9 +7,10 @@ from urlparse import urljoin
 
 import requests
 
-from data.database import CloseForLongOperation
 from data import model
+from data.database import CloseForLongOperation, TagManifest, Image
 from data.model.storage import get_storage_locations
+from data.registry_model.datatypes import Manifest, LegacyImage
 from util.abchelpers import nooper
 from util.failover import failover, FailoverException
 from util.secscan.validator import SecurityConfigValidator
@@ -61,6 +62,12 @@ _API_METHOD_PING = 'metrics'
 
 def compute_layer_id(layer):
   """ Returns the ID for the layer in the security scanner. """
+  # NOTE: this is temporary until we switch to Clair V3.
+  if isinstance(layer, Manifest):
+    layer = TagManifest.get(id=layer._db_id).tag.image
+  elif isinstance(layer, LegacyImage):
+    layer = Image.get(id=layer._db_id)
+
   return '%s.%s' % (layer.docker_image_id, layer.storage.uuid)
 
 

util/test/test_imagetree.py

@@ -1,82 +0,0 @@
-from data import model
-from util.imagetree import ImageTree
-
-from test.fixtures import *
-
-NAMESPACE = 'devtable'
-SIMPLE_REPO = 'simple'
-COMPLEX_REPO = 'complex'
-
-def _get_base_image(all_images):
-  for image in all_images:
-    if image.ancestors == '/':
-      return image
-
-  return None
-
-def test_longest_path_simple_repo(initialized_db):
-  all_images = list(model.image.get_repository_images(NAMESPACE, SIMPLE_REPO))
-  all_tags = list(model.tag.list_repository_tags(NAMESPACE, SIMPLE_REPO))
-  tree = ImageTree(all_images, all_tags)
-
-  base_image = _get_base_image(all_images)
-  tag_image = all_tags[0].image
-
-  def checker(index, image):
-    return True
-
-  ancestors = tag_image.ancestors.split('/')[2:-1] # Skip the first image.
-  result = tree.find_longest_path(base_image.id, checker)
-  assert len(result) == 3
-  for index in range(0, 2):
-    assert result[index].id == int(ancestors[index])
-
-  assert tree.tag_containing_image(result[-1]) == 'latest'
-
-def test_longest_path_complex_repo(initialized_db):
-  all_images = list(model.image.get_repository_images(NAMESPACE, COMPLEX_REPO))
-  all_tags = list(model.tag.list_repository_tags(NAMESPACE, COMPLEX_REPO))
-  tree = ImageTree(all_images, all_tags)
-
-  base_image = _get_base_image(all_images)
-
-  def checker(index, image):
-    return True
-
-  result = tree.find_longest_path(base_image.id, checker)
-  assert len(result) == 5
-  assert tree.tag_containing_image(result[-1]) == 'prod'
-
-def test_filtering(initialized_db):
-  all_images = list(model.image.get_repository_images(NAMESPACE, COMPLEX_REPO))
-  all_tags = list(model.tag.list_repository_tags(NAMESPACE, COMPLEX_REPO))
-  tree = ImageTree(all_images, all_tags, base_filter=1245)
-
-  base_image = _get_base_image(all_images)
-
-  def checker(index, image):
-    return True
-
-  result = tree.find_longest_path(base_image.id, checker)
-  assert len(result) == 0
-
-def test_longest_path_simple_repo_direct_lookup(initialized_db):
-  repository = model.repository.get_repository(NAMESPACE, SIMPLE_REPO)
-  all_images = list(model.image.get_repository_images(NAMESPACE, SIMPLE_REPO))
-  all_tags = list(model.tag.list_repository_tags(NAMESPACE, SIMPLE_REPO))
-
-  base_image = _get_base_image(all_images)
-
-  def checker(index, image):
-    return True
-
-  filtered_images = model.image.get_repository_images_without_placements(
-    repository,
-    with_ancestor=base_image)
-  assert set([a.id for a in all_images]) == set([f.id for f in filtered_images])
-
-  tree = ImageTree(filtered_images, all_tags)
-
-  result = tree.find_longest_path(base_image.id, checker)
-  assert len(result) == 3
-  assert tree.tag_containing_image(result[-1]) == 'latest'