quay/data/model/tag.py

import logging

from uuid import uuid4

from data.model import (image, db_transaction, DataModelException, _basequery,
                        InvalidManifestException)
from data.database import (RepositoryTag, Repository, Image, ImageStorage, Namespace, TagManifest,
                           RepositoryNotification, Label, TagManifestLabel, get_epoch_timestamp,
                           db_for_update)


logger = logging.getLogger(__name__)


def _tag_alive(query, now_ts=None):
  if now_ts is None:
    now_ts = get_epoch_timestamp()
  return query.where((RepositoryTag.lifetime_end_ts >> None) |
                     (RepositoryTag.lifetime_end_ts > now_ts))


def get_matching_tags(docker_image_id, storage_uuid, *args):
  """ Returns a query pointing to all tags that contain the image with the
      given docker_image_id and storage_uuid. """
  image_query = image.get_repository_image_and_deriving(docker_image_id, storage_uuid)

  return _tag_alive(RepositoryTag
                    .select(*args)
                    .distinct()
                    .join(Image)
                    .join(ImageStorage)
                    .where(Image.id << image_query, RepositoryTag.hidden == False))


def get_tags_for_image(image_id, *args):
  return _tag_alive(RepositoryTag
                    .select(*args)
                    .distinct()
                    .where(RepositoryTag.image == image_id,
                           RepositoryTag.hidden == False))


def filter_tags_have_repository_event(query, event):
  return (query
          .switch(RepositoryTag)
          .join(Repository)
          .join(RepositoryNotification)
          .where(RepositoryNotification.event == event))

def list_repository_tags(namespace_name, repository_name, include_hidden=False,
                         include_storage=False):
  to_select = (RepositoryTag, Image)
  if include_storage:
    to_select = (RepositoryTag, Image, ImageStorage)

  query = _tag_alive(RepositoryTag
                     .select(*to_select)
                     .join(Repository)
                     .join(Namespace, on=(Repository.namespace_user == Namespace.id))
                     .switch(RepositoryTag)
                     .join(Image)
                     .where(Repository.name == repository_name,
                            Namespace.username == namespace_name))

  if not include_hidden:
    query = query.where(RepositoryTag.hidden == False)

  if include_storage:
    query = query.switch(Image).join(ImageStorage)

  return query


def create_or_update_tag(namespace_name, repository_name, tag_name, tag_docker_image_id,
                         reversion=False):
  try:
    repo = _basequery.get_existing_repository(namespace_name, repository_name)
  except Repository.DoesNotExist:
    raise DataModelException('Invalid repository %s/%s' % (namespace_name, repository_name))

  now_ts = get_epoch_timestamp()

  with db_transaction():
    try:
      tag = db_for_update(_tag_alive(RepositoryTag
                                     .select()
                                     .where(RepositoryTag.repository == repo,
                                            RepositoryTag.name == tag_name), now_ts)).get()
      tag.lifetime_end_ts = now_ts
      tag.save()
    except RepositoryTag.DoesNotExist:
      pass

    try:
      image_obj = Image.get(Image.docker_image_id == tag_docker_image_id, Image.repository == repo)
    except Image.DoesNotExist:
      raise DataModelException('Invalid image with id: %s' % tag_docker_image_id)

    return RepositoryTag.create(repository=repo, image=image_obj, name=tag_name,
                                lifetime_start_ts=now_ts, reversion=reversion)


def create_temporary_hidden_tag(repo, image_obj, expiration_s):
  """ Create a tag with a defined timeline, that will not appear in the UI or CLI. Returns the name
      of the temporary tag. """
  now_ts = get_epoch_timestamp()
  expire_ts = now_ts + expiration_s
  tag_name = str(uuid4())
  RepositoryTag.create(repository=repo, image=image_obj, name=tag_name, lifetime_start_ts=now_ts,
                       lifetime_end_ts=expire_ts, hidden=True)
  return tag_name


def delete_tag(namespace_name, repository_name, tag_name):
  now_ts = get_epoch_timestamp()
  with db_transaction():
    try:
      query = _tag_alive(RepositoryTag
                         .select(RepositoryTag, Repository)
                         .join(Repository)
                         .join(Namespace, on=(Repository.namespace_user == Namespace.id))
                         .where(Repository.name == repository_name,
                                Namespace.username == namespace_name,
                                RepositoryTag.name == tag_name), now_ts)
      found = db_for_update(query).get()
    except RepositoryTag.DoesNotExist:
      msg = ('Invalid repository tag \'%s\' on repository \'%s/%s\'' %
             (tag_name, namespace_name, repository_name))
      raise DataModelException(msg)

    found.lifetime_end_ts = now_ts
    found.save()


def garbage_collect_tags(repo):
  expired_time = get_epoch_timestamp() - repo.namespace_user.removed_tag_expiration_s

  tags_to_delete = list(RepositoryTag
                        .select(RepositoryTag.id)
                        .where(RepositoryTag.repository == repo,
                               ~(RepositoryTag.lifetime_end_ts >> None),
                               (RepositoryTag.lifetime_end_ts <= expired_time))
                        .order_by(RepositoryTag.id))

  if len(tags_to_delete) > 0:
    with db_transaction():
      manifests_to_delete = list(TagManifest
                                 .select(TagManifest.id)
                                 .join(RepositoryTag)
                                 .where(RepositoryTag.id << tags_to_delete))

      num_deleted_manifests = 0
      if len(manifests_to_delete) > 0:
        # Find the set of IDs for all the labels to delete.
        labels = list(TagManifestLabel
                      .select(TagManifestLabel.id)
                      .where(TagManifestLabel.annotated << manifests_to_delete))

        if len(labels) > 0:
          # Delete the mapping entries.
          (TagManifestLabel
           .delete()
           .where(TagManifestLabel.annotated << manifests_to_delete)
           .execute())

          # Delete the labels themselves.
          Label.delete().where(Label.id << [label.id for label in labels]).execute()

        # Delete the tag manifests themselves.
        num_deleted_manifests = (TagManifest
                                 .delete()
                                 .where(TagManifest.id << manifests_to_delete)
                                 .execute())

      num_deleted_tags = (RepositoryTag
                          .delete()
                          .where(RepositoryTag.id << tags_to_delete)
                          .execute())

      logger.debug('Removed %s tags with %s manifests', num_deleted_tags, num_deleted_manifests)


def _get_repo_tag_image(tag_name, include_storage, modifier):
  query = Image.select().join(RepositoryTag)

  if include_storage:
    query = (Image.select(Image, ImageStorage)
                  .join(ImageStorage)
                  .switch(Image)
                  .join(RepositoryTag))

  images = _tag_alive(modifier(query.where(RepositoryTag.name == tag_name)))
  if not images:
    raise DataModelException('Unable to find image for tag.')
  else:
    return images[0]


def get_repo_tag_image(repo, tag_name, include_storage=False):
  def modifier(query):
    return query.where(RepositoryTag.repository == repo)

  return _get_repo_tag_image(tag_name, include_storage, modifier)


def get_tag_image(namespace_name, repository_name, tag_name, include_storage=False):
  def modifier(query):
    return (query.switch(RepositoryTag)
                 .join(Repository)
                 .join(Namespace)
                 .where(Namespace.username == namespace_name, Repository.name == repository_name))

  return _get_repo_tag_image(tag_name, include_storage, modifier)


def list_repository_tag_history(repo_obj, page=1, size=100, specific_tag=None):
  query = (RepositoryTag
           .select(RepositoryTag, Image)
           .join(Image)
           .where(RepositoryTag.repository == repo_obj)
           .where(RepositoryTag.hidden == False)
           .order_by(RepositoryTag.lifetime_start_ts.desc(), RepositoryTag.name)
           .limit(size + 1)
           .offset(size * (page - 1)))

  if specific_tag:
    query = query.where(RepositoryTag.name == specific_tag)

  tags = list(query)
  return tags[0:size], len(tags) > size


def revert_tag(repo_obj, tag_name, docker_image_id):
  """ Reverts a tag to a specific image ID. """
  # Verify that the image ID already existed under this repository under the
  # tag.
  try:
    (RepositoryTag
     .select()
     .join(Image)
     .where(RepositoryTag.repository == repo_obj)
     .where(RepositoryTag.name == tag_name)
     .where(Image.docker_image_id == docker_image_id)
     .get())
  except RepositoryTag.DoesNotExist:
    raise DataModelException('Cannot revert to unknown or invalid image')

  return create_or_update_tag(repo_obj.namespace_user.username, repo_obj.name, tag_name,
                              docker_image_id, reversion=True)


def store_tag_manifest(namespace, repo_name, tag_name, docker_image_id, manifest_digest,
                       manifest_data):
  """ Stores a tag manifest for a specific tag name in the database. Returns the TagManifest
      object, as well as a boolean indicating whether the TagManifest was created or updated.
  """
  with db_transaction():
    tag = create_or_update_tag(namespace, repo_name, tag_name, docker_image_id)

    try:
      manifest = TagManifest.get(digest=manifest_digest)
      if manifest.tag == tag:
        return manifest, False

      manifest.tag = tag
      manifest.save()
      return manifest, True
    except TagManifest.DoesNotExist:
      return TagManifest.create(tag=tag, digest=manifest_digest, json_data=manifest_data), True


def get_active_tag(namespace, repo_name, tag_name):
  return _tag_alive(RepositoryTag
                    .select()
                    .join(Repository)
                    .join(Namespace, on=(Repository.namespace_user == Namespace.id))
                    .where(RepositoryTag.name == tag_name, Repository.name == repo_name,
                           Namespace.username == namespace)).get()


def associate_generated_tag_manifest(namespace, repo_name, tag_name, manifest_digest,
                                     manifest_data):
  tag = get_active_tag(namespace, repo_name, tag_name)
  return TagManifest.create(tag=tag, digest=manifest_digest, json_data=manifest_data)


def load_tag_manifest(namespace, repo_name, tag_name):
  try:
    return (_load_repo_manifests(namespace, repo_name)
            .where(RepositoryTag.name == tag_name)
            .get())
  except TagManifest.DoesNotExist:
    msg = 'Manifest not found for tag {0} in repo {1}/{2}'.format(tag_name, namespace, repo_name)
    raise InvalidManifestException(msg)


def load_manifest_by_digest(namespace, repo_name, digest):
  try:
    return (_load_repo_manifests(namespace, repo_name)
            .where(TagManifest.digest == digest)
            .get())
  except TagManifest.DoesNotExist:
    msg = 'Manifest not found with digest {0} in repo {1}/{2}'.format(digest, namespace, repo_name)
    raise InvalidManifestException(msg)


def _load_repo_manifests(namespace, repo_name):
    return _tag_alive(TagManifest
                      .select(TagManifest, RepositoryTag, Repository)
                      .join(RepositoryTag)
                      .join(Repository)
                      .join(Namespace, on=(Namespace.id == Repository.namespace_user))
                      .where(Repository.name == repo_name, Namespace.username == namespace))