Basic labels support

Adds basic labels support to the registry code (V2), and the API. Note that this does not yet add any UI related support.

data/model/__init__.py

@@ -5,9 +5,18 @@ class DataModelException(Exception):
   pass
 
 
+class InvalidLabelKeyException(DataModelException):
+  pass
+
+
+class InvalidMediaTypeException(DataModelException):
+  pass
+
+
 class BlobDoesNotExist(DataModelException):
   pass
 
+
 class TorrentInfoDoesNotExist(DataModelException):
   pass
 
@@ -105,7 +114,23 @@ config = Config()
 
 # There MUST NOT be any circular dependencies between these subsections. If there are fix it by
 # moving the minimal number of things to _basequery
-# TODO document the methods and modules for each one of the submodules below.
-from data.model import (blob, build, image, log, notification, oauth, organization, permission,
-                        repository, service_keys, storage, tag, team, token, user, release,
-                        modelutil)
+from data.model import (
+  blob,
+  build,
+  image,
+  label,
+  log,
+  modelutil,
+  notification,
+  oauth,
+  organization,
+  permission,
+  release,
+  repository,
+  service_keys,
+  storage,
+  tag,
+  team,
+  token,
+  user,
+)

data/model/label.py

@@ -0,0 +1,121 @@
+import logging
+
+from data.database import Label, TagManifestLabel, MediaType, LabelSourceType, db_transaction
+from data.model import InvalidLabelKeyException, InvalidMediaTypeException, DataModelException
+from data.model._basequery import prefix_search
+from util.validation import validate_label_key
+from util.validation import is_json
+from cachetools import lru_cache
+
+logger = logging.getLogger(__name__)
+
+
+@lru_cache(maxsize=1)
+def get_label_source_types():
+  source_type_map = {}
+  for kind in LabelSourceType.select():
+    source_type_map[kind.id] = kind.name
+    source_type_map[kind.name] = kind.id
+
+  return source_type_map
+
+
+@lru_cache(maxsize=1)
+def get_media_types():
+  media_type_map = {}
+  for kind in MediaType.select():
+    media_type_map[kind.id] = kind.name
+    media_type_map[kind.name] = kind.id
+
+  return media_type_map
+
+
+def _get_label_source_type_id(name):
+  kinds = get_label_source_types()
+  return kinds[name]
+
+
+def _get_media_type_id(name):
+  kinds = get_media_types()
+  return kinds[name]
+
+
+def create_manifest_label(tag_manifest, key, value, source_type_name, media_type_name=None):
+  """ Creates a new manifest label on a specific tag manifest. """
+  if not key:
+    raise InvalidLabelKeyException()
+
+  # Note that we don't prevent invalid label names coming from the manifest to be stored, as Docker
+  # does not currently prevent them from being put into said manifests.
+  if not validate_label_key(key) and source_type_name != 'manifest':
+    raise InvalidLabelKeyException()
+
+  # Find the matching media type. If none specified, we infer.
+  if media_type_name is None:
+    media_type_name = 'text/plain'
+    if is_json(value):
+      media_type_name = 'application/json'
+
+  media_type_id = _get_media_type_id(media_type_name)
+  if media_type_id is None:
+    raise InvalidMediaTypeException
+
+  source_type_id = _get_label_source_type_id(source_type_name)
+
+  with db_transaction():
+    label = Label.create(key=key, value=value, source_type=source_type_id, media_type=media_type_id)
+    TagManifestLabel.create(annotated=tag_manifest, label=label,
+                            repository=tag_manifest.tag.repository)
+
+  return label
+
+
+def list_manifest_labels(tag_manifest, prefix_filter=None):
+  """ Lists all labels found on the given tag manifest. """
+  query = (Label.select(Label, MediaType)
+           .join(MediaType)
+           .switch(Label)
+           .join(LabelSourceType)
+           .switch(Label)
+           .join(TagManifestLabel)
+           .where(TagManifestLabel.annotated == tag_manifest))
+
+  if prefix_filter is not None:
+    query = query.where(prefix_search(Label.key, prefix_filter))
+
+  return query
+
+
+def get_manifest_label(label_uuid, tag_manifest):
+  """ Retrieves the manifest label on the tag manifest with the given ID. """
+  try:
+    return (Label.select(Label, LabelSourceType)
+            .join(LabelSourceType)
+            .where(Label.uuid == label_uuid)
+            .switch(Label)
+            .join(TagManifestLabel)
+            .where(TagManifestLabel.annotated == tag_manifest)
+            .get())
+  except Label.DoesNotExist:
+    return None
+
+
+def delete_manifest_label(label_uuid, tag_manifest):
+  """ Deletes the manifest label on the tag manifest with the given ID. """
+
+  # Find the label itself.
+  label = get_manifest_label(label_uuid, tag_manifest)
+  if label is None:
+    return None
+
+  if not label.source_type.mutable:
+    raise DataModelException('Cannot delete immutable label')
+
+  # Delete the mapping record and label.
+  deleted_count = TagManifestLabel.delete().where(TagManifestLabel.label == label).execute()
+  if deleted_count != 1:
+    logger.warning('More than a single label deleted for matching label %s', label_uuid)
+
+  label.delete_instance(recursive=False)
+  return label
+

data/model/repository.py

@@ -10,7 +10,8 @@ from data.model import (DataModelException, tag, db_transaction, storage, permis
 from data.database import (Repository, Namespace, RepositoryTag, Star, Image, User,
                            Visibility, RepositoryPermission, RepositoryActionCount,
                            Role, RepositoryAuthorizedEmail, TagManifest, DerivedStorageForImage,
-                           get_epoch_timestamp, db_random_func)
+                           Label, TagManifestLabel, db_for_update, get_epoch_timestamp,
+                           db_random_func)
 
 
 logger = logging.getLogger(__name__)
@@ -50,11 +51,24 @@ def _purge_all_repository_tags(namespace_name, repository_name):
     raise DataModelException('Invalid repository \'%s/%s\'' %
                              (namespace_name, repository_name))
 
-  # Delete all manifests.
+  # Finds all the tags to delete.
   repo_tags = list(RepositoryTag.select().where(RepositoryTag.repository == repo.id))
   if not repo_tags:
     return
 
+  # Find all labels to delete.
+  labels = list(TagManifestLabel
+                .select(TagManifestLabel.label)
+                .where(TagManifestLabel.repository == repo))
+
+  # Delete all the mapping entries.
+  TagManifestLabel.delete().where(TagManifestLabel.repository == repo).execute()
+
+  # Delete all the matching labels.
+  if labels:
+    Label.delete().where(Label.id << [label.id for label in labels]).execute()
+
+  # Delete all the manifests.
   TagManifest.delete().where(TagManifest.tag << repo_tags).execute()
 
   # Delete all tags.

data/model/tag.py

@@ -5,7 +5,8 @@ from uuid import uuid4
 from data.model import (image, db_transaction, DataModelException, _basequery,
                         InvalidManifestException)
 from data.database import (RepositoryTag, Repository, Image, ImageStorage, Namespace, TagManifest,
-                           RepositoryNotification, get_epoch_timestamp, db_for_update)
+                           RepositoryNotification, Label, TagManifestLabel, get_epoch_timestamp,
+                           db_for_update)
 
 
 logger = logging.getLogger(__name__)
@@ -150,6 +151,22 @@ def garbage_collect_tags(repo):
 
       num_deleted_manifests = 0
       if len(manifests_to_delete) > 0:
+        # Find the set of IDs for all the labels to delete.
+        labels = list(TagManifestLabel
+                      .select(TagManifestLabel.id)
+                      .where(TagManifestLabel.annotated << manifests_to_delete))
+
+        if len(labels) > 0:
+          # Delete the mapping entries.
+          (TagManifestLabel
+           .delete()
+           .where(TagManifestLabel.annotated << manifests_to_delete)
+           .execute())
+
+          # Delete the labels themselves.
+          Label.delete().where(Label.id << [label.id for label in labels]).execute()
+
+        # Delete the tag manifests themselves.
         num_deleted_manifests = (TagManifest
                                  .delete()
                                  .where(TagManifest.id << manifests_to_delete)
@@ -234,15 +251,22 @@ def revert_tag(repo_obj, tag_name, docker_image_id):
 
 def store_tag_manifest(namespace, repo_name, tag_name, docker_image_id, manifest_digest,
                        manifest_data):
+  """ Stores a tag manifest for a specific tag name in the database. Returns the TagManifest
+      object, as well as a boolean indicating whether the TagManifest was created or updated.
+  """
   with db_transaction():
     tag = create_or_update_tag(namespace, repo_name, tag_name, docker_image_id)
 
     try:
       manifest = TagManifest.get(digest=manifest_digest)
+      if manifest.tag == tag:
+        return manifest, False
+
       manifest.tag = tag
       manifest.save()
+      return manifest, True
     except TagManifest.DoesNotExist:
-      return TagManifest.create(tag=tag, digest=manifest_digest, json_data=manifest_data)
+      return TagManifest.create(tag=tag, digest=manifest_digest, json_data=manifest_data), True
 
 
 def get_active_tag(namespace, repo_name, tag_name):
@@ -282,7 +306,7 @@ def load_manifest_by_digest(namespace, repo_name, digest):
 
 def _load_repo_manifests(namespace, repo_name):
     return _tag_alive(TagManifest
-                      .select(TagManifest, RepositoryTag)
+                      .select(TagManifest, RepositoryTag, Repository)
                       .join(RepositoryTag)
                       .join(Repository)
                       .join(Namespace, on=(Namespace.id == Repository.namespace_user))