From 254f06e63446d23e77e1be77ff2efbf336d4a015 Mon Sep 17 00:00:00 2001 From: Joseph Schorr Date: Fri, 17 Aug 2018 18:34:10 -0400 Subject: [PATCH] Implement legacy image portion of the data model This also makes use of the newly created input system --- data/registry_model/datatype.py | 49 ++++++++++++++ data/registry_model/datatypes.py | 65 ++++++++++++------- data/registry_model/interface.py | 13 ++++ data/registry_model/registry_pre_oci_model.py | 43 +++++++++++- .../registry_model/test/test_pre_oci_model.py | 41 ++++++++++++ 5 files changed, 188 insertions(+), 23 deletions(-) create mode 100644 data/registry_model/datatype.py diff --git a/data/registry_model/datatype.py b/data/registry_model/datatype.py new file mode 100644 index 000000000..6571e61b4 --- /dev/null +++ b/data/registry_model/datatype.py @@ -0,0 +1,49 @@ +# pylint: disable=protected-access + +from functools import wraps, total_ordering + +def datatype(name, static_fields): + """ Defines a base class for a datatype that will represent a row from the database, + in an abstracted form. + """ + @total_ordering + class DataType(object): + __name__ = name + + def __init__(self, **kwargs): + self._db_id = kwargs.pop('db_id', None) + self._inputs = kwargs.pop('inputs', None) + self._fields = kwargs + + for name in static_fields: + assert name in self._fields, 'Missing field %s' % name + + def __eq__(self, other): + return self._db_id == other._db_id + + def __lt__(self, other): + return self._db_id < other._db_id + + def __getattr__(self, name): + if name in static_fields: + return self._fields[name] + + return None + + return DataType + + +def requiresinput(input_name): + """ Marks a property on the data type as requiring an input to be invoked. """ + def inner(func): + @wraps(func) + def wrapper(self, *args, **kwargs): + if self._inputs.get(input_name) is None: + raise Exception('Cannot invoke function with missing input `%s`', input_name) + + kwargs[input_name] = self._inputs[input_name] + result = func(self, *args, **kwargs) + return result + + return wrapper + return inner diff --git a/data/registry_model/datatypes.py b/data/registry_model/datatypes.py index 8a65f4ec8..7487d1564 100644 --- a/data/registry_model/datatypes.py +++ b/data/registry_model/datatypes.py @@ -1,25 +1,4 @@ -def datatype(name, static_fields): - """ Defines a base class for a datatype that will represent a row from the database, - in an abstracted form. - """ - class DataType(object): - __name__ = name - - def __init__(self, *args, **kwargs): - self._db_id = kwargs.pop('db_id', None) - self._fields = kwargs - - for name in static_fields: - assert name in self._fields, 'Missing field %s' % name - - def __getattr__(self, name): - if name in static_fields: - return self._fields[name] - - return None - - return DataType - +from data.registry_model.datatype import datatype, requiresinput class RepositoryReference(datatype('Repository', [])): """ RepositoryReference is a reference to a repository, passed to registry interface methods. """ @@ -49,3 +28,45 @@ class Manifest(datatype('Manifest', ['digest'])): return None return Manifest(db_id=tag_manifest.id, digest=tag_manifest.digest) + + +class LegacyImage(datatype('LegacyImage', ['docker_image_id', 'created', 'comment', 'command', + 'image_size', 'uploading'])): + """ LegacyImage represents a Docker V1-style image found in a repository. """ + @classmethod + def for_image(cls, image, images_map=None, tags_map=None): + if image is None: + return None + + return LegacyImage(db_id=image.id, + inputs=dict(images_map=images_map, tags_map=tags_map, + ancestor_id_list=image.ancestor_id_list()), + docker_image_id=image.docker_image_id, + created=image.created, + comment=image.comment, + command=image.command, + image_size=image.storage.image_size, + uploading=image.storage.uploading) + + @property + @requiresinput('images_map') + @requiresinput('ancestor_id_list') + def parents(self, images_map, ancestor_id_list): + """ Returns the parent images for this image. Raises an exception if the parents have + not been loaded before this property is invoked. + """ + return [LegacyImage.for_image(images_map[ancestor_id], images_map=images_map) + for ancestor_id in ancestor_id_list + if images_map.get(ancestor_id)] + + @property + @requiresinput('tags_map') + def tags(self, tags_map): + """ Returns the tags pointing to this image. Raises an exception if the tags have + not been loaded before this property is invoked. + """ + tags = tags_map.get(self._db_id) + if not tags: + return [] + + return [Tag.for_repository_tag(tag) for tag in tags] diff --git a/data/registry_model/interface.py b/data/registry_model/interface.py index d92ccdea9..9961f7dda 100644 --- a/data/registry_model/interface.py +++ b/data/registry_model/interface.py @@ -37,3 +37,16 @@ class RegistryDataInterface(object): @abstractmethod def create_manifest_label(self, manifest, key, value, source_type_name, media_type_name=None): """ Creates a label on the manifest with the given key and value. """ + + @abstractmethod + def get_legacy_images(self, repository_ref): + """ + Returns an iterator of all the LegacyImage's defined in the matching repository. + """ + + @abstractmethod + def get_legacy_image(self, repository_ref, docker_image_id, include_parents=False): + """ + Returns the matching LegacyImages under the matching repository, if any. If none, + returns None. + """ diff --git a/data/registry_model/registry_pre_oci_model.py b/data/registry_model/registry_pre_oci_model.py index 0249955cb..6e37814de 100644 --- a/data/registry_model/registry_pre_oci_model.py +++ b/data/registry_model/registry_pre_oci_model.py @@ -1,9 +1,11 @@ # pylint: disable=protected-access +from collections import defaultdict + from data import database from data import model from data.registry_model.interface import RegistryDataInterface -from data.registry_model.datatypes import Tag, RepositoryReference, Manifest +from data.registry_model.datatypes import Tag, RepositoryReference, Manifest, LegacyImage class PreOCIModel(RegistryDataInterface): @@ -62,5 +64,44 @@ class PreOCIModel(RegistryDataInterface): model.label.create_manifest_label(tag_manifest, key, value, source_type_name, media_type_name) + def get_legacy_images(self, repository_ref): + """ + Returns an iterator of all the LegacyImage's defined in the matching repository. + """ + repo = model.repository.lookup_repository(repository_ref._db_id) + if repo is None: + return None + + all_images = model.image.get_repository_images_without_placements(repo) + all_images_map = {image.id: image for image in all_images} + + all_tags = model.tag.list_repository_tags(repo.namespace_user.username, repo.name) + tags_by_image_id = defaultdict(list) + for tag in all_tags: + tags_by_image_id[tag.image_id].append(tag) + + return [LegacyImage.for_image(image, images_map=all_images_map, tags_map=tags_by_image_id) + for image in all_images] + + def get_legacy_image(self, repository_ref, docker_image_id, include_parents=False): + """ + Returns the matching LegacyImages under the matching repository, if any. If none, + returns None. + """ + repo = model.repository.lookup_repository(repository_ref._db_id) + if repo is None: + return None + + image = model.image.get_image(repository_ref._db_id, docker_image_id) + if image is None: + return None + + parent_images_map = None + if include_parents: + parent_images = model.image.get_parent_images(repo.namespace_user.username, repo.name, image) + parent_images_map = {image.id: image for image in parent_images} + + return LegacyImage.for_image(image, images_map=parent_images_map) + pre_oci_model = PreOCIModel() diff --git a/data/registry_model/test/test_pre_oci_model.py b/data/registry_model/test/test_pre_oci_model.py index 7e46b8cc6..78b5ca545 100644 --- a/data/registry_model/test/test_pre_oci_model.py +++ b/data/registry_model/test/test_pre_oci_model.py @@ -75,3 +75,44 @@ def test_create_manifest_label(pre_oci_model): found_manifest = pre_oci_model.get_manifest_for_tag(found_tag) pre_oci_model.create_manifest_label(found_manifest, 'foo', 'bar', 'internal') + + +@pytest.mark.parametrize('repo_namespace, repo_name', [ + ('devtable', 'simple'), + ('devtable', 'complex'), + ('devtable', 'history'), + ('buynlarge', 'orgrepo'), +]) +def test_legacy_images(repo_namespace, repo_name, pre_oci_model): + repository_ref = pre_oci_model.lookup_repository(repo_namespace, repo_name) + legacy_images = pre_oci_model.get_legacy_images(repository_ref) + assert len(legacy_images) + + found_tags = set() + for image in legacy_images: + found_image = pre_oci_model.get_legacy_image(repository_ref, image.docker_image_id, + include_parents=True) + + assert found_image.docker_image_id == image.docker_image_id + assert found_image.parents == image.parents + + # Check that the tags list can be retrieved. + assert image.tags is not None + found_tags.update({tag.name for tag in image.tags}) + + # Check against the actual DB row. + model_image = model.image.get_image(repository_ref._db_id, found_image.docker_image_id) + assert model_image.id == found_image._db_id + assert ([pid for pid in model_image.ancestor_id_list()] == + [p._db_id for p in found_image.parents]) + + # Try without parents and ensure it raises an exception. + found_image = pre_oci_model.get_legacy_image(repository_ref, image.docker_image_id, + include_parents=False) + with pytest.raises(Exception): + assert not found_image.parents + + assert found_tags + + unknown = pre_oci_model.get_legacy_image(repository_ref, 'unknown', include_parents=True) + assert unknown is None