From c5d8b5f86baaa61ace97147913d3c9f5e505fe37 Mon Sep 17 00:00:00 2001 From: Joseph Schorr Date: Mon, 19 Jun 2017 19:03:10 -0400 Subject: [PATCH] Add support for tag expiration based on a `quay.expires-after` label --- endpoints/v2/labelhandlers.py | 36 ++++++++++++++++++++++++++++++++ endpoints/v2/manifest.py | 7 +++---- endpoints/v2/models_interface.py | 8 +++++++ endpoints/v2/models_pre_oci.py | 8 +++++++ test/registry_tests.py | 33 +++++++++++++++++++++++++++++ 5 files changed, 88 insertions(+), 4 deletions(-) create mode 100644 endpoints/v2/labelhandlers.py diff --git a/endpoints/v2/labelhandlers.py b/endpoints/v2/labelhandlers.py new file mode 100644 index 000000000..d179ff4bc --- /dev/null +++ b/endpoints/v2/labelhandlers.py @@ -0,0 +1,36 @@ +import logging + +from app import app +from endpoints.v2.models_pre_oci import pre_oci_model as model +from util.timedeltastring import convert_to_timedelta + +logger = logging.getLogger(__name__) + +min_expire_sec = convert_to_timedelta(app.config.get('LABELED_EXPIRATION_MINIMUM', '1h')) +max_expire_sec = convert_to_timedelta(app.config.get('LABELED_EXPIRATION_MAXIMUM', '104w')) + +def _expires_after(value, namespace_name, repo_name, digest): + """ Sets the expiration of a manifest based on the quay.expires-in label. """ + try: + timedelta = convert_to_timedelta(value) + except ValueError: + logger.exception('Could not convert %s to timedeltastring for %s/%s@%s', value, namespace_name, + repo_name, digest) + return + + total_seconds = min(max(timedelta.total_seconds(), min_expire_sec.total_seconds()), + max_expire_sec.total_seconds()) + + logger.debug('Labeling manifest %s/%s@%s with expiration of %s', namespace_name, repo_name, + digest, total_seconds) + model.set_manifest_expires_after(namespace_name, repo_name, digest, total_seconds) + + +_LABEL_HANDLES = { + 'quay.expires-after': _expires_after, +} + +def handle_label(key, value, namespace_name, repo_name, digest): + handler = _LABEL_HANDLES.get(key) + if handler is not None: + handler(value, namespace_name, repo_name, digest) diff --git a/endpoints/v2/manifest.py b/endpoints/v2/manifest.py index 5d480472b..d35c46556 100644 --- a/endpoints/v2/manifest.py +++ b/endpoints/v2/manifest.py @@ -17,6 +17,7 @@ from endpoints.v2.errors import ( BlobUnknown, ManifestInvalid, ManifestUnknown, TagInvalid, NameInvalid) from endpoints.v2.models_interface import Label from endpoints.v2.models_pre_oci import data_model as model +from endpoints.v2.labelhandlers import handle_label from image.docker import ManifestException from image.docker.schema1 import DockerSchema1Manifest, DockerSchema1ManifestBuilder from image.docker.schema2 import DOCKER_SCHEMA2_CONTENT_TYPES @@ -188,6 +189,8 @@ def _write_manifest(namespace_name, repo_name, manifest): for key, value in manifest.layers[-1].v1_metadata.labels.iteritems(): media_type = 'application/json' if is_json(value) else 'text/plain' labels.append(Label(key=key, value=value, source_type='manifest', media_type=media_type)) + handle_label(key, value, namespace_name, repo_name, manifest.digest) + model.create_manifest_labels(namespace_name, repo_name, manifest.digest, labels) return repo, storage_map @@ -268,7 +271,3 @@ def _generate_and_store_manifest(namespace_name, repo_name, tag_name): model.create_manifest_and_update_tag(namespace_name, repo_name, tag_name, manifest.digest, manifest.bytes) return manifest - - -def _determine_media_type(value): - media_type_name = 'application/json' if is_json(value) else 'text/plain' diff --git a/endpoints/v2/models_interface.py b/endpoints/v2/models_interface.py index bbfd51b2c..66118a95f 100644 --- a/endpoints/v2/models_interface.py +++ b/endpoints/v2/models_interface.py @@ -256,3 +256,11 @@ class DockerRegistryV2DataInterface(object): Once everything is moved over, this could be in util.registry and not even touch the database. """ pass + + @abstractmethod + def set_manifest_expires_after(self, namespace_name, repo_name, digest, expires_after_sec): + """ + Sets that the manifest with given digest expires after the number of seconds from *now*. + """ + pass + diff --git a/endpoints/v2/models_pre_oci.py b/endpoints/v2/models_pre_oci.py index a241c7259..264bf149d 100644 --- a/endpoints/v2/models_pre_oci.py +++ b/endpoints/v2/models_pre_oci.py @@ -250,6 +250,14 @@ class PreOCIModel(DockerRegistryV2DataInterface): blob_record = model.storage.get_storage_by_uuid(blob.uuid) return model.storage.get_layer_path(blob_record) + def set_manifest_expires_after(self, namespace_name, repo_name, digest, expires_after_sec): + try: + manifest = model.tag.load_manifest_by_digest(namespace_name, repo_name, digest) + manifest.tag.lifetime_end_ts = manifest.tag.lifetime_start_ts + expires_after_sec + manifest.tag.save() + except model.InvalidManifestException: + return + def _docker_v1_metadata(namespace_name, repo_name, repo_image): """ diff --git a/test/registry_tests.py b/test/registry_tests.py index ad6eda27c..5a069e34a 100644 --- a/test/registry_tests.py +++ b/test/registry_tests.py @@ -42,6 +42,7 @@ from image.docker.schema1 import DockerSchema1ManifestBuilder from initdb import wipe_database, initialize_database, populate_database from jsonschema import validate as validate_schema from util.security.registry_jwt import decode_bearer_header +from util.timedeltastring import convert_to_timedelta try: @@ -1535,6 +1536,38 @@ class V2RegistryTests(V2RegistryPullMixin, V2RegistryPushMixin, RegistryTestsMix self.assertTrue('text/plain' in media_types) self.assertTrue('application/json' in media_types) + def test_expiration_label(self): + # Push a new repo with the latest tag. + images = [{ + 'id': 'someid', + 'config': {'Labels': {'quay.expires-after': '1d'}}, + 'contents': 'somecontent' + }] + + (_, manifests) = self.do_push('devtable', 'newrepo', 'devtable', 'password', images=images) + + self.conduct_api_login('devtable', 'password') + tags = self.conduct('GET', '/api/v1/repository/devtable/newrepo/tag').json() + tag = tags['tags'][0] + + self.assertEqual(tag['end_ts'], tag['start_ts'] + convert_to_timedelta('1d').total_seconds()) + + def test_invalid_expiration_label(self): + # Push a new repo with the latest tag. + images = [{ + 'id': 'someid', + 'config': {'Labels': {'quay.expires-after': 'blahblah'}}, + 'contents': 'somecontent' + }] + + (_, manifests) = self.do_push('devtable', 'newrepo', 'devtable', 'password', images=images) + + self.conduct_api_login('devtable', 'password') + tags = self.conduct('GET', '/api/v1/repository/devtable/newrepo/tag').json() + tag = tags['tags'][0] + + self.assertIsNone(tag.get('end_ts')) + def test_invalid_manifest_type(self): namespace = 'devtable' repository = 'somerepo'