diff --git a/data/migrations/versions/d8989249f8f6_add_change_tag_expiration_log_type.py b/data/migrations/versions/d8989249f8f6_add_change_tag_expiration_log_type.py new file mode 100644 index 000000000..9ae981008 --- /dev/null +++ b/data/migrations/versions/d8989249f8f6_add_change_tag_expiration_log_type.py @@ -0,0 +1,25 @@ +"""Add change_tag_expiration log type + +Revision ID: d8989249f8f6 +Revises: dc4af11a5f90 +Create Date: 2017-06-21 21:18:25.948689 + +""" + +# revision identifiers, used by Alembic. +revision = 'd8989249f8f6' +down_revision = 'dc4af11a5f90' + +from alembic import op + +def upgrade(tables): + op.bulk_insert(tables.logentrykind, [ + {'name': 'change_tag_expiration'}, + ]) + + +def downgrade(tables): + op.execute(tables + .logentrykind + .delete() + .where(tables.logentrykind.c.name == op.inline_literal('change_tag_expiration'))) diff --git a/data/model/tag.py b/data/model/tag.py index 24926174b..d62ee2bcb 100644 --- a/data/model/tag.py +++ b/data/model/tag.py @@ -1,13 +1,17 @@ import logging +import time +from calendar import timegm from uuid import uuid4 from peewee import IntegrityError, JOIN_LEFT_OUTER, fn from data.model import (image, db_transaction, DataModelException, _basequery, - InvalidManifestException, TagAlreadyCreatedException, StaleTagException) + InvalidManifestException, TagAlreadyCreatedException, StaleTagException, + config) from data.database import (RepositoryTag, Repository, Image, ImageStorage, Namespace, TagManifest, RepositoryNotification, Label, TagManifestLabel, get_epoch_timestamp, db_for_update) +from util.timedeltastring import convert_to_timedelta logger = logging.getLogger(__name__) @@ -526,6 +530,15 @@ def get_active_tag(namespace, repo_name, tag_name): Namespace.username == namespace)).get() +def get_possibly_expired_tag(namespace, repo_name, tag_name): + return (RepositoryTag + .select() + .join(Repository) + .join(Namespace, on=(Repository.namespace_user == Namespace.id)) + .where(RepositoryTag.name == tag_name, Repository.name == repo_name, + Namespace.username == namespace)).get() + + def associate_generated_tag_manifest(namespace, repo_name, tag_name, manifest_digest, manifest_data): tag = get_active_tag(namespace, repo_name, tag_name) @@ -570,3 +583,41 @@ def _load_repo_manifests(namespace, repo_name): .join(Repository) .join(Namespace, on=(Namespace.id == Repository.namespace_user)) .where(Repository.name == repo_name, Namespace.username == namespace)) + + +def change_repository_tag_expiration(namespace_name, repo_name, tag_name, expiration_date): + """ Changes the expiration of the tag with the given name to the given expiration datetime. If + the expiration datetime is None, then the tag is marked as not expiring. + """ + try: + tag = get_active_tag(namespace_name, repo_name, tag_name) + return change_tag_expiration(tag, expiration_date) + except RepositoryTag.DoesNotExist: + return (None, False) + + +def change_tag_expiration(tag, expiration_date): + """ Changes the expiration of the given tag to the given expiration datetime. If + the expiration datetime is None, then the tag is marked as not expiring. + """ + end_ts = None + min_expire_sec = convert_to_timedelta(config.app_config.get('LABELED_EXPIRATION_MINIMUM', '1h')) + max_expire_sec = convert_to_timedelta(config.app_config.get('LABELED_EXPIRATION_MAXIMUM', '104w')) + + if expiration_date is not None: + offset = timegm(expiration_date.utctimetuple()) - tag.lifetime_start_ts + offset = min(max(offset, min_expire_sec.total_seconds()), max_expire_sec.total_seconds()) + end_ts = tag.lifetime_start_ts + offset + + if end_ts == tag.lifetime_end_ts: + return (None, True) + + # Note: We check not just the ID of the tag but also its lifetime_end_ts, to ensure that it has + # not changed while we were updatings it expiration. + result = (RepositoryTag + .update(lifetime_end_ts=end_ts) + .where(RepositoryTag.id == tag.id, + RepositoryTag.lifetime_end_ts == tag.lifetime_end_ts) + .execute()) + + return (tag.lifetime_end_ts, result > 0) diff --git a/data/model/test/test_tag.py b/data/model/test/test_tag.py index 8fa0eb852..6cfa7366d 100644 --- a/data/model/test/test_tag.py +++ b/data/model/test/test_tag.py @@ -1,12 +1,16 @@ import pytest +from datetime import datetime from mock import patch +from time import time from data.database import Image, RepositoryTag, ImageStorage, Repository from data.model.repository import create_repository from data.model.tag import (list_active_repo_tags, create_or_update_tag, delete_tag, - get_matching_tags, _tag_alive, get_matching_tags_for_images) + get_matching_tags, _tag_alive, get_matching_tags_for_images, + change_tag_expiration, get_active_tag) from data.model.image import find_create_or_link_image +from util.timedeltastring import convert_to_timedelta from test.fixtures import * @@ -112,7 +116,7 @@ def assert_tags(repository, *args): for tag in tags: assert not tag.name in tags_dict assert not tag.hidden - assert not tag.lifetime_end_ts + assert not tag.lifetime_end_ts or tag.lifetime_end_ts > time() tags_dict[tag.name] = tag @@ -145,6 +149,13 @@ def test_list_active_tags(initialized_db): # Make sure they are returned. assert_tags(repository, 'foo', 'bar') + # Mark as a tag as expiring in the far future, and make sure it is still returned. + footag.lifetime_end_ts = footag.lifetime_start_ts + 10000000 + footag.save() + + # Make sure they are returned. + assert_tags(repository, 'foo', 'bar') + # Delete a tag and make sure it isn't returned. footag = delete_tag('devtable', 'somenewrepo', 'foo') footag.lifetime_end_ts -= 4 @@ -159,6 +170,44 @@ def test_list_active_tags(initialized_db): assert_tags(repository, 'foo', 'bar') + # Mark as a tag as expiring in the far future, and make sure it is still returned. + footag.lifetime_end_ts = footag.lifetime_start_ts + 10000000 + footag.save() + + # Make sure they are returned. + assert_tags(repository, 'foo', 'bar') + # "Move" foo by updating it and make sure we don't get duplicates. create_or_update_tag('devtable', 'somenewrepo', 'foo', image2.docker_image_id) assert_tags(repository, 'foo', 'bar') + + +@pytest.mark.parametrize('expiration_offset, expected_offset', [ + (None, None), + ('0s', '1h'), + ('30m', '1h'), + ('2h', '2h'), + ('2w', '2w'), + ('200w', '104w'), +]) +def test_change_tag_expiration(expiration_offset, expected_offset, initialized_db): + repository = create_repository('devtable', 'somenewrepo', None) + image1 = find_create_or_link_image('foobarimage1', repository, None, {}, 'local_us') + footag = create_or_update_tag('devtable', 'somenewrepo', 'foo', image1.docker_image_id) + + expiration_date = None + if expiration_offset is not None: + expiration_date = datetime.utcnow() + convert_to_timedelta(expiration_offset) + + assert change_tag_expiration(footag, expiration_date) + + # Lookup the tag again. + footag_updated = get_active_tag('devtable', 'somenewrepo', 'foo') + + if expected_offset is None: + assert footag_updated.lifetime_end_ts is None + else: + start_date = datetime.utcfromtimestamp(footag_updated.lifetime_start_ts) + end_date = datetime.utcfromtimestamp(footag_updated.lifetime_end_ts) + expected_end_date = start_date + convert_to_timedelta(expected_offset) + assert end_date == expected_end_date diff --git a/endpoints/api/repository.py b/endpoints/api/repository.py index eb740ef44..f8f41de44 100644 --- a/endpoints/api/repository.py +++ b/endpoints/api/repository.py @@ -334,6 +334,10 @@ class Repository(RepositoryParamResource): last_modified = format_date(datetime.fromtimestamp(tag.lifetime_start_ts)) tag_info['last_modified'] = last_modified + if tag.lifetime_end_ts: + expiration = format_date(datetime.fromtimestamp(tag.lifetime_end_ts)) + tag_info['expiration'] = expiration + if tag.tagmanifest is not None: tag_info['manifest_digest'] = tag.tagmanifest.digest @@ -498,11 +502,11 @@ class RepositoryTrust(RepositoryParamResource): repo = model.repository.get_repository(namespace, repository) if not repo: raise NotFound() - + tags, _ = tuf_metadata_api.get_default_tags_with_expiration(namespace, repository) if tags and not tuf_metadata_api.delete_metadata(namespace, repository): raise DownstreamIssue({'message': 'Unable to delete downstream trust metadata'}) - + values = request.get_json() model.repository.set_trust(repo, values['trust_enabled']) diff --git a/endpoints/api/tag.py b/endpoints/api/tag.py index 8e65257a5..106d051f0 100644 --- a/endpoints/api/tag.py +++ b/endpoints/api/tag.py @@ -1,5 +1,6 @@ """ Manage the tags of a repository. """ +from datetime import datetime, timedelta from flask import request, abort from auth.auth_context import get_authenticated_user @@ -9,7 +10,7 @@ from endpoints.api import (resource, nickname, require_repo_read, require_repo_w parse_args, query_param, truthy_bool, disallow_for_app_repositories) from endpoints.api.tag_models_interface import Repository from endpoints.api.tag_models_pre_oci import pre_oci_model as model -from endpoints.exception import NotFound +from endpoints.exception import NotFound, InvalidRequest from endpoints.v2.manifest import _generate_and_store_manifest from util.names import TAG_ERROR, TAG_REGEX @@ -52,14 +53,17 @@ class ListRepositoryTags(RepositoryParamResource): class RepositoryTag(RepositoryParamResource): """ Resource for managing repository tags. """ schemas = { - 'MoveTag': { + 'ChangeTag': { 'type': 'object', - 'description': 'Description of to which image a new or existing tag should point', - 'required': ['image',], + 'description': 'Makes changes to a specific tag', 'properties': { 'image': { - 'type': 'string', - 'description': 'Image identifier to which the tag should point', + 'type': ['string', 'null'], + 'description': '(If specified) Image identifier to which the tag should point', + }, + 'expiration': { + 'type': ['number', 'null'], + 'description': '(If specified) The expiration for the image', }, }, }, @@ -67,33 +71,64 @@ class RepositoryTag(RepositoryParamResource): @require_repo_write @disallow_for_app_repositories - @nickname('changeTagImage') - @validate_json_request('MoveTag') + @nickname('changeTag') + @validate_json_request('ChangeTag') def put(self, namespace, repository, tag): """ Change which image a tag points to or create a new tag.""" if not TAG_REGEX.match(tag): abort(400, TAG_ERROR) - image_id = request.get_json()['image'] - repo = model.get_repo(namespace, repository, image_id) + repo = model.get_repo(namespace, repository) if not repo: raise NotFound() - original_image_id = model.get_repo_tag_image(repo, tag) - model.create_or_update_tag(namespace, repository, tag, image_id) + if 'expiration' in request.get_json(): + expiration = request.get_json().get('expiration') + expiration_date = None + if expiration is not None: + try: + expiration_date = datetime.utcfromtimestamp(float(expiration)) + except ValueError: + abort(400) - username = get_authenticated_user().username - log_action('move_tag' if original_image_id else 'create_tag', namespace, { - 'username': username, - 'repo': repository, - 'tag': tag, - 'namespace': namespace, - 'image': image_id, - 'original_image': original_image_id - }, repo_name=repository) + if expiration_date <= datetime.now(): + abort(400) - _generate_and_store_manifest(namespace, repository, tag) + existing_end_ts, ok = model.change_repository_tag_expiration(namespace, repository, tag, + expiration_date) + if ok: + if not (existing_end_ts is None and expiration_date is None): + log_action('change_tag_expiration', namespace, { + 'username': get_authenticated_user().username, + 'repo': repository, + 'tag': tag, + 'namespace': namespace, + 'expiration_date': expiration_date, + 'old_expiration_date': existing_end_ts + }, repo_name=repository) + else: + raise InvalidRequest('Could not update tag expiration; Tag has probably changed') + + if 'image' in request.get_json(): + image_id = request.get_json()['image'] + image = model.get_repository_image(namespace, repository, image_id) + if image is None: + raise NotFound() + + original_image_id = model.get_repo_tag_image(repo, tag) + model.create_or_update_tag(namespace, repository, tag, image_id) + + username = get_authenticated_user().username + log_action('move_tag' if original_image_id else 'create_tag', namespace, { + 'username': username, + 'repo': repository, + 'tag': tag, + 'namespace': namespace, + 'image': image_id, + 'original_image': original_image_id + }, repo_name=repository) + _generate_and_store_manifest(namespace, repository, tag) return 'Updated', 201 diff --git a/endpoints/api/tag_models_interface.py b/endpoints/api/tag_models_interface.py index c54a4cd86..a005aed9b 100644 --- a/endpoints/api/tag_models_interface.py +++ b/endpoints/api/tag_models_interface.py @@ -61,7 +61,7 @@ class Repository(namedtuple('Repository', ['namespace_name', 'repository_name']) class Image( namedtuple('Image', [ 'docker_image_id', 'created', 'comment', 'command', 'storage_image_size', - 'storage_uploading', 'ancestor_length', 'ancestor_id_list' + 'storage_uploading', 'ancestor_id_list' ])): """ Image @@ -71,7 +71,6 @@ class Image( :type command: string :type storage_image_size: int :type storage_uploading: boolean - :type ancestor_length: int :type ancestor_id_list: [int] """ @@ -91,7 +90,7 @@ class Image( 'command': json.loads(command) if command else None, 'size': self.storage_image_size, 'uploading': self.storage_uploading, - 'sort_index': self.ancestor_length, + 'sort_index': len(self.ancestor_id_list), } if include_ancestors: @@ -116,9 +115,9 @@ class TagDataInterface(object): """ @abstractmethod - def get_repo(self, namespace_name, repository_name, docker_image_id): + def get_repo(self, namespace_name, repository_name): """ - Returns a repository associated with the given namespace, repository, and docker_image_id + Returns a repository associated with the given namespace and repository name. """ @abstractmethod @@ -157,6 +156,13 @@ class TagDataInterface(object): Returns the repository associated with the namespace_name and repository_name """ + @abstractmethod + def get_repository_image(self, namespace_name, repository_name, docker_image_id): + """ + Returns the repository image associated with the namespace_name, repository_name, and docker + image ID. + """ + @abstractmethod def restore_tag_to_manifest(self, repository_name, tag_name, manifest_digest): """ @@ -170,3 +176,11 @@ class TagDataInterface(object): Returns the existing repo tag image if it exists or else returns None Side effects include adding the tag with associated name to the image with the associated id in the named repo. """ + + @abstractmethod + def change_repository_tag_expiration(self, namespace_name, repository_name, tag_name, + expiration_date): + """ Sets the expiration date of the tag under the matching repository to that given. If the + expiration date is None, then the tag will not expire. Returns a tuple of the previous + expiration timestamp in seconds (if any), and whether the operation succeeded. + """ diff --git a/endpoints/api/tag_models_pre_oci.py b/endpoints/api/tag_models_pre_oci.py index 1002c78c8..99adf9a57 100644 --- a/endpoints/api/tag_models_pre_oci.py +++ b/endpoints/api/tag_models_pre_oci.py @@ -27,12 +27,12 @@ class PreOCIModel(TagDataInterface): return RepositoryTagHistory(tags=repository_tag_history, more=more) - def get_repo(self, namespace_name, repository_name, docker_image_id): - image = model.image.get_repo_image(namespace_name, repository_name, docker_image_id) - if image is None: + def get_repo(self, namespace_name, repository_name): + repo = model.repository.get_repository(namespace_name, repository_name) + if repo is None: return None - return Repository(image.repository.namespace_user, image.repository.name) + return Repository(repo.namespace_user, repo.name) def get_repo_tag_image(self, repository, tag_name): repo = model.repository.get_repository(str(repository.namespace_name), str(repository.repository_name)) @@ -73,6 +73,13 @@ class PreOCIModel(TagDataInterface): new_tags.append(convert_tag(tag)) return new_tags + def get_repository_image(self, namespace_name, repository_name, docker_image_id): + image = model.image.get_repo_image(namespace_name, repository_name, docker_image_id) + if image is None: + return None + + return convert_image(image) + def get_repository(self, namespace_name, repository_name): repo = model.repository.get_repository(namespace_name, repository_name) if repo is None: @@ -102,13 +109,17 @@ class PreOCIModel(TagDataInterface): return convert_image(image) + def change_repository_tag_expiration(self, namespace_name, repository_name, tag_name, + expiration_date): + return model.tag.change_repository_tag_expiration(namespace_name, repository_name, tag_name, + expiration_date) + def convert_image(database_image): return Image(docker_image_id=database_image.docker_image_id, created=database_image.created, comment=database_image.comment, command=database_image.command, storage_image_size=database_image.storage.image_size, storage_uploading=database_image.storage.uploading, - ancestor_length=len(database_image.ancestors), ancestor_id_list=database_image.ancestor_id_list()) diff --git a/endpoints/api/test/test_tag.py b/endpoints/api/test/test_tag.py index 1d13d0eae..abc0adf9e 100644 --- a/endpoints/api/test/test_tag.py +++ b/endpoints/api/test/test_tag.py @@ -124,6 +124,43 @@ def find_no_repo_tag_history(): yield +@pytest.mark.parametrize('expiration_time, expected_status', [ + (None, 201), + ('aksdjhasd', 400), +]) +def test_change_tag_expiration_default(expiration_time, expected_status, client, app): + with client_with_identity('devtable', client) as cl: + params = { + 'repository': 'devtable/simple', + 'tag': 'latest', + } + + request_body = { + 'expiration': expiration_time, + } + + conduct_api_call(cl, RepositoryTag, 'put', params, request_body, expected_status) + + +def test_change_tag_expiration(client, app): + with client_with_identity('devtable', client) as cl: + params = { + 'repository': 'devtable/simple', + 'tag': 'latest', + } + + tag = model.tag.get_active_tag('devtable', 'simple', 'latest') + updated_expiration = tag.lifetime_start_ts + 60*60*24 + + request_body = { + 'expiration': updated_expiration, + } + + conduct_api_call(cl, RepositoryTag, 'put', params, request_body, 201) + tag = model.tag.get_active_tag('devtable', 'simple', 'latest') + assert tag.lifetime_end_ts == updated_expiration + + @pytest.mark.parametrize('test_image,test_tag,expected_status', [ ('image1', '-INVALID-TAG-NAME', 400), ('image1', '.INVALID-TAG-NAME', 400), diff --git a/endpoints/api/test/test_tag_models_pre_oci.py b/endpoints/api/test/test_tag_models_pre_oci.py index b7c750854..5ddb34ab3 100644 --- a/endpoints/api/test/test_tag_models_pre_oci.py +++ b/endpoints/api/test/test_tag_models_pre_oci.py @@ -106,20 +106,18 @@ def test_list_repository_tag_history(expected, namespace_name, repository_name, specific_tag) == expected -def get_repo_image_mock(monkeypatch, return_value): - def return_return_value(namespace_name, repository_name, image_id): +def get_repo_mock(monkeypatch, return_value): + def return_return_value(namespace_name, repository_name): return return_value - monkeypatch.setattr(model.image, 'get_repo_image', return_return_value) + monkeypatch.setattr(model.repository, 'get_repository', return_return_value) def test_get_repo_not_exists(get_monkeypatch): namespace_name = 'namespace_name' repository_name = 'repository_name' - image_id = 'image_id' - get_repo_image_mock(get_monkeypatch, None) - - repo = pre_oci_model.get_repo(namespace_name, repository_name, image_id) + get_repo_mock(get_monkeypatch, None) + repo = pre_oci_model.get_repo(namespace_name, repository_name) assert repo is None @@ -127,14 +125,13 @@ def test_get_repo_not_exists(get_monkeypatch): def test_get_repo_exists(get_monkeypatch): namespace_name = 'namespace_name' repository_name = 'repository_name' - image_id = 'image_id' mock = Mock() mock.namespace_user = namespace_name mock.name = repository_name mock.repository = mock - get_repo_image_mock(get_monkeypatch, mock) + get_repo_mock(get_monkeypatch, mock) - repo = pre_oci_model.get_repo(namespace_name, repository_name, image_id) + repo = pre_oci_model.get_repo(namespace_name, repository_name) assert repo is not None assert repo.repository_name == repository_name diff --git a/endpoints/api/user.py b/endpoints/api/user.py index 093824e13..f359c16d5 100644 --- a/endpoints/api/user.py +++ b/endpoints/api/user.py @@ -20,7 +20,7 @@ from auth.permissions import (AdministerOrganizationPermission, CreateRepository UserAdminPermission, UserReadPermission, SuperUserPermission) from data import model from data.billing import get_plan -from data.database import Repository as RepositoryTable, UserPromptTypes +from data.database import Repository as RepositoryTable from endpoints.api import (ApiResource, nickname, resource, validate_json_request, request_error, log_action, internal_only, require_user_admin, parse_args, query_param, require_scope, format_date, show_if, diff --git a/endpoints/v2/errors.py b/endpoints/v2/errors.py index 0ae998106..40d7b9529 100644 --- a/endpoints/v2/errors.py +++ b/endpoints/v2/errors.py @@ -57,6 +57,13 @@ class ManifestUnknown(V2RegistryException): def __init__(self, detail=None): super(ManifestUnknown, self).__init__('MANIFEST_UNKNOWN', 'manifest unknown', detail, 404) +class TagExpired(V2RegistryException): + def __init__(self, message=None, detail=None): + super(TagExpired, self).__init__('TAG_EXPIRED', + message or 'Tag has expired', + detail, + 404) + class ManifestUnverified(V2RegistryException): def __init__(self, detail=None): diff --git a/endpoints/v2/labelhandlers.py b/endpoints/v2/labelhandlers.py new file mode 100644 index 000000000..67596f404 --- /dev/null +++ b/endpoints/v2/labelhandlers.py @@ -0,0 +1,36 @@ +import logging + +from app import app +from endpoints.v2.models_pre_oci import data_model as model +from util.timedeltastring import convert_to_timedelta + +logger = logging.getLogger(__name__) + +min_expire_sec = convert_to_timedelta(app.config.get('LABELED_EXPIRATION_MINIMUM', '1h')) +max_expire_sec = convert_to_timedelta(app.config.get('LABELED_EXPIRATION_MAXIMUM', '104w')) + +def _expires_after(value, namespace_name, repo_name, digest): + """ Sets the expiration of a manifest based on the quay.expires-in label. """ + try: + timedelta = convert_to_timedelta(value) + except ValueError: + logger.exception('Could not convert %s to timedeltastring for %s/%s@%s', value, namespace_name, + repo_name, digest) + return + + total_seconds = min(max(timedelta.total_seconds(), min_expire_sec.total_seconds()), + max_expire_sec.total_seconds()) + + logger.debug('Labeling manifest %s/%s@%s with expiration of %s', namespace_name, repo_name, + digest, total_seconds) + model.set_manifest_expires_after(namespace_name, repo_name, digest, total_seconds) + + +_LABEL_HANDLES = { + 'quay.expires-after': _expires_after, +} + +def handle_label(key, value, namespace_name, repo_name, digest): + handler = _LABEL_HANDLES.get(key) + if handler is not None: + handler(value, namespace_name, repo_name, digest) diff --git a/endpoints/v2/manifest.py b/endpoints/v2/manifest.py index 5d480472b..b79111f39 100644 --- a/endpoints/v2/manifest.py +++ b/endpoints/v2/manifest.py @@ -13,10 +13,11 @@ from endpoints.common import parse_repository_name from endpoints.decorators import anon_protect from endpoints.notificationhelper import spawn_notification from endpoints.v2 import v2_bp, require_repo_read, require_repo_write -from endpoints.v2.errors import ( - BlobUnknown, ManifestInvalid, ManifestUnknown, TagInvalid, NameInvalid) from endpoints.v2.models_interface import Label from endpoints.v2.models_pre_oci import data_model as model +from endpoints.v2.errors import (BlobUnknown, ManifestInvalid, ManifestUnknown, TagInvalid, + NameInvalid, TagExpired) +from endpoints.v2.labelhandlers import handle_label from image.docker import ManifestException from image.docker.schema1 import DockerSchema1Manifest, DockerSchema1ManifestBuilder from image.docker.schema2 import DOCKER_SCHEMA2_CONTENT_TYPES @@ -42,7 +43,14 @@ def fetch_manifest_by_tagname(namespace_name, repo_name, manifest_ref): if manifest is None: has_tag = model.has_active_tag(namespace_name, repo_name, manifest_ref) if not has_tag: - raise ManifestUnknown() + has_expired_tag = model.has_tag(namespace_name, repo_name, manifest_ref) + if has_expired_tag: + logger.debug('Found expired tag %s for repository %s/%s', manifest_ref, namespace_name, + repo_name) + msg = 'Tag %s was deleted or has expired. To pull, revive via time machine' % manifest_ref + raise TagExpired(msg) + else: + raise ManifestUnknown() manifest = _generate_and_store_manifest(namespace_name, repo_name, manifest_ref) if manifest is None: @@ -188,6 +196,8 @@ def _write_manifest(namespace_name, repo_name, manifest): for key, value in manifest.layers[-1].v1_metadata.labels.iteritems(): media_type = 'application/json' if is_json(value) else 'text/plain' labels.append(Label(key=key, value=value, source_type='manifest', media_type=media_type)) + handle_label(key, value, namespace_name, repo_name, manifest.digest) + model.create_manifest_labels(namespace_name, repo_name, manifest.digest, labels) return repo, storage_map @@ -268,7 +278,3 @@ def _generate_and_store_manifest(namespace_name, repo_name, tag_name): model.create_manifest_and_update_tag(namespace_name, repo_name, tag_name, manifest.digest, manifest.bytes) return manifest - - -def _determine_media_type(value): - media_type_name = 'application/json' if is_json(value) else 'text/plain' diff --git a/endpoints/v2/models_interface.py b/endpoints/v2/models_interface.py index bbfd51b2c..66118a95f 100644 --- a/endpoints/v2/models_interface.py +++ b/endpoints/v2/models_interface.py @@ -256,3 +256,11 @@ class DockerRegistryV2DataInterface(object): Once everything is moved over, this could be in util.registry and not even touch the database. """ pass + + @abstractmethod + def set_manifest_expires_after(self, namespace_name, repo_name, digest, expires_after_sec): + """ + Sets that the manifest with given digest expires after the number of seconds from *now*. + """ + pass + diff --git a/endpoints/v2/models_pre_oci.py b/endpoints/v2/models_pre_oci.py index a241c7259..fa2e7dc49 100644 --- a/endpoints/v2/models_pre_oci.py +++ b/endpoints/v2/models_pre_oci.py @@ -37,6 +37,13 @@ class PreOCIModel(DockerRegistryV2DataInterface): except database.RepositoryTag.DoesNotExist: return False + def has_tag(self, namespace_name, repo_name, tag_name): + try: + model.tag.get_possibly_expired_tag(namespace_name, repo_name, tag_name) + return True + except database.RepositoryTag.DoesNotExist: + return False + def get_manifest_by_tag(self, namespace_name, repo_name, tag_name): try: manifest = model.tag.load_tag_manifest(namespace_name, repo_name, tag_name) @@ -250,6 +257,14 @@ class PreOCIModel(DockerRegistryV2DataInterface): blob_record = model.storage.get_storage_by_uuid(blob.uuid) return model.storage.get_layer_path(blob_record) + def set_manifest_expires_after(self, namespace_name, repo_name, digest, expires_after_sec): + try: + manifest = model.tag.load_manifest_by_digest(namespace_name, repo_name, digest) + manifest.tag.lifetime_end_ts = manifest.tag.lifetime_start_ts + expires_after_sec + manifest.tag.save() + except model.InvalidManifestException: + return + def _docker_v1_metadata(namespace_name, repo_name, repo_image): """ diff --git a/initdb.py b/initdb.py index 387cc50f8..c6b36e18e 100644 --- a/initdb.py +++ b/initdb.py @@ -350,6 +350,8 @@ def initialize_database(): LogEntryKind.create(name='manifest_label_add') LogEntryKind.create(name='manifest_label_delete') + LogEntryKind.create(name='change_tag_expiration') + ImageStorageLocation.create(name='local_eu') ImageStorageLocation.create(name='local_us') diff --git a/static/css/directives/ui/service-keys-manager.css b/static/css/directives/ui/service-keys-manager.css index c7b1e54ae..b63b377f3 100644 --- a/static/css/directives/ui/service-keys-manager.css +++ b/static/css/directives/ui/service-keys-manager.css @@ -20,30 +20,10 @@ color: #777; } -.service-keys-manager-element .expired a { - color: #D64456; -} - -.service-keys-manager-element .critical a { - color: #F77454; -} - -.service-keys-manager-element .warning a { - color: #FCA657; -} - -.service-keys-manager-element .info a { - color: #2FC98E; -} - .service-keys-manager-element .rotation { color: #777; } -.service-keys-manager-element .no-expiration { - color: #128E72; -} - .service-keys-manager-element .approval-automatic { font-size: 12px; color: #777; diff --git a/static/directives/datetime-picker.html b/static/directives/datetime-picker.html index 25f57d701..653c3869a 100644 --- a/static/directives/datetime-picker.html +++ b/static/directives/datetime-picker.html @@ -1,3 +1,3 @@ - + \ No newline at end of file diff --git a/static/directives/repo-view/repo-panel-tags.html b/static/directives/repo-view/repo-panel-tags.html index 0e572a568..20fe43daa 100644 --- a/static/directives/repo-view/repo-panel-tags.html +++ b/static/directives/repo-view/repo-panel-tags.html @@ -57,6 +57,12 @@ Delete Tags +
  • + + Change Tags Expiration + +
    • @@ -105,6 +111,11 @@ style="width: 80px;"> Size + + Expires + @@ -133,12 +144,16 @@ ng-if="repository.trust_enabled"> + + Unknown + + + + + + + + + + + + + + @@ -254,6 +282,10 @@ ng-class="repository.tag_operations_disabled ? 'disabled-option' : ''"> Delete Tag + + Change Expiration + @@ -261,7 +293,7 @@ - +
    diff --git a/static/directives/service-keys-manager.html b/static/directives/service-keys-manager.html index 1935846dc..cfd82507f 100644 --- a/static/directives/service-keys-manager.html +++ b/static/directives/service-keys-manager.html @@ -113,20 +113,14 @@ - + Automatically rotated - - - - - Expiresd - - - - - Does not expire + + + + diff --git a/static/directives/tag-operations-dialog.html b/static/directives/tag-operations-dialog.html index 5f5f7b2be..12e1b7d21 100644 --- a/static/directives/tag-operations-dialog.html +++ b/static/directives/tag-operations-dialog.html @@ -111,6 +111,29 @@ + +
    +
    + +
      +
    • + {{ tag_info.name }} +
      • +
    + + + + + If specified, the date and time that the key expires. If set to none, the tag(s) will not expire. + +
    +
    +
    + + + + + + + Never + + \ No newline at end of file diff --git a/static/js/directives/ui/expiration-status-view/expiration-status-view.component.ts b/static/js/directives/ui/expiration-status-view/expiration-status-view.component.ts new file mode 100644 index 000000000..21092b951 --- /dev/null +++ b/static/js/directives/ui/expiration-status-view/expiration-status-view.component.ts @@ -0,0 +1,40 @@ +import { Input, Component, Inject } from 'ng-metadata/core'; +import * as moment from "moment"; +import './expiration-status-view.component.css'; + +type expirationInfo = { + className: string; + icon: string; +}; + +/** + * A component that displays expiration status. + */ +@Component({ + selector: 'expiration-status-view', + templateUrl: '/static/js/directives/ui/expiration-status-view/expiration-status-view.component.html', +}) +export class ExpirationStatusViewComponent { + @Input('<') public expirationDate: Date; + + private getExpirationInfo(expirationDate): expirationInfo|null { + if (!expirationDate) { + return null; + } + + const expiration = moment(expirationDate); + if (moment().isAfter(expiration)) { + return {'className': 'expired', 'icon': 'fa-warning'}; + } + + if (moment().add(1, 'week').isAfter(expiration)) { + return {'className': 'critical', 'icon': 'fa-warning'}; + } + + if (moment().add(1, 'month').isAfter(expiration)) { + return {'className': 'warning', 'icon': 'fa-warning'}; + } + + return {'className': 'info', 'icon': 'fa-clock-o'}; + } +} diff --git a/static/js/directives/ui/logs-view.js b/static/js/directives/ui/logs-view.js index 13b26e0d6..a239d0cc0 100644 --- a/static/js/directives/ui/logs-view.js +++ b/static/js/directives/ui/logs-view.js @@ -271,6 +271,18 @@ angular.module('quay').directive('logsView', function () { 'manifest_label_add': 'Label {key} added to manifest {manifest_digest} under repository {namespace}/{repo}', 'manifest_label_delete': 'Label {key} deleted from manifest {manifest_digest} under repository {namespace}/{repo}', + 'change_tag_expiration': function(metadata) { + if (metadata.expiration_date && metadata.old_expiration_date) { + return 'Tag {tag} set to expire on {expiration_date} (previously {old_expiration_date})'; + } else if (metadata.expiration_date) { + return 'Tag {tag} set to expire on {expiration_date}'; + } else if (metadata.old_expiration_date) { + return 'Tag {tag} set to no longer expire (previously {old_expiration_date})'; + } else { + return 'Tag {tag} set to no longer expire'; + } + }, + // Note: These are deprecated. 'add_repo_webhook': 'Add webhook in repository {repo}', 'delete_repo_webhook': 'Delete webhook in repository {repo}' @@ -332,6 +344,7 @@ angular.module('quay').directive('logsView', function () { 'take_ownership': 'Take Namespace Ownership', 'manifest_label_add': 'Add Manifest Label', 'manifest_label_delete': 'Delete Manifest Label', + 'change_tag_expiration': 'Change tag expiration', // Note: these are deprecated. 'add_repo_webhook': 'Add webhook', diff --git a/static/js/directives/ui/service-keys-manager.js b/static/js/directives/ui/service-keys-manager.js index d6a4da38f..636385019 100644 --- a/static/js/directives/ui/service-keys-manager.js +++ b/static/js/directives/ui/service-keys-manager.js @@ -80,32 +80,19 @@ angular.module('quay').directive('serviceKeysManager', function () { return moment(key.created_date).add(key.rotation_duration, 's').format('LLL'); }; - $scope.getExpirationInfo = function(key) { + $scope.willRotate = function(key) { if (!key.expiration_date) { - return ''; + return false; } if (key.rotation_duration) { var rotate_date = moment(key.created_date).add(key.rotation_duration, 's') if (moment().isBefore(rotate_date)) { - return {'className': 'rotation', 'icon': 'fa-refresh', 'willRotate': true}; + return true; } } - expiration_date = moment(key.expiration_date); - if (moment().isAfter(expiration_date)) { - return {'className': 'expired', 'icon': 'fa-warning'}; - } - - if (moment().add(1, 'week').isAfter(expiration_date)) { - return {'className': 'critical', 'icon': 'fa-warning'}; - } - - if (moment().add(1, 'month').isAfter(expiration_date)) { - return {'className': 'warning', 'icon': 'fa-warning'}; - } - - return {'className': 'info', 'icon': 'fa-check'}; + return false; }; $scope.showChangeName = function(key) { diff --git a/static/js/directives/ui/tag-operations-dialog.js b/static/js/directives/ui/tag-operations-dialog.js index 186713108..65ebb988d 100644 --- a/static/js/directives/ui/tag-operations-dialog.js +++ b/static/js/directives/ui/tag-operations-dialog.js @@ -18,6 +18,7 @@ angular.module('quay').directive('tagOperationsDialog', function () { }, controller: function($scope, $element, $timeout, ApiService) { $scope.addingTag = false; + $scope.changeTagsExpirationInfo = null; var markChanged = function(added, removed) { // Reload the repository. @@ -81,13 +82,58 @@ angular.module('quay').directive('tagOperationsDialog', function () { $element.find('#createOrMoveTagModal').modal('hide'); }); - ApiService.changeTagImage(data, params).then(function(resp) { + ApiService.changeTag(data, params).then(function(resp) { $element.find('#createOrMoveTagModal').modal('hide'); $scope.addingTag = false; markChanged([tag], []); }, errorHandler); }; + $scope.changeTagsExpiration = function(tags, expiration_date, callback) { + if (!$scope.repository.can_write) { return; } + + var count = tags.length; + var perform = function(index) { + if (index >= count) { + callback(true); + markChanged(tags, []); + return; + } + + var tag_info = tags[index]; + if (!tag_info) { return; } + + $scope.changeTagExpiration(tag_info.name, expiration_date, function(result) { + if (!result) { + callback(false); + return; + } + + perform(index + 1); + }, true); + }; + + perform(0); + }; + + $scope.changeTagExpiration = function(tag, expiration_date, callback) { + if (!$scope.repository.can_write) { return; } + + var params = { + 'repository': $scope.repository.namespace + '/' + $scope.repository.name, + 'tag': tag + }; + + var data = { + 'expiration': expiration_date + }; + + var errorHandler = ApiService.errorDisplay('Cannot change tag expiration', callback); + ApiService.changeTag(data, params).then(function() { + callback(true); + }, errorHandler); + }; + $scope.deleteMultipleTags = function(tags, callback) { if (!$scope.repository.can_write) { return; } @@ -296,6 +342,19 @@ angular.module('quay').directive('tagOperationsDialog', function () { }, ApiService.errorDisplay('Could not load manifest labels')); }, + 'askChangeTagsExpiration': function(tags) { + if ($scope.alertOnTagOpsDisabled()) { + return; + } + + var expiration_date = null; + expiration_date = tags[0].expiration_date ? tags[0].expiration_date / 1000 : null; + $scope.changeTagsExpirationInfo = { + 'tags': tags, + 'expiration_date': expiration_date + }; + }, + 'askRestoreTag': function(tag, image_id, opt_manifest_digest) { if ($scope.alertOnTagOpsDisabled()) { return; diff --git a/static/js/quay.module.ts b/static/js/quay.module.ts index 07f697bd7..014f2efec 100644 --- a/static/js/quay.module.ts +++ b/static/js/quay.module.ts @@ -31,6 +31,7 @@ import { MarkdownToolbarComponent } from './directives/ui/markdown/markdown-tool import { MarkdownEditorComponent } from './directives/ui/markdown/markdown-editor.component'; import { DockerfileCommandComponent } from './directives/ui/dockerfile-command/dockerfile-command.component'; import { ImageCommandComponent } from './directives/ui/image-command/image-command.component'; +import { ExpirationStatusViewComponent } from './directives/ui/expiration-status-view/expiration-status-view.component'; import { BrowserPlatform, browserPlatform } from './constants/platform.constant'; import { ManageTriggerComponent } from './directives/ui/manage-trigger/manage-trigger.component'; import { ClipboardCopyDirective } from './directives/ui/clipboard-copy/clipboard-copy.directive'; @@ -74,6 +75,7 @@ import * as Clipboard from 'clipboard'; ImageCommandComponent, TypeaheadDirective, ManageTriggerComponent, + ExpirationStatusViewComponent, ClipboardCopyDirective, TriggerDescriptionComponent, ], diff --git a/test/data/test.db b/test/data/test.db index 2dde19399..e109172d6 100644 Binary files a/test/data/test.db and b/test/data/test.db differ diff --git a/test/registry_tests.py b/test/registry_tests.py index ad6eda27c..5a069e34a 100644 --- a/test/registry_tests.py +++ b/test/registry_tests.py @@ -42,6 +42,7 @@ from image.docker.schema1 import DockerSchema1ManifestBuilder from initdb import wipe_database, initialize_database, populate_database from jsonschema import validate as validate_schema from util.security.registry_jwt import decode_bearer_header +from util.timedeltastring import convert_to_timedelta try: @@ -1535,6 +1536,38 @@ class V2RegistryTests(V2RegistryPullMixin, V2RegistryPushMixin, RegistryTestsMix self.assertTrue('text/plain' in media_types) self.assertTrue('application/json' in media_types) + def test_expiration_label(self): + # Push a new repo with the latest tag. + images = [{ + 'id': 'someid', + 'config': {'Labels': {'quay.expires-after': '1d'}}, + 'contents': 'somecontent' + }] + + (_, manifests) = self.do_push('devtable', 'newrepo', 'devtable', 'password', images=images) + + self.conduct_api_login('devtable', 'password') + tags = self.conduct('GET', '/api/v1/repository/devtable/newrepo/tag').json() + tag = tags['tags'][0] + + self.assertEqual(tag['end_ts'], tag['start_ts'] + convert_to_timedelta('1d').total_seconds()) + + def test_invalid_expiration_label(self): + # Push a new repo with the latest tag. + images = [{ + 'id': 'someid', + 'config': {'Labels': {'quay.expires-after': 'blahblah'}}, + 'contents': 'somecontent' + }] + + (_, manifests) = self.do_push('devtable', 'newrepo', 'devtable', 'password', images=images) + + self.conduct_api_login('devtable', 'password') + tags = self.conduct('GET', '/api/v1/repository/devtable/newrepo/tag').json() + tag = tags['tags'][0] + + self.assertIsNone(tag.get('end_ts')) + def test_invalid_manifest_type(self): namespace = 'devtable' repository = 'somerepo'