From 99d7fde8ee7a6d4c1e6346ca3543a1383d92c5b7 Mon Sep 17 00:00:00 2001 From: Joseph Schorr Date: Wed, 21 Jun 2017 21:33:26 -0400 Subject: [PATCH] Add UI for viewing and changing the expiration of tags --- ...f8f6_add_change_tag_expiration_log_type.py | 25 +++++++ data/model/tag.py | 44 +++++++++++- data/model/test/test_tag.py | 36 +++++++++- endpoints/api/repository.py | 8 ++- endpoints/api/tag.py | 67 ++++++++++++++----- endpoints/api/test/test_tag.py | 37 ++++++++++ endpoints/api/user.py | 2 +- initdb.py | 2 + .../directives/repo-view/repo-panel-tags.html | 34 +++++++++- static/directives/tag-operations-dialog.html | 22 ++++++ .../directives/repo-view/repo-panel-tags.js | 7 +- static/js/directives/ui/logs-view.js | 13 ++++ .../js/directives/ui/tag-operations-dialog.js | 58 +++++++++++++++- 13 files changed, 329 insertions(+), 26 deletions(-) create mode 100644 data/migrations/versions/d8989249f8f6_add_change_tag_expiration_log_type.py diff --git a/data/migrations/versions/d8989249f8f6_add_change_tag_expiration_log_type.py b/data/migrations/versions/d8989249f8f6_add_change_tag_expiration_log_type.py new file mode 100644 index 000000000..9ae981008 --- /dev/null +++ b/data/migrations/versions/d8989249f8f6_add_change_tag_expiration_log_type.py @@ -0,0 +1,25 @@ +"""Add change_tag_expiration log type + +Revision ID: d8989249f8f6 +Revises: dc4af11a5f90 +Create Date: 2017-06-21 21:18:25.948689 + +""" + +# revision identifiers, used by Alembic. +revision = 'd8989249f8f6' +down_revision = 'dc4af11a5f90' + +from alembic import op + +def upgrade(tables): + op.bulk_insert(tables.logentrykind, [ + {'name': 'change_tag_expiration'}, + ]) + + +def downgrade(tables): + op.execute(tables + .logentrykind + .delete() + .where(tables.logentrykind.c.name == op.inline_literal('change_tag_expiration'))) diff --git a/data/model/tag.py b/data/model/tag.py index 24926174b..57faea80c 100644 --- a/data/model/tag.py +++ b/data/model/tag.py @@ -1,13 +1,17 @@ import logging +import time +from calendar import timegm from uuid import uuid4 from peewee import IntegrityError, JOIN_LEFT_OUTER, fn from data.model import (image, db_transaction, DataModelException, _basequery, - InvalidManifestException, TagAlreadyCreatedException, StaleTagException) + InvalidManifestException, TagAlreadyCreatedException, StaleTagException, + config) from data.database import (RepositoryTag, Repository, Image, ImageStorage, Namespace, TagManifest, RepositoryNotification, Label, TagManifestLabel, get_epoch_timestamp, db_for_update) +from util.timedeltastring import convert_to_timedelta logger = logging.getLogger(__name__) @@ -570,3 +574,41 @@ def _load_repo_manifests(namespace, repo_name): .join(Repository) .join(Namespace, on=(Namespace.id == Repository.namespace_user)) .where(Repository.name == repo_name, Namespace.username == namespace)) + + +def change_repository_tag_expiration(namespace_name, repo_name, tag_name, expiration_date): + """ Changes the expiration of the tag with the given name to the given expiration datetime. If + the expiration datetime is None, then the tag is marked as not expiring. + """ + try: + tag = get_active_tag(namespace_name, repo_name, tag_name) + return change_tag_expiration(tag, expiration_date) + except RepositoryTag.DoesNotExist: + return (None, False) + + +def change_tag_expiration(tag, expiration_date): + """ Changes the expiration of the given tag to the given expiration datetime. If + the expiration datetime is None, then the tag is marked as not expiring. + """ + end_ts = None + min_expire_sec = convert_to_timedelta(config.app_config.get('LABELED_EXPIRATION_MINIMUM', '1h')) + max_expire_sec = convert_to_timedelta(config.app_config.get('LABELED_EXPIRATION_MAXIMUM', '104w')) + + if expiration_date is not None: + offset = timegm(expiration_date.utctimetuple()) - tag.lifetime_start_ts + offset = min(max(offset, min_expire_sec.total_seconds()), max_expire_sec.total_seconds()) + end_ts = tag.lifetime_start_ts + offset + + if end_ts == tag.lifetime_end_ts: + return (None, True) + + # Note: We check not just the ID of the tag but also its lifetime_end_ts, to ensure that it has + # not changed while we were updatings it expiration. + result = (RepositoryTag + .update(lifetime_end_ts=end_ts) + .where(RepositoryTag.id == tag.id, + RepositoryTag.lifetime_end_ts == tag.lifetime_end_ts) + .execute()) + + return (tag.lifetime_end_ts, result > 0) diff --git a/data/model/test/test_tag.py b/data/model/test/test_tag.py index 1e7177310..6cfa7366d 100644 --- a/data/model/test/test_tag.py +++ b/data/model/test/test_tag.py @@ -1,13 +1,16 @@ import pytest +from datetime import datetime from mock import patch from time import time from data.database import Image, RepositoryTag, ImageStorage, Repository from data.model.repository import create_repository from data.model.tag import (list_active_repo_tags, create_or_update_tag, delete_tag, - get_matching_tags, _tag_alive, get_matching_tags_for_images) + get_matching_tags, _tag_alive, get_matching_tags_for_images, + change_tag_expiration, get_active_tag) from data.model.image import find_create_or_link_image +from util.timedeltastring import convert_to_timedelta from test.fixtures import * @@ -177,3 +180,34 @@ def test_list_active_tags(initialized_db): # "Move" foo by updating it and make sure we don't get duplicates. create_or_update_tag('devtable', 'somenewrepo', 'foo', image2.docker_image_id) assert_tags(repository, 'foo', 'bar') + + +@pytest.mark.parametrize('expiration_offset, expected_offset', [ + (None, None), + ('0s', '1h'), + ('30m', '1h'), + ('2h', '2h'), + ('2w', '2w'), + ('200w', '104w'), +]) +def test_change_tag_expiration(expiration_offset, expected_offset, initialized_db): + repository = create_repository('devtable', 'somenewrepo', None) + image1 = find_create_or_link_image('foobarimage1', repository, None, {}, 'local_us') + footag = create_or_update_tag('devtable', 'somenewrepo', 'foo', image1.docker_image_id) + + expiration_date = None + if expiration_offset is not None: + expiration_date = datetime.utcnow() + convert_to_timedelta(expiration_offset) + + assert change_tag_expiration(footag, expiration_date) + + # Lookup the tag again. + footag_updated = get_active_tag('devtable', 'somenewrepo', 'foo') + + if expected_offset is None: + assert footag_updated.lifetime_end_ts is None + else: + start_date = datetime.utcfromtimestamp(footag_updated.lifetime_start_ts) + end_date = datetime.utcfromtimestamp(footag_updated.lifetime_end_ts) + expected_end_date = start_date + convert_to_timedelta(expected_offset) + assert end_date == expected_end_date diff --git a/endpoints/api/repository.py b/endpoints/api/repository.py index eb740ef44..f8f41de44 100644 --- a/endpoints/api/repository.py +++ b/endpoints/api/repository.py @@ -334,6 +334,10 @@ class Repository(RepositoryParamResource): last_modified = format_date(datetime.fromtimestamp(tag.lifetime_start_ts)) tag_info['last_modified'] = last_modified + if tag.lifetime_end_ts: + expiration = format_date(datetime.fromtimestamp(tag.lifetime_end_ts)) + tag_info['expiration'] = expiration + if tag.tagmanifest is not None: tag_info['manifest_digest'] = tag.tagmanifest.digest @@ -498,11 +502,11 @@ class RepositoryTrust(RepositoryParamResource): repo = model.repository.get_repository(namespace, repository) if not repo: raise NotFound() - + tags, _ = tuf_metadata_api.get_default_tags_with_expiration(namespace, repository) if tags and not tuf_metadata_api.delete_metadata(namespace, repository): raise DownstreamIssue({'message': 'Unable to delete downstream trust metadata'}) - + values = request.get_json() model.repository.set_trust(repo, values['trust_enabled']) diff --git a/endpoints/api/tag.py b/endpoints/api/tag.py index 8e65257a5..42e375de2 100644 --- a/endpoints/api/tag.py +++ b/endpoints/api/tag.py @@ -1,5 +1,6 @@ """ Manage the tags of a repository. """ +from datetime import datetime, timedelta from flask import request, abort from auth.auth_context import get_authenticated_user @@ -54,12 +55,15 @@ class RepositoryTag(RepositoryParamResource): schemas = { 'MoveTag': { 'type': 'object', - 'description': 'Description of to which image a new or existing tag should point', - 'required': ['image',], + 'description': 'Makes changes to a specific tag', 'properties': { 'image': { - 'type': 'string', - 'description': 'Image identifier to which the tag should point', + 'type': ['string', 'null'], + 'description': '(If specified) Image identifier to which the tag should point', + }, + 'image': { + 'type': ['number', 'null'], + 'description': '(If specified) The expiration for the image', }, }, }, @@ -75,25 +79,52 @@ class RepositoryTag(RepositoryParamResource): if not TAG_REGEX.match(tag): abort(400, TAG_ERROR) - image_id = request.get_json()['image'] - repo = model.get_repo(namespace, repository, image_id) + repo = model.get_repo(namespace, repository) if not repo: raise NotFound() - original_image_id = model.get_repo_tag_image(repo, tag) - model.create_or_update_tag(namespace, repository, tag, image_id) + if 'expiration' in request.get_json(): + expiration = request.get_json().get('expiration') + expiration_date = None + if expiration is not None: + try: + expiration_date = datetime.utcfromtimestamp(float(expiration)) + except ValueError: + abort(400) - username = get_authenticated_user().username - log_action('move_tag' if original_image_id else 'create_tag', namespace, { - 'username': username, - 'repo': repository, - 'tag': tag, - 'namespace': namespace, - 'image': image_id, - 'original_image': original_image_id - }, repo_name=repository) + if expiration_date <= datetime.now(): + abort(400) - _generate_and_store_manifest(namespace, repository, tag) + existing_end_ts, ok = model.change_repository_tag_expiration(namespace, repository, tag, + expiration_date) + if ok: + if not (existing_end_ts is None and expiration_date is None): + log_action('change_tag_expiration', namespace, { + 'username': get_authenticated_user().username, + 'repo': repository, + 'tag': tag, + 'namespace': namespace, + 'expiration_date': expiration_date, + 'old_expiration_date': existing_end_ts + }, repo=repo) + else: + abort(400, 'Could not update tag expiration; Tag has probably changed') + + if 'image' in request.get_json(): + image_id = request.get_json()['image'] + original_image_id = model.get_repo_tag_image(repo, tag) + model.create_or_update_tag(namespace, repository, tag, image_id) + + username = get_authenticated_user().username + log_action('move_tag' if original_image_id else 'create_tag', namespace, { + 'username': username, + 'repo': repository, + 'tag': tag, + 'namespace': namespace, + 'image': image_id, + 'original_image': original_image_id + }, repo_name=repository) + _generate_and_store_manifest(namespace, repository, tag) return 'Updated', 201 diff --git a/endpoints/api/test/test_tag.py b/endpoints/api/test/test_tag.py index 1d13d0eae..abc0adf9e 100644 --- a/endpoints/api/test/test_tag.py +++ b/endpoints/api/test/test_tag.py @@ -124,6 +124,43 @@ def find_no_repo_tag_history(): yield +@pytest.mark.parametrize('expiration_time, expected_status', [ + (None, 201), + ('aksdjhasd', 400), +]) +def test_change_tag_expiration_default(expiration_time, expected_status, client, app): + with client_with_identity('devtable', client) as cl: + params = { + 'repository': 'devtable/simple', + 'tag': 'latest', + } + + request_body = { + 'expiration': expiration_time, + } + + conduct_api_call(cl, RepositoryTag, 'put', params, request_body, expected_status) + + +def test_change_tag_expiration(client, app): + with client_with_identity('devtable', client) as cl: + params = { + 'repository': 'devtable/simple', + 'tag': 'latest', + } + + tag = model.tag.get_active_tag('devtable', 'simple', 'latest') + updated_expiration = tag.lifetime_start_ts + 60*60*24 + + request_body = { + 'expiration': updated_expiration, + } + + conduct_api_call(cl, RepositoryTag, 'put', params, request_body, 201) + tag = model.tag.get_active_tag('devtable', 'simple', 'latest') + assert tag.lifetime_end_ts == updated_expiration + + @pytest.mark.parametrize('test_image,test_tag,expected_status', [ ('image1', '-INVALID-TAG-NAME', 400), ('image1', '.INVALID-TAG-NAME', 400), diff --git a/endpoints/api/user.py b/endpoints/api/user.py index 093824e13..f359c16d5 100644 --- a/endpoints/api/user.py +++ b/endpoints/api/user.py @@ -20,7 +20,7 @@ from auth.permissions import (AdministerOrganizationPermission, CreateRepository UserAdminPermission, UserReadPermission, SuperUserPermission) from data import model from data.billing import get_plan -from data.database import Repository as RepositoryTable, UserPromptTypes +from data.database import Repository as RepositoryTable from endpoints.api import (ApiResource, nickname, resource, validate_json_request, request_error, log_action, internal_only, require_user_admin, parse_args, query_param, require_scope, format_date, show_if, diff --git a/initdb.py b/initdb.py index 387cc50f8..c6b36e18e 100644 --- a/initdb.py +++ b/initdb.py @@ -350,6 +350,8 @@ def initialize_database(): LogEntryKind.create(name='manifest_label_add') LogEntryKind.create(name='manifest_label_delete') + LogEntryKind.create(name='change_tag_expiration') + ImageStorageLocation.create(name='local_eu') ImageStorageLocation.create(name='local_us') diff --git a/static/directives/repo-view/repo-panel-tags.html b/static/directives/repo-view/repo-panel-tags.html index 0e572a568..20fe43daa 100644 --- a/static/directives/repo-view/repo-panel-tags.html +++ b/static/directives/repo-view/repo-panel-tags.html @@ -57,6 +57,12 @@ Delete Tags +
  • + + Change Tags Expiration + +
    • @@ -105,6 +111,11 @@ style="width: 80px;"> Size + + Expires + @@ -133,12 +144,16 @@ ng-if="repository.trust_enabled"> + + Unknown + + + + + + + + + + + + + + @@ -254,6 +282,10 @@ ng-class="repository.tag_operations_disabled ? 'disabled-option' : ''"> Delete Tag + + Change Expiration + @@ -261,7 +293,7 @@ - +
    diff --git a/static/directives/tag-operations-dialog.html b/static/directives/tag-operations-dialog.html index 5f5f7b2be..e6a4ef480 100644 --- a/static/directives/tag-operations-dialog.html +++ b/static/directives/tag-operations-dialog.html @@ -111,6 +111,28 @@ + +
    +
    + +
      +
    • + {{ tag_info.name }} +
      • +
    + + + + + If specified, the date and time that the key expires. If set to none, the tag(s) will not expire. + +
    +
    +
    = count) { + callback(true); + markChanged(tags, []); + return; + } + + var tag_info = tags[index]; + if (!tag_info) { return; } + + $scope.changeTagExpiration(tag_info.name, expiration_date, function(result) { + if (!result) { + callback(false); + return; + } + + perform(index + 1); + }, true); + }; + + perform(0); + }; + + $scope.changeTagExpiration = function(tag, expiration_date, callback) { + if (!$scope.repository.can_write) { return; } + + var params = { + 'repository': $scope.repository.namespace + '/' + $scope.repository.name, + 'tag': tag + }; + + var data = { + 'expiration': expiration_date + }; + + var errorHandler = ApiService.errorDisplay('Cannot change tag expiration', callback); + ApiService.changeTag(data, params).then(function() { + callback(true); + }, errorHandler); + }; + $scope.deleteMultipleTags = function(tags, callback) { if (!$scope.repository.can_write) { return; } @@ -296,6 +341,17 @@ angular.module('quay').directive('tagOperationsDialog', function () { }, ApiService.errorDisplay('Could not load manifest labels')); }, + 'askChangeTagsExpiration': function(tags) { + if ($scope.alertOnTagOpsDisabled()) { + return; + } + + $scope.changeTagsExpirationInfo ={ + 'tags': tags, + 'expiration_date': null + }; + }, + 'askRestoreTag': function(tag, image_id, opt_manifest_digest) { if ($scope.alertOnTagOpsDisabled()) { return;