Merge pull request #2718 from coreos-inc/tag-expiration
Formal tag expiration support
This commit is contained in:
commit
a6db05e8b5
32 changed files with 621 additions and 117 deletions
|
@ -334,6 +334,10 @@ class Repository(RepositoryParamResource):
|
|||
last_modified = format_date(datetime.fromtimestamp(tag.lifetime_start_ts))
|
||||
tag_info['last_modified'] = last_modified
|
||||
|
||||
if tag.lifetime_end_ts:
|
||||
expiration = format_date(datetime.fromtimestamp(tag.lifetime_end_ts))
|
||||
tag_info['expiration'] = expiration
|
||||
|
||||
if tag.tagmanifest is not None:
|
||||
tag_info['manifest_digest'] = tag.tagmanifest.digest
|
||||
|
||||
|
@ -498,11 +502,11 @@ class RepositoryTrust(RepositoryParamResource):
|
|||
repo = model.repository.get_repository(namespace, repository)
|
||||
if not repo:
|
||||
raise NotFound()
|
||||
|
||||
|
||||
tags, _ = tuf_metadata_api.get_default_tags_with_expiration(namespace, repository)
|
||||
if tags and not tuf_metadata_api.delete_metadata(namespace, repository):
|
||||
raise DownstreamIssue({'message': 'Unable to delete downstream trust metadata'})
|
||||
|
||||
|
||||
values = request.get_json()
|
||||
model.repository.set_trust(repo, values['trust_enabled'])
|
||||
|
||||
|
|
|
@ -1,5 +1,6 @@
|
|||
""" Manage the tags of a repository. """
|
||||
|
||||
from datetime import datetime, timedelta
|
||||
from flask import request, abort
|
||||
|
||||
from auth.auth_context import get_authenticated_user
|
||||
|
@ -9,7 +10,7 @@ from endpoints.api import (resource, nickname, require_repo_read, require_repo_w
|
|||
parse_args, query_param, truthy_bool, disallow_for_app_repositories)
|
||||
from endpoints.api.tag_models_interface import Repository
|
||||
from endpoints.api.tag_models_pre_oci import pre_oci_model as model
|
||||
from endpoints.exception import NotFound
|
||||
from endpoints.exception import NotFound, InvalidRequest
|
||||
from endpoints.v2.manifest import _generate_and_store_manifest
|
||||
from util.names import TAG_ERROR, TAG_REGEX
|
||||
|
||||
|
@ -52,14 +53,17 @@ class ListRepositoryTags(RepositoryParamResource):
|
|||
class RepositoryTag(RepositoryParamResource):
|
||||
""" Resource for managing repository tags. """
|
||||
schemas = {
|
||||
'MoveTag': {
|
||||
'ChangeTag': {
|
||||
'type': 'object',
|
||||
'description': 'Description of to which image a new or existing tag should point',
|
||||
'required': ['image',],
|
||||
'description': 'Makes changes to a specific tag',
|
||||
'properties': {
|
||||
'image': {
|
||||
'type': 'string',
|
||||
'description': 'Image identifier to which the tag should point',
|
||||
'type': ['string', 'null'],
|
||||
'description': '(If specified) Image identifier to which the tag should point',
|
||||
},
|
||||
'expiration': {
|
||||
'type': ['number', 'null'],
|
||||
'description': '(If specified) The expiration for the image',
|
||||
},
|
||||
},
|
||||
},
|
||||
|
@ -67,33 +71,64 @@ class RepositoryTag(RepositoryParamResource):
|
|||
|
||||
@require_repo_write
|
||||
@disallow_for_app_repositories
|
||||
@nickname('changeTagImage')
|
||||
@validate_json_request('MoveTag')
|
||||
@nickname('changeTag')
|
||||
@validate_json_request('ChangeTag')
|
||||
def put(self, namespace, repository, tag):
|
||||
""" Change which image a tag points to or create a new tag."""
|
||||
|
||||
if not TAG_REGEX.match(tag):
|
||||
abort(400, TAG_ERROR)
|
||||
|
||||
image_id = request.get_json()['image']
|
||||
repo = model.get_repo(namespace, repository, image_id)
|
||||
repo = model.get_repo(namespace, repository)
|
||||
if not repo:
|
||||
raise NotFound()
|
||||
|
||||
original_image_id = model.get_repo_tag_image(repo, tag)
|
||||
model.create_or_update_tag(namespace, repository, tag, image_id)
|
||||
if 'expiration' in request.get_json():
|
||||
expiration = request.get_json().get('expiration')
|
||||
expiration_date = None
|
||||
if expiration is not None:
|
||||
try:
|
||||
expiration_date = datetime.utcfromtimestamp(float(expiration))
|
||||
except ValueError:
|
||||
abort(400)
|
||||
|
||||
username = get_authenticated_user().username
|
||||
log_action('move_tag' if original_image_id else 'create_tag', namespace, {
|
||||
'username': username,
|
||||
'repo': repository,
|
||||
'tag': tag,
|
||||
'namespace': namespace,
|
||||
'image': image_id,
|
||||
'original_image': original_image_id
|
||||
}, repo_name=repository)
|
||||
if expiration_date <= datetime.now():
|
||||
abort(400)
|
||||
|
||||
_generate_and_store_manifest(namespace, repository, tag)
|
||||
existing_end_ts, ok = model.change_repository_tag_expiration(namespace, repository, tag,
|
||||
expiration_date)
|
||||
if ok:
|
||||
if not (existing_end_ts is None and expiration_date is None):
|
||||
log_action('change_tag_expiration', namespace, {
|
||||
'username': get_authenticated_user().username,
|
||||
'repo': repository,
|
||||
'tag': tag,
|
||||
'namespace': namespace,
|
||||
'expiration_date': expiration_date,
|
||||
'old_expiration_date': existing_end_ts
|
||||
}, repo_name=repository)
|
||||
else:
|
||||
raise InvalidRequest('Could not update tag expiration; Tag has probably changed')
|
||||
|
||||
if 'image' in request.get_json():
|
||||
image_id = request.get_json()['image']
|
||||
image = model.get_repository_image(namespace, repository, image_id)
|
||||
if image is None:
|
||||
raise NotFound()
|
||||
|
||||
original_image_id = model.get_repo_tag_image(repo, tag)
|
||||
model.create_or_update_tag(namespace, repository, tag, image_id)
|
||||
|
||||
username = get_authenticated_user().username
|
||||
log_action('move_tag' if original_image_id else 'create_tag', namespace, {
|
||||
'username': username,
|
||||
'repo': repository,
|
||||
'tag': tag,
|
||||
'namespace': namespace,
|
||||
'image': image_id,
|
||||
'original_image': original_image_id
|
||||
}, repo_name=repository)
|
||||
_generate_and_store_manifest(namespace, repository, tag)
|
||||
|
||||
return 'Updated', 201
|
||||
|
||||
|
|
|
@ -61,7 +61,7 @@ class Repository(namedtuple('Repository', ['namespace_name', 'repository_name'])
|
|||
class Image(
|
||||
namedtuple('Image', [
|
||||
'docker_image_id', 'created', 'comment', 'command', 'storage_image_size',
|
||||
'storage_uploading', 'ancestor_length', 'ancestor_id_list'
|
||||
'storage_uploading', 'ancestor_id_list'
|
||||
])):
|
||||
"""
|
||||
Image
|
||||
|
@ -71,7 +71,6 @@ class Image(
|
|||
:type command: string
|
||||
:type storage_image_size: int
|
||||
:type storage_uploading: boolean
|
||||
:type ancestor_length: int
|
||||
:type ancestor_id_list: [int]
|
||||
"""
|
||||
|
||||
|
@ -91,7 +90,7 @@ class Image(
|
|||
'command': json.loads(command) if command else None,
|
||||
'size': self.storage_image_size,
|
||||
'uploading': self.storage_uploading,
|
||||
'sort_index': self.ancestor_length,
|
||||
'sort_index': len(self.ancestor_id_list),
|
||||
}
|
||||
|
||||
if include_ancestors:
|
||||
|
@ -116,9 +115,9 @@ class TagDataInterface(object):
|
|||
"""
|
||||
|
||||
@abstractmethod
|
||||
def get_repo(self, namespace_name, repository_name, docker_image_id):
|
||||
def get_repo(self, namespace_name, repository_name):
|
||||
"""
|
||||
Returns a repository associated with the given namespace, repository, and docker_image_id
|
||||
Returns a repository associated with the given namespace and repository name.
|
||||
"""
|
||||
|
||||
@abstractmethod
|
||||
|
@ -157,6 +156,13 @@ class TagDataInterface(object):
|
|||
Returns the repository associated with the namespace_name and repository_name
|
||||
"""
|
||||
|
||||
@abstractmethod
|
||||
def get_repository_image(self, namespace_name, repository_name, docker_image_id):
|
||||
"""
|
||||
Returns the repository image associated with the namespace_name, repository_name, and docker
|
||||
image ID.
|
||||
"""
|
||||
|
||||
@abstractmethod
|
||||
def restore_tag_to_manifest(self, repository_name, tag_name, manifest_digest):
|
||||
"""
|
||||
|
@ -170,3 +176,11 @@ class TagDataInterface(object):
|
|||
Returns the existing repo tag image if it exists or else returns None
|
||||
Side effects include adding the tag with associated name to the image with the associated id in the named repo.
|
||||
"""
|
||||
|
||||
@abstractmethod
|
||||
def change_repository_tag_expiration(self, namespace_name, repository_name, tag_name,
|
||||
expiration_date):
|
||||
""" Sets the expiration date of the tag under the matching repository to that given. If the
|
||||
expiration date is None, then the tag will not expire. Returns a tuple of the previous
|
||||
expiration timestamp in seconds (if any), and whether the operation succeeded.
|
||||
"""
|
||||
|
|
|
@ -27,12 +27,12 @@ class PreOCIModel(TagDataInterface):
|
|||
|
||||
return RepositoryTagHistory(tags=repository_tag_history, more=more)
|
||||
|
||||
def get_repo(self, namespace_name, repository_name, docker_image_id):
|
||||
image = model.image.get_repo_image(namespace_name, repository_name, docker_image_id)
|
||||
if image is None:
|
||||
def get_repo(self, namespace_name, repository_name):
|
||||
repo = model.repository.get_repository(namespace_name, repository_name)
|
||||
if repo is None:
|
||||
return None
|
||||
|
||||
return Repository(image.repository.namespace_user, image.repository.name)
|
||||
return Repository(repo.namespace_user, repo.name)
|
||||
|
||||
def get_repo_tag_image(self, repository, tag_name):
|
||||
repo = model.repository.get_repository(str(repository.namespace_name), str(repository.repository_name))
|
||||
|
@ -73,6 +73,13 @@ class PreOCIModel(TagDataInterface):
|
|||
new_tags.append(convert_tag(tag))
|
||||
return new_tags
|
||||
|
||||
def get_repository_image(self, namespace_name, repository_name, docker_image_id):
|
||||
image = model.image.get_repo_image(namespace_name, repository_name, docker_image_id)
|
||||
if image is None:
|
||||
return None
|
||||
|
||||
return convert_image(image)
|
||||
|
||||
def get_repository(self, namespace_name, repository_name):
|
||||
repo = model.repository.get_repository(namespace_name, repository_name)
|
||||
if repo is None:
|
||||
|
@ -102,13 +109,17 @@ class PreOCIModel(TagDataInterface):
|
|||
|
||||
return convert_image(image)
|
||||
|
||||
def change_repository_tag_expiration(self, namespace_name, repository_name, tag_name,
|
||||
expiration_date):
|
||||
return model.tag.change_repository_tag_expiration(namespace_name, repository_name, tag_name,
|
||||
expiration_date)
|
||||
|
||||
|
||||
def convert_image(database_image):
|
||||
return Image(docker_image_id=database_image.docker_image_id, created=database_image.created,
|
||||
comment=database_image.comment, command=database_image.command,
|
||||
storage_image_size=database_image.storage.image_size,
|
||||
storage_uploading=database_image.storage.uploading,
|
||||
ancestor_length=len(database_image.ancestors),
|
||||
ancestor_id_list=database_image.ancestor_id_list())
|
||||
|
||||
|
||||
|
|
|
@ -124,6 +124,43 @@ def find_no_repo_tag_history():
|
|||
yield
|
||||
|
||||
|
||||
@pytest.mark.parametrize('expiration_time, expected_status', [
|
||||
(None, 201),
|
||||
('aksdjhasd', 400),
|
||||
])
|
||||
def test_change_tag_expiration_default(expiration_time, expected_status, client, app):
|
||||
with client_with_identity('devtable', client) as cl:
|
||||
params = {
|
||||
'repository': 'devtable/simple',
|
||||
'tag': 'latest',
|
||||
}
|
||||
|
||||
request_body = {
|
||||
'expiration': expiration_time,
|
||||
}
|
||||
|
||||
conduct_api_call(cl, RepositoryTag, 'put', params, request_body, expected_status)
|
||||
|
||||
|
||||
def test_change_tag_expiration(client, app):
|
||||
with client_with_identity('devtable', client) as cl:
|
||||
params = {
|
||||
'repository': 'devtable/simple',
|
||||
'tag': 'latest',
|
||||
}
|
||||
|
||||
tag = model.tag.get_active_tag('devtable', 'simple', 'latest')
|
||||
updated_expiration = tag.lifetime_start_ts + 60*60*24
|
||||
|
||||
request_body = {
|
||||
'expiration': updated_expiration,
|
||||
}
|
||||
|
||||
conduct_api_call(cl, RepositoryTag, 'put', params, request_body, 201)
|
||||
tag = model.tag.get_active_tag('devtable', 'simple', 'latest')
|
||||
assert tag.lifetime_end_ts == updated_expiration
|
||||
|
||||
|
||||
@pytest.mark.parametrize('test_image,test_tag,expected_status', [
|
||||
('image1', '-INVALID-TAG-NAME', 400),
|
||||
('image1', '.INVALID-TAG-NAME', 400),
|
||||
|
|
|
@ -106,20 +106,18 @@ def test_list_repository_tag_history(expected, namespace_name, repository_name,
|
|||
specific_tag) == expected
|
||||
|
||||
|
||||
def get_repo_image_mock(monkeypatch, return_value):
|
||||
def return_return_value(namespace_name, repository_name, image_id):
|
||||
def get_repo_mock(monkeypatch, return_value):
|
||||
def return_return_value(namespace_name, repository_name):
|
||||
return return_value
|
||||
|
||||
monkeypatch.setattr(model.image, 'get_repo_image', return_return_value)
|
||||
monkeypatch.setattr(model.repository, 'get_repository', return_return_value)
|
||||
|
||||
|
||||
def test_get_repo_not_exists(get_monkeypatch):
|
||||
namespace_name = 'namespace_name'
|
||||
repository_name = 'repository_name'
|
||||
image_id = 'image_id'
|
||||
get_repo_image_mock(get_monkeypatch, None)
|
||||
|
||||
repo = pre_oci_model.get_repo(namespace_name, repository_name, image_id)
|
||||
get_repo_mock(get_monkeypatch, None)
|
||||
repo = pre_oci_model.get_repo(namespace_name, repository_name)
|
||||
|
||||
assert repo is None
|
||||
|
||||
|
@ -127,14 +125,13 @@ def test_get_repo_not_exists(get_monkeypatch):
|
|||
def test_get_repo_exists(get_monkeypatch):
|
||||
namespace_name = 'namespace_name'
|
||||
repository_name = 'repository_name'
|
||||
image_id = 'image_id'
|
||||
mock = Mock()
|
||||
mock.namespace_user = namespace_name
|
||||
mock.name = repository_name
|
||||
mock.repository = mock
|
||||
get_repo_image_mock(get_monkeypatch, mock)
|
||||
get_repo_mock(get_monkeypatch, mock)
|
||||
|
||||
repo = pre_oci_model.get_repo(namespace_name, repository_name, image_id)
|
||||
repo = pre_oci_model.get_repo(namespace_name, repository_name)
|
||||
|
||||
assert repo is not None
|
||||
assert repo.repository_name == repository_name
|
||||
|
|
|
@ -20,7 +20,7 @@ from auth.permissions import (AdministerOrganizationPermission, CreateRepository
|
|||
UserAdminPermission, UserReadPermission, SuperUserPermission)
|
||||
from data import model
|
||||
from data.billing import get_plan
|
||||
from data.database import Repository as RepositoryTable, UserPromptTypes
|
||||
from data.database import Repository as RepositoryTable
|
||||
from endpoints.api import (ApiResource, nickname, resource, validate_json_request, request_error,
|
||||
log_action, internal_only, require_user_admin, parse_args,
|
||||
query_param, require_scope, format_date, show_if,
|
||||
|
|
|
@ -57,6 +57,13 @@ class ManifestUnknown(V2RegistryException):
|
|||
def __init__(self, detail=None):
|
||||
super(ManifestUnknown, self).__init__('MANIFEST_UNKNOWN', 'manifest unknown', detail, 404)
|
||||
|
||||
class TagExpired(V2RegistryException):
|
||||
def __init__(self, message=None, detail=None):
|
||||
super(TagExpired, self).__init__('TAG_EXPIRED',
|
||||
message or 'Tag has expired',
|
||||
detail,
|
||||
404)
|
||||
|
||||
|
||||
class ManifestUnverified(V2RegistryException):
|
||||
def __init__(self, detail=None):
|
||||
|
|
36
endpoints/v2/labelhandlers.py
Normal file
36
endpoints/v2/labelhandlers.py
Normal file
|
@ -0,0 +1,36 @@
|
|||
import logging
|
||||
|
||||
from app import app
|
||||
from endpoints.v2.models_pre_oci import data_model as model
|
||||
from util.timedeltastring import convert_to_timedelta
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
min_expire_sec = convert_to_timedelta(app.config.get('LABELED_EXPIRATION_MINIMUM', '1h'))
|
||||
max_expire_sec = convert_to_timedelta(app.config.get('LABELED_EXPIRATION_MAXIMUM', '104w'))
|
||||
|
||||
def _expires_after(value, namespace_name, repo_name, digest):
|
||||
""" Sets the expiration of a manifest based on the quay.expires-in label. """
|
||||
try:
|
||||
timedelta = convert_to_timedelta(value)
|
||||
except ValueError:
|
||||
logger.exception('Could not convert %s to timedeltastring for %s/%s@%s', value, namespace_name,
|
||||
repo_name, digest)
|
||||
return
|
||||
|
||||
total_seconds = min(max(timedelta.total_seconds(), min_expire_sec.total_seconds()),
|
||||
max_expire_sec.total_seconds())
|
||||
|
||||
logger.debug('Labeling manifest %s/%s@%s with expiration of %s', namespace_name, repo_name,
|
||||
digest, total_seconds)
|
||||
model.set_manifest_expires_after(namespace_name, repo_name, digest, total_seconds)
|
||||
|
||||
|
||||
_LABEL_HANDLES = {
|
||||
'quay.expires-after': _expires_after,
|
||||
}
|
||||
|
||||
def handle_label(key, value, namespace_name, repo_name, digest):
|
||||
handler = _LABEL_HANDLES.get(key)
|
||||
if handler is not None:
|
||||
handler(value, namespace_name, repo_name, digest)
|
|
@ -13,10 +13,11 @@ from endpoints.common import parse_repository_name
|
|||
from endpoints.decorators import anon_protect
|
||||
from endpoints.notificationhelper import spawn_notification
|
||||
from endpoints.v2 import v2_bp, require_repo_read, require_repo_write
|
||||
from endpoints.v2.errors import (
|
||||
BlobUnknown, ManifestInvalid, ManifestUnknown, TagInvalid, NameInvalid)
|
||||
from endpoints.v2.models_interface import Label
|
||||
from endpoints.v2.models_pre_oci import data_model as model
|
||||
from endpoints.v2.errors import (BlobUnknown, ManifestInvalid, ManifestUnknown, TagInvalid,
|
||||
NameInvalid, TagExpired)
|
||||
from endpoints.v2.labelhandlers import handle_label
|
||||
from image.docker import ManifestException
|
||||
from image.docker.schema1 import DockerSchema1Manifest, DockerSchema1ManifestBuilder
|
||||
from image.docker.schema2 import DOCKER_SCHEMA2_CONTENT_TYPES
|
||||
|
@ -42,7 +43,14 @@ def fetch_manifest_by_tagname(namespace_name, repo_name, manifest_ref):
|
|||
if manifest is None:
|
||||
has_tag = model.has_active_tag(namespace_name, repo_name, manifest_ref)
|
||||
if not has_tag:
|
||||
raise ManifestUnknown()
|
||||
has_expired_tag = model.has_tag(namespace_name, repo_name, manifest_ref)
|
||||
if has_expired_tag:
|
||||
logger.debug('Found expired tag %s for repository %s/%s', manifest_ref, namespace_name,
|
||||
repo_name)
|
||||
msg = 'Tag %s was deleted or has expired. To pull, revive via time machine' % manifest_ref
|
||||
raise TagExpired(msg)
|
||||
else:
|
||||
raise ManifestUnknown()
|
||||
|
||||
manifest = _generate_and_store_manifest(namespace_name, repo_name, manifest_ref)
|
||||
if manifest is None:
|
||||
|
@ -188,6 +196,8 @@ def _write_manifest(namespace_name, repo_name, manifest):
|
|||
for key, value in manifest.layers[-1].v1_metadata.labels.iteritems():
|
||||
media_type = 'application/json' if is_json(value) else 'text/plain'
|
||||
labels.append(Label(key=key, value=value, source_type='manifest', media_type=media_type))
|
||||
handle_label(key, value, namespace_name, repo_name, manifest.digest)
|
||||
|
||||
model.create_manifest_labels(namespace_name, repo_name, manifest.digest, labels)
|
||||
|
||||
return repo, storage_map
|
||||
|
@ -268,7 +278,3 @@ def _generate_and_store_manifest(namespace_name, repo_name, tag_name):
|
|||
model.create_manifest_and_update_tag(namespace_name, repo_name, tag_name, manifest.digest,
|
||||
manifest.bytes)
|
||||
return manifest
|
||||
|
||||
|
||||
def _determine_media_type(value):
|
||||
media_type_name = 'application/json' if is_json(value) else 'text/plain'
|
||||
|
|
|
@ -256,3 +256,11 @@ class DockerRegistryV2DataInterface(object):
|
|||
Once everything is moved over, this could be in util.registry and not even touch the database.
|
||||
"""
|
||||
pass
|
||||
|
||||
@abstractmethod
|
||||
def set_manifest_expires_after(self, namespace_name, repo_name, digest, expires_after_sec):
|
||||
"""
|
||||
Sets that the manifest with given digest expires after the number of seconds from *now*.
|
||||
"""
|
||||
pass
|
||||
|
||||
|
|
|
@ -37,6 +37,13 @@ class PreOCIModel(DockerRegistryV2DataInterface):
|
|||
except database.RepositoryTag.DoesNotExist:
|
||||
return False
|
||||
|
||||
def has_tag(self, namespace_name, repo_name, tag_name):
|
||||
try:
|
||||
model.tag.get_possibly_expired_tag(namespace_name, repo_name, tag_name)
|
||||
return True
|
||||
except database.RepositoryTag.DoesNotExist:
|
||||
return False
|
||||
|
||||
def get_manifest_by_tag(self, namespace_name, repo_name, tag_name):
|
||||
try:
|
||||
manifest = model.tag.load_tag_manifest(namespace_name, repo_name, tag_name)
|
||||
|
@ -250,6 +257,14 @@ class PreOCIModel(DockerRegistryV2DataInterface):
|
|||
blob_record = model.storage.get_storage_by_uuid(blob.uuid)
|
||||
return model.storage.get_layer_path(blob_record)
|
||||
|
||||
def set_manifest_expires_after(self, namespace_name, repo_name, digest, expires_after_sec):
|
||||
try:
|
||||
manifest = model.tag.load_manifest_by_digest(namespace_name, repo_name, digest)
|
||||
manifest.tag.lifetime_end_ts = manifest.tag.lifetime_start_ts + expires_after_sec
|
||||
manifest.tag.save()
|
||||
except model.InvalidManifestException:
|
||||
return
|
||||
|
||||
|
||||
def _docker_v1_metadata(namespace_name, repo_name, repo_image):
|
||||
"""
|
||||
|
|
Reference in a new issue