vbatts/quay
Archived
1
0
Fork 0
Code Issues Pull requests Releases Wiki Activity

Merge pull request #2718 from coreos-inc/tag-expiration

Browse source 
Formal tag expiration support
This commit is contained in:
josephschorr 2017-07-19 17:48:11 -04:00 • committed by GitHub
commit a6db05e8b5
32 changed files with 621 additions and 117 deletions
Download patch file Download diff file Expand all files Collapse all files

8
endpoints/api/repository.py
View file

@ -334,6 +334,10 @@ class Repository(RepositoryParamResource):
last_modified = format_date(datetime.fromtimestamp(tag.lifetime_start_ts))
tag_info['last_modified'] = last_modified
if tag.lifetime_end_ts:
expiration = format_date(datetime.fromtimestamp(tag.lifetime_end_ts))
tag_info['expiration'] = expiration
if tag.tagmanifest is not None:
tag_info['manifest_digest'] = tag.tagmanifest.digest
@ -498,11 +502,11 @@ class RepositoryTrust(RepositoryParamResource):
repo = model.repository.get_repository(namespace, repository)
if not repo:
raise NotFound()
tags, _ = tuf_metadata_api.get_default_tags_with_expiration(namespace, repository)
if tags and not tuf_metadata_api.delete_metadata(namespace, repository):
raise DownstreamIssue({'message': 'Unable to delete downstream trust metadata'})
values = request.get_json()
model.repository.set_trust(repo, values['trust_enabled'])

79
endpoints/api/tag.py
View file

@ -1,5 +1,6 @@
""" Manage the tags of a repository. """
from datetime import datetime, timedelta
from flask import request, abort
from auth.auth_context import get_authenticated_user
@ -9,7 +10,7 @@ from endpoints.api import (resource, nickname, require_repo_read, require_repo_w
parse_args, query_param, truthy_bool, disallow_for_app_repositories)
from endpoints.api.tag_models_interface import Repository
from endpoints.api.tag_models_pre_oci import pre_oci_model as model
from endpoints.exception import NotFound
from endpoints.exception import NotFound, InvalidRequest
from endpoints.v2.manifest import _generate_and_store_manifest
from util.names import TAG_ERROR, TAG_REGEX
@ -52,14 +53,17 @@ class ListRepositoryTags(RepositoryParamResource):
class RepositoryTag(RepositoryParamResource):
""" Resource for managing repository tags. """
schemas = {
'MoveTag': {
'ChangeTag': {
'type': 'object',
'description': 'Description of to which image a new or existing tag should point',
'required': ['image',],
'description': 'Makes changes to a specific tag',
'properties': {
'image': {
'type': 'string',
'description': 'Image identifier to which the tag should point',
'type': ['string', 'null'],
'description': '(If specified) Image identifier to which the tag should point',
},
'expiration': {
'type': ['number', 'null'],
'description': '(If specified) The expiration for the image',
},
},
},
@ -67,33 +71,64 @@ class RepositoryTag(RepositoryParamResource):
@require_repo_write
@disallow_for_app_repositories
@nickname('changeTagImage')
@validate_json_request('MoveTag')
@nickname('changeTag')
@validate_json_request('ChangeTag')
def put(self, namespace, repository, tag):
""" Change which image a tag points to or create a new tag."""
if not TAG_REGEX.match(tag):
abort(400, TAG_ERROR)
image_id = request.get_json()['image']
repo = model.get_repo(namespace, repository, image_id)
repo = model.get_repo(namespace, repository)
if not repo:
raise NotFound()
original_image_id = model.get_repo_tag_image(repo, tag)
model.create_or_update_tag(namespace, repository, tag, image_id)
if 'expiration' in request.get_json():
expiration = request.get_json().get('expiration')
expiration_date = None
if expiration is not None:
try:
expiration_date = datetime.utcfromtimestamp(float(expiration))
except ValueError:
abort(400)
username = get_authenticated_user().username
log_action('move_tag' if original_image_id else 'create_tag', namespace, {
'username': username,
'repo': repository,
'tag': tag,
'namespace': namespace,
'image': image_id,
'original_image': original_image_id
}, repo_name=repository)
if expiration_date <= datetime.now():
abort(400)
_generate_and_store_manifest(namespace, repository, tag)
existing_end_ts, ok = model.change_repository_tag_expiration(namespace, repository, tag,
expiration_date)
if ok:
if not (existing_end_ts is None and expiration_date is None):
log_action('change_tag_expiration', namespace, {
'username': get_authenticated_user().username,
'repo': repository,
'tag': tag,
'namespace': namespace,
'expiration_date': expiration_date,
'old_expiration_date': existing_end_ts
}, repo_name=repository)
else:
raise InvalidRequest('Could not update tag expiration; Tag has probably changed')
if 'image' in request.get_json():
image_id = request.get_json()['image']
image = model.get_repository_image(namespace, repository, image_id)
if image is None:
raise NotFound()
original_image_id = model.get_repo_tag_image(repo, tag)
model.create_or_update_tag(namespace, repository, tag, image_id)
username = get_authenticated_user().username
log_action('move_tag' if original_image_id else 'create_tag', namespace, {
'username': username,
'repo': repository,
'tag': tag,
'namespace': namespace,
'image': image_id,
'original_image': original_image_id
}, repo_name=repository)
_generate_and_store_manifest(namespace, repository, tag)
return 'Updated', 201

24
endpoints/api/tag_models_interface.py
View file

@ -61,7 +61,7 @@ class Repository(namedtuple('Repository', ['namespace_name', 'repository_name'])
class Image(
namedtuple('Image', [
'docker_image_id', 'created', 'comment', 'command', 'storage_image_size',
'storage_uploading', 'ancestor_length', 'ancestor_id_list'
'storage_uploading', 'ancestor_id_list'
])):
"""
Image
@ -71,7 +71,6 @@ class Image(
:type command: string
:type storage_image_size: int
:type storage_uploading: boolean
:type ancestor_length: int
:type ancestor_id_list: [int]
"""
@ -91,7 +90,7 @@ class Image(
'command': json.loads(command) if command else None,
'size': self.storage_image_size,
'uploading': self.storage_uploading,
'sort_index': self.ancestor_length,
'sort_index': len(self.ancestor_id_list),
}
if include_ancestors:
@ -116,9 +115,9 @@ class TagDataInterface(object):
"""
@abstractmethod
def get_repo(self, namespace_name, repository_name, docker_image_id):
def get_repo(self, namespace_name, repository_name):
"""
Returns a repository associated with the given namespace, repository, and docker_image_id
Returns a repository associated with the given namespace and repository name.
"""
@abstractmethod
@ -157,6 +156,13 @@ class TagDataInterface(object):
Returns the repository associated with the namespace_name and repository_name
"""
@abstractmethod
def get_repository_image(self, namespace_name, repository_name, docker_image_id):
"""
Returns the repository image associated with the namespace_name, repository_name, and docker
image ID.
"""
@abstractmethod
def restore_tag_to_manifest(self, repository_name, tag_name, manifest_digest):
"""
@ -170,3 +176,11 @@ class TagDataInterface(object):
Returns the existing repo tag image if it exists or else returns None
Side effects include adding the tag with associated name to the image with the associated id in the named repo.
"""
@abstractmethod
def change_repository_tag_expiration(self, namespace_name, repository_name, tag_name,
expiration_date):
""" Sets the expiration date of the tag under the matching repository to that given. If the
expiration date is None, then the tag will not expire. Returns a tuple of the previous
expiration timestamp in seconds (if any), and whether the operation succeeded.
"""

21
endpoints/api/tag_models_pre_oci.py
View file

@ -27,12 +27,12 @@ class PreOCIModel(TagDataInterface):
return RepositoryTagHistory(tags=repository_tag_history, more=more)
def get_repo(self, namespace_name, repository_name, docker_image_id):
image = model.image.get_repo_image(namespace_name, repository_name, docker_image_id)
if image is None:
def get_repo(self, namespace_name, repository_name):
repo = model.repository.get_repository(namespace_name, repository_name)
if repo is None:
return None
return Repository(image.repository.namespace_user, image.repository.name)
return Repository(repo.namespace_user, repo.name)
def get_repo_tag_image(self, repository, tag_name):
repo = model.repository.get_repository(str(repository.namespace_name), str(repository.repository_name))
@ -73,6 +73,13 @@ class PreOCIModel(TagDataInterface):
new_tags.append(convert_tag(tag))
return new_tags
def get_repository_image(self, namespace_name, repository_name, docker_image_id):
image = model.image.get_repo_image(namespace_name, repository_name, docker_image_id)
if image is None:
return None
return convert_image(image)
def get_repository(self, namespace_name, repository_name):
repo = model.repository.get_repository(namespace_name, repository_name)
if repo is None:
@ -102,13 +109,17 @@ class PreOCIModel(TagDataInterface):
return convert_image(image)
def change_repository_tag_expiration(self, namespace_name, repository_name, tag_name,
expiration_date):
return model.tag.change_repository_tag_expiration(namespace_name, repository_name, tag_name,
expiration_date)
def convert_image(database_image):
return Image(docker_image_id=database_image.docker_image_id, created=database_image.created,
comment=database_image.comment, command=database_image.command,
storage_image_size=database_image.storage.image_size,
storage_uploading=database_image.storage.uploading,
ancestor_length=len(database_image.ancestors),
ancestor_id_list=database_image.ancestor_id_list())

37
endpoints/api/test/test_tag.py
View file

@ -124,6 +124,43 @@ def find_no_repo_tag_history():
yield
@pytest.mark.parametrize('expiration_time, expected_status', [
(None, 201),
('aksdjhasd', 400),
])
def test_change_tag_expiration_default(expiration_time, expected_status, client, app):
with client_with_identity('devtable', client) as cl:
params = {
'repository': 'devtable/simple',
'tag': 'latest',
}
request_body = {
'expiration': expiration_time,
}
conduct_api_call(cl, RepositoryTag, 'put', params, request_body, expected_status)
def test_change_tag_expiration(client, app):
with client_with_identity('devtable', client) as cl:
params = {
'repository': 'devtable/simple',
'tag': 'latest',
}
tag = model.tag.get_active_tag('devtable', 'simple', 'latest')
updated_expiration = tag.lifetime_start_ts + 60*60*24
request_body = {
'expiration': updated_expiration,
}
conduct_api_call(cl, RepositoryTag, 'put', params, request_body, 201)
tag = model.tag.get_active_tag('devtable', 'simple', 'latest')
assert tag.lifetime_end_ts == updated_expiration
@pytest.mark.parametrize('test_image,test_tag,expected_status', [
('image1', '-INVALID-TAG-NAME', 400),
('image1', '.INVALID-TAG-NAME', 400),

17
endpoints/api/test/test_tag_models_pre_oci.py
View file

@ -106,20 +106,18 @@ def test_list_repository_tag_history(expected, namespace_name, repository_name,
specific_tag) == expected
def get_repo_image_mock(monkeypatch, return_value):
def return_return_value(namespace_name, repository_name, image_id):
def get_repo_mock(monkeypatch, return_value):
def return_return_value(namespace_name, repository_name):
return return_value
monkeypatch.setattr(model.image, 'get_repo_image', return_return_value)
monkeypatch.setattr(model.repository, 'get_repository', return_return_value)
def test_get_repo_not_exists(get_monkeypatch):
namespace_name = 'namespace_name'
repository_name = 'repository_name'
image_id = 'image_id'
get_repo_image_mock(get_monkeypatch, None)
repo = pre_oci_model.get_repo(namespace_name, repository_name, image_id)
get_repo_mock(get_monkeypatch, None)
repo = pre_oci_model.get_repo(namespace_name, repository_name)
assert repo is None
@ -127,14 +125,13 @@ def test_get_repo_not_exists(get_monkeypatch):
def test_get_repo_exists(get_monkeypatch):
namespace_name = 'namespace_name'
repository_name = 'repository_name'
image_id = 'image_id'
mock = Mock()
mock.namespace_user = namespace_name
mock.name = repository_name
mock.repository = mock
get_repo_image_mock(get_monkeypatch, mock)
get_repo_mock(get_monkeypatch, mock)
repo = pre_oci_model.get_repo(namespace_name, repository_name, image_id)
repo = pre_oci_model.get_repo(namespace_name, repository_name)
assert repo is not None
assert repo.repository_name == repository_name

2
endpoints/api/user.py
View file

@ -20,7 +20,7 @@ from auth.permissions import (AdministerOrganizationPermission, CreateRepository
UserAdminPermission, UserReadPermission, SuperUserPermission)
from data import model
from data.billing import get_plan
from data.database import Repository as RepositoryTable, UserPromptTypes
from data.database import Repository as RepositoryTable
from endpoints.api import (ApiResource, nickname, resource, validate_json_request, request_error,
log_action, internal_only, require_user_admin, parse_args,
query_param, require_scope, format_date, show_if,

7
endpoints/v2/errors.py
View file

@ -57,6 +57,13 @@ class ManifestUnknown(V2RegistryException):
def __init__(self, detail=None):
super(ManifestUnknown, self).__init__('MANIFEST_UNKNOWN', 'manifest unknown', detail, 404)
class TagExpired(V2RegistryException):
def __init__(self, message=None, detail=None):
super(TagExpired, self).__init__('TAG_EXPIRED',
message or 'Tag has expired',
detail,
404)
class ManifestUnverified(V2RegistryException):
def __init__(self, detail=None):

36
endpoints/v2/labelhandlers.py Normal file
View file

@ -0,0 +1,36 @@
import logging
from app import app
from endpoints.v2.models_pre_oci import data_model as model
from util.timedeltastring import convert_to_timedelta
logger = logging.getLogger(__name__)
min_expire_sec = convert_to_timedelta(app.config.get('LABELED_EXPIRATION_MINIMUM', '1h'))
max_expire_sec = convert_to_timedelta(app.config.get('LABELED_EXPIRATION_MAXIMUM', '104w'))
def _expires_after(value, namespace_name, repo_name, digest):
""" Sets the expiration of a manifest based on the quay.expires-in label. """
try:
timedelta = convert_to_timedelta(value)
except ValueError:
logger.exception('Could not convert %s to timedeltastring for %s/%s@%s', value, namespace_name,
repo_name, digest)
return
total_seconds = min(max(timedelta.total_seconds(), min_expire_sec.total_seconds()),
max_expire_sec.total_seconds())
logger.debug('Labeling manifest %s/%s@%s with expiration of %s', namespace_name, repo_name,
digest, total_seconds)
model.set_manifest_expires_after(namespace_name, repo_name, digest, total_seconds)
_LABEL_HANDLES = {
'quay.expires-after': _expires_after,
}
def handle_label(key, value, namespace_name, repo_name, digest):
handler = _LABEL_HANDLES.get(key)
if handler is not None:
handler(value, namespace_name, repo_name, digest)

20
endpoints/v2/manifest.py
View file

@ -13,10 +13,11 @@ from endpoints.common import parse_repository_name
from endpoints.decorators import anon_protect
from endpoints.notificationhelper import spawn_notification
from endpoints.v2 import v2_bp, require_repo_read, require_repo_write
from endpoints.v2.errors import (
BlobUnknown, ManifestInvalid, ManifestUnknown, TagInvalid, NameInvalid)
from endpoints.v2.models_interface import Label
from endpoints.v2.models_pre_oci import data_model as model
from endpoints.v2.errors import (BlobUnknown, ManifestInvalid, ManifestUnknown, TagInvalid,
NameInvalid, TagExpired)
from endpoints.v2.labelhandlers import handle_label
from image.docker import ManifestException
from image.docker.schema1 import DockerSchema1Manifest, DockerSchema1ManifestBuilder
from image.docker.schema2 import DOCKER_SCHEMA2_CONTENT_TYPES
@ -42,7 +43,14 @@ def fetch_manifest_by_tagname(namespace_name, repo_name, manifest_ref):
if manifest is None:
has_tag = model.has_active_tag(namespace_name, repo_name, manifest_ref)
if not has_tag:
raise ManifestUnknown()
has_expired_tag = model.has_tag(namespace_name, repo_name, manifest_ref)
if has_expired_tag:
logger.debug('Found expired tag %s for repository %s/%s', manifest_ref, namespace_name,
repo_name)
msg = 'Tag %s was deleted or has expired. To pull, revive via time machine' % manifest_ref
raise TagExpired(msg)
else:
raise ManifestUnknown()
manifest = _generate_and_store_manifest(namespace_name, repo_name, manifest_ref)
if manifest is None:
@ -188,6 +196,8 @@ def _write_manifest(namespace_name, repo_name, manifest):
for key, value in manifest.layers[-1].v1_metadata.labels.iteritems():
media_type = 'application/json' if is_json(value) else 'text/plain'
labels.append(Label(key=key, value=value, source_type='manifest', media_type=media_type))
handle_label(key, value, namespace_name, repo_name, manifest.digest)
model.create_manifest_labels(namespace_name, repo_name, manifest.digest, labels)
return repo, storage_map
@ -268,7 +278,3 @@ def _generate_and_store_manifest(namespace_name, repo_name, tag_name):
model.create_manifest_and_update_tag(namespace_name, repo_name, tag_name, manifest.digest,
manifest.bytes)
return manifest
def _determine_media_type(value):
media_type_name = 'application/json' if is_json(value) else 'text/plain'

8
endpoints/v2/models_interface.py
View file

@ -256,3 +256,11 @@ class DockerRegistryV2DataInterface(object):
Once everything is moved over, this could be in util.registry and not even touch the database.
"""
pass
@abstractmethod
def set_manifest_expires_after(self, namespace_name, repo_name, digest, expires_after_sec):
"""
Sets that the manifest with given digest expires after the number of seconds from *now*.
"""
pass

15
endpoints/v2/models_pre_oci.py
View file

@ -37,6 +37,13 @@ class PreOCIModel(DockerRegistryV2DataInterface):
except database.RepositoryTag.DoesNotExist:
return False
def has_tag(self, namespace_name, repo_name, tag_name):
try:
model.tag.get_possibly_expired_tag(namespace_name, repo_name, tag_name)
return True
except database.RepositoryTag.DoesNotExist:
return False
def get_manifest_by_tag(self, namespace_name, repo_name, tag_name):
try:
manifest = model.tag.load_tag_manifest(namespace_name, repo_name, tag_name)
@ -250,6 +257,14 @@ class PreOCIModel(DockerRegistryV2DataInterface):
blob_record = model.storage.get_storage_by_uuid(blob.uuid)
return model.storage.get_layer_path(blob_record)
def set_manifest_expires_after(self, namespace_name, repo_name, digest, expires_after_sec):
try:
manifest = model.tag.load_manifest_by_digest(namespace_name, repo_name, digest)
manifest.tag.lifetime_end_ts = manifest.tag.lifetime_start_ts + expires_after_sec
manifest.tag.save()
except model.InvalidManifestException:
return
def _docker_v1_metadata(namespace_name, repo_name, repo_image):
"""