From 897a091692736e0be5f239be96692363376150e8 Mon Sep 17 00:00:00 2001 From: Charlton Austin Date: Mon, 10 Jul 2017 09:46:02 -0400 Subject: [PATCH] style(data+endpoints): ran yapf ### Description of Changes ran yapf for the branch [TESTING->locally using docker compose] Issue: https://coreosdev.atlassian.net/browse/QUAY-632 ## Reviewer Checklist - [ ] It works! - [ ] Comments provide sufficient explanations for the next contributor - [ ] Tests cover changes and corner cases - [ ] Follows Quay syntax patterns and format --- data/model/log.py | 40 ++++------- endpoints/api/image.py | 7 +- endpoints/api/tag.py | 11 +-- endpoints/api/tag_models_interface.py | 16 +++-- endpoints/api/tag_models_pre_oci.py | 12 ++-- endpoints/api/test/test_models_pre_oci.py | 48 +++++++++---- endpoints/api/test/test_tag.py | 88 +++++++++++++++-------- 7 files changed, 135 insertions(+), 87 deletions(-) diff --git a/data/model/log.py b/data/model/log.py index a1cd195e2..6d8edd6ba 100644 --- a/data/model/log.py +++ b/data/model/log.py @@ -14,11 +14,10 @@ logger = logging.getLogger(__name__) ACTIONS_ALLOWED_WITHOUT_AUDIT_LOGGING = ['pull_repo'] + def _logs_query(selections, start_time, end_time, performer=None, repository=None, namespace=None, ignore=None): - joined = (LogEntry - .select(*selections) - .switch(LogEntry) + joined = (LogEntry.select(*selections).switch(LogEntry) .where(LogEntry.datetime >= start_time, LogEntry.datetime < end_time)) if repository: @@ -75,14 +74,12 @@ def get_logs_query(start_time, end_time, performer=None, repository=None, namesp selections.append(Account) query = _logs_query(selections, start_time, end_time, performer, repository, namespace, ignore) - query = (query.switch(LogEntry) - .join(Performer, JOIN_LEFT_OUTER, - on=(LogEntry.performer == Performer.id).alias('performer'))) + query = (query.switch(LogEntry).join(Performer, JOIN_LEFT_OUTER, + on=(LogEntry.performer == Performer.id).alias('performer'))) if namespace is None and repository is None: - query = (query.switch(LogEntry) - .join(Account, JOIN_LEFT_OUTER, - on=(LogEntry.account == Account.id).alias('account'))) + query = (query.switch(LogEntry).join(Account, JOIN_LEFT_OUTER, + on=(LogEntry.account == Account.id).alias('account'))) return query @@ -94,8 +91,8 @@ def _json_serialize(obj): return obj -def log_action(kind_name, user_or_organization_name, performer=None, repository=None, - ip=None, metadata={}, timestamp=None): +def log_action(kind_name, user_or_organization_name, performer=None, repository=None, ip=None, + metadata={}, timestamp=None): if not timestamp: timestamp = datetime.today() @@ -112,7 +109,8 @@ def log_action(kind_name, user_or_organization_name, performer=None, repository= if repository is not None: if hasattr(repository, 'namespace_name') and hasattr(repository, 'repository_name'): - maybe_repo = data.model.repository.get_repository(repository.namespace_name, repository.repository_name) + maybe_repo = data.model.repository.get_repository(repository.namespace_name, + repository.repository_name) repository = maybe_repo.id if maybe_repo else None elif hasattr(repository, 'id'): repository = repository.id @@ -139,15 +137,10 @@ def log_action(kind_name, user_or_organization_name, performer=None, repository= raise - def get_stale_logs_start_id(): """ Gets the oldest log entry. """ try: - return (LogEntry - .select(LogEntry.id) - .order_by(LogEntry.id) - .limit(1) - .tuples())[0][0] + return (LogEntry.select(LogEntry.id).order_by(LogEntry.id).limit(1).tuples())[0][0] except IndexError: return None @@ -155,9 +148,7 @@ def get_stale_logs_start_id(): def get_stale_logs_cutoff_id(cutoff_date): """ Gets the most recent ID created before the cutoff_date. """ try: - return (LogEntry - .select(fn.Max(LogEntry.id)) - .where(LogEntry.datetime <= cutoff_date) + return (LogEntry.select(fn.Max(LogEntry.id)).where(LogEntry.datetime <= cutoff_date) .tuples())[0][0] except IndexError: return None @@ -184,12 +175,11 @@ def get_repositories_action_sums(repository_ids): # Filter the join to recent entries only. last_week = datetime.now() - timedelta(weeks=1) - tuples = (RepositoryActionCount - .select(RepositoryActionCount.repository, fn.Sum(RepositoryActionCount.count)) + tuples = (RepositoryActionCount.select(RepositoryActionCount.repository, + fn.Sum(RepositoryActionCount.count)) .where(RepositoryActionCount.repository << repository_ids) .where(RepositoryActionCount.date >= last_week) - .group_by(RepositoryActionCount.repository) - .tuples()) + .group_by(RepositoryActionCount.repository).tuples()) action_count_map = {} for record in tuples: diff --git a/endpoints/api/image.py b/endpoints/api/image.py index f1b67e6d8..117ddbd81 100644 --- a/endpoints/api/image.py +++ b/endpoints/api/image.py @@ -74,6 +74,7 @@ def historical_image_view(image, image_map): @path_param('repository', 'The full path of the repository. e.g. namespace/name') class RepositoryImageList(RepositoryParamResource): """ Resource for listing repository images. """ + @require_repo_read @nickname('listRepositoryImages') @disallow_for_app_repositories @@ -105,9 +106,7 @@ class RepositoryImageList(RepositoryParamResource): image_json['tags'] = tags_by_docker_id[image_json['id']] return image_json - return { - 'images': [add_tags(image_view(image, image_map)) for image in filtered_images] - } + return {'images': [add_tags(image_view(image, image_map)) for image in filtered_images]} @resource('/v1/repository//image/') @@ -115,6 +114,7 @@ class RepositoryImageList(RepositoryParamResource): @path_param('image_id', 'The Docker image ID') class RepositoryImage(RepositoryParamResource): """ Resource for handling repository images. """ + @require_repo_read @nickname('getImage') @disallow_for_app_repositories @@ -130,4 +130,3 @@ class RepositoryImage(RepositoryParamResource): image_map[current_image.id] = current_image return historical_image_view(image, image_map) - diff --git a/endpoints/api/tag.py b/endpoints/api/tag.py index 2298de8aa..b261db74a 100644 --- a/endpoints/api/tag.py +++ b/endpoints/api/tag.py @@ -53,8 +53,8 @@ class ListRepositoryTags(RepositoryParamResource): limit = min(100, max(1, parsed_args.get('limit', 50))) tag_history = model.list_repository_tag_history(namespace_name=namespace, - repository_name=repository, page=page, - size=limit, specific_tag=specific_tag) + repository_name=repository, page=page, + size=limit, specific_tag=specific_tag) if not tag_history: raise NotFound() @@ -75,7 +75,7 @@ class RepositoryTag(RepositoryParamResource): 'MoveTag': { 'type': 'object', 'description': 'Description of to which image a new or existing tag should point', - 'required': ['image', ], + 'required': ['image',], 'properties': { 'image': { 'type': 'string', @@ -149,7 +149,8 @@ class RepositoryTagImages(RepositoryParamResource): def get(self, namespace, repository, tag, parsed_args): """ List the images for the specified repository tag. """ try: - tag_image = model.get_repo_tag_image(Repository(namespace_name=namespace, repository_name=repository), tag) + tag_image = model.get_repo_tag_image( + Repository(namespace_name=namespace, repository_name=repository), tag) except DataModelException: raise NotFound() @@ -195,7 +196,7 @@ class RestoreTag(RepositoryParamResource): 'RestoreTag': { 'type': 'object', 'description': 'Restores a tag to a specific image', - 'required': ['image', ], + 'required': ['image',], 'properties': { 'image': { 'type': 'string', diff --git a/endpoints/api/tag_models_interface.py b/endpoints/api/tag_models_interface.py index f93ce0b6f..aa21dfc61 100644 --- a/endpoints/api/tag_models_interface.py +++ b/endpoints/api/tag_models_interface.py @@ -5,8 +5,10 @@ from six import add_metaclass class Tag( - namedtuple('Tag', ['name', 'image', 'reversion', 'lifetime_start_ts', 'lifetime_end_ts', - 'manifest_list', 'docker_image_id'])): + namedtuple('Tag', [ + 'name', 'image', 'reversion', 'lifetime_start_ts', 'lifetime_end_ts', 'manifest_list', + 'docker_image_id' + ])): """ Tag represents a name to an image. :type name: string @@ -35,8 +37,11 @@ class Repository(namedtuple('Repository', ['namespace_name', 'repository_name']) """ -class Image(namedtuple('Image', ['docker_image_id', 'created', 'comment', 'command', 'storage_image_size', - 'storage_uploading', 'ancestor_length', 'ancestor_id_list'])): +class Image( + namedtuple('Image', [ + 'docker_image_id', 'created', 'comment', 'command', 'storage_image_size', + 'storage_uploading', 'ancestor_length', 'ancestor_id_list' + ])): """ Image :type docker_image_id: string @@ -58,7 +63,8 @@ class TagDataInterface(object): """ @abstractmethod - def list_repository_tag_history(self, namespace_name, repository_name, page=1, size=100, specific_tag=None): + def list_repository_tag_history(self, namespace_name, repository_name, page=1, size=100, + specific_tag=None): """ Returns a RepositoryTagHistory with a list of historic tags and whether there are more tags then returned. """ diff --git a/endpoints/api/tag_models_pre_oci.py b/endpoints/api/tag_models_pre_oci.py index 65fc77496..00276a0ca 100644 --- a/endpoints/api/tag_models_pre_oci.py +++ b/endpoints/api/tag_models_pre_oci.py @@ -9,7 +9,8 @@ class PreOCIModel(TagDataInterface): before it was changed to support the OCI specification. """ - def list_repository_tag_history(self, namespace_name, repository_name, page=1, size=100, specific_tag=None): + def list_repository_tag_history(self, namespace_name, repository_name, page=1, size=100, + specific_tag=None): repository = model.repository.get_repository(namespace_name, repository_name) if repository is None: return None @@ -46,7 +47,8 @@ class PreOCIModel(TagDataInterface): return convert_image(image) def create_or_update_tag(self, namespace_name, repository_name, tag_name, docker_image_id): - return model.tag.create_or_update_tag(namespace_name, repository_name, tag_name, docker_image_id) + return model.tag.create_or_update_tag(namespace_name, repository_name, tag_name, + docker_image_id) def delete_tag(self, namespace_name, repository_name, tag_name): return model.tag.delete_tag(namespace_name, repository_name, tag_name) @@ -104,8 +106,10 @@ class PreOCIModel(TagDataInterface): def convert_image(database_image): return Image(docker_image_id=database_image.docker_image_id, created=database_image.created, comment=database_image.comment, command=database_image.command, - storage_image_size=database_image.storage.image_size, storage_uploading=database_image.storage.uploading, - ancestor_length=len(database_image.ancestors), ancestor_id_list=database_image.ancestor_id_list()) + storage_image_size=database_image.storage.image_size, + storage_uploading=database_image.storage.uploading, + ancestor_length=len(database_image.ancestors), + ancestor_id_list=database_image.ancestor_id_list()) def convert_tag(tag, manifest_list=None): diff --git a/endpoints/api/test/test_models_pre_oci.py b/endpoints/api/test/test_models_pre_oci.py index ecf343fb6..b7c750854 100644 --- a/endpoints/api/test/test_models_pre_oci.py +++ b/endpoints/api/test/test_models_pre_oci.py @@ -168,7 +168,8 @@ def test_get_repo_tag_image_with_repo_and_repo_tag(get_monkeypatch): get_repository_mock(get_monkeypatch, mock_image) get_repo_tag_image_mock(get_monkeypatch, mock_image) - image = pre_oci_model.get_repo_tag_image(Repository('namespace_name', 'repository_name'), 'tag_name') + image = pre_oci_model.get_repo_tag_image( + Repository('namespace_name', 'repository_name'), 'tag_name') assert image is not None assert image.docker_image_id == 'some docker image id' @@ -177,7 +178,8 @@ def test_get_repo_tag_image_with_repo_and_repo_tag(get_monkeypatch): def test_get_repo_tag_image_without_repo(get_monkeypatch): get_repository_mock(get_monkeypatch, None) - image = pre_oci_model.get_repo_tag_image(Repository('namespace_name', 'repository_name'), 'tag_name') + image = pre_oci_model.get_repo_tag_image( + Repository('namespace_name', 'repository_name'), 'tag_name') assert image is None @@ -192,7 +194,8 @@ def test_get_repo_tag_image_without_repo_tag_image(get_monkeypatch): get_monkeypatch.setattr(model.tag, 'get_repo_tag_image', raise_exception) - image = pre_oci_model.get_repo_tag_image(Repository('namespace_name', 'repository_name'), 'tag_name') + image = pre_oci_model.get_repo_tag_image( + Repository('namespace_name', 'repository_name'), 'tag_name') assert image is None @@ -201,7 +204,8 @@ def test_create_or_update_tag(get_monkeypatch): mock = Mock() get_monkeypatch.setattr(model.tag, 'create_or_update_tag', mock) - pre_oci_model.create_or_update_tag('namespace_name', 'repository_name', 'tag_name', 'docker_image_id') + pre_oci_model.create_or_update_tag('namespace_name', 'repository_name', 'tag_name', + 'docker_image_id') assert mock.call_count == 1 assert mock.call_args == call('namespace_name', 'repository_name', 'tag_name', 'docker_image_id') @@ -249,14 +253,30 @@ def test_get_parent_images(get_monkeypatch): def fake_list(): return [] - image_one = AttrDict( - {'docker_image_id': 'docker_image_id', 'created': 'created_one', 'comment': 'comment_one', 'command': 'command_one', - 'storage': AttrDict({'image_size': 'image_size_one', 'uploading': 'uploading_one'}), 'ancestors': 'one/two/three', - 'ancestor_id_list': fake_list}) - image_two = AttrDict( - {'docker_image_id': 'docker_image_id_two', 'created': 'created', 'comment': 'comment', 'command': 'command', - 'storage': AttrDict({'image_size': 'image_size', 'uploading': 'uploading'}), 'ancestors': 'four/five/six', - 'ancestor_id_list': fake_list}) + image_one = AttrDict({ + 'docker_image_id': 'docker_image_id', + 'created': 'created_one', + 'comment': 'comment_one', + 'command': 'command_one', + 'storage': AttrDict({ + 'image_size': 'image_size_one', + 'uploading': 'uploading_one' + }), + 'ancestors': 'one/two/three', + 'ancestor_id_list': fake_list + }) + image_two = AttrDict({ + 'docker_image_id': 'docker_image_id_two', + 'created': 'created', + 'comment': 'comment', + 'command': 'command', + 'storage': AttrDict({ + 'image_size': 'image_size', + 'uploading': 'uploading' + }), + 'ancestors': 'four/five/six', + 'ancestor_id_list': fake_list + }) get_parent_images_mock = Mock(return_value=[image_one, image_two]) get_monkeypatch.setattr(model.image, 'get_parent_images', get_parent_images_mock) @@ -303,7 +323,8 @@ def test_tag_to_manifest(get_monkeypatch): get_monkeypatch.setattr(model.tag, 'restore_tag_to_manifest', restore_tag_mock) get_monkeypatch.setattr(model.repository, 'get_repository', get_repository_mock) - pre_oci_model.restore_tag_to_manifest(Repository('namespace', 'repository'), 'tag_name', 'manifest_digest') + pre_oci_model.restore_tag_to_manifest( + Repository('namespace', 'repository'), 'tag_name', 'manifest_digest') get_repository_mock.assert_called_once_with('namespace', 'repository') restore_tag_mock.assert_called_once_with(repo_mock, 'tag_name', 'manifest_digest') @@ -314,7 +335,6 @@ def test__tag_to_image(get_monkeypatch): restore_tag_mock = Mock(return_value=None) get_repository_mock = Mock(return_value=repo_mock) - get_monkeypatch.setattr(model.tag, 'restore_tag_to_image', restore_tag_mock) get_monkeypatch.setattr(model.repository, 'get_repository', get_repository_mock) diff --git a/endpoints/api/test/test_tag.py b/endpoints/api/test/test_tag.py index 30f6dd76e..1d13d0eae 100644 --- a/endpoints/api/test/test_tag.py +++ b/endpoints/api/test/test_tag.py @@ -23,13 +23,15 @@ def get_repo_image(): img = Mock(repository=mock, docker_image_id=12) if image_id == 'image1' else None return img - with patch('endpoints.api.tag_models_pre_oci.model.image.get_repo_image', side_effect=mock_callable) as mk: + with patch('endpoints.api.tag_models_pre_oci.model.image.get_repo_image', + side_effect=mock_callable) as mk: yield mk @pytest.fixture() def get_repository(): - with patch('endpoints.api.tag_models_pre_oci.model.image.get_repo_image', return_value='mock_repo') as mk: + with patch('endpoints.api.tag_models_pre_oci.model.image.get_repo_image', + return_value='mock_repo') as mk: yield mk @@ -42,8 +44,9 @@ def get_repo_tag_image(): return [] if tag == 'existing-tag': - return Mock(docker_image_id='mock_docker_image_id', created=12345, comment='comment', command='command', - storage=storage_mock, ancestors=[], ancestor_id_list=fake_ancestor_id_list) + return Mock(docker_image_id='mock_docker_image_id', created=12345, comment='comment', + command='command', storage=storage_mock, ancestors=[], + ancestor_id_list=fake_ancestor_id_list) else: raise DataModelException('Unable to find image for tag.') @@ -100,12 +103,14 @@ def authd_client(client): def list_repository_tag_history(): def list_repository_tag_history(namespace_name, repository_name, page, size, specific_tag): return RepositoryTagHistory(tags=[ - Tag(name='First Tag', image='image', reversion=False, lifetime_start_ts=0, lifetime_end_ts=0, manifest_list=[], - docker_image_id='first docker image id'), - Tag(name='Second Tag', image='second image', reversion=True, lifetime_start_ts=10, lifetime_end_ts=100, - manifest_list=[], docker_image_id='second docker image id')], more=False) + Tag(name='First Tag', image='image', reversion=False, lifetime_start_ts=0, lifetime_end_ts=0, + manifest_list=[], docker_image_id='first docker image id'), + Tag(name='Second Tag', image='second image', reversion=True, lifetime_start_ts=10, + lifetime_end_ts=100, manifest_list=[], docker_image_id='second docker image id') + ], more=False) - with patch('endpoints.api.tag.model.list_repository_tag_history', side_effect=list_repository_tag_history): + with patch('endpoints.api.tag.model.list_repository_tag_history', + side_effect=list_repository_tag_history): yield @@ -114,7 +119,8 @@ def find_no_repo_tag_history(): def list_repository_tag_history(namespace_name, repository_name, page, size, specific_tag): return None - with patch('endpoints.api.tag.model.list_repository_tag_history', side_effect=list_repository_tag_history): + with patch('endpoints.api.tag.model.list_repository_tag_history', + side_effect=list_repository_tag_history): yield @@ -140,23 +146,39 @@ def test_move_tag(test_image, test_tag, expected_status, get_repo_image, get_rep conduct_api_call(authd_client, RepositoryTag, 'put', params, request_body, expected_status) -@pytest.mark.parametrize('namespace, repository, specific_tag, page, limit, expected_response_code, expected', [ - ('devtable', 'simple', None, 1, 10, 200, {'has_additional': False}), - ('devtable', 'simple', None, 1, 10, 200, {'page': 1}), - ('devtable', 'simple', None, 1, 10, 200, {'tags': [{'docker_image_id': 'first docker image id', - 'name': 'First Tag', - 'reversion': False}, - {'docker_image_id': 'second docker image id', - 'end_ts': 100, - 'name': 'Second Tag', - 'reversion': True, - 'start_ts': 10}]}), -]) +@pytest.mark.parametrize( + 'namespace, repository, specific_tag, page, limit, expected_response_code, expected', [ + ('devtable', 'simple', None, 1, 10, 200, { + 'has_additional': False + }), + ('devtable', 'simple', None, 1, 10, 200, { + 'page': 1 + }), + ('devtable', 'simple', None, 1, 10, 200, { + 'tags': [{ + 'docker_image_id': 'first docker image id', + 'name': 'First Tag', + 'reversion': False + }, { + 'docker_image_id': 'second docker image id', + 'end_ts': 100, + 'name': 'Second Tag', + 'reversion': True, + 'start_ts': 10 + }] + }), + ]) def test_list_repository_tags_view_is_correct(namespace, repository, specific_tag, page, limit, - list_repository_tag_history, expected_response_code, expected, - authd_client): - params = {'repository': namespace + '/' + repository, 'specificTag': specific_tag, 'page': page, 'limit': limit} - response = conduct_api_call(authd_client, ListRepositoryTags, 'get', params, expected_code=expected_response_code) + list_repository_tag_history, expected_response_code, + expected, authd_client): + params = { + 'repository': namespace + '/' + repository, + 'specificTag': specific_tag, + 'page': page, + 'limit': limit + } + response = conduct_api_call(authd_client, ListRepositoryTags, 'get', params, + expected_code=expected_response_code) compare_list_history_tags_response(expected, response.json) @@ -183,17 +205,23 @@ def test_no_repo_tag_history(find_no_repo_tag_history, authd_client): ('specific_tag', -1, 101, 'specific_tag', 1, 100), ('specific_tag', 0, 0, 'specific_tag', 1, 1), ]) -def test_repo_tag_history_param_parse(specific_tag, page, limit, expected_specific_tag, expected_page, expected_limit, - authd_client): +def test_repo_tag_history_param_parse(specific_tag, page, limit, expected_specific_tag, + expected_page, expected_limit, authd_client): mock = MagicMock() mock.return_value = RepositoryTagHistory(tags=[], more=False) with patch('endpoints.api.tag.model.list_repository_tag_history', side_effect=mock): - params = {'repository': 'devtable/simple', 'specificTag': specific_tag, 'page': page, 'limit': limit} + params = { + 'repository': 'devtable/simple', + 'specificTag': specific_tag, + 'page': page, + 'limit': limit + } conduct_api_call(authd_client, ListRepositoryTags, 'get', params) assert mock.call_args == call(namespace_name='devtable', repository_name='simple', - page=expected_page, size=expected_limit, specific_tag=expected_specific_tag) + page=expected_page, size=expected_limit, + specific_tag=expected_specific_tag) @pytest.mark.parametrize('test_manifest,test_tag,manifest_generated,expected_status', [