diff --git a/conf/init/service/repositoryactioncounter/log/run b/conf/init/service/repositoryactioncounter/log/run new file mode 100755 index 000000000..d86d5766f --- /dev/null +++ b/conf/init/service/repositoryactioncounter/log/run @@ -0,0 +1,2 @@ +#!/bin/sh +exec logger -i -t repositoryactioncounter \ No newline at end of file diff --git a/conf/init/service/repositoryactioncounter/run b/conf/init/service/repositoryactioncounter/run new file mode 100755 index 000000000..08e0e3164 --- /dev/null +++ b/conf/init/service/repositoryactioncounter/run @@ -0,0 +1,8 @@ +#! /bin/bash + +echo 'Starting repository action count worker' + +cd / +venv/bin/python -m workers.repositoryactioncounter 2>&1 + +echo 'Repository action worker exited' \ No newline at end of file diff --git a/data/database.py b/data/database.py index 837309a42..d1c701c70 100644 --- a/data/database.py +++ b/data/database.py @@ -299,7 +299,7 @@ class Repository(BaseModel): # Therefore, we define our own deletion order here and use the dependency system to verify it. ordered_dependencies = [RepositoryAuthorizedEmail, RepositoryTag, Image, LogEntry, RepositoryBuild, RepositoryBuildTrigger, RepositoryNotification, - RepositoryPermission, AccessToken, Star] + RepositoryPermission, AccessToken, Star, RepositoryActionCount] for query, fk in self.dependencies(search_nullable=True): model = fk.model_class @@ -498,6 +498,7 @@ class RepositoryTag(BaseModel): lifetime_start_ts = IntegerField(default=get_epoch_timestamp) lifetime_end_ts = IntegerField(null=True, index=True) hidden = BooleanField(default=False) + reversion = BooleanField(default=False) class Meta: database = db @@ -561,6 +562,20 @@ class LogEntry(BaseModel): metadata_json = TextField(default='{}') +class RepositoryActionCount(BaseModel): + repository = ForeignKeyField(Repository, index=True) + count = IntegerField() + date = DateField(index=True) + + class Meta: + database = db + read_slaves = (read_slave,) + indexes = ( + # create a unique index on repository and date + (('repository', 'date'), True), + ) + + class OAuthApplication(BaseModel): client_id = CharField(index=True, default=random_string_generator(length=20)) client_secret = CharField(default=random_string_generator(length=40)) @@ -646,4 +661,4 @@ all_models = [User, Repository, Image, AccessToken, Role, RepositoryPermission, ExternalNotificationEvent, ExternalNotificationMethod, RepositoryNotification, RepositoryAuthorizedEmail, ImageStorageTransformation, DerivedImageStorage, TeamMemberInvite, ImageStorageSignature, ImageStorageSignatureKind, - AccessTokenKind, Star] + AccessTokenKind, Star, RepositoryActionCount] diff --git a/data/migrations/versions/1c3decf6b9c4_add_revert_tag_log_entry_kind.py b/data/migrations/versions/1c3decf6b9c4_add_revert_tag_log_entry_kind.py new file mode 100644 index 000000000..6e2bccb68 --- /dev/null +++ b/data/migrations/versions/1c3decf6b9c4_add_revert_tag_log_entry_kind.py @@ -0,0 +1,29 @@ +"""Add revert_tag log entry kind + +Revision ID: 1c3decf6b9c4 +Revises: 4ce2169efd3b +Create Date: 2015-04-16 17:14:11.154856 + +""" + +# revision identifiers, used by Alembic. +revision = '1c3decf6b9c4' +down_revision = '4ce2169efd3b' + +from alembic import op +import sqlalchemy as sa + + +def upgrade(tables): + op.bulk_insert(tables.logentrykind, + [ + {'id': 47, 'name':'revert_tag'}, + ]) + + +def downgrade(tables): + op.execute( + (tables.logentrykind.delete() + .where(tables.logentrykind.c.name == op.inline_literal('revert_tag'))) + + ) \ No newline at end of file diff --git a/data/migrations/versions/214350b6a8b1_add_private_key_to_build_triggers.py b/data/migrations/versions/214350b6a8b1_add_private_key_to_build_triggers.py index 2b4401e1f..dc7e052bc 100644 --- a/data/migrations/versions/214350b6a8b1_add_private_key_to_build_triggers.py +++ b/data/migrations/versions/214350b6a8b1_add_private_key_to_build_triggers.py @@ -8,7 +8,7 @@ Create Date: 2015-03-19 14:23:52.604505 # revision identifiers, used by Alembic. revision = '214350b6a8b1' -down_revision = '2b4dc0818a5e' +down_revision = '67eb43c778b' from alembic import op import sqlalchemy as sa diff --git a/data/migrations/versions/30c044b75632_add_repositoryactioncount_table.py b/data/migrations/versions/30c044b75632_add_repositoryactioncount_table.py new file mode 100644 index 000000000..8df45958e --- /dev/null +++ b/data/migrations/versions/30c044b75632_add_repositoryactioncount_table.py @@ -0,0 +1,36 @@ +"""Add RepositoryActionCount table + +Revision ID: 30c044b75632 +Revises: 2b4dc0818a5e +Create Date: 2015-04-13 13:21:18.159602 + +""" + +# revision identifiers, used by Alembic. +revision = '30c044b75632' +down_revision = '2b4dc0818a5e' + +from alembic import op +import sqlalchemy as sa + + +def upgrade(tables): + ### commands auto generated by Alembic - please adjust! ### + op.create_table('repositoryactioncount', + sa.Column('id', sa.Integer(), nullable=False), + sa.Column('repository_id', sa.Integer(), nullable=False), + sa.Column('count', sa.Integer(), nullable=False), + sa.Column('date', sa.Date(), nullable=False), + sa.ForeignKeyConstraint(['repository_id'], ['repository.id'], name=op.f('fk_repositoryactioncount_repository_id_repository')), + sa.PrimaryKeyConstraint('id', name=op.f('pk_repositoryactioncount')) + ) + op.create_index('repositoryactioncount_date', 'repositoryactioncount', ['date'], unique=False) + op.create_index('repositoryactioncount_repository_id', 'repositoryactioncount', ['repository_id'], unique=False) + op.create_index('repositoryactioncount_repository_id_date', 'repositoryactioncount', ['repository_id', 'date'], unique=True) + ### end Alembic commands ### + + +def downgrade(tables): + ### commands auto generated by Alembic - please adjust! ### + op.drop_table('repositoryactioncount') + ### end Alembic commands ### diff --git a/data/migrations/versions/4ce2169efd3b_add_reversion_column_to_the_tags_table.py b/data/migrations/versions/4ce2169efd3b_add_reversion_column_to_the_tags_table.py new file mode 100644 index 000000000..733971ef1 --- /dev/null +++ b/data/migrations/versions/4ce2169efd3b_add_reversion_column_to_the_tags_table.py @@ -0,0 +1,26 @@ +"""Add reversion column to the tags table + +Revision ID: 4ce2169efd3b +Revises: 30c044b75632 +Create Date: 2015-04-16 17:10:16.039835 + +""" + +# revision identifiers, used by Alembic. +revision = '4ce2169efd3b' +down_revision = '30c044b75632' + +from alembic import op +import sqlalchemy as sa + + +def upgrade(tables): + ### commands auto generated by Alembic - please adjust! ### + op.add_column('repositorytag', sa.Column('reversion', sa.Boolean(), nullable=False)) + ### end Alembic commands ### + + +def downgrade(tables): + ### commands auto generated by Alembic - please adjust! ### + op.drop_column('repositorytag', 'reversion') + ### end Alembic commands ### diff --git a/data/migrations/versions/67eb43c778b_add_index_for_repository_datetime_to_.py b/data/migrations/versions/67eb43c778b_add_index_for_repository_datetime_to_.py new file mode 100644 index 000000000..00ff374e4 --- /dev/null +++ b/data/migrations/versions/67eb43c778b_add_index_for_repository_datetime_to_.py @@ -0,0 +1,26 @@ +"""add index for repository+datetime to logentry + +Revision ID: 67eb43c778b +Revises: 1c3decf6b9c4 +Create Date: 2015-04-19 16:00:39.126289 + +""" + +# revision identifiers, used by Alembic. +revision = '67eb43c778b' +down_revision = '1c3decf6b9c4' + +from alembic import op +import sqlalchemy as sa + + +def upgrade(tables): + ### commands auto generated by Alembic - please adjust! ### + op.create_index('logentry_repository_id_datetime', 'logentry', ['repository_id', 'datetime'], unique=False) + ### end Alembic commands ### + + +def downgrade(tables): + ### commands auto generated by Alembic - please adjust! ### + op.drop_index('logentry_repository_id_datetime', table_name='logentry') + ### end Alembic commands ### diff --git a/data/model/legacy.py b/data/model/legacy.py index 7ec27eed9..813fe0e67 100644 --- a/data/model/legacy.py +++ b/data/model/legacy.py @@ -18,7 +18,7 @@ from data.database import (User, Repository, Image, AccessToken, Role, Repositor DerivedImageStorage, ImageStorageTransformation, random_string_generator, db, BUILD_PHASE, QuayUserField, ImageStorageSignature, QueueItem, ImageStorageSignatureKind, validate_database_url, db_for_update, - AccessTokenKind, Star, get_epoch_timestamp) + AccessTokenKind, Star, get_epoch_timestamp, RepositoryActionCount) from peewee import JOIN_LEFT_OUTER, fn from util.validation import (validate_username, validate_email, validate_password, INVALID_PASSWORD_MESSAGE) @@ -995,20 +995,19 @@ def get_sorted_matching_repositories(prefix, only_public, checker, limit=10): """ Returns repositories matching the given prefix string and passing the given checker function. """ - last_week = datetime.now() - timedelta(weeks=1) results = [] existing_ids = [] - def get_search_results(search_clause, with_count): + def get_search_results(search_clause, with_count=False): if len(results) >= limit: return - selected = [Repository, Namespace] + select_items = [Repository, Namespace] if with_count: - selected.append(fn.Count(LogEntry.id).alias('count')) + select_items.append(fn.Sum(RepositoryActionCount.count).alias('count')) - query = (Repository.select(*selected) + query = (Repository.select(*select_items) .join(Namespace, JOIN_LEFT_OUTER, on=(Namespace.id == Repository.namespace_user)) .switch(Repository) .where(search_clause) @@ -1021,9 +1020,10 @@ def get_sorted_matching_repositories(prefix, only_public, checker, limit=10): query = query.where(~(Repository.id << existing_ids)) if with_count: - query = (query.join(LogEntry, JOIN_LEFT_OUTER) - .where(LogEntry.datetime >= last_week) - .order_by(fn.Count(LogEntry.id).desc())) + query = (query.switch(Repository) + .join(RepositoryActionCount) + .where(RepositoryActionCount.date >= last_week) + .order_by(fn.Sum(RepositoryActionCount.count).desc())) for result in query: if len(results) >= limit: @@ -1042,13 +1042,13 @@ def get_sorted_matching_repositories(prefix, only_public, checker, limit=10): existing_ids.append(result.id) # For performance reasons, we conduct the repo name and repo namespace searches on their - # own, and with and without counts on their own. This also affords us the ability to give - # higher precedence to repository names matching over namespaces, which is semantically correct. - get_search_results((Repository.name ** (prefix + '%')), with_count=True) - get_search_results((Repository.name ** (prefix + '%')), with_count=False) + # own. This also affords us the ability to give higher precedence to repository names matching + # over namespaces, which is semantically correct. + get_search_results(Repository.name ** (prefix + '%'), with_count=True) + get_search_results(Repository.name ** (prefix + '%'), with_count=False) - get_search_results((Namespace.username ** (prefix + '%')), with_count=True) - get_search_results((Namespace.username ** (prefix + '%')), with_count=False) + get_search_results(Namespace.username ** (prefix + '%'), with_count=True) + get_search_results(Namespace.username ** (prefix + '%'), with_count=False) return results @@ -1762,13 +1762,18 @@ def _tag_alive(query, now_ts=None): (RepositoryTag.lifetime_end_ts > now_ts)) -def list_repository_tag_history(repository, limit=100): +def list_repository_tag_history(repository, limit=100, specific_tag=None): query = (RepositoryTag .select(RepositoryTag, Image) .join(Image) .where(RepositoryTag.repository == repository) + .where(RepositoryTag.hidden == False) .order_by(RepositoryTag.lifetime_start_ts.desc()) .limit(limit)) + + if specific_tag: + query = query.where(RepositoryTag.name == specific_tag) + return query @@ -1990,7 +1995,7 @@ def get_parent_images(namespace_name, repository_name, image_obj): def create_or_update_tag(namespace_name, repository_name, tag_name, - tag_docker_image_id): + tag_docker_image_id, reversion=False): try: repo = _get_repository(namespace_name, repository_name) except Repository.DoesNotExist: @@ -2015,7 +2020,7 @@ def create_or_update_tag(namespace_name, repository_name, tag_name, raise DataModelException('Invalid image with id: %s' % tag_docker_image_id) return RepositoryTag.create(repository=repo, image=image, name=tag_name, - lifetime_start_ts=now_ts) + lifetime_start_ts=now_ts, reversion=reversion) def delete_tag(namespace_name, repository_name, tag_name): now_ts = get_epoch_timestamp() @@ -2823,3 +2828,22 @@ def repository_is_starred(user, repository): return True except Star.DoesNotExist: return False + + +def revert_tag(repository, tag_name, docker_image_id): + """ Reverts a tag to a specific image ID. """ + # Verify that the image ID already existed under this repository under the + # tag. + try: + (RepositoryTag.select() + .join(Image) + .where(RepositoryTag.repository == repository) + .where(RepositoryTag.name == tag_name) + .where(Image.docker_image_id == docker_image_id) + .get()) + except RepositoryTag.DoesNotExist: + raise DataModelException('Cannot revert to unknown or invalid image') + + return create_or_update_tag(repository.namespace_user.username, repository.name, + tag_name, docker_image_id, reversion=True) + diff --git a/data/model/sqlalchemybridge.py b/data/model/sqlalchemybridge.py index 8b7d8b664..43248b55a 100644 --- a/data/model/sqlalchemybridge.py +++ b/data/model/sqlalchemybridge.py @@ -1,7 +1,7 @@ from sqlalchemy import (Table, MetaData, Column, ForeignKey, Integer, String, Boolean, Text, - DateTime, BigInteger, Index) + DateTime, Date, BigInteger, Index) from peewee import (PrimaryKeyField, CharField, BooleanField, DateTimeField, TextField, - ForeignKeyField, BigIntegerField, IntegerField) + ForeignKeyField, BigIntegerField, IntegerField, DateField) OPTIONS_TO_COPY = [ @@ -42,6 +42,8 @@ def gen_sqlalchemy_metadata(peewee_model_list): alchemy_type = Boolean elif isinstance(field, DateTimeField): alchemy_type = DateTime + elif isinstance(field, DateField): + alchemy_type = Date elif isinstance(field, TextField): alchemy_type = Text elif isinstance(field, ForeignKeyField): diff --git a/endpoints/api/tag.py b/endpoints/api/tag.py index f698be851..9ca880e3f 100644 --- a/endpoints/api/tag.py +++ b/endpoints/api/tag.py @@ -2,7 +2,7 @@ from flask import request, abort from endpoints.api import (resource, nickname, require_repo_read, require_repo_write, RepositoryParamResource, log_action, NotFound, validate_json_request, - path_param, format_date) + path_param, format_date, parse_args, query_param) from endpoints.api.image import image_view from data import model from auth.auth_context import get_authenticated_user @@ -17,8 +17,11 @@ class ListRepositoryTags(RepositoryParamResource): """ Resource for listing repository tags. """ @require_repo_write + @parse_args + @query_param('specificTag', 'Filters the tags to the specific tag.', type=str, default='') + @query_param('limit', 'Limit to the number of results to return. Max 100.', type=int, default=50) @nickname('listRepoTags') - def get(self, namespace, repository): + def get(self, args, namespace, repository): repo = model.get_repository(namespace, repository) if not repo: abort(404) @@ -27,6 +30,7 @@ class ListRepositoryTags(RepositoryParamResource): tag_info = { 'name': tag.name, 'docker_image_id': tag.image.docker_image_id, + 'reversion': tag.reversion, } if tag.lifetime_start_ts > 0: @@ -37,7 +41,9 @@ class ListRepositoryTags(RepositoryParamResource): return tag_info - tags = model.list_repository_tag_history(repo, limit=100) + specific_tag = args.get('specificTag') or None + limit = min(100, max(1, args.get('limit', 50))) + tags = model.list_repository_tag_history(repo, limit=limit, specific_tag=specific_tag) return {'tags': [tag_view(tag) for tag in tags]} @@ -134,3 +140,54 @@ class RepositoryTagImages(RepositoryParamResource): return { 'images': [image_view(image, image_map) for image in all_images] } + + + +@resource('/v1/repository//tag//revert') +@path_param('repository', 'The full path of the repository. e.g. namespace/name') +@path_param('tag', 'The name of the tag') +class RevertTag(RepositoryParamResource): + """ Resource for reverting a repository tag back to a previous image. """ + schemas = { + 'RevertTag': { + 'id': 'RevertTag', + 'type': 'object', + 'description': 'Reverts a tag to a specific image', + 'required': [ + 'image', + ], + 'properties': { + 'image': { + 'type': 'string', + 'description': 'Image identifier to which the tag should point', + }, + }, + }, + } + + @require_repo_write + @nickname('revertTag') + @validate_json_request('RevertTag') + def post(self, namespace, repository, tag): + """ Reverts a repository tag back to a previous image in the repository. """ + try: + tag_image = model.get_tag_image(namespace, repository, tag) + except model.DataModelException: + raise NotFound() + + # Revert the tag back to the previous image. + image_id = request.get_json()['image'] + model.revert_tag(tag_image.repository, tag, image_id) + model.garbage_collect_repository(namespace, repository) + + # Log the reversion. + username = get_authenticated_user().username + log_action('revert_tag', namespace, + {'username': username, 'repo': repository, 'tag': tag, + 'image': image_id, 'original_image': tag_image.docker_image_id}, + repo=model.get_repository(namespace, repository)) + + return { + 'image_id': image_id, + 'original_image_id': tag_image.docker_image_id + } diff --git a/initdb.py b/initdb.py index 026c6b5f3..6267bb9b2 100644 --- a/initdb.py +++ b/initdb.py @@ -16,6 +16,8 @@ from data import model from data.model import oauth from app import app, storage as store +from workers import repositoryactioncounter + logger = logging.getLogger(__name__) @@ -221,6 +223,7 @@ def initialize_database(): LogEntryKind.create(name='create_tag') LogEntryKind.create(name='move_tag') LogEntryKind.create(name='delete_tag') + LogEntryKind.create(name='revert_tag') LogEntryKind.create(name='add_repo_permission') LogEntryKind.create(name='change_repo_permission') LogEntryKind.create(name='delete_repo_permission') @@ -589,6 +592,9 @@ def populate_database(): 'trigger_id': trigger.uuid, 'config': json.loads(trigger.config), 'service': trigger.service.name}) + while repositoryactioncounter.count_repository_actions(): + pass + if __name__ == '__main__': log_level = getattr(logging, app.config['LOGGING_LEVEL']) logging.basicConfig(level=log_level) diff --git a/static/css/directives/repo-view/repo-panel-tags.css b/static/css/directives/repo-view/repo-panel-tags.css index 1a81a5604..c515b9190 100644 --- a/static/css/directives/repo-view/repo-panel-tags.css +++ b/static/css/directives/repo-view/repo-panel-tags.css @@ -65,134 +65,22 @@ padding-left: 20px; } -.repo-panel-tags-element .options-col .fa-download { +.repo-panel-tags-element .options-col .fa-download, .repo-panel-tags-element .options-col .fa-history { color: #999; cursor: pointer; } -.repo-panel-tags-element .history-list { - margin: 10px; - border-left: 2px solid #eee; -} - -.repo-panel-tags-element .history-entry { - position:relative; - margin-top: 20px; - padding-left: 26px; - - transition: all 350ms ease-in-out; -} - -.repo-panel-tags-element .history-entry .history-text { - transition: transform 350ms ease-in-out, opacity 350ms ease-in-out; - overflow: hidden; - height: 40px; -} - -.repo-panel-tags-element .history-entry.filtered-mismatch { - margin-top: 10px; -} - -.repo-panel-tags-element .history-entry.filtered-mismatch .history-text { - height: 18px; - opacity: 0; -} - -.repo-panel-tags-element .history-entry.filtered-mismatch .history-icon { - opacity: 0.5; - transform: scale(0.5, 0.5); -} - -.repo-panel-tags-element .history-entry .history-date-break { - font-size: 16px; -} - -.repo-panel-tags-element .history-entry .history-date-break:before { - content: ""; - position: absolute; - border-radius: 50%; - width: 12px; - height: 12px; - background: #ccc; - top: 4px; - left: -7px; -} - -.repo-panel-tags-element .history-entry .history-icon { - border-radius: 50%; - width: 32px; - height: 32px; - line-height: 33px; - text-align: center; - font-size: 20px; - color: white; - background: #ccc; - - position: absolute; - left: -17px; - top: -4px; - display: inline-block; - - transition: all 350ms ease-in-out; -} - -.repo-panel-tags-element .history-entry.move .history-icon:before { - content: "\f061"; - font-family: FontAwesome; -} - -.repo-panel-tags-element .history-entry.create .history-icon:before { - content: "\f02b"; - font-family: FontAwesome; -} - -.repo-panel-tags-element .history-entry.delete .history-icon:before { - content: "\f014"; - font-family: FontAwesome; -} - -.repo-panel-tags-element .history-entry.move .history-icon { - background-color: #1f77b4; -} - -.repo-panel-tags-element .history-entry.create .history-icon { - background-color: #98df8a; -} - -.repo-panel-tags-element .history-entry.delete .history-icon { - background-color: #ff9896; -} - -.repo-panel-tags-element .history-entry .history-icon .fa-tag { - margin-right: 0px; -} - -.repo-panel-tags-element .history-entry .tag-span { - display: inline-block; - border-radius: 4px; - padding: 2px; - background: #eee; - padding-right: 6px; - color: black; - cursor: pointer; -} - -.repo-panel-tags-element .history-entry .tag-span.checked { - background: #F6FCFF; -} - -.repo-panel-tags-element .history-entry .tag-span:before { - content: "\f02b"; - font-family: FontAwesome; - margin-left: 4px; - margin-right: 4px; -} - -.repo-panel-tags-element .history-entry .history-description { - color: #777; -} - -.repo-panel-tags-element .history-entry .history-datetime { +.repo-panel-tags-element .tag-image-history-item .image-id { + font-family: Consolas, "Lucida Console", Monaco, monospace; font-size: 12px; +} + +.repo-panel-tags-element .tag-image-history-item .image-apply-time { color: #ccc; + font-size: 11px; + padding-left: 20px; +} + +.repo-panel-tags-element .tag-image-history-item .fa-circle-o { + margin-right: 2px; } \ No newline at end of file diff --git a/static/css/directives/ui/repo-tag-history.css b/static/css/directives/ui/repo-tag-history.css new file mode 100644 index 000000000..a0a2c2df7 --- /dev/null +++ b/static/css/directives/ui/repo-tag-history.css @@ -0,0 +1,141 @@ +.repo-tag-history-element .history-list { + margin: 10px; + border-left: 2px solid #eee; + margin-right: 150px; +} + +.repo-tag-history-element .history-entry { + position:relative; + margin-top: 20px; + padding-left: 26px; + + transition: all 350ms ease-in-out; +} + +.repo-tag-history-element .history-entry .history-text { + transition: transform 350ms ease-in-out, opacity 350ms ease-in-out; + overflow: hidden; + height: 40px; +} + +.repo-tag-history-element .history-entry.filtered-mismatch { + margin-top: 10px; +} + +.repo-tag-history-element .history-entry.filtered-mismatch .history-text { + height: 18px; + opacity: 0; +} + +.repo-tag-history-element .history-entry.filtered-mismatch .history-icon { + opacity: 0.5; + transform: scale(0.5, 0.5); +} + +.repo-tag-history-element .history-entry.filtered-mismatch.current .history-icon { + background-color: #ccc !important; +} + +.repo-tag-history-element .history-entry .history-date-break { + font-size: 16px; +} + +.repo-tag-history-element .history-entry .history-date-break:before { + content: "\f073"; + font-family: FontAwesome; + + position: absolute; + width: 12px; + height: 12px; + top: 1px; + left: -9px; + background: white; +} + +.repo-tag-history-element .history-entry .history-icon { + position: absolute; + left: -17px; + top: -4px; + + border-radius: 50%; + width: 32px; + height: 32px; + line-height: 32px; + text-align: center; + font-size: 20px; + color: white; + background: #ccc; + + display: inline-block; + transition: all 350ms ease-in-out; +} + +.repo-tag-history-element .history-entry.move .history-icon:before { + content: "\f061"; + font-family: FontAwesome; +} + +.repo-tag-history-element .history-entry.create .history-icon:before { + content: "\f02b"; + font-family: FontAwesome; +} + +.repo-tag-history-element .history-entry.revert .history-icon:before { + content: "\f0e2"; + font-family: FontAwesome; +} + +.repo-tag-history-element .history-entry.delete .history-icon:before { + content: "\f014"; + font-family: FontAwesome; +} + +.repo-tag-history-element .history-entry.current.revert .history-icon { + background-color: #F0C577; +} + +.repo-tag-history-element .history-entry.current.move .history-icon { + background-color: #77BFF0; +} + +.repo-tag-history-element .history-entry.current.create .history-icon { + background-color: #98df8a; +} + +.repo-tag-history-element .history-entry.current.delete .history-icon { + background-color: #ff9896; +} + +.repo-tag-history-element .history-entry .history-icon .fa-tag { + margin-right: 0px; +} + +.repo-tag-history-element .history-entry .tag-span { + display: inline-block; + border-radius: 4px; + padding: 2px; + background: #eee; + padding-right: 6px; + color: black; + cursor: pointer; +} + +.repo-tag-history-element .history-entry .tag-span.checked { + background: #F6FCFF; +} + +.repo-tag-history-element .history-entry .tag-span:before { + content: "\f02b"; + font-family: FontAwesome; + margin-left: 4px; + margin-right: 4px; +} + +.repo-tag-history-element .history-entry .history-description { + color: #777; +} + +.repo-tag-history-element .history-entry .history-datetime { + font-size: 12px; + color: #ccc; +} \ No newline at end of file diff --git a/static/css/directives/ui/tag-operations-dialog.css b/static/css/directives/ui/tag-operations-dialog.css new file mode 100644 index 000000000..1bbccaaff --- /dev/null +++ b/static/css/directives/ui/tag-operations-dialog.css @@ -0,0 +1,4 @@ +.tag-operations-dialog .image-id { + font-family: Consolas, "Lucida Console", Monaco, monospace; + font-size: 12px; +} \ No newline at end of file diff --git a/static/directives/new-header-bar.html b/static/directives/new-header-bar.html index f0f11f5f6..94aa60710 100644 --- a/static/directives/new-header-bar.html +++ b/static/directives/new-header-bar.html @@ -185,7 +185,8 @@ {{ result.namespace.name }}/{{ result.name }}
- {{ result.description }} +
diff --git a/static/directives/repo-tag-history.html b/static/directives/repo-tag-history.html new file mode 100644 index 000000000..16d8e5e7c --- /dev/null +++ b/static/directives/repo-tag-history.html @@ -0,0 +1,51 @@ +
+
+ + + + + +
+
+
No recent tag activity.
+
There has not been any recent tag activity on this repository.
+
+ +
+
+ {{ entry.date | amDateFormat:'dddd, MMMM Do YYYY' }} +
+
+
+
+
+ {{ entry.tag_name }} + + + was created pointing to image + + + was deleted + + + was moved to image + + from image + + + + was reverted to image + + from image + + + +
+
{{ entry.time | amDateFormat:'dddd, MMMM Do YYYY, h:mm:ss a' }}
+
+
+
+
+
\ No newline at end of file diff --git a/static/directives/repo-view/repo-panel-tags.html b/static/directives/repo-view/repo-panel-tags.html index 575d91dcc..ee54bf013 100644 --- a/static/directives/repo-view/repo-panel-tags.html +++ b/static/directives/repo-view/repo-panel-tags.html @@ -1,5 +1,5 @@
-
+
+ + +
+ +
+ This will change the image to which the tag points. +
+ + Are you sure you want to revert tag + {{ revertTagInfo.tag.name }} to image + {{ revertTagInfo.image_id.substr(0, 12) }}? +
+
\ No newline at end of file diff --git a/static/js/directives/repo-view/repo-panel-tags.js b/static/js/directives/repo-view/repo-panel-tags.js index 283ce2dde..97b63250b 100644 --- a/static/js/directives/repo-view/repo-panel-tags.js +++ b/static/js/directives/repo-view/repo-panel-tags.js @@ -24,6 +24,7 @@ angular.module('quay').directive('repoPanelTags', function () { }; $scope.iterationState = {}; + $scope.tagHistory = {}; $scope.tagActionHandler = null; $scope.showingHistory = false; @@ -84,6 +85,9 @@ angular.module('quay').directive('repoPanelTags', function () { 'count': imageMap[image_id].length, 'tags': imageMap[image_id] }); + + imageMap[image_id]['color'] = colors(index); + ++index; } }); @@ -119,136 +123,11 @@ angular.module('quay').directive('repoPanelTags', function () { // Process each of the tags. setTagState(); - - if ($scope.showingHistory) { - loadTimeline(); - } }); - var loadTimeline = function() { - var params = { - 'repository': $scope.repository.namespace + '/' + $scope.repository.name - }; - - ApiService.listRepoTags(null, params).then(function(resp) { - var tagData = []; - var currentTags = {}; - - resp.tags.forEach(function(tag) { - var tagName = tag.name; - var imageId = tag.docker_image_id; - var oldImageId = null; - - if (tag.end_ts) { - var action = 'delete'; - - // If the end time matches the existing start time for this tag, then this is a move - // instead of a delete. - var currentTime = tag.end_ts * 1000; - if (currentTags[tagName] && currentTags[tagName].start_ts == tag.end_ts) { - action = 'move'; - - // Remove the create. - var index = tagData.indexOf(currentTags[tagName]); - var createEntry = tagData.splice(index, 1)[0]; - - imageId = createEntry.docker_image_id; - oldImageId = tag.docker_image_id; - } - - // Add the delete/move. - tagData.push({ - 'tag_name': tagName, - 'action': action, - 'start_ts': tag.start_ts, - 'end_ts': tag.end_ts, - 'time': currentTime, - 'docker_image_id': imageId, - 'old_docker_image_id': oldImageId - }) - } - - if (tag.start_ts) { - var currentTime = tag.start_ts * 1000; - var create = { - 'tag_name': tagName, - 'action': 'create', - 'start_ts': tag.start_ts, - 'end_ts': tag.end_ts, - 'time': currentTime, - 'docker_image_id': tag.docker_image_id, - 'old_docker_image_id': '' - }; - - tagData.push(create); - currentTags[tagName] = create; - } - }); - - tagData.sort(function(a, b) { - return b.time - a.time; - }); - - for (var i = tagData.length - 1; i >= 1; --i) { - var current = tagData[i]; - var next = tagData[i - 1]; - - if (new Date(current.time).getDate() != new Date(next.time).getDate()) { - tagData.splice(i - 1, 0, { - 'date_break': true, - 'date': new Date(current.time) - }); - i--; - } - } - - if (tagData.length > 0) { - tagData.splice(0, 0, { - 'date_break': true, - 'date': new Date(tagData[0].time) - }); - } - - $scope.tagHistoryData = tagData; - }); - }; - - $scope.getEntryClasses = function(entry, historyFilter) { - var classes = entry.action + ' '; - if (!historyFilter || !entry.action) { - return classes; - } - - var parts = (historyFilter || '').split(','); - var isMatch = parts.some(function(part) { - if (part && entry.tag_name) { - isMatch = entry.tag_name.indexOf(part) >= 0; - isMatch = isMatch || entry.docker_image_id.indexOf(part) >= 0; - isMatch = isMatch || entry.old_docker_image_id.indexOf(part) >= 0; - return isMatch; - } - }); - - classes += isMatch ? 'filtered-match' : 'filtered-mismatch'; - return classes; - }; - $scope.showHistory = function(value, opt_tagname) { - if (opt_tagname) { - $scope.options.historyFilter = opt_tagname; - } else { - $scope.options.historyFilter = ''; - } - - if ($scope.showingHistory == value) { - return; - } - + $scope.options.historyFilter = opt_tagname ? opt_tagname : ''; $scope.showingHistory = value; - - if ($scope.showingHistory) { - loadTimeline(); - } }; $scope.toggleHistory = function() { @@ -350,12 +229,22 @@ angular.module('quay').directive('repoPanelTags', function () { return names.join(','); }; - $scope.isChecked = function(tagName, checked) { - return checked.some(function(tag) { - if (tag.name == tagName) { - return true; - } - }); + $scope.loadTagHistory = function(tag) { + delete $scope.tagHistory[tag.name]; + + var params = { + 'repository': $scope.repository.namespace + '/' + $scope.repository.name, + 'specificTag': tag.name, + 'limit': 5 + }; + + ApiService.listRepoTags(null, params).then(function(resp) { + $scope.tagHistory[tag.name] = resp.tags; + }, ApiService.errorDisplay('Could not load tag history')); + }; + + $scope.askRevertTag = function(tag, image_id) { + $scope.tagActionHandler.askRevertTag(tag, image_id); }; } }; diff --git a/static/js/directives/ui/logs-view.js b/static/js/directives/ui/logs-view.js index 8f85c3261..9a353b2c8 100644 --- a/static/js/directives/ui/logs-view.js +++ b/static/js/directives/ui/logs-view.js @@ -98,6 +98,7 @@ angular.module('quay').directive('logsView', function () { return 'Remove permission for token {token} from repository {repo}'; } }, + 'revert_tag': 'Tag {tag} reverted to image {image} from image {original_image}', 'delete_tag': 'Tag {tag} deleted in repository {repo} by user {username}', 'create_tag': 'Tag {tag} created in repository {repo} on image {image} by user {username}', 'move_tag': 'Tag {tag} moved from image {original_image} to image {image} in repository {repo} by user {username}', @@ -213,6 +214,7 @@ angular.module('quay').directive('logsView', function () { 'delete_tag': 'Delete Tag', 'create_tag': 'Create Tag', 'move_tag': 'Move Tag', + 'revert_tag':' Revert Tag', 'org_create_team': 'Create team', 'org_delete_team': 'Delete team', 'org_add_team_member': 'Add team member', diff --git a/static/js/directives/ui/repo-tag-history.js b/static/js/directives/ui/repo-tag-history.js new file mode 100644 index 000000000..c6ba6120d --- /dev/null +++ b/static/js/directives/ui/repo-tag-history.js @@ -0,0 +1,157 @@ +/** + * An element which displays its contents wrapped in an tag, but only if the href is not null. + */ +angular.module('quay').directive('repoTagHistory', function () { + var directiveDefinitionObject = { + priority: 0, + templateUrl: '/static/directives/repo-tag-history.html', + replace: false, + transclude: true, + restrict: 'C', + scope: { + 'repository': '=repository', + 'filter': '=filter', + 'isEnabled': '=isEnabled' + }, + controller: function($scope, $element, ApiService) { + $scope.tagHistoryData = null; + $scope.tagHistoryLeaves = {}; + + var loadTimeline = function() { + if (!$scope.repository || !$scope.isEnabled) { return; } + + var params = { + 'repository': $scope.repository.namespace + '/' + $scope.repository.name + }; + + ApiService.listRepoTags(null, params).then(function(resp) { + processTags(resp.tags); + }); + }; + + $scope.$watch('isEnabled', loadTimeline); + $scope.$watch('repository', loadTimeline); + + var processTags = function(tags) { + var entries = []; + var tagEntries = {}; + + // For each tag, turn the tag into create, move, delete, restore, etc entries. + tags.forEach(function(tag) { + var tagName = tag.name; + var dockerImageId = tag.docker_image_id; + + if (!tagEntries[tagName]) { + tagEntries[tagName] = []; + } + + var removeEntry = function(entry) { + entries.splice(entries.indexOf(entry), 1); + tagEntries[entry.tag_name].splice(tagEntries[entry.tag_name].indexOf(entry), 1); + }; + + var addEntry = function(action, time, opt_docker_id, opt_old_docker_id) { + var entry = { + 'tag_name': tagName, + 'action': action, + 'start_ts': tag.start_ts, + 'end_ts': tag.end_ts, + 'reversion': tag.reversion, + 'time': time * 1000, // JS expects ms, not s since epoch. + 'docker_image_id': opt_docker_id || dockerImageId, + 'old_docker_image_id': opt_old_docker_id || '' + }; + + tagEntries[tagName].push(entry); + entries.push(entry); + }; + + // If the tag has an end time, it was either deleted or moved. + if (tag.end_ts) { + // If a future entry exists with a start time equal to the end time for this tag, + // then the action was a move, rather than a delete and a create. + var currentEntries = tagEntries[tagName]; + var futureEntry = currentEntries.length > 0 ? currentEntries[currentEntries.length - 1] : {}; + if (futureEntry.start_ts == tag.end_ts) { + removeEntry(futureEntry); + addEntry(futureEntry.reversion ? 'revert': 'move', tag.end_ts, + futureEntry.docker_image_id, dockerImageId); + } else { + addEntry('delete', tag.end_ts) + } + } + + // If the tag has a start time, it was created. + if (tag.start_ts) { + addEntry('create', tag.start_ts); + } + }); + + // Sort the overall entries by datetime descending. + entries.sort(function(a, b) { + return b.time - a.time; + }); + + // Sort the tag entries by datetime descending. + Object.keys(tagEntries).forEach(function(name) { + var te = tagEntries[name]; + te.sort(function(a, b) { + return b.time - a.time; + }); + }); + + // Add date dividers in. + for (var i = entries.length - 1; i >= 1; --i) { + var current = entries[i]; + var next = entries[i - 1]; + + if (new Date(current.time).getDate() != new Date(next.time).getDate()) { + entries.splice(i, 0, { + 'date_break': true, + 'date': new Date(current.time) + }); + i--; + } + } + + // Add the top-level date divider. + if (entries.length > 0) { + entries.splice(0, 0, { + 'date_break': true, + 'date': new Date(entries[0].time) + }); + } + + $scope.historyEntries = entries; + $scope.historyEntryMap = tagEntries; + }; + + $scope.getEntryClasses = function(entry, historyFilter) { + if (!entry.action) { return ''; } + + var classes = entry.action + ' '; + if ($scope.historyEntryMap[entry.tag_name][0] == entry) { + classes += ' current '; + } + + if (!historyFilter || !entry.action) { + return classes; + } + + var parts = (historyFilter || '').split(','); + var isMatch = parts.some(function(part) { + if (part && entry.tag_name) { + isMatch = entry.tag_name.indexOf(part) >= 0; + isMatch = isMatch || entry.docker_image_id.indexOf(part) >= 0; + isMatch = isMatch || entry.old_docker_image_id.indexOf(part) >= 0; + return isMatch; + } + }); + + classes += isMatch ? 'filtered-match' : 'filtered-mismatch'; + return classes; + }; + } + }; + return directiveDefinitionObject; +}); \ No newline at end of file diff --git a/static/js/directives/ui/tag-operations-dialog.js b/static/js/directives/ui/tag-operations-dialog.js index 4a5aa1877..3fc3ea080 100644 --- a/static/js/directives/ui/tag-operations-dialog.js +++ b/static/js/directives/ui/tag-operations-dialog.js @@ -121,6 +121,25 @@ angular.module('quay').directive('tagOperationsDialog', function () { }, errorHandler); }; + $scope.revertTag = function(tag, image_id, callback) { + if (!$scope.repository.can_write) { return; } + + var params = { + 'repository': $scope.repository.namespace + '/' + $scope.repository.name, + 'tag': tag.name + }; + + var data = { + 'image': image_id + }; + + var errorHandler = ApiService.errorDisplay('Cannot revert tag', callback); + ApiService.revertTag(data, params).then(function() { + callback(true); + markChanged([], [tag]); + }, errorHandler); + }; + $scope.actionHandler = { 'askDeleteTag': function(tag) { $scope.deleteTagInfo = { @@ -140,6 +159,20 @@ angular.module('quay').directive('tagOperationsDialog', function () { $scope.addingTag = false; $scope.addTagForm.$setPristine(); $element.find('#createOrMoveTagModal').modal('show'); + }, + + 'askRevertTag': function(tag, image_id) { + if (tag.image_id == image_id) { + bootbox.alert('This is the current image for the tag'); + return; + } + + $scope.revertTagInfo = { + 'tag': tag, + 'image_id': image_id + }; + + $element.find('#revertTagModal').modal('show'); } }; } diff --git a/test/data/test.db b/test/data/test.db index dabe25344..e1ff8597b 100644 Binary files a/test/data/test.db and b/test/data/test.db differ diff --git a/test/test_api_security.py b/test/test_api_security.py index 979986c8c..bc65ecdfe 100644 --- a/test/test_api_security.py +++ b/test/test_api_security.py @@ -11,7 +11,7 @@ from initdb import setup_database_for_testing, finished_database_for_testing from endpoints.api import api_bp, api from endpoints.api.team import TeamMember, TeamMemberList, OrganizationTeam, TeamMemberInvite -from endpoints.api.tag import RepositoryTagImages, RepositoryTag, ListRepositoryTags +from endpoints.api.tag import RepositoryTagImages, RepositoryTag, ListRepositoryTags, RevertTag from endpoints.api.search import FindRepositories, EntitySearch from endpoints.api.image import RepositoryImageChanges, RepositoryImage, RepositoryImageList from endpoints.api.build import (FileDropResource, RepositoryBuildStatus, RepositoryBuildLogs, @@ -2481,6 +2481,61 @@ class TestRepositoryImage5avqBuynlargeOrgrepo(ApiTestCase): self._run_test('GET', 404, 'devtable', None) +class TestRevertTagHp8rPublicPublicrepo(ApiTestCase): + def setUp(self): + ApiTestCase.setUp(self) + self._set_url(RevertTag, tag="HP8R", repository="public/publicrepo") + + def test_post_anonymous(self): + self._run_test('POST', 401, None, {u'image': 'WXNG'}) + + def test_post_freshuser(self): + self._run_test('POST', 403, 'freshuser', {u'image': 'WXNG'}) + + def test_post_reader(self): + self._run_test('POST', 403, 'reader', {u'image': 'WXNG'}) + + def test_post_devtable(self): + self._run_test('POST', 403, 'devtable', {u'image': 'WXNG'}) + + +class TestRevertTagHp8rDevtableShared(ApiTestCase): + def setUp(self): + ApiTestCase.setUp(self) + self._set_url(RevertTag, tag="HP8R", repository="devtable/shared") + + def test_post_anonymous(self): + self._run_test('POST', 401, None, {u'image': 'WXNG'}) + + def test_post_freshuser(self): + self._run_test('POST', 403, 'freshuser', {u'image': 'WXNG'}) + + def test_post_reader(self): + self._run_test('POST', 403, 'reader', {u'image': 'WXNG'}) + + def test_post_devtable(self): + self._run_test('POST', 404, 'devtable', {u'image': 'WXNG'}) + + +class TestRevertTagHp8rBuynlargeOrgrepo(ApiTestCase): + def setUp(self): + ApiTestCase.setUp(self) + self._set_url(RevertTag, tag="HP8R", repository="buynlarge/orgrepo") + + def test_post_anonymous(self): + self._run_test('POST', 401, None, {u'image': 'WXNG'}) + + def test_post_freshuser(self): + self._run_test('POST', 403, 'freshuser', {u'image': 'WXNG'}) + + def test_post_reader(self): + self._run_test('POST', 403, 'reader', {u'image': 'WXNG'}) + + def test_post_devtable(self): + self._run_test('POST', 404, 'devtable', {u'image': 'WXNG'}) + + + class TestRepositoryTagHp8rPublicPublicrepo(ApiTestCase): def setUp(self): ApiTestCase.setUp(self) diff --git a/test/test_api_usage.py b/test/test_api_usage.py index e994fe205..561794174 100644 --- a/test/test_api_usage.py +++ b/test/test_api_usage.py @@ -14,7 +14,7 @@ from initdb import setup_database_for_testing, finished_database_for_testing from data import model, database from endpoints.api.team import TeamMember, TeamMemberList, TeamMemberInvite, OrganizationTeam -from endpoints.api.tag import RepositoryTagImages, RepositoryTag +from endpoints.api.tag import RepositoryTagImages, RepositoryTag, RevertTag, ListRepositoryTags from endpoints.api.search import FindRepositories, EntitySearch, ConductSearch from endpoints.api.image import RepositoryImage, RepositoryImageList from endpoints.api.build import (RepositoryBuildStatus, RepositoryBuildLogs, RepositoryBuildList, @@ -1746,6 +1746,45 @@ class TestGetImageChanges(ApiTestCase): # image_id=image_id)) +class TestRevertTag(ApiTestCase): + def test_reverttag_invalidtag(self): + self.login(ADMIN_ACCESS_USER) + + self.postResponse(RevertTag, + params=dict(repository=ADMIN_ACCESS_USER + '/history', tag='invalidtag'), + data=dict(image='invalid_image'), + expected_code=404) + + def test_reverttag_invalidimage(self): + self.login(ADMIN_ACCESS_USER) + + self.postResponse(RevertTag, + params=dict(repository=ADMIN_ACCESS_USER + '/history', tag='latest'), + data=dict(image='invalid_image'), + expected_code=400) + + def test_reverttag(self): + self.login(ADMIN_ACCESS_USER) + + json = self.getJsonResponse(ListRepositoryTags, + params=dict(repository=ADMIN_ACCESS_USER + '/history', tag='latest')) + + self.assertEquals(2, len(json['tags'])) + self.assertFalse('end_ts' in json['tags'][0]) + + previous_image_id = json['tags'][1]['docker_image_id'] + + self.postJsonResponse(RevertTag, + params=dict(repository=ADMIN_ACCESS_USER + '/history', tag='latest'), + data=dict(image=previous_image_id)) + + json = self.getJsonResponse(ListRepositoryTags, + params=dict(repository=ADMIN_ACCESS_USER + '/history', tag='latest')) + self.assertEquals(3, len(json['tags'])) + self.assertFalse('end_ts' in json['tags'][0]) + self.assertEquals(previous_image_id, json['tags'][0]['docker_image_id']) + + class TestListAndDeleteTag(ApiTestCase): def test_listdeletecreateandmovetag(self): self.login(ADMIN_ACCESS_USER) diff --git a/workers/repositoryactioncounter.py b/workers/repositoryactioncounter.py new file mode 100644 index 000000000..5a73c194b --- /dev/null +++ b/workers/repositoryactioncounter.py @@ -0,0 +1,52 @@ +import logging + +from apscheduler.schedulers.blocking import BlockingScheduler + +from app import app +from data.database import Repository, LogEntry, RepositoryActionCount, db_random_func, fn +from datetime import date, datetime, timedelta + +POLL_PERIOD_SECONDS = 30 + +logger = logging.getLogger(__name__) +sched = BlockingScheduler() + +@sched.scheduled_job(trigger='interval', seconds=10) +def count_repository_actions(): + """ Counts actions for a random repository for the previous day. """ + + try: + # Get a random repository to count. + today = date.today() + yesterday = today - timedelta(days=1) + has_yesterday_actions = (RepositoryActionCount.select(RepositoryActionCount.repository) + .where(RepositoryActionCount.date == yesterday)) + + to_count = (Repository.select() + .where(~(Repository.id << (has_yesterday_actions))) + .order_by(db_random_func()).get()) + + logger.debug('Counting: %s', to_count.id) + + actions = (LogEntry.select() + .where(LogEntry.repository == to_count, + LogEntry.datetime >= yesterday, + LogEntry.datetime < today) + .count()) + + # Create the row. + try: + RepositoryActionCount.create(repository=to_count, date=yesterday, count=actions) + except: + logger.exception('Exception when writing count') + + return True + + except Repository.DoesNotExist: + logger.debug('No further repositories to count') + return False + + +if __name__ == "__main__": + logging.basicConfig(level=logging.DEBUG) + sched.start()