From f4f67c8c62dc3801303d13e2f3e220679148076a Mon Sep 17 00:00:00 2001 From: Jimmy Zelinskie Date: Wed, 10 May 2017 14:40:11 -0400 Subject: [PATCH 1/5] app-public-view: add Audit Logs tab --- .../app-public-view/app-public-view.component.html | 14 +++++++++++--- .../app-public-view/app-public-view.component.ts | 5 +++++ 2 files changed, 16 insertions(+), 3 deletions(-) diff --git a/static/js/directives/ui/app-public-view/app-public-view.component.html b/static/js/directives/ui/app-public-view/app-public-view.component.html index 1375b2d85..753ac6293 100644 --- a/static/js/directives/ui/app-public-view/app-public-view.component.html +++ b/static/js/directives/ui/app-public-view/app-public-view.component.html @@ -21,8 +21,10 @@ - + + + + @@ -85,6 +87,12 @@ + + +
+ + +
@@ -123,4 +131,4 @@ - \ No newline at end of file + diff --git a/static/js/directives/ui/app-public-view/app-public-view.component.ts b/static/js/directives/ui/app-public-view/app-public-view.component.ts index 6b1f75258..104249a66 100644 --- a/static/js/directives/ui/app-public-view/app-public-view.component.ts +++ b/static/js/directives/ui/app-public-view/app-public-view.component.ts @@ -11,6 +11,7 @@ import { Input, Component, Inject } from 'ng-metadata/core'; export class AppPublicViewComponent { @Input('<') public repository: any; private settingsShown: number = 0; + private logsShown: number = 0; constructor(@Inject('Config') private Config: any) { this.updateDescription = this.updateDescription.bind(this); @@ -24,4 +25,8 @@ export class AppPublicViewComponent { public showSettings(): void { this.settingsShown++; } + + public showLogs(): void { + this.logsShown++; + } } From 4db789b656fa7f774e327f03db68beb857e75f8f Mon Sep 17 00:00:00 2001 From: Jimmy Zelinskie Date: Thu, 11 May 2017 13:33:18 -0400 Subject: [PATCH 2/5] add audit logging to app registry endpoints --- data/interfaces/appr.py | 30 +++++++++++++++++++---- endpoints/appr/registry.py | 22 ++++++++++++++--- endpoints/v1/index.py | 12 ++++----- endpoints/v1/tag.py | 4 +-- endpoints/v2/manifest.py | 2 +- endpoints/verbs/__init__.py | 2 +- endpoints/trackhelper.py => util/audit.py | 2 +- 7 files changed, 54 insertions(+), 20 deletions(-) rename endpoints/trackhelper.py => util/audit.py (98%) diff --git a/data/interfaces/appr.py b/data/interfaces/appr.py index 03d7a1077..a8772ce18 100644 --- a/data/interfaces/appr.py +++ b/data/interfaces/appr.py @@ -10,8 +10,11 @@ from six import add_metaclass from app import storage, authentication from data import model, oci_model from data.database import Tag, Manifest, MediaType, Blob, Repository, Channel +from util.audit import track_and_log +from util.morecollections import AttrDict from util.names import parse_robot_username + class BlobDescriptor(namedtuple('Blob', ['mediaType', 'size', 'digest', 'urls'])): """ BlobDescriptor describes a blob with its mediatype, size and digest. A BlobDescriptor is used to retrieves the actual blob. @@ -55,10 +58,6 @@ class AppRegistryDataInterface(object): """ Interface that represents all data store interactions required by a App Registry. """ - @abstractmethod - def _application(self, package_name): - pass - @abstractmethod def list_applications(self, namespace=None, media_type=None, search=None, username=None, with_channels=False): @@ -175,6 +174,11 @@ class AppRegistryDataInterface(object): Raises: ChannelNotFound, PackageNotFound """ + @abstractmethod + def log_action(self, event_name, namespace_name, repo_name=None, analytics_name=None, + analytics_sample=1, **kwargs): + """ Logs an action to the audit log. """ + def _split_package_name(package): """ Returns the namespace and package-name """ @@ -200,6 +204,22 @@ class OCIAppModel(AppRegistryDataInterface): raise_package_not_found(package) return repo + def log_action(self, event_name, namespace_name, repo_name=None, analytics_name=None, + analytics_sample=1, metadata=None): + metadata = {} if metadata is None else metadata + + repo = None + if repo_name is not None: + db_repo = model.repository.get_repository(namespace_name, repo_name, + kind_filter='application') + repo = AttrDict({ + 'id': db_repo.id, + 'name': db_repo.name, + 'namespace_name': db_repo.namespace_user.username, + }) + track_and_log(event_name, repo, analytics_name=analytics_name, + analytics_sample=analytics_sample, **metadata) + def list_applications(self, namespace=None, media_type=None, search=None, username=None, with_channels=False): """ Lists all repositories that contain applications, with optional filtering to a specific @@ -248,7 +268,7 @@ class OCIAppModel(AppRegistryDataInterface): def create_application(self, package_name, visibility, owner): """ Create a new app repository, owner is the user who creates it """ ns, name = _split_package_name(package_name) - model.repository.create_repository(ns, name, owner, visibility, "application") + model.repository.create_repository(ns, name, owner, visibility, 'application') def application_exists(self, package_name): """ Create a new app repository, owner is the user who creates it """ diff --git a/endpoints/appr/registry.py b/endpoints/appr/registry.py index a77b72104..970998e44 100644 --- a/endpoints/appr/registry.py +++ b/endpoints/appr/registry.py @@ -13,6 +13,7 @@ from flask import jsonify, request from auth.auth_context import get_authenticated_user from auth.decorators import process_auth from auth.permissions import (CreateRepositoryPermission, ModifyRepositoryPermission) +from data.interfaces.appr import oci_app_model as model from endpoints.appr import (appr_bp, require_app_repo_read, require_app_repo_write) from endpoints.appr.cnr_backend import Blob, Channel, Package, User from endpoints.appr.decorators import disallow_for_image_repository @@ -102,6 +103,8 @@ def list_packages(): def delete_package(namespace, package_name, release, media_type): reponame = repo_name(namespace, package_name) result = cnr_registry.delete_package(reponame, release, media_type, package_class=Package) + model.log_action('delete_tag', namespace, repo_name=package_name, + metadata={'release': release, 'mediatype': media_type}) return jsonify(result) @@ -136,7 +139,7 @@ def show_package_releases(namespace, package_name): @process_auth @require_app_repo_read @anon_protect -def show_package_releasse_manifests(namespace, package_name, release): +def show_package_release_manifests(namespace, package_name, release): reponame = repo_name(namespace, package_name) result = cnr_registry.show_package_manifests(reponame, release, package_class=Package) return jsonify(result) @@ -153,6 +156,8 @@ def pull(namespace, package_name, release, media_type): reponame = repo_name(namespace, package_name) logger.info("pull %s", reponame) data = cnr_registry.pull(reponame, release, media_type, Package, blob_class=Blob) + model.log_action('pull_repo', namespace, repo_name=package_name, + metadata={'release': release, 'mediatype': media_type}) return _pull(data) @@ -178,6 +183,7 @@ def push(namespace, package_name): {"package": reponame, "scopes": ['create']}) Package.create_repository(reponame, private, owner) + model.log_action('create_repo', namespace, repo_name=package_name) if not ModifyRepositoryPermission(namespace, package_name).can(): raise Forbidden("Unauthorized access for: %s" % reponame, @@ -194,6 +200,8 @@ def push(namespace, package_name): blob = Blob(reponame, values['blob']) app_release = cnr_registry.push(reponame, release_version, media_type, blob, force, package_class=Package, user=owner, visibility=private) + model.log_action('push_repo', namespace, repo_name=package_name, + metadata={'release': release_version}) return jsonify(app_release) @@ -246,6 +254,8 @@ def add_channel_release(namespace, package_name, channel_name, release): reponame = repo_name(namespace, package_name) result = cnr_registry.add_channel_release(reponame, channel_name, release, channel_class=Channel, package_class=Package) + model.log_action('create_tag', namespace, repo_name=package_name, + metadata={'channel': channel_name, 'release': release}) return jsonify(result) @@ -254,13 +264,13 @@ def _check_channel_name(channel_name, release=None): logger.debug('Found invalid channel name CNR add channel release: %s', channel_name) raise InvalidUsage("Found invalid channelname %s" % release, {'name': channel_name, - "release": release}) + 'release': release}) if release is not None and not TAG_REGEX.match(release): logger.debug('Found invalid release name CNR add channel release: %s', release) - raise InvalidUsage("Found invalid channel release name %s" % release, + raise InvalidUsage('Found invalid channel release name %s' % release, {'name': channel_name, - "release": release}) + 'release': release}) @appr_bp.route( @@ -275,6 +285,8 @@ def delete_channel_release(namespace, package_name, channel_name, release): reponame = repo_name(namespace, package_name) result = cnr_registry.delete_channel_release(reponame, channel_name, release, channel_class=Channel, package_class=Package) + model.log_action('delete_tag', namespace, repo_name=package_name, + metadata={'channel': channel_name, 'release': release}) return jsonify(result) @@ -289,4 +301,6 @@ def delete_channel(namespace, package_name, channel_name): _check_channel_name(channel_name) reponame = repo_name(namespace, package_name) result = cnr_registry.delete_channel(reponame, channel_name, channel_class=Channel) + model.log_action('delete_tag', namespace, repo_name=package_name, + metadata={'channel': channel_name}) return jsonify(result) diff --git a/endpoints/v1/index.py b/endpoints/v1/index.py index f579e5f15..3708178f3 100644 --- a/endpoints/v1/index.py +++ b/endpoints/v1/index.py @@ -6,7 +6,6 @@ from functools import wraps from flask import request, make_response, jsonify, session -from data.interfaces.v1 import pre_oci_model as model from app import authentication, userevents, metric_queue from auth.auth_context import get_authenticated_user, get_validated_token, get_validated_oauth_token from auth.decorators import process_auth @@ -14,13 +13,14 @@ from auth.permissions import (ModifyRepositoryPermission, UserAdminPermission, ReadRepositoryPermission, CreateRepositoryPermission, repository_read_grant, repository_write_grant) from auth.signedgrant import generate_signed_token +from data.interfaces.v1 import pre_oci_model as model +from endpoints.common import parse_repository_name +from endpoints.decorators import anon_protect, anon_allowed +from endpoints.notificationhelper import spawn_notification +from endpoints.v1 import v1_bp +from util.audit import track_and_log from util.http import abort from util.names import REPOSITORY_NAME_REGEX -from endpoints.common import parse_repository_name -from endpoints.v1 import v1_bp -from endpoints.trackhelper import track_and_log -from endpoints.notificationhelper import spawn_notification -from endpoints.decorators import anon_protect, anon_allowed logger = logging.getLogger(__name__) diff --git a/endpoints/v1/tag.py b/endpoints/v1/tag.py index ce3726374..00f6d3bcb 100644 --- a/endpoints/v1/tag.py +++ b/endpoints/v1/tag.py @@ -4,7 +4,6 @@ import json from flask import abort, request, jsonify, make_response, session -from util.names import TAG_ERROR, TAG_REGEX from auth.decorators import process_auth from auth.permissions import (ReadRepositoryPermission, ModifyRepositoryPermission) @@ -13,7 +12,8 @@ from data.interfaces.v1 import pre_oci_model as model from endpoints.common import parse_repository_name from endpoints.decorators import anon_protect from endpoints.v1 import v1_bp -from endpoints.trackhelper import track_and_log +from util.audit import track_and_log +from util.names import TAG_ERROR, TAG_REGEX logger = logging.getLogger(__name__) diff --git a/endpoints/v2/manifest.py b/endpoints/v2/manifest.py index a4155add2..732403598 100644 --- a/endpoints/v2/manifest.py +++ b/endpoints/v2/manifest.py @@ -15,11 +15,11 @@ from endpoints.decorators import anon_protect from endpoints.v2 import v2_bp, require_repo_read, require_repo_write from endpoints.v2.errors import (BlobUnknown, ManifestInvalid, ManifestUnknown, TagInvalid, NameInvalid) -from endpoints.trackhelper import track_and_log from endpoints.notificationhelper import spawn_notification from image.docker import ManifestException from image.docker.schema1 import DockerSchema1Manifest, DockerSchema1ManifestBuilder from image.docker.schema2 import DOCKER_SCHEMA2_CONTENT_TYPES +from util.audit import track_and_log from util.names import VALID_TAG_PATTERN from util.registry.replication import queue_replication_batch from util.validation import is_json diff --git a/endpoints/verbs/__init__.py b/endpoints/verbs/__init__.py index 35a919cbb..27a5f2330 100644 --- a/endpoints/verbs/__init__.py +++ b/endpoints/verbs/__init__.py @@ -13,11 +13,11 @@ from data import database from data.interfaces.verbs import pre_oci_model as model from endpoints.common import route_show_if, parse_repository_name from endpoints.decorators import anon_protect -from endpoints.trackhelper import track_and_log from endpoints.v2.blob import BLOB_DIGEST_ROUTE from image.appc import AppCImageFormatter from image.docker.squashed import SquashedDockerImageFormatter from storage import Storage +from util.audit import track_and_log from util.http import exact_abort from util.registry.filelike import wrap_with_handler from util.registry.queuefile import QueueFile diff --git a/endpoints/trackhelper.py b/util/audit.py similarity index 98% rename from endpoints/trackhelper.py rename to util/audit.py index 0aa66cefa..3a6132d27 100644 --- a/endpoints/trackhelper.py +++ b/util/audit.py @@ -15,7 +15,7 @@ logger = logging.getLogger(__name__) def track_and_log(event_name, repo_obj, analytics_name=None, analytics_sample=1, **kwargs): repo_name = repo_obj.name - namespace_name = repo_obj.namespace_name, + namespace_name = repo_obj.namespace_name metadata = { 'repo': repo_name, 'namespace': namespace_name, From d7564fd627333f32825d145df7a53e43a68c5048 Mon Sep 17 00:00:00 2001 From: Jimmy Zelinskie Date: Thu, 11 May 2017 13:33:36 -0400 Subject: [PATCH 3/5] add app metadata fields to usage logs component --- static/js/directives/ui/logs-view.js | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/static/js/directives/ui/logs-view.js b/static/js/directives/ui/logs-view.js index d1054b026..284a355a8 100644 --- a/static/js/directives/ui/logs-view.js +++ b/static/js/directives/ui/logs-view.js @@ -61,6 +61,8 @@ angular.module('quay').directive('logsView', function () { 'push_repo': function(metadata) { if (metadata.tag) { return 'Push of {tag} to repository {namespace}/{repo}'; + } else if (metadata.release) { + return 'Push of {release} to repository {namespace}/{repo}'; } else { return 'Repository push to {namespace}/{repo}'; } @@ -91,6 +93,15 @@ angular.module('quay').directive('logsView', function () { description = 'tag {tag} from repository {namespace}/{repo}'; } else if (metadata.manifest_digest) { description = 'digest {manifest_digest} from repository {namespace}/{repo}'; + } else if (metadata.release) { + description = 'release {release}'; + if (metadata.channel) { + description += ' via channel {channel}'; + } + if (metadata.mediatype) { + description += ' for {mediatype}'; + } + description += ' from repository {namespace}/{repo}'; } if (metadata.token) { From 74440555110d3093e842872b78e990ce3fa41639 Mon Sep 17 00:00:00 2001 From: Jimmy Zelinskie Date: Mon, 15 May 2017 20:41:43 -0400 Subject: [PATCH 4/5] auth: remove relative imports --- auth/registry_jwt_auth.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/auth/registry_jwt_auth.py b/auth/registry_jwt_auth.py index a9e912e8b..4208a8462 100644 --- a/auth/registry_jwt_auth.py +++ b/auth/registry_jwt_auth.py @@ -7,10 +7,10 @@ from flask import request, url_for from flask_principal import identity_changed, Identity from app import app, get_app_url, instance_keys -from .auth_context import set_grant_context, get_grant_context -from .permissions import repository_read_grant, repository_write_grant, repository_admin_grant -from util.names import parse_namespace_repository +from auth.auth_context import (set_grant_context, get_grant_context) +from auth.permissions import repository_read_grant, repository_write_grant, repository_admin_grant from util.http import abort +from util.names import parse_namespace_repository from util.security.registry_jwt import (ANONYMOUS_SUB, decode_bearer_header, InvalidBearerTokenException) from data import model @@ -18,8 +18,10 @@ from data import model logger = logging.getLogger(__name__) + CONTEXT_KINDS = ['user', 'token', 'oauth'] + ACCESS_SCHEMA = { 'type': 'array', 'description': 'List of access granted to the subject', From e2c25ce9bc6baf21e41d42bc8b57b57c0a38bddc Mon Sep 17 00:00:00 2001 From: Jimmy Zelinskie Date: Tue, 16 May 2017 17:05:31 -0400 Subject: [PATCH 5/5] registry tests: assert audit log metadata --- test/registry_tests.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/test/registry_tests.py b/test/registry_tests.py index e241a71f0..34df38e93 100644 --- a/test/registry_tests.py +++ b/test/registry_tests.py @@ -1015,6 +1015,8 @@ class RegistryTestsMixin(object): self.assertEquals(1, len(logs)) self.assertEquals('push_repo', logs[0]['kind']) + self.assertEquals('public', logs[0]['metadata']['namespace']) + self.assertEquals('newrepo', logs[0]['metadata']['repo']) self.assertEquals('public', logs[0]['performer']['name']) # Pull the repository. @@ -1044,6 +1046,8 @@ class RegistryTestsMixin(object): self.assertEquals(1, len(logs)) self.assertEquals('push_repo', logs[0]['kind']) + self.assertEquals('buynlarge', logs[0]['metadata']['namespace']) + self.assertEquals('newrepo', logs[0]['metadata']['repo']) self.assertEquals('buynlarge+ownerbot', logs[0]['performer']['name']) # Pull the repository. @@ -1055,6 +1059,8 @@ class RegistryTestsMixin(object): self.assertEquals(2, len(logs)) self.assertEquals('pull_repo', logs[0]['kind']) + self.assertEquals('buynlarge', logs[0]['metadata']['namespace']) + self.assertEquals('newrepo', logs[0]['metadata']['repo']) self.assertEquals('buynlarge+ownerbot', logs[0]['performer']['name']) @@ -1074,6 +1080,8 @@ class RegistryTestsMixin(object): logs = result.json()['logs'] self.assertEquals('pull_repo', logs[0]['kind']) + self.assertEquals('devtable', logs[0]['metadata']['namespace']) + self.assertEquals('newrepo', logs[0]['metadata']['repo']) self.assertEquals('my-new-token', logs[0]['metadata']['token']) @@ -1091,6 +1099,8 @@ class RegistryTestsMixin(object): self.assertEquals(2, len(logs)) self.assertEquals('pull_repo', logs[0]['kind']) + self.assertEquals('devtable', logs[0]['metadata']['namespace']) + self.assertEquals('newrepo', logs[0]['metadata']['repo']) self.assertEquals('devtable', logs[0]['performer']['name']) self.assertEquals(1, logs[0]['metadata']['oauth_token_id'])