diff --git a/app.py b/app.py index d631e0780..2c96969af 100644 --- a/app.py +++ b/app.py @@ -37,7 +37,7 @@ from util.saas.metricqueue import MetricQueue from util.config.provider import get_config_provider from util.config.configutil import generate_secret_key from util.config.superusermanager import SuperUserManager -from util.secscan.secscanendpoint import SecurityScanEndpoint +from util.secscan.api import SecurityScannerAPI OVERRIDE_CONFIG_DIRECTORY = 'conf/stack/' OVERRIDE_CONFIG_YAML_FILENAME = 'conf/stack/config.yaml' @@ -160,7 +160,9 @@ image_replication_queue = WorkQueue(app.config['REPLICATION_QUEUE_NAME'], tf) dockerfile_build_queue = WorkQueue(app.config['DOCKERFILE_BUILD_QUEUE_NAME'], tf, reporter=MetricQueueReporter(metric_queue)) notification_queue = WorkQueue(app.config['NOTIFICATION_QUEUE_NAME'], tf) -secscan_endpoint = SecurityScanEndpoint(app, config_provider) +secscan_notification_queue = WorkQueue(app.config['SECSCAN_NOTIFICATION_QUEUE_NAME'], tf) + +secscan_api = SecurityScannerAPI(app, config_provider) # Check for a key in config. If none found, generate a new signing key for Docker V2 manifests. _v2_key_path = os.path.join(OVERRIDE_CONFIG_DIRECTORY, DOCKER_V2_SIGNINGKEY_FILENAME) diff --git a/conf/init/service/security_notification_worker/log/run b/conf/init/service/security_notification_worker/log/run new file mode 100755 index 000000000..262fed98e --- /dev/null +++ b/conf/init/service/security_notification_worker/log/run @@ -0,0 +1,2 @@ +#!/bin/sh +exec logger -i -t securitynotificationworker diff --git a/conf/init/service/security_notification_worker/run b/conf/init/service/security_notification_worker/run new file mode 100755 index 000000000..83c94e686 --- /dev/null +++ b/conf/init/service/security_notification_worker/run @@ -0,0 +1,8 @@ +#! /bin/bash + +echo 'Starting security scanner notification worker' + +cd / +venv/bin/python -m workers.security_notification_worker 2>&1 + +echo 'Security scanner notification worker exited' diff --git a/conf/init/service/securityworker/log/run b/conf/init/service/securityworker/log/run old mode 100644 new mode 100755 diff --git a/conf/init/service/securityworker/run b/conf/init/service/securityworker/run old mode 100644 new mode 100755 diff --git a/config.py b/config.py index bf08c5d5f..6fa139e7d 100644 --- a/config.py +++ b/config.py @@ -50,7 +50,7 @@ class DefaultConfig(object): CONTACT_INFO = [ 'mailto:support@quay.io', - 'irc://chat.freenode.net:6665/quayio', + 'irc://chat.freenode.net:6665/quay', 'tel:+1-888-930-3475', 'https://twitter.com/quayio', ] @@ -130,6 +130,7 @@ class DefaultConfig(object): DIFFS_QUEUE_NAME = 'imagediff' DOCKERFILE_BUILD_QUEUE_NAME = 'dockerfilebuild' REPLICATION_QUEUE_NAME = 'imagestoragereplication' + SECSCAN_NOTIFICATION_QUEUE_NAME = 'secscan_notification' # Super user config. Note: This MUST BE an empty list for the default config. SUPER_USERS = [] @@ -257,7 +258,7 @@ class DefaultConfig(object): # Security scanner FEATURE_SECURITY_SCANNER = False SECURITY_SCANNER = { - 'ENDPOINT': 'http://192.168.99.100:6060', + 'ENDPOINT': 'http://192.168.99.101:6060', 'ENGINE_VERSION_TARGET': 1, 'API_VERSION': 'v1', 'API_TIMEOUT_SECONDS': 10, diff --git a/data/migrations/versions/50925110da8c_add_event_specific_config.py b/data/migrations/versions/50925110da8c_add_event_specific_config.py index 4a7672b70..8b67fe51a 100644 --- a/data/migrations/versions/50925110da8c_add_event_specific_config.py +++ b/data/migrations/versions/50925110da8c_add_event_specific_config.py @@ -14,7 +14,6 @@ from alembic import op import sqlalchemy as sa from util.migrate import UTF8LongText - def upgrade(tables): ### commands auto generated by Alembic - please adjust! ### op.add_column('repositorynotification', sa.Column('event_config_json', UTF8LongText, nullable=False)) diff --git a/data/model/image.py b/data/model/image.py index 469f7606f..87ae77fc5 100644 --- a/data/model/image.py +++ b/data/model/image.py @@ -1,7 +1,7 @@ import logging import dateutil.parser -from peewee import JOIN_LEFT_OUTER, fn +from peewee import JOIN_LEFT_OUTER, fn, SQL from datetime import datetime from data.model import (DataModelException, db_transaction, _basequery, storage, @@ -13,6 +13,25 @@ from data.database import (Image, Repository, ImageStoragePlacement, Namespace, logger = logging.getLogger(__name__) +def get_repository_image_and_deriving(docker_image_id, storage_uuid): + """ Returns all matching images with the given docker image ID and storage uuid, along with any + images which have the image ID as parents. + """ + try: + image_found = (Image + .select() + .join(ImageStorage) + .where(Image.docker_image_id == docker_image_id, + ImageStorage.uuid == storage_uuid) + .get()) + except Image.DoesNotExist: + return Image.select().where(Image.id < 0) # Empty query + + ancestors_pattern = '%s%s/%%' % (image_found.ancestors, image_found.id) + return Image.select().where((Image.ancestors ** ancestors_pattern) | + (Image.id == image_found.id)) + + def get_parent_images(namespace_name, repository_name, image_obj): """ Returns a list of parent Image objects starting with the most recent parent and ending with the base layer. diff --git a/data/model/tag.py b/data/model/tag.py index cddc95ebd..1a7932347 100644 --- a/data/model/tag.py +++ b/data/model/tag.py @@ -13,6 +13,20 @@ def _tag_alive(query, now_ts=None): (RepositoryTag.lifetime_end_ts > now_ts)) +def get_matching_tags(docker_image_id, storage_uuid, *args): + """ Returns a query pointing to all tags that contain the image with the + given docker_image_id and storage_uuid. """ + image_query = image.get_repository_image_and_deriving(docker_image_id, storage_uuid) + + return (RepositoryTag + .select(*args) + .distinct() + .join(Image) + .join(ImageStorage) + .where(Image.id << image_query, RepositoryTag.lifetime_end_ts >> None, + RepositoryTag.hidden == False)) + + def list_repository_tags(namespace_name, repository_name, include_hidden=False, include_storage=False): to_select = (RepositoryTag, Image) diff --git a/endpoints/api/repositorynotification.py b/endpoints/api/repositorynotification.py index 30c71cf54..538bbe25e 100644 --- a/endpoints/api/repositorynotification.py +++ b/endpoints/api/repositorynotification.py @@ -22,12 +22,19 @@ def notification_view(note): except: config = {} + event_config = {} + try: + event_config = json.loads(note.event_config_json) + except: + event_config = {} + return { 'uuid': note.uuid, 'event': note.event.name, 'method': note.method.name, 'config': config, 'title': note.title, + 'event_config': event_config, } @@ -160,7 +167,7 @@ class TestRepositoryNotification(RepositoryParamResource): raise NotFound() event_info = NotificationEvent.get_event(test_note.event.name) - sample_data = event_info.get_sample_data(repository=test_note.repository) + sample_data = event_info.get_sample_data(test_note) notification_data = build_notification_data(test_note, sample_data) notification_queue.put([test_note.repository.namespace_user.username, repository, test_note.event.name], json.dumps(notification_data)) diff --git a/endpoints/api/secscan.py b/endpoints/api/secscan.py index ab3f73051..b4b3681fc 100644 --- a/endpoints/api/secscan.py +++ b/endpoints/api/secscan.py @@ -5,7 +5,7 @@ import features import json import requests -from app import secscan_endpoint +from app import secscan_api from data import model from endpoints.api import (require_repo_read, NotFound, DownstreamIssue, path_param, RepositoryParamResource, resource, nickname, show_if, parse_args, @@ -15,10 +15,17 @@ from endpoints.api import (require_repo_read, NotFound, DownstreamIssue, path_pa logger = logging.getLogger(__name__) +class SCAN_STATUS(object): + """ Security scan status enum """ + SCANNED = 'scanned' + FAILED = 'failed' + QUEUED = 'queued' + + def _call_security_api(relative_url, *args, **kwargs): """ Issues an HTTP call to the sec API at the given relative URL. """ try: - response = secscan_endpoint.call_api(relative_url, *args, **kwargs) + response = secscan_api.call(relative_url, None, *args, **kwargs) except requests.exceptions.Timeout: raise DownstreamIssue(payload=dict(message='API call timed out')) except requests.exceptions.ConnectionError: @@ -39,37 +46,44 @@ def _call_security_api(relative_url, *args, **kwargs): return response_data +def _get_status(repo_image): + if repo_image.security_indexed_engine is not None and repo_image.security_indexed_engine >= 0: + return SCAN_STATUS.SCANNED if repo_image.security_indexed else SCAN_STATUS.FAILED + + return SCAN_STATUS.QUEUED + + @show_if(features.SECURITY_SCANNER) -@resource('/v1/repository//tag//vulnerabilities') +@resource('/v1/repository//image//vulnerabilities') @path_param('repository', 'The full path of the repository. e.g. namespace/name') -@path_param('tag', 'The name of the tag') -class RepositoryTagVulnerabilities(RepositoryParamResource): - """ Operations for managing the vulnerabilities in a repository tag. """ +@path_param('imageid', 'The image ID') +class RepositoryImageVulnerabilities(RepositoryParamResource): + """ Operations for managing the vulnerabilities in a repository image. """ @require_repo_read - @nickname('getRepoTagVulnerabilities') + @nickname('getRepoImageVulnerabilities') @parse_args @query_param('minimumPriority', 'Minimum vulnerability priority', type=str, default='Low') - def get(self, args, namespace, repository, tag): + def get(self, args, namespace, repository, imageid): """ Fetches the vulnerabilities (if any) for a repository tag. """ - try: - tag_image = model.tag.get_tag_image(namespace, repository, tag) - except model.DataModelException: + repo_image = model.image.get_repo_image(namespace, repository, imageid) + if repo_image is None: raise NotFound() - if not tag_image.security_indexed: - logger.debug('Image %s for tag %s under repository %s/%s not security indexed', - tag_image.docker_image_id, tag, namespace, repository) + if not repo_image.security_indexed: + logger.debug('Image %s under repository %s/%s not security indexed', + repo_image.docker_image_id, namespace, repository) return { - 'security_indexed': False + 'status': _get_status(repo_image), } - data = _call_security_api('layers/%s/vulnerabilities', tag_image.docker_image_id, + layer_id = '%s.%s' % (repo_image.docker_image_id, repo_image.storage.uuid) + data = _call_security_api('layers/%s/vulnerabilities', layer_id, minimumPriority=args.minimumPriority) return { - 'security_indexed': True, + 'status': _get_status(repo_image), 'data': data, } @@ -91,13 +105,14 @@ class RepositoryImagePackages(RepositoryParamResource): if not repo_image.security_indexed: return { - 'security_indexed': False + 'status': _get_status(repo_image), } - data = _call_security_api('layers/%s/packages/diff', repo_image.docker_image_id) + layer_id = '%s.%s' % (repo_image.docker_image_id, repo_image.storage.uuid) + data = _call_security_api('layers/%s/packages', layer_id) return { - 'security_indexed': True, + 'status': _get_status(repo_image), 'data': data, } diff --git a/endpoints/common.py b/endpoints/common.py index 6e3abb4ac..16ce3eb5f 100644 --- a/endpoints/common.py +++ b/endpoints/common.py @@ -22,6 +22,7 @@ from werkzeug.routing import BaseConverter from functools import wraps from config import frontend_visible_config from external_libraries import get_external_javascript, get_external_css +from util.secscan.api import PRIORITY_LEVELS import features @@ -183,6 +184,7 @@ def render_page_template(name, **kwargs): config_set=json.dumps(frontend_visible_config(app.config)), oauth_set=json.dumps(get_oauth_config()), scope_set=json.dumps(scopes.app_scopes(app.config)), + vuln_priority_set=json.dumps(PRIORITY_LEVELS), mixpanel_key=app.config.get('MIXPANEL_KEY', ''), google_analytics_key=app.config.get('GOOGLE_ANALYTICS_KEY', ''), sentry_public_dsn=app.config.get('SENTRY_PUBLIC_DSN', ''), diff --git a/endpoints/notificationevent.py b/endpoints/notificationevent.py index ebd7e10b6..365b815d3 100644 --- a/endpoints/notificationevent.py +++ b/endpoints/notificationevent.py @@ -1,9 +1,11 @@ import logging import time +import json from datetime import datetime from notificationhelper import build_event_data from util.jinjautil import get_template_env +from util.secscan.api import PRIORITY_LEVELS, get_priority_for_index template_env = get_template_env("events") logger = logging.getLogger(__name__) @@ -37,13 +39,18 @@ class NotificationEvent(object): 'notification_data': notification_data }) - def get_sample_data(self, repository=None): + def get_sample_data(self, notification): """ - Returns sample data for testing the raising of this notification, with an optional - repository. + Returns sample data for testing the raising of this notification, with an example notification. """ raise NotImplementedError + def should_perform(self, event_data, notification_data): + """ + Whether a notification for this event should be performed. By default returns True. + """ + return True + @classmethod def event_name(cls): """ @@ -71,8 +78,8 @@ class RepoPushEvent(NotificationEvent): def get_summary(self, event_data, notification_data): return 'Repository %s updated' % (event_data['repository']) - def get_sample_data(self, repository): - return build_event_data(repository, { + def get_sample_data(self, notification): + return build_event_data(notification.repository, { 'updated_tags': {'latest': 'someimageid', 'foo': 'anotherimage'}, 'pruned_image_count': 3 }) @@ -99,18 +106,27 @@ class VulnerabilityFoundEvent(NotificationEvent): return 'info' - def get_sample_data(self, repository): - return build_event_data(repository, { + def get_sample_data(self, notification): + event_config = json.loads(notification.event_config_json) + + return build_event_data(notification.repository, { 'tags': ['latest', 'prod'], 'image': 'some-image-id', 'vulnerability': { 'id': 'CVE-FAKE-CVE', 'description': 'A futurist vulnerability', 'link': 'https://security-tracker.debian.org/tracker/CVE-FAKE-CVE', - 'priority': 'Critical', + 'priority': get_priority_for_index(event_config['level']) }, }) + def should_perform(self, event_data, notification_data): + event_config = json.loads(notification_data.event_config_json) + expected_level_index = event_config['level'] + priority = PRIORITY_LEVELS[event_data['vulnerability']['priority']] + actual_level_index = priority['index'] + return expected_level_index <= actual_level_index + def get_summary(self, event_data, notification_data): msg = '%s vulnerability detected in repository %s in tags %s' return msg % (event_data['vulnerability']['priority'], @@ -126,10 +142,10 @@ class BuildQueueEvent(NotificationEvent): def get_level(self, event_data, notification_data): return 'info' - def get_sample_data(self, repository): + def get_sample_data(self, notification): build_uuid = 'fake-build-id' - return build_event_data(repository, { + return build_event_data(notification.repository, { 'is_manual': False, 'build_id': build_uuid, 'build_name': 'some-fake-build', @@ -165,10 +181,10 @@ class BuildStartEvent(NotificationEvent): def get_level(self, event_data, notification_data): return 'info' - def get_sample_data(self, repository): + def get_sample_data(self, notification): build_uuid = 'fake-build-id' - return build_event_data(repository, { + return build_event_data(notification.repository, { 'build_id': build_uuid, 'build_name': 'some-fake-build', 'docker_tags': ['latest', 'foo', 'bar'], @@ -193,10 +209,10 @@ class BuildSuccessEvent(NotificationEvent): def get_level(self, event_data, notification_data): return 'success' - def get_sample_data(self, repository): + def get_sample_data(self, notification): build_uuid = 'fake-build-id' - return build_event_data(repository, { + return build_event_data(notification.repository, { 'build_id': build_uuid, 'build_name': 'some-fake-build', 'docker_tags': ['latest', 'foo', 'bar'], @@ -222,10 +238,10 @@ class BuildFailureEvent(NotificationEvent): def get_level(self, event_data, notification_data): return 'error' - def get_sample_data(self, repository): + def get_sample_data(self, notification): build_uuid = 'fake-build-id' - return build_event_data(repository, { + return build_event_data(notification.repository, { 'build_id': build_uuid, 'build_name': 'some-fake-build', 'docker_tags': ['latest', 'foo', 'bar'], diff --git a/endpoints/secscan.py b/endpoints/secscan.py new file mode 100644 index 000000000..4326ea621 --- /dev/null +++ b/endpoints/secscan.py @@ -0,0 +1,25 @@ +import logging +import json + +import features + +from app import secscan_notification_queue +from flask import request, make_response, Blueprint +from endpoints.common import route_show_if + +logger = logging.getLogger(__name__) +secscan = Blueprint('secscan', __name__) + +@route_show_if(features.SECURITY_SCANNER) +@secscan.route('/notification', methods=['POST']) +def secscan_notification(): + data = request.get_json() + logger.debug('Got notification from Clair: %s', data) + + content = data['Content'] + layer_ids = content.get('NewIntroducingLayersIDs', content.get('IntroducingLayersIDs', [])) + if not layer_ids: + return make_response('Okay') + + secscan_notification_queue.put(['notification', data['Name']], json.dumps(data)) + return make_response('Okay') diff --git a/endpoints/v1/registry.py b/endpoints/v1/registry.py index 925e8d3c4..d62cada5b 100644 --- a/endpoints/v1/registry.py +++ b/endpoints/v1/registry.py @@ -7,7 +7,7 @@ from functools import wraps from datetime import datetime from time import time -from app import storage as store, image_diff_queue, image_replication_queue, app +from app import storage as store, image_replication_queue, app from auth.auth import process_auth, extract_namespace_repo_from_session from auth.auth_context import get_authenticated_user, get_grant_user_context from digest import checksums @@ -41,20 +41,9 @@ def _finish_image(namespace, repository, repo_image): # Checksum is ok, we remove the marker set_uploading_flag(repo_image, False) - image_id = repo_image.docker_image_id - - # The layer is ready for download, send a job to the work queue to - # process it. - logger.debug('Adding layer to diff queue') - repo = model.repository.get_repository(namespace, repository) - image_diff_queue.put([repo.namespace_user.username, repository, image_id], json.dumps({ - 'namespace_user_id': repo.namespace_user.id, - 'repository': repository, - 'image_id': image_id, - })) - # Send a job to the work queue to replicate the image layer. if features.STORAGE_REPLICATION: + repo = model.repository.get_repository(namespace, repository) image_replication_queue.put([repo_image.storage.uuid], json.dumps({ 'namespace_user_id': repo.namespace_user.id, 'storage_id': repo_image.storage.uuid, diff --git a/pylintrc b/pylintrc index e1d21d338..7cecd0de7 100644 --- a/pylintrc +++ b/pylintrc @@ -9,7 +9,7 @@ # --enable=similarities". If you want to run only the classes checker, but have # no Warning level messages displayed, use"--disable=all --enable=classes # --disable=W" -disable=missing-docstring +disable=missing-docstring,invalid-name,too-many-locals [TYPECHECK] diff --git a/static/css/directives/repo-view/repo-panel-info.css b/static/css/directives/repo-view/repo-panel-info.css index 1b39b3a6b..032fd7d4b 100644 --- a/static/css/directives/repo-view/repo-panel-info.css +++ b/static/css/directives/repo-view/repo-panel-info.css @@ -3,10 +3,56 @@ float: right; } +.repo-panel-info-element .right-sec-controls { + border: 1px solid #ddd; + padding: 20px; + border-radius: 4px; + max-width: 400px; +} + +.repo-panel-info-element .right-sec-controls { + color: #333; + font-weight: 300; + padding-left: 70px; + position: relative; +} + +.repo-panel-info-element .right-sec-controls .sec-logo { + position: absolute; + top: 17px; + left: 15px; +} + +.repo-panel-info-element .right-sec-controls .sec-logo .lock { + position: absolute; + top: 5px; + right: 10px; +} + +.repo-panel-info-element .right-sec-controls b { + color: #333; + font-weight: normal; + margin-bottom: 20px; + display: block; +} + +.repo-panel-info-element .right-sec-controls .configure-alerts { + margin-top: 20px; + font-weight: normal; +} + +.repo-panel-info-element .right-sec-controls .configure-alerts .fa { + margin-right: 6px; +} + +.repo-panel-info-element .right-sec-controls .repository-events-summary { + margin-top: 20px; +} + .repo-panel-info-element .right-controls .copy-box { width: 400px; - display: inline-block; - margin-left: 10px; + margin-top: 10px; + margin-bottom: 20px; } .repo-panel-info-element .stat-col { diff --git a/static/css/directives/repo-view/repo-panel-tags.css b/static/css/directives/repo-view/repo-panel-tags.css index 267daf9e9..c0a1073c4 100644 --- a/static/css/directives/repo-view/repo-panel-tags.css +++ b/static/css/directives/repo-view/repo-panel-tags.css @@ -85,6 +85,37 @@ margin-right: 2px; } +.repo-panel-tags-element .security-scan-col span { + cursor: pointer; +} + +.repo-panel-tags-element .security-scan-col i.fa { + margin-right: 4px; +} + +.repo-panel-tags-element .security-scan-col .scanning, +.repo-panel-tags-element .security-scan-col .failed-scan, +.repo-panel-tags-element .security-scan-col .vuln-load-error { + color: #9B9B9B; + font-size: 12px; +} + +.repo-panel-tags-element .security-scan-col .no-vulns a { + color: #2FC98E; +} + +.repo-panel-tags-element .security-scan-col .vuln-link, +.repo-panel-tags-element .security-scan-col .vuln-link span { + text-decoration: none !important +} + +.repo-panel-tags-element .security-scan-col .has-vulns.Critical .highest-vuln, +.repo-panel-tags-element .security-scan-col .has-vulns.Defcon1 .highest-vuln { +} + +.repo-panel-tags-element .other-vulns { + color: black; +} @media (max-width: 767px) { .repo-panel-tags-element .tag-span { diff --git a/static/css/directives/ui/filter-box.css b/static/css/directives/ui/filter-box.css index 82e43c9c6..604930a0d 100644 --- a/static/css/directives/ui/filter-box.css +++ b/static/css/directives/ui/filter-box.css @@ -15,4 +15,35 @@ margin-right: 10px; margin-bottom: 10px; color: #ccc; +} + +.filter-box.floating { + float: right; + min-width: 300px; + margin-top: 15px; + position: relative; +} + +.filter-box.floating .filter-message { + position: absolute; + left: -200px; + top: 7px; +} + + +@media (max-width: 767px) { + .filter-box.floating { + float: none; + width: 100%; + display: block; + margin-top: 10px; + } + + .filter-box.floating .form-control { + max-width: 100%; + } + + .filter-box.floating .filter-message { + display: none; + } } \ No newline at end of file diff --git a/static/css/directives/ui/repository-events-summary.css b/static/css/directives/ui/repository-events-summary.css new file mode 100644 index 000000000..1ff4d60b2 --- /dev/null +++ b/static/css/directives/ui/repository-events-summary.css @@ -0,0 +1,21 @@ +.repository-events-summary-element .summary-list { + padding: 0px; + margin: 0px; + list-style: none; +} + +.repository-events-summary-element .summary-list li { + margin-bottom: 6px; +} + +.repository-events-summary-element .summary-list li i.fa { + margin-right: 4px; +} + +.repository-events-summary-element .notification-event-fields { + display: inline-block; + padding: 0px; + margin: 0px; + list-style: none; + padding-left: 24px; +} \ No newline at end of file diff --git a/static/css/directives/ui/repository-events-table.css b/static/css/directives/ui/repository-events-table.css index 82909d022..f471a997c 100644 --- a/static/css/directives/ui/repository-events-table.css +++ b/static/css/directives/ui/repository-events-table.css @@ -1,3 +1,13 @@ .repository-events-table-element .notification-row i.fa { margin-right: 6px; +} + +.repository-events-table-element .notification-event-fields { + list-style: none; + padding: 0px; + margin-left: 28px; + margin-top: 3px; + font-size: 13px; + color: #888; + margin-bottom: 0px; } \ No newline at end of file diff --git a/static/css/directives/ui/vulnerability-priority-view.css b/static/css/directives/ui/vulnerability-priority-view.css new file mode 100644 index 000000000..88dd4d27c --- /dev/null +++ b/static/css/directives/ui/vulnerability-priority-view.css @@ -0,0 +1,19 @@ +.vulnerability-priority-view-element i.fa { + margin-right: 4px; +} + +.vulnerability-priority-view-element.Unknown, +.vulnerability-priority-view-element.Low, +.vulnerability-priority-view-element.Negligible { + color: #9B9B9B; +} + +.vulnerability-priority-view-element.Medium { + color: #FCA657; +} + +.vulnerability-priority-view-element.High, +.vulnerability-priority-view-element.Critical, +.vulnerability-priority-view-element.Defcon1 { + color: #D64456; +} diff --git a/static/css/pages/image-view.css b/static/css/pages/image-view.css index f91ceacc9..a1cb55bab 100644 --- a/static/css/pages/image-view.css +++ b/static/css/pages/image-view.css @@ -21,5 +21,50 @@ } .image-view .co-tab-content h3 { + margin-bottom: 30px; +} + +.image-view .fa-bug { + margin-right: 4px; +} + +.image-view .co-filter-box { + float: right; + min-width: 300px; + margin-bottom: 10px; +} + +.image-view .co-filter-box .current-filtered { + display: inline-block; + margin-right: 10px; + color: #999; +} + +.image-view .co-filter-box input { + display: inline-block; +} + +.image-view .level-col h4 { + margin-top: 0px; margin-bottom: 20px; } + +.image-view .levels { + list-style: none; + padding: 0px; + margin: 0px; +} + +.image-view .levels li { + margin-bottom: 20px; +} + +.image-view .levels li .description { + margin-top: 6px; + font-size: 14px; + color: #999; +} + +.image-view .level-col { + padding: 20px; +} \ No newline at end of file diff --git a/static/directives/header-bar.html b/static/directives/header-bar.html index ddab29e3f..ebcb33c52 100644 --- a/static/directives/header-bar.html +++ b/static/directives/header-bar.html @@ -24,14 +24,16 @@
  • Tour
  • Tutorial
  • Pricing
    • -
  • Docs
    • +
  • Docs
    • +
  • Blog
    • diff --git a/static/directives/repo-view/repo-panel-info.html b/static/directives/repo-view/repo-panel-info.html index 5fdab5a4c..be8286968 100644 --- a/static/directives/repo-view/repo-panel-info.html +++ b/static/directives/repo-view/repo-panel-info.html @@ -65,7 +65,25 @@

    Description

    diff --git a/static/directives/repo-view/repo-panel-tags.html b/static/directives/repo-view/repo-panel-tags.html index e1d94db67..567abce3a 100644 --- a/static/directives/repo-view/repo-panel-tags.html +++ b/static/directives/repo-view/repo-panel-tags.html @@ -81,6 +81,12 @@ style="min-width: 120px;"> Last Modified + + Security Scan + @@ -107,6 +113,61 @@ Unknown + + + + Could not load security information + + + + Queued for scan + + + + + Failed to scan + + + + + + + Passed + + + + + + + + + {{ getTagVulnerabilities(tag).highestVulnerability.Count }} + + + + + + {{ getTagVulnerabilities(tag).vulnerabilities.length - getTagVulnerabilities(tag).highestVulnerability.Count }} others + + + + More Info + + + + diff --git a/static/directives/repository-events-summary.html b/static/directives/repository-events-summary.html new file mode 100644 index 000000000..d5f5c45f2 --- /dev/null +++ b/static/directives/repository-events-summary.html @@ -0,0 +1,22 @@ +
    +
    +
      +
    • + + {{ getMethodInfo(notification).title }} for + +
        +
      • + {{ field.title }} of + + + {{ findEnumValue(field.values, notification.event_config[field.name]).title }} + + +
        • +
      +
      • +
    +
    +
    \ No newline at end of file diff --git a/static/directives/repository-events-table.html b/static/directives/repository-events-table.html index 3a463f54c..fd40d9789 100644 --- a/static/directives/repository-events-table.html +++ b/static/directives/repository-events-table.html @@ -44,6 +44,17 @@ {{ getEventInfo(notification).title }} + +
      +
    • + {{ field.title }}: + + + {{ findEnumValue(field.values, notification.event_config[field.name]).title }} + + +
      • +
    @@ -89,5 +100,6 @@
    diff --git a/static/directives/vulnerability-priority-view.html b/static/directives/vulnerability-priority-view.html new file mode 100644 index 000000000..23bcce345 --- /dev/null +++ b/static/directives/vulnerability-priority-view.html @@ -0,0 +1,5 @@ + + + + {{ priority }} + \ No newline at end of file diff --git a/static/img/lock.svg b/static/img/lock.svg new file mode 100644 index 000000000..748961f52 --- /dev/null +++ b/static/img/lock.svg @@ -0,0 +1,17 @@ + + + + Artboard 1 + Created with Sketch. + + + + + + + + + + + + \ No newline at end of file diff --git a/static/img/scan.svg b/static/img/scan.svg new file mode 100644 index 000000000..91ea19e43 --- /dev/null +++ b/static/img/scan.svg @@ -0,0 +1,14 @@ + + + + + + + + + + + diff --git a/static/js/directives/repo-view/repo-panel-info.js b/static/js/directives/repo-view/repo-panel-info.js index 2902699ec..f11894732 100644 --- a/static/js/directives/repo-view/repo-panel-info.js +++ b/static/js/directives/repo-view/repo-panel-info.js @@ -10,7 +10,8 @@ angular.module('quay').directive('repoPanelInfo', function () { restrict: 'C', scope: { 'repository': '=repository', - 'builds': '=builds' + 'builds': '=builds', + 'isEnabled': '=isEnabled' }, controller: function($scope, $element, ApiService, Config) { $scope.$watch('repository', function(repository) { diff --git a/static/js/directives/repo-view/repo-panel-tags.js b/static/js/directives/repo-view/repo-panel-tags.js index e7618bb35..582ed3cdc 100644 --- a/static/js/directives/repo-view/repo-panel-tags.js +++ b/static/js/directives/repo-view/repo-panel-tags.js @@ -18,7 +18,7 @@ angular.module('quay').directive('repoPanelTags', function () { 'getImages': '&getImages' }, - controller: function($scope, $element, $filter, $location, ApiService, UIService) { + controller: function($scope, $element, $filter, $location, ApiService, UIService, VulnerabilityService) { var orderBy = $filter('orderBy'); $scope.checkedTags = UIService.createCheckStateController([], 'name'); @@ -34,7 +34,8 @@ angular.module('quay').directive('repoPanelTags', function () { $scope.tagHistory = {}; $scope.tagActionHandler = null; $scope.showingHistory = false; - $scope.tagsPerPage = 50; + $scope.tagsPerPage = 25; + $scope.imageVulnerabilities = {}; var setTagState = function() { if (!$scope.repository || !$scope.selectedTags) { return; } @@ -149,6 +150,68 @@ angular.module('quay').directive('repoPanelTags', function () { setTagState(); }); + $scope.loadImageVulnerabilities = function(image_id, imageData) { + var params = { + 'imageid': image_id, + 'repository': $scope.repository.namespace + '/' + $scope.repository.name, + }; + + ApiService.getRepoImageVulnerabilities(null, params).then(function(resp) { + imageData.loading = false; + imageData.status = resp['status']; + + if (imageData.status == 'scanned') { + var vulnerabilities = resp.data.Vulnerabilities; + + imageData.hasVulnerabilities = !!vulnerabilities.length; + imageData.vulnerabilities = vulnerabilities; + + var highest = { + 'Priority': 'Unknown', + 'Count': 0, + 'index': 100000 + }; + + resp.data.Vulnerabilities.forEach(function(v) { + if (VulnerabilityService.LEVELS[v.Priority].index < highest.index) { + highest = { + 'Priority': v.Priority, + 'Count': 1, + 'index': VulnerabilityService.LEVELS[v.Priority].index + } + } else if (VulnerabilityService.LEVELS[v.Priority].index == highest.index) { + highest['Count']++; + } + }); + + imageData.highestVulnerability = highest; + } + }, function() { + imageData.loading = false; + imageData.hasError = true; + }); + }; + + $scope.getTagVulnerabilities = function(tag) { + return $scope.getImageVulnerabilities(tag.image_id); + }; + + $scope.getImageVulnerabilities = function(image_id) { + if (!$scope.repository) { + return + } + + if (!$scope.imageVulnerabilities[image_id]) { + $scope.imageVulnerabilities[image_id] = { + 'loading': true + }; + + $scope.loadImageVulnerabilities(image_id, $scope.imageVulnerabilities[image_id]); + } + + return $scope.imageVulnerabilities[image_id]; + }; + $scope.clearSelectedTags = function() { $scope.checkedTags.setChecked([]); }; diff --git a/static/js/directives/ui/create-external-notification-dialog.js b/static/js/directives/ui/create-external-notification-dialog.js index cbf8696e3..874769216 100644 --- a/static/js/directives/ui/create-external-notification-dialog.js +++ b/static/js/directives/ui/create-external-notification-dialog.js @@ -11,7 +11,8 @@ angular.module('quay').directive('createExternalNotificationDialog', function () scope: { 'repository': '=repository', 'counter': '=counter', - 'notificationCreated': '¬ificationCreated' + 'notificationCreated': '¬ificationCreated', + 'defaultData': '=defaultData' }, controller: function($scope, $element, ExternalNotificationData, ApiService, $timeout, StringBuilderService) { $scope.currentEvent = null; @@ -98,6 +99,13 @@ angular.module('quay').directive('createExternalNotificationDialog', function () ApiService.createRepoNotification(data, params).then(function(resp) { $scope.status = ''; $scope.notificationCreated({'notification': resp}); + + // Used by repository-events-summary. + if (!$scope.repository._notificationCounter) { + $scope.repository._notificationCounter = 0; + } + + $scope.repository._notificationCounter++; $('#createNotificationModal').modal('hide'); }); }; @@ -154,6 +162,13 @@ angular.module('quay').directive('createExternalNotificationDialog', function () $scope.currentEvent = null; $scope.currentMethod = null; $scope.unauthorizedEmail = false; + + $timeout(function() { + if ($scope.defaultData && $scope.defaultData['currentEvent']) { + $scope.setEvent($scope.defaultData['currentEvent']); + } + }, 100); + $('#createNotificationModal').modal({}); } }); diff --git a/static/js/directives/ui/repository-events-summary.js b/static/js/directives/ui/repository-events-summary.js new file mode 100644 index 000000000..50d667334 --- /dev/null +++ b/static/js/directives/ui/repository-events-summary.js @@ -0,0 +1,77 @@ +/** + * An element which displays a summary of events on a repository of a particular type. + */ +angular.module('quay').directive('repositoryEventsSummary', function () { + var directiveDefinitionObject = { + priority: 0, + templateUrl: '/static/directives/repository-events-summary.html', + replace: false, + transclude: false, + restrict: 'C', + scope: { + 'repository': '=repository', + 'isEnabled': '=isEnabled', + 'eventFilter': '@eventFilter', + 'hasEvents': '=hasEvents' + }, + controller: function($scope, ApiService, ExternalNotificationData) { + var loadNotifications = function() { + if (!$scope.repository || !$scope.isEnabled || !$scope.eventFilter || $scope.notificationsResource) { + return; + } + + var params = { + 'repository': $scope.repository.namespace + '/' + $scope.repository.name + }; + + $scope.notificationsResource = ApiService.listRepoNotificationsAsResource(params).get( + function(resp) { + var notifications = []; + resp.notifications.forEach(function(notification) { + if (notification.event == $scope.eventFilter) { + notifications.push(notification); + } + }); + + $scope.notifications = notifications; + $scope.hasEvents = !!notifications.length; + return $scope.notifications; + }); + }; + + $scope.$watch('repository', loadNotifications); + $scope.$watch('isEnabled', loadNotifications); + $scope.$watch('eventFilter', loadNotifications); + + // Watch _notificationCounter, which is set by create-external-notification-dialog. We use this + // to invalidate and reload. + $scope.$watch('repository._notificationCounter', function() { + $scope.notificationsResource = null; + loadNotifications(); + }); + + loadNotifications(); + + $scope.findEnumValue = function(values, index) { + var found = null; + Object.keys(values).forEach(function(key) { + if (values[key]['index'] == index) { + found = values[key]; + return + } + }); + + return found + }; + + $scope.getEventInfo = function(notification) { + return ExternalNotificationData.getEventInfo(notification.event); + }; + + $scope.getMethodInfo = function(notification) { + return ExternalNotificationData.getMethodInfo(notification.method); + }; + } + }; + return directiveDefinitionObject; +}); \ No newline at end of file diff --git a/static/js/directives/ui/repository-events-table.js b/static/js/directives/ui/repository-events-table.js index 05b3e3226..8aca804b6 100644 --- a/static/js/directives/ui/repository-events-table.js +++ b/static/js/directives/ui/repository-events-table.js @@ -13,11 +13,29 @@ angular.module('quay').directive('repositoryEventsTable', function () { 'repository': '=repository', 'isEnabled': '=isEnabled' }, - controller: function($scope, $element, ApiService, Restangular, UtilService, ExternalNotificationData) { + controller: function($scope, $element, $timeout, ApiService, Restangular, UtilService, ExternalNotificationData, $location) { $scope.showNewNotificationCounter = 0; + $scope.newNotificationData = {}; var loadNotifications = function() { - if (!$scope.repository || $scope.notificationsResource || !$scope.isEnabled) { return; } + if (!$scope.repository || !$scope.isEnabled) { return; } + + var add_event = $location.search()['add_event']; + if (add_event) { + $timeout(function() { + $scope.newNotificationData = { + 'currentEvent': ExternalNotificationData.getEventInfo(add_event) + }; + + $scope.askCreateNotification(); + }, 100); + + $location.search('add_event', null); + } + + if ($scope.notificationsResource) { + return; + } var params = { 'repository': $scope.repository.namespace + '/' + $scope.repository.name @@ -43,6 +61,18 @@ angular.module('quay').directive('repositoryEventsTable', function () { $scope.showNewNotificationCounter++; }; + $scope.findEnumValue = function(values, index) { + var found = null; + Object.keys(values).forEach(function(key) { + if (values[key]['index'] == index) { + found = values[key]; + return + } + }); + + return found + }; + $scope.getEventInfo = function(notification) { return ExternalNotificationData.getEventInfo(notification.event); }; @@ -61,6 +91,13 @@ angular.module('quay').directive('repositoryEventsTable', function () { var index = $.inArray(notification, $scope.notifications); if (index < 0) { return; } $scope.notifications.splice(index, 1); + + if (!$scope.repository._notificationCounter) { + $scope.repository._notificationCounter = 0; + } + + $scope.repository._notificationCounter++; + }, ApiService.errorDisplay('Cannot delete notification')); }; diff --git a/static/js/directives/ui/vulnerability-priority-view.js b/static/js/directives/ui/vulnerability-priority-view.js new file mode 100644 index 000000000..a257218f5 --- /dev/null +++ b/static/js/directives/ui/vulnerability-priority-view.js @@ -0,0 +1,18 @@ +/** + * An element which displays a priority triangle for vulnerabilities. + */ +angular.module('quay').directive('vulnerabilityPriorityView', function () { + var directiveDefinitionObject = { + priority: 0, + templateUrl: '/static/directives/vulnerability-priority-view.html', + replace: false, + transclude: true, + restrict: 'C', + scope: { + 'priority': '=priority' + }, + controller: function($scope, $element) { + } + }; + return directiveDefinitionObject; +}); \ No newline at end of file diff --git a/static/js/pages/image-view.js b/static/js/pages/image-view.js index d8d3adc22..2783c49a6 100644 --- a/static/js/pages/image-view.js +++ b/static/js/pages/image-view.js @@ -10,11 +10,16 @@ }) }]); - function ImageViewCtrl($scope, $routeParams, $rootScope, $timeout, ApiService, ImageMetadataService) { + function ImageViewCtrl($scope, $routeParams, $rootScope, $timeout, ApiService, ImageMetadataService, VulnerabilityService, Features) { var namespace = $routeParams.namespace; var name = $routeParams.name; var imageid = $routeParams.image; + $scope.options = { + 'vulnFilter': '', + 'packageFilter': '' + }; + var loadImage = function() { var params = { 'repository': namespace + '/' + name, @@ -40,6 +45,46 @@ loadImage(); loadRepository(); + $scope.downloadPackages = function() { + if (!Features.SECURITY_SCANNER || $scope.packagesResource) { return; } + + var params = { + 'repository': namespace + '/' + name, + 'imageid': imageid + }; + + $scope.packagesResource = ApiService.getRepoImagePackagesAsResource(params).get(function(packages) { + $scope.packages = packages; + return packages; + }); + }; + + $scope.loadImageVulnerabilities = function() { + if (!Features.SECURITY_SCANNER || $scope.vulnerabilitiesResource) { return; } + + $scope.VulnerabilityLevels = VulnerabilityService.getLevels(); + + var params = { + 'repository': namespace + '/' + name, + 'imageid': imageid + }; + + $scope.vulnerabilitiesResource = ApiService.getRepoImageVulnerabilitiesAsResource(params).get(function(resp) { + $scope.vulnerabilityInfo = resp; + $scope.vulnerabilities = []; + + if (resp.data && resp.data.Vulnerabilities) { + resp.data.Vulnerabilities.forEach(function(vuln) { + vuln_copy = jQuery.extend({}, vuln); + vuln_copy['index'] = VulnerabilityService.LEVELS[vuln['Priority']]['index']; + $scope.vulnerabilities.push(vuln_copy); + }); + } + + return resp; + }); + }; + $scope.downloadChanges = function() { if ($scope.changesResource) { return; } diff --git a/static/js/pages/repo-view.js b/static/js/pages/repo-view.js index c5ae7f093..e7088d399 100644 --- a/static/js/pages/repo-view.js +++ b/static/js/pages/repo-view.js @@ -17,6 +17,7 @@ var imageLoader = ImageLoaderService.getLoader($scope.namespace, $scope.name); // Tab-enabled counters. + $scope.infoShown = 0; $scope.tagsShown = 0; $scope.logsShown = 0; $scope.buildsShown = 0; @@ -119,6 +120,10 @@ $scope.viewScope.selectedTags = $.unique(tagNames.split(',')); }; + $scope.showInfo = function() { + $scope.infoShown++; + }; + $scope.showBuilds = function() { $scope.buildsShown++; }; diff --git a/static/js/services/external-notification-data.js b/static/js/services/external-notification-data.js index 14de1d24e..b9356f671 100644 --- a/static/js/services/external-notification-data.js +++ b/static/js/services/external-notification-data.js @@ -47,12 +47,12 @@ function(Config, Features, VulnerabilityService) { events.push({ 'id': 'vulnerability_found', 'title': 'Package Vulnerability Found', - 'icon': 'fa-flag', + 'icon': 'fa-bug', 'fields': [ { 'name': 'level', 'type': 'enum', - 'title': 'Minimum Severity Level', + 'title': 'Minimum Priority Level', 'values': VulnerabilityService.LEVELS, } ] diff --git a/static/js/services/notification-service.js b/static/js/services/notification-service.js index bd4c8b70c..31ff5af2c 100644 --- a/static/js/services/notification-service.js +++ b/static/js/services/notification-service.js @@ -129,7 +129,8 @@ function($rootScope, $interval, UserService, ApiService, StringBuilderService, P 'message': 'A {vulnerability.priority} vulnerability was detected in repository {repository}', 'page': function(metadata) { return '/repository/' + metadata.repository + '?tab=tags'; - } + }, + 'dismissable': true } }; diff --git a/static/js/services/vulnerability-service.js b/static/js/services/vulnerability-service.js index 752861adb..12c1a172f 100644 --- a/static/js/services/vulnerability-service.js +++ b/static/js/services/vulnerability-service.js @@ -3,89 +3,7 @@ */ angular.module('quay').factory('VulnerabilityService', ['Config', function(Config) { var vulnService = {}; - - // NOTE: This objects are used directly in the external-notification-data service, so make sure - // to update that code if the format here is changed. - vulnService.LEVELS = { - 'Unknown': { - 'title': 'Unknown', - 'index': '6', - 'level': 'info', - - 'description': 'Unknown is either a security problem that has not been assigned ' + - 'to a priority yet or a priority that our system did not recognize', - 'banner_required': false - }, - - 'Negligible': { - 'title': 'Negligible', - 'index': '5', - 'level': 'info', - - 'description': 'Negligible is technically a security problem, but is only theoretical ' + - 'in nature, requires a very special situation, has almost no install base, ' + - 'or does no real damage.', - 'banner_required': false - }, - - 'Low': { - 'title': 'Low', - 'index': '4', - 'level': 'warning', - - 'description': 'Low is a security problem, but is hard to exploit due to environment, ' + - 'requires a user-assisted attack, a small install base, or does very ' + - 'little damage.', - 'banner_required': false - }, - - 'Medium': { - 'title': 'Medium', - 'value': 'Medium', - 'index': '3', - 'level': 'warning', - - 'description': 'Medium is a real security problem, and is exploitable for many people. ' + - 'Includes network daemon denial of service attacks, cross-site scripting, ' + - 'and gaining user privileges.', - 'banner_required': false - }, - - 'High': { - 'title': 'High', - 'value': 'High', - 'index': '2', - 'level': 'warning', - - 'description': 'High is a real problem, exploitable for many people in a default installation. ' + - 'Includes serious remote denial of services, local root privilege escalations, ' + - 'or data loss.', - 'banner_required': false - }, - - 'Critical': { - 'title': 'Critical', - 'value': 'Critical', - 'index': '1', - 'level': 'error', - - 'description': 'Critical is a world-burning problem, exploitable for nearly all people in ' + - 'a installation of the package. Includes remote root privilege escalations, ' + - 'or massive data loss.', - 'banner_required': true - }, - - 'Defcon1': { - 'title': 'Defcon 1', - 'value': 'Defcon1', - 'index': '0', - 'level': 'error', - - 'description': 'Defcon1 is a Critical problem which has been manually highlighted ' + - 'by the Quay team. It requires immediate attention.', - 'banner_required': true - } - }; + vulnService.LEVELS = window.__vuln_priority; vulnService.getLevels = function() { return Object.keys(vulnService.LEVELS).map(function(key) { diff --git a/static/partials/image-view.html b/static/partials/image-view.html index 181938afc..0693097cb 100644 --- a/static/partials/image-view.html +++ b/static/partials/image-view.html @@ -25,6 +25,16 @@ tab-init="downloadChanges()"> + + + + + +
    @@ -53,6 +63,118 @@
    + + +
    +
    +
    +
    + +

    Image Security

    +
    +
    This image has not been indexed yet
    +
    + Please try again in a few minutes. +
    +
    + +
    +
    This image could not be indexed
    +
    + Our security scanner was unable to index this image. +
    +
    + +
    +
    This image contains no recognized security vulnerabilities
    +
    + Quay currently indexes Debian, Red Hat and Ubuntu packages. +
    +
    + +
    + + + + + + + + + + + +
    VulnerabilityPriorityDescription
    {{ vulnerability.ID }} + + {{ vulnerability.Description }}
    + +
    +
    No matching vulnerabilities found
    +
    + Please adjust your filter above. +
    +
    +
    +
    + + +
    +
    + + +
    +
    +
    + +

    Image Packages

    +
    +
    This image has not been indexed yet
    +
    + Please try again in a few minutes. +
    +
    + +
    +
    This image could not be indexed
    +
    + Our security scanner was unable to index this image. +
    +
    + + + + + + + + + + + + + +
    Package NamePackage VersionOS
    {{ package.Name }}{{ package.Version }}{{ package.OS }}
    + +
    +
    No matching packages found
    +
    + Please adjust your filter above. +
    +
    +
    +
    diff --git a/static/partials/repo-view.html b/static/partials/repo-view.html index cd3cbca2b..e16ba85ed 100644 --- a/static/partials/repo-view.html +++ b/static/partials/repo-view.html @@ -17,7 +17,8 @@
    - + @@ -56,7 +57,8 @@
    + builds="viewScope.builds" + is-enabled="infoShown">
    diff --git a/templates/base.html b/templates/base.html index c3f09c8ce..1eca42bde 100644 --- a/templates/base.html +++ b/templates/base.html @@ -38,6 +38,7 @@ window.__config = {{ config_set|safe }}; window.__oauth = {{ oauth_set|safe }}; window.__auth_scopes = {{ scope_set|safe }}; + window.__vuln_priority = {{ vuln_priority_set|safe }} window.__token = '{{ csrf_token() }}'; @@ -106,7 +107,6 @@ mixpanel.init("{{ mixpanel_key }}", { track_pageview : false, debug: {{ is_debug
    • ©2015 CoreOS, Inc.
      • -
    • Blog
    • Terms
    • Privacy
    • Security
      • diff --git a/test/data/test.db b/test/data/test.db index 6b2b75174..33cbdddfe 100644 Binary files a/test/data/test.db and b/test/data/test.db differ diff --git a/test/test_api_security.py b/test/test_api_security.py index 11b33f71a..a50a21bda 100644 --- a/test/test_api_security.py +++ b/test/test_api_security.py @@ -49,8 +49,7 @@ from endpoints.api.superuser import (SuperUserLogs, SuperUserList, SuperUserMana SuperUserSendRecoveryEmail, ChangeLog, SuperUserOrganizationManagement, SuperUserOrganizationList, SuperUserAggregateLogs) - -from endpoints.api.secscan import RepositoryImagePackages, RepositoryTagVulnerabilities +from endpoints.api.secscan import RepositoryImagePackages, RepositoryImageVulnerabilities try: @@ -4225,10 +4224,10 @@ class TestOrganizationInvoiceField(ApiTestCase): self._run_test('DELETE', 201, 'devtable', None) -class TestRepositoryTagVulnerabilities(ApiTestCase): +class TestRepositoryImageVulnerabilities(ApiTestCase): def setUp(self): ApiTestCase.setUp(self) - self._set_url(RepositoryTagVulnerabilities, repository='devtable/simple', tag='latest') + self._set_url(RepositoryImageVulnerabilities, repository='devtable/simple', imageid='fake') def test_get_anonymous(self): self._run_test('GET', 401, None, None) @@ -4240,7 +4239,7 @@ class TestRepositoryTagVulnerabilities(ApiTestCase): self._run_test('GET', 403, 'reader', None) def test_get_devtable(self): - self._run_test('GET', 200, 'devtable', None) + self._run_test('GET', 404, 'devtable', None) class TestRepositoryImagePackages(ApiTestCase): diff --git a/util/migrate/allocator.py b/util/migrate/allocator.py index 6e531d251..e154c9e23 100644 --- a/util/migrate/allocator.py +++ b/util/migrate/allocator.py @@ -126,6 +126,7 @@ def yield_random_entries(batch_query, primary_key_field, batch_size, max_id): an "id" field which can be inspected. """ + max_id = max(max_id, 1) allocator = CompletedKeys(max_id + 1) try: diff --git a/util/secscan/api.py b/util/secscan/api.py new file mode 100644 index 000000000..5041ec8ff --- /dev/null +++ b/util/secscan/api.py @@ -0,0 +1,224 @@ +import features +import logging +import requests + +from data.database import CloseForLongOperation +from urlparse import urljoin + +logger = logging.getLogger(__name__) + +# NOTE: This objects are used directly in the external-notification-data and vulnerability-service +# on the frontend, so be careful with changing their existing keys. +PRIORITY_LEVELS = { + 'Unknown': { + 'title': 'Unknown', + 'index': '6', + 'level': 'info', + + 'description': 'Unknown is either a security problem that has not been assigned ' + + 'to a priority yet or a priority that our system did not recognize', + 'banner_required': False + }, + + 'Negligible': { + 'title': 'Negligible', + 'index': '5', + 'level': 'info', + + 'description': 'Negligible is technically a security problem, but is only theoretical ' + + 'in nature, requires a very special situation, has almost no install base, ' + + 'or does no real damage.', + 'banner_required': False + }, + + 'Low': { + 'title': 'Low', + 'index': '4', + 'level': 'warning', + + 'description': 'Low is a security problem, but is hard to exploit due to environment, ' + + 'requires a user-assisted attack, a small install base, or does very ' + + 'little damage.', + 'banner_required': False + }, + + 'Medium': { + 'title': 'Medium', + 'value': 'Medium', + 'index': '3', + 'level': 'warning', + + 'description': 'Medium is a real security problem, and is exploitable for many people. ' + + 'Includes network daemon denial of service attacks, cross-site scripting, ' + + 'and gaining user privileges.', + 'banner_required': False + }, + + 'High': { + 'title': 'High', + 'value': 'High', + 'index': '2', + 'level': 'warning', + + 'description': 'High is a real problem, exploitable for many people in a default installation. ' + + 'Includes serious remote denial of services, local root privilege escalations, ' + + 'or data loss.', + 'banner_required': False + }, + + 'Critical': { + 'title': 'Critical', + 'value': 'Critical', + 'index': '1', + 'level': 'error', + + 'description': 'Critical is a world-burning problem, exploitable for nearly all people in ' + + 'a installation of the package. Includes remote root privilege escalations, ' + + 'or massive data loss.', + 'banner_required': True + }, + + 'Defcon1': { + 'title': 'Defcon 1', + 'value': 'Defcon1', + 'index': '0', + 'level': 'error', + + 'description': 'Defcon1 is a Critical problem which has been manually highlighted ' + + 'by the Quay team. It requires immediate attention.', + 'banner_required': True + } +} + + +def get_priority_for_index(index): + for priority in PRIORITY_LEVELS: + if PRIORITY_LEVELS[priority]['index'] == index: + return priority + + return 'Unknown' + +class SecurityConfigValidator(object): + def __init__(self, app, config_provider): + self._config_provider = config_provider + + if not features.SECURITY_SCANNER: + return + + self._security_config = app.config['SECURITY_SCANNER'] + if self._security_config is None: + return + + self._certificate = self._get_filepath('CA_CERTIFICATE_FILENAME') or False + self._public_key = self._get_filepath('PUBLIC_KEY_FILENAME') + self._private_key = self._get_filepath('PRIVATE_KEY_FILENAME') + + if self._public_key and self._private_key: + self._keys = (self._public_key, self._private_key) + else: + self._keys = None + + def _get_filepath(self, key): + config = self._security_config + + if key in config: + with self._config_provider.get_volume_file(config[key]) as f: + return f.name + + return None + + def cert(self): + return self._certificate + + def keypair(self): + return self._keys + + def valid(self): + if not features.SECURITY_SCANNER: + return False + + if not self._security_config: + logger.debug('Missing SECURITY_SCANNER block in configuration') + return False + + if not 'ENDPOINT' in self._security_config: + logger.debug('Missing ENDPOINT field in SECURITY_SCANNER configuration') + return False + + endpoint = self._security_config['ENDPOINT'] or '' + if not endpoint.startswith('http://') and not endpoint.startswith('https://'): + logger.debug('ENDPOINT field in SECURITY_SCANNER configuration must start with http or https') + return False + + if endpoint.startswith('https://') and (self._certificate is False or self._keys is None): + logger.debug('Certificate and key pair required for talking to security worker over HTTPS') + return False + + return True + + +class SecurityScannerAPI(object): + """ Helper class for talking to the Security Scan service (Clair). """ + def __init__(self, app, config_provider): + self.app = app + self.config_provider = config_provider + self._security_config = None + + config_validator = SecurityConfigValidator(app, config_provider) + if not config_validator.valid(): + logger.warning('Invalid config provided to SecurityScannerAPI') + return + + self._security_config = app.config.get('SECURITY_SCANNER') + self._certificate = config_validator.cert() + self._keys = config_validator.keypair() + + def check_layer_vulnerable(self, layer_id, cve_id): + """ Checks with Clair whether the given layer is vulnerable to the given CVE. """ + try: + body = { + 'LayersIDs': [layer_id] + } + response = self.call('vulnerabilities/%s/affected-layers', body, cve_id) + except requests.exceptions.RequestException: + logger.exception('Got exception when trying to call Clair endpoint') + return False + + if response.status_code != 200: + return False + + try: + response_data = response.json() + except ValueError: + logger.exception('Got exception when trying to parse Clair response') + return False + + if (not layer_id in response_data or + not response_data[layer_id].get('Vulnerable', False)): + return False + + return True + + def call(self, relative_url, body=None, *args, **kwargs): + """ Issues an HTTP call to the sec API at the given relative URL. + This function disconnects from the database while awaiting a response + from the API server. + """ + security_config = self._security_config + if security_config is None: + raise Exception('Cannot call unconfigured security system') + + api_url = urljoin(security_config['ENDPOINT'], '/' + security_config['API_VERSION']) + '/' + url = urljoin(api_url, relative_url % args) + + client = self.app.config['HTTPCLIENT'] + timeout = security_config.get('API_TIMEOUT_SECONDS', 1) + logger.debug('Looking up sec information: %s', url) + + with CloseForLongOperation(self.app.config): + if body is not None: + return client.post(url, json=body, params=kwargs, timeout=timeout, cert=self._keys, + verify=self._certificate) + else: + return client.get(url, params=kwargs, timeout=timeout, cert=self._keys, + verify=self._certificate) diff --git a/util/secscan/secscanendpoint.py b/util/secscan/secscanendpoint.py deleted file mode 100644 index 7f759219b..000000000 --- a/util/secscan/secscanendpoint.py +++ /dev/null @@ -1,50 +0,0 @@ -import features -import logging -import requests -import json - -from urlparse import urljoin - -logger = logging.getLogger(__name__) - -class SecurityScanEndpoint(object): - """ Helper class for talking to the Security Scan service (Clair). """ - def __init__(self, app, config_provider): - self.app = app - self.config_provider = config_provider - - if not features.SECURITY_SCANNER: - return - - self.security_config = app.config['SECURITY_SCANNER'] - - self.certificate = self._getfilepath('CA_CERTIFICATE_FILENAME') or False - self.public_key = self._getfilepath('PUBLIC_KEY_FILENAME') - self.private_key = self._getfilepath('PRIVATE_KEY_FILENAME') - - if self.public_key and self.private_key: - self.keys = (self.public_key, self.private_key) - else: - self.keys = None - - def _getfilepath(self, config_key): - security_config = self.security_config - - if config_key in security_config: - with self.config_provider.get_volume_file(security_config[config_key]) as f: - return f.name - - return None - - def call_api(self, relative_url, *args, **kwargs): - """ Issues an HTTP call to the sec API at the given relative URL. """ - security_config = self.security_config - api_url = urljoin(security_config['ENDPOINT'], '/' + security_config['API_VERSION']) + '/' - url = urljoin(api_url, relative_url % args) - - client = self.app.config['HTTPCLIENT'] - timeout = security_config.get('API_TIMEOUT_SECONDS', 1) - logger.debug('Looking up sec information: %s', url) - - return client.get(url, params=kwargs, timeout=timeout, cert=self.keys, - verify=self.certificate) \ No newline at end of file diff --git a/web.py b/web.py index 96457d5c9..445c2fa5b 100644 --- a/web.py +++ b/web.py @@ -11,6 +11,7 @@ from endpoints.oauthlogin import oauthlogin from endpoints.githubtrigger import githubtrigger from endpoints.gitlabtrigger import gitlabtrigger from endpoints.bitbuckettrigger import bitbuckettrigger +from endpoints.secscan import secscan if os.environ.get('DEBUGLOG') == 'true': logging.config.fileConfig('conf/logging_debug.conf', disable_existing_loggers=False) @@ -23,3 +24,4 @@ application.register_blueprint(bitbuckettrigger, url_prefix='/oauth1') application.register_blueprint(api_bp, url_prefix='/api') application.register_blueprint(webhooks, url_prefix='/webhooks') application.register_blueprint(realtime, url_prefix='/realtime') +application.register_blueprint(secscan, url_prefix='/secscan') diff --git a/workers/notificationworker.py b/workers/notificationworker.py index 3d3e2f80f..9c747303f 100644 --- a/workers/notificationworker.py +++ b/workers/notificationworker.py @@ -34,7 +34,8 @@ class NotificationWorker(QueueWorker): logger.exception('Cannot find notification event: %s', ex.message) raise JobException('Cannot find notification event: %s' % ex.message) - method_handler.perform(notification, event_handler, job_details) + if event_handler.should_perform(job_details['event_data'], notification): + method_handler.perform(notification, event_handler, job_details) if __name__ == "__main__": diff --git a/workers/security_notification_worker.py b/workers/security_notification_worker.py new file mode 100644 index 000000000..e54847abf --- /dev/null +++ b/workers/security_notification_worker.py @@ -0,0 +1,91 @@ +import json +import logging +import time + +from collections import defaultdict + +import features + +from app import secscan_notification_queue, secscan_api +from data import model +from data.database import (Image, ImageStorage, ExternalNotificationEvent, + Repository, RepositoryNotification, RepositoryTag) +from endpoints.notificationhelper import spawn_notification +from workers.queueworker import QueueWorker + + +logger = logging.getLogger(__name__) + + +class SecurityNotificationWorker(QueueWorker): + def process_queue_item(self, data): + cve_id = data['Name'] + vulnerability = data['Content']['Vulnerability'] + priority = vulnerability['Priority'] + + # Lookup the external event for when we have vulnerabilities. + event = ExternalNotificationEvent.get(name='vulnerability_found') + + # For each layer, retrieving the matching tags and join with repository to determine which + # require new notifications. + tag_map = defaultdict(set) + repository_map = {} + + # Find all tags that contain the layer(s) introducing the vulnerability. + content = data['Content'] + layer_ids = content.get('NewIntroducingLayersIDs', content.get('IntroducingLayersIDs', [])) + for layer_id in layer_ids: + (docker_image_id, storage_uuid) = layer_id.split('.', 2) + tags = model.tag.get_matching_tags(docker_image_id, storage_uuid, RepositoryTag, + Repository, Image, ImageStorage) + + # Additionally filter to tags only in repositories that have the event setup. + matching = list(tags + .switch(RepositoryTag) + .join(Repository) + .join(RepositoryNotification) + .where(RepositoryNotification.event == event)) + + check_map = {} + for tag in matching: + # Verify that the tag's root image has the vulnerability. + tag_layer_id = '%s.%s' % (tag.image.docker_image_id, tag.image.storage.uuid) + logger.debug('Checking if layer %s is vulnerable to %s', tag_layer_id, cve_id) + + if not tag_layer_id in check_map: + is_vulerable = secscan_api.check_layer_vulnerable(tag_layer_id, cve_id) + check_map[tag_layer_id] = is_vulerable + + logger.debug('Result of layer %s is vulnerable to %s check: %s', tag_layer_id, cve_id, + check_map[tag_layer_id]) + + if check_map[tag_layer_id]: + # Add the vulnerable tag to the list. + tag_map[tag.repository_id].add(tag.name) + repository_map[tag.repository_id] = tag.repository + + # For each of the tags found, issue a notification. + for repository_id in tag_map: + tags = tag_map[repository_id] + event_data = { + 'tags': list(tags), + 'vulnerability': { + 'id': data['Name'], + 'description': vulnerability['Description'], + 'link': vulnerability['Link'], + 'priority': priority, + }, + } + + spawn_notification(repository_map[repository_id], 'vulnerability_found', event_data) + + +if __name__ == '__main__': + if not features.SECURITY_SCANNER: + logger.debug('Security scanner disabled; skipping SecurityNotificationWorker') + while True: + time.sleep(100000) + + worker = SecurityNotificationWorker(secscan_notification_queue, poll_period_seconds=30, + reservation_seconds=30, retry_after_seconds=30) + worker.start() diff --git a/workers/securityworker.py b/workers/securityworker.py index e6ccd369b..2ce3fac9e 100644 --- a/workers/securityworker.py +++ b/workers/securityworker.py @@ -1,217 +1,312 @@ import logging +import logging.config + import requests import features import time import os import random -from sys import exc_info +from endpoints.notificationhelper import spawn_notification +from collections import defaultdict from peewee import JOIN_LEFT_OUTER -from app import app, storage, OVERRIDE_CONFIG_DIRECTORY +from app import app, config_provider, storage, OVERRIDE_CONFIG_DIRECTORY, secscan_api from workers.worker import Worker -from data.database import Image, ImageStorage, ImageStorageLocation, ImageStoragePlacement, db_random_func, UseThenDisconnect +from data.database import (Image, ImageStorage, ImageStorageLocation, ImageStoragePlacement, + db_random_func, UseThenDisconnect, RepositoryTag, Repository, + ExternalNotificationEvent, RepositoryNotification) +from util.secscan.api import SecurityConfigValidator logger = logging.getLogger(__name__) BATCH_SIZE = 20 INDEXING_INTERVAL = 10 -API_METHOD_INSERT = '/layers' -API_METHOD_VERSION = '/versions/engine' +API_METHOD_INSERT = '/v1/layers' +API_METHOD_VERSION = '/v1/versions/engine' -def _get_image_to_export(version): +def _get_images_to_export_list(version): Parent = Image.alias() ParentImageStorage = ImageStorage.alias() rimages = [] - # Without parent + # Collect the images without parents candidates = (Image - .select(Image.docker_image_id, ImageStorage.uuid, ImageStorage.checksum) - .join(ImageStorage) - .where(Image.security_indexed_engine < version, Image.parent_id >> None, ImageStorage.uploading == False, ImageStorage.checksum != '') - .limit(BATCH_SIZE*10) - .alias('candidates')) + .select(Image.id, Image.docker_image_id, ImageStorage.uuid) + .join(ImageStorage) + .where(Image.security_indexed_engine < version, + Image.parent_id >> None, + ImageStorage.uploading == False) + .limit(BATCH_SIZE*10) + .alias('candidates')) images = (Image - .select(candidates.c.docker_image_id, candidates.c.uuid, candidates.c.checksum) - .from_(candidates) - .order_by(db_random_func()) - .tuples() - .limit(BATCH_SIZE)) + .select(candidates.c.id, candidates.c.docker_image_id, candidates.c.uuid) + .from_(candidates) + .order_by(db_random_func()) + .tuples() + .limit(BATCH_SIZE)) for image in images: - rimages.append({'docker_image_id': image[0], 'storage_uuid': image[1], 'storage_checksum': image[2], 'parent_docker_image_id': None, 'parent_storage_uuid': None}) + rimages.append({'image_id': image[0], + 'docker_image_id': image[1], + 'storage_uuid': image[2], + 'parent_docker_image_id': None, + 'parent_storage_uuid': None}) - # With analyzed parent + # Collect the images with analyzed parents. candidates = (Image - .select(Image.docker_image_id, ImageStorage.uuid, ImageStorage.checksum, Parent.docker_image_id.alias('parent_docker_image_id'), ParentImageStorage.uuid.alias('parent_storage_uuid')) - .join(Parent, on=(Image.parent_id == Parent.id)) - .join(ParentImageStorage, on=(ParentImageStorage.id == Parent.storage)) - .switch(Image) - .join(ImageStorage) - .where(Image.security_indexed_engine < version, Parent.security_indexed == True, Parent.security_indexed_engine >= version, ImageStorage.uploading == False, ImageStorage.checksum != '') - .limit(BATCH_SIZE*10) - .alias('candidates')) + .select(Image.id, + Image.docker_image_id, + ImageStorage.uuid, + Parent.docker_image_id.alias('parent_docker_image_id'), + ParentImageStorage.uuid.alias('parent_storage_uuid')) + .join(Parent, on=(Image.parent_id == Parent.id)) + .join(ParentImageStorage, on=(ParentImageStorage.id == Parent.storage)) + .switch(Image) + .join(ImageStorage) + .where(Image.security_indexed_engine < version, + Parent.security_indexed == True, + Parent.security_indexed_engine >= version, + ImageStorage.uploading == False) + .limit(BATCH_SIZE*10) + .alias('candidates')) images = (Image - .select(candidates.c.docker_image_id, candidates.c.uuid, candidates.c.checksum, candidates.c.parent_docker_image_id, candidates.c.parent_storage_uuid) - .from_(candidates) - .order_by(db_random_func()) - .tuples() - .limit(BATCH_SIZE)) + .select(candidates.c.id, + candidates.c.docker_image_id, + candidates.c.uuid, + candidates.c.parent_docker_image_id, + candidates.c.parent_storage_uuid) + .from_(candidates) + .order_by(db_random_func()) + .tuples() + .limit(BATCH_SIZE)) for image in images: - rimages.append({'docker_image_id': image[0], 'storage_uuid': image[1], 'storage_checksum': image[2], 'parent_docker_image_id': image[3], 'parent_storage_uuid': image[4]}) + rimages.append({'image_id': image[0], + 'docker_image_id': image[1], + 'storage_uuid': image[2], + 'parent_docker_image_id': image[3], + 'parent_storage_uuid': image[4]}) - # Re-shuffle, otherwise the images without parents will always be on the top + # Shuffle the images, otherwise the images without parents will always be on the top random.shuffle(rimages) - return rimages def _get_storage_locations(uuid): query = (ImageStoragePlacement - .select() - .join(ImageStorageLocation) - .switch(ImageStoragePlacement) - .join(ImageStorage, JOIN_LEFT_OUTER) - .where(ImageStorage.uuid == uuid)) + .select() + .join(ImageStorageLocation) + .switch(ImageStoragePlacement) + .join(ImageStorage, JOIN_LEFT_OUTER) + .where(ImageStorage.uuid == uuid)) - locations = list() - for location in query: - locations.append(location.location.name) - - return locations + return [location.location.name for location in query] def _update_image(image, indexed, version): query = (Image - .select() - .join(ImageStorage) - .where(Image.docker_image_id == image['docker_image_id'], ImageStorage.uuid == image['storage_uuid'])) + .select() + .join(ImageStorage) + .where(Image.docker_image_id == image['docker_image_id'], + ImageStorage.uuid == image['storage_uuid'])) - updated_images = list() - for image in query: - updated_images.append(image.id) + ids_to_update = [row.id for row in query] + if not ids_to_update: + return + + (Image + .update(security_indexed=indexed, security_indexed_engine=version) + .where(Image.id << ids_to_update) + .execute()) - query = (Image - .update(security_indexed=indexed, security_indexed_engine=version) - .where(Image.id << updated_images)) - query.execute() class SecurityWorker(Worker): def __init__(self): super(SecurityWorker, self).__init__() - if self._load_configuration(): + validator = SecurityConfigValidator(app, config_provider) + if validator.valid(): + secscan_config = app.config.get('SECURITY_SCANNER') + self._api = secscan_config['ENDPOINT'] + self._target_version = secscan_config['ENGINE_VERSION_TARGET'] + self._default_storage_locations = app.config['DISTRIBUTED_STORAGE_PREFERENCE'] + self._cert = validator.cert() + self._keys = validator.keypair() + self.add_operation(self._index_images, INDEXING_INTERVAL) + logger.warning('Failed to validate security scan configuration') - def _load_configuration(self): - # Load configuration - config = app.config.get('SECURITY_SCANNER') + def _get_image_url(self, image): + """ Gets the download URL for an image and if the storage doesn't exist, + marks the image as unindexed. """ + path = storage.image_layer_path(image['storage_uuid']) + locations = self._default_storage_locations - if not config or not 'ENDPOINT' in config or not 'ENGINE_VERSION_TARGET' in config or not 'DISTRIBUTED_STORAGE_PREFERENCE' in app.config: - logger.exception('No configuration found for the security worker') - return False - self._api = config['ENDPOINT'] - self._target_version = config['ENGINE_VERSION_TARGET'] - self._default_storage_locations = app.config['DISTRIBUTED_STORAGE_PREFERENCE'] + if not storage.exists(locations, path): + locations = _get_storage_locations(image['storage_uuid']) - self._ca_verification = False - self._cert = None - if 'CA_CERTIFICATE_FILENAME' in config: - self._ca_verification = os.path.join(OVERRIDE_CONFIG_DIRECTORY, config['CA_CERTIFICATE_FILENAME']) - if not os.path.isfile(self._ca_verification): - logger.exception('Could not find configured CA file') - return False - if 'PRIVATE_KEY_FILENAME' in config and 'PUBLIC_KEY_FILENAME' in config: - self._cert = ( - os.path.join(OVERRIDE_CONFIG_DIRECTORY, config['PUBLIC_KEY_FILENAME']), - os.path.join(OVERRIDE_CONFIG_DIRECTORY, config['PRIVATE_KEY_FILENAME']), - ) - if not os.path.isfile(self._cert[0]) or not os.path.isfile(self._cert[1]): - logger.exception('Could not find configured key pair files') - return False + if not locations or not storage.exists(locations, path): + logger.warning('Could not find a valid location to download layer %s', + image['docker_image_id']+'.'+image['storage_uuid']) + _update_image(image, False, self._target_version) + return None - return True + uri = storage.get_direct_download_url(locations, path) + if uri is None: + # Handle local storage + local_storage_enabled = False + for storage_type, _ in app.config.get('DISTRIBUTED_STORAGE_CONFIG', {}).values(): + if storage_type == 'LocalStorage': + local_storage_enabled = True + + if local_storage_enabled: + uri = path + else: + logger.warning('Could not get image URL and local storage was not enabled') + return None + + return uri + + def _new_request(self, image): + url = self._get_image_url(image) + if url is None: + return None + + request = { + 'ID': '%s.%s' % (image['docker_image_id'], image['storage_uuid']), + 'Path': url, + } + + if image['parent_docker_image_id'] is not None and image['parent_storage_uuid'] is not None: + request['ParentID'] = '%s.%s' % (image['parent_docker_image_id'], + image['parent_storage_uuid']) + + return request + + def _analyze_image(self, image): + """ Analyzes an image by passing it to Clair. Returns the vulnerabilities detected + (if any) or None on error. + """ + request = self._new_request(image) + if request is None: + return None + + # Analyze the image. + try: + logger.info('Analyzing %s', request['ID']) + # Using invalid certificates doesn't return proper errors because of + # https://github.com/shazow/urllib3/issues/556 + httpResponse = requests.post(self._api + API_METHOD_INSERT, json=request, + cert=self._keys, verify=self._cert) + jsonResponse = httpResponse.json() + except (requests.exceptions.RequestException, ValueError): + logger.exception('An exception occurred when analyzing layer ID %s', request['ID']) + return None + + # Handle any errors from the security scanner. + if httpResponse.status_code != 201: + if 'OS and/or package manager are not supported' in jsonResponse.get('Message', ''): + # The current engine could not index this layer + logger.warning('A warning event occurred when analyzing layer ID %s : %s', + request['ID'], jsonResponse['Message']) + + # Hopefully, there is no version lower than the target one running + _update_image(image, False, self._target_version) + else: + logger.warning('Got non-201 when analyzing layer ID %s: %s', request['ID'], jsonResponse) + + return None + + # Verify that the version matches. + api_version = jsonResponse['Version'] + if api_version < self._target_version: + logger.warning('An engine runs on version %d but the target version is %d') + + # Mark the image as analyzed. + logger.debug('Layer %s analyzed successfully; Loading vulnerabilities for layer', + image['image_id']) + _update_image(image, True, api_version) + + # Lookup the vulnerabilities for the image, now that it is analyzed. + try: + response = secscan_api.call('layers/%s/vulnerabilities', None, request['ID']) + logger.debug('Got response %s for vulnerabilities for layer %s', + response.status_code, image['image_id']) + if response.status_code == 404: + return None + except (requests.exceptions.RequestException, ValueError): + logger.exception('Failed to get vulnerability response for %s', image['image_id']) + return None + + return response.json() def _index_images(self): + logger.debug('Started indexing') + with UseThenDisconnect(app.config): while True: - # Get images to analyze + # Lookup the images to index. + images = [] try: - images = _get_image_to_export(self._target_version) + logger.debug('Looking up images to index') + images = _get_images_to_export_list(self._target_version) except Image.DoesNotExist: - logger.debug('No more image to analyze') + pass + + if not images: + logger.debug('No more images left to analyze') return - for img in images: - # Get layer storage URL - path = storage.image_layer_path(img['storage_uuid']) - locations = self._default_storage_locations - if not storage.exists(locations, path): - locations = _get_storage_locations(img['storage_uuid']) - if not storage.exists(locations, path): - logger.warning('Could not find a valid location to download layer %s', img['docker_image_id']+'.'+img['storage_uuid']) - # Mark as analyzed because that error is most likely to occur during the pre-process, with the database copy - # when images are actually removed on the real database (and therefore in S3) - _update_image(img, False, self._target_version) + logger.debug('Found %d images to index', len(images)) + for image in images: + # Analyze the image, retrieving the vulnerabilities (if any). + sec_data = self._analyze_image(image) + if sec_data is None: continue - uri = storage.get_direct_download_url(locations, path) - if uri == None: - # Local storage hack - uri = path - # Forge request - request = { - 'ID': img['docker_image_id']+'.'+img['storage_uuid'], - 'TarSum': img['storage_checksum'], - 'Path': uri - } + if not sec_data.get('Vulnerabilities'): + continue - if img['parent_docker_image_id'] is not None and img['parent_storage_uuid'] is not None: - request['ParentID'] = img['parent_docker_image_id']+'.'+img['parent_storage_uuid'] + # Dispatch events for any detected vulnerabilities + logger.debug('Got vulnerabilities for layer %s: %s', image['image_id'], sec_data) + event = ExternalNotificationEvent.get(name='vulnerability_found') + matching = (RepositoryTag + .select(RepositoryTag, Repository) + .distinct() + .join(Repository) + .join(RepositoryNotification) + .where(RepositoryNotification.event == event, + RepositoryTag.image == image['image_id'], + RepositoryTag.hidden == False, + RepositoryTag.lifetime_end_ts >> None)) - # Post request - try: - logger.info('Analyzing %s', request['ID']) - # Using invalid certificates doesn't return proper errors because of - # https://github.com/shazow/urllib3/issues/556 - httpResponse = requests.post(self._api + API_METHOD_INSERT, json=request, cert=self._cert, verify=self._ca_verification) - except: - logger.exception('An exception occurred when analyzing layer ID %s : %s', request['ID'], exc_info()[0]) - return - try: - jsonResponse = httpResponse.json() - except: - logger.exception('An exception occurred when analyzing layer ID %s : the response is not valid JSON (%s)', request['ID'], httpResponse.text) - return + repository_map = defaultdict(list) + + for tag in matching: + repository_map[tag.repository_id].append(tag) + + for repository_id in repository_map: + tags = repository_map[repository_id] + + for vuln in sec_data['Vulnerabilities']: + event_data = { + 'tags': [tag.name for tag in tags], + 'vulnerability': { + 'id': vuln['ID'], + 'description': vuln['Description'], + 'link': vuln['Link'], + 'priority': vuln['Priority'], + }, + } + + spawn_notification(tags[0].repository, 'vulnerability_found', event_data) - if httpResponse.status_code == 201: - # The layer has been successfully indexed - api_version = jsonResponse['Version'] - if api_version < self._target_version: - logger.warning('An engine runs on version %d but the target version is %d') - _update_image(img, True, api_version) - logger.info('Layer ID %s : analyzed successfully', request['ID']) - else: - if 'Message' in jsonResponse: - if 'OS and/or package manager are not supported' in jsonResponse['Message']: - # The current engine could not index this layer - logger.warning('A warning event occurred when analyzing layer ID %s : %s', request['ID'], jsonResponse['Message']) - # Hopefully, there is no version lower than the target one running - _update_image(img, False, self._target_version) - else: - logger.exception('An exception occurred when analyzing layer ID %s : %d %s', request['ID'], httpResponse.status_code, jsonResponse['Message']) - return - else: - logger.exception('An exception occurred when analyzing layer ID %s : %d', request['ID'], httpResponse.status_code) - return if __name__ == '__main__': - logging.getLogger('requests').setLevel(logging.WARNING) - logging.getLogger('apscheduler').setLevel(logging.CRITICAL) - if not features.SECURITY_SCANNER: - logger.debug('Security scanner disabled; skipping') + logger.debug('Security scanner disabled; skipping SecurityWorker') while True: time.sleep(100000) + logging.config.fileConfig('conf/logging_debug.conf', disable_existing_loggers=False) worker = SecurityWorker() worker.start() diff --git a/workers/worker.py b/workers/worker.py index a9ea5d219..47dcaf9ef 100644 --- a/workers/worker.py +++ b/workers/worker.py @@ -61,7 +61,7 @@ class Worker(object): pass def start(self): - logging.config.fileConfig('conf/logging.conf', disable_existing_loggers=False) + logging.config.fileConfig('conf/logging_debug.conf', disable_existing_loggers=False) if not app.config.get('SETUP_COMPLETE', False): logger.info('Product setup is not yet complete; skipping worker startup')