diff --git a/data/model/tag.py b/data/model/tag.py index fcaa7f342..002be14b2 100644 --- a/data/model/tag.py +++ b/data/model/tag.py @@ -22,7 +22,8 @@ def get_matching_tags(docker_image_id, storage_uuid, *args): .distinct() .join(Image) .join(ImageStorage) - .where(Image.id << image_query, RepositoryTag.lifetime_end_ts >> None)) + .where(Image.id << image_query, RepositoryTag.lifetime_end_ts >> None, + RepositoryTag.hidden == False)) def list_repository_tags(namespace_name, repository_name, include_hidden=False, diff --git a/util/secscan/api.py b/util/secscan/api.py index 277fd3f6d..5041ec8ff 100644 --- a/util/secscan/api.py +++ b/util/secscan/api.py @@ -134,12 +134,24 @@ class SecurityConfigValidator(object): return self._keys def valid(self): - if (not features.SECURITY_SCANNER - or not self._security_config - or not 'ENDPOINT' in self._security_config - or not 'ENGINE_VERSION_TARGET' in self._security_config - or not 'DISTRIBUTED_STORAGE_PREFERENCE' in self._security_config - or (self._certificate is False and self._keys is None)): + if not features.SECURITY_SCANNER: + return False + + if not self._security_config: + logger.debug('Missing SECURITY_SCANNER block in configuration') + return False + + if not 'ENDPOINT' in self._security_config: + logger.debug('Missing ENDPOINT field in SECURITY_SCANNER configuration') + return False + + endpoint = self._security_config['ENDPOINT'] or '' + if not endpoint.startswith('http://') and not endpoint.startswith('https://'): + logger.debug('ENDPOINT field in SECURITY_SCANNER configuration must start with http or https') + return False + + if endpoint.startswith('https://') and (self._certificate is False or self._keys is None): + logger.debug('Certificate and key pair required for talking to security worker over HTTPS') return False return True @@ -150,6 +162,7 @@ class SecurityScannerAPI(object): def __init__(self, app, config_provider): self.app = app self.config_provider = config_provider + self._security_config = None config_validator = SecurityConfigValidator(app, config_provider) if not config_validator.valid(): @@ -192,6 +205,9 @@ class SecurityScannerAPI(object): from the API server. """ security_config = self._security_config + if security_config is None: + raise Exception('Cannot call unconfigured security system') + api_url = urljoin(security_config['ENDPOINT'], '/' + security_config['API_VERSION']) + '/' url = urljoin(api_url, relative_url % args) diff --git a/workers/security_notification_worker.py b/workers/security_notification_worker.py index 1679e80b6..e54847abf 100644 --- a/workers/security_notification_worker.py +++ b/workers/security_notification_worker.py @@ -18,9 +18,7 @@ logger = logging.getLogger(__name__) class SecurityNotificationWorker(QueueWorker): - def process_queue_item(self, queueitem): - data = json.loads(queueitem.body) - + def process_queue_item(self, data): cve_id = data['Name'] vulnerability = data['Content']['Vulnerability'] priority = vulnerability['Priority'] diff --git a/workers/securityworker.py b/workers/securityworker.py index 664e91472..f6e053c40 100644 --- a/workers/securityworker.py +++ b/workers/securityworker.py @@ -119,7 +119,7 @@ def _update_image(image, indexed, version): class SecurityWorker(Worker): def __init__(self): super(SecurityWorker, self).__init__() - validator = SecurityConfigValidator(app.config, config_provider) + validator = SecurityConfigValidator(app, config_provider) if validator.valid(): secscan_config = app.config.get('SECURITY_SCANNER') self._api = secscan_config['ENDPOINT'] @@ -143,10 +143,7 @@ class SecurityWorker(Worker): if not storage.exists(locations, path): logger.warning('Could not find a valid location to download layer %s', image['docker_image_id']+'.'+image['storage_uuid']) - try: - _update_image(image, False, self._target_version) - except: - logger.exception('Failed to update unindexed image') + _update_image(image, False, self._target_version) return None uri = storage.get_direct_download_url(locations, path) @@ -182,10 +179,14 @@ class SecurityWorker(Worker): return request def _analyze_image(self, image): + """ Analyzes an image by passing it to Clair. Returns the vulnerabilities detected + (if any) or None on error. + """ request = self._new_request(image) if request is None: return None + # Analyze the image. try: logger.info('Analyzing %s', request['ID']) # Using invalid certificates doesn't return proper errors because of @@ -199,43 +200,36 @@ class SecurityWorker(Worker): # Handle any errors from the security scanner. if httpResponse.status_code != 201: - if 'Message' in jsonResponse: - if 'OS and/or package manager are not supported' in jsonResponse['Message']: - # The current engine could not index this layer - logger.warning('A warning event occurred when analyzing layer ID %s : %s', - request['ID'], jsonResponse['Message']) + if 'OS and/or package manager are not supported' in jsonResponse.get('Message', ''): + # The current engine could not index this layer + logger.warning('A warning event occurred when analyzing layer ID %s : %s', + request['ID'], jsonResponse['Message']) - # Hopefully, there is no version lower than the target one running - try: - _update_image(image, False, self._target_version) - except: - logger.exception('Failed to update image to be unindexed') - else: - logger.warning('Failed to handle JSON message "%s" when analyzing layer ID %s', - jsonResponse['Message'], request['ID']) - return None + # Hopefully, there is no version lower than the target one running + _update_image(image, False, self._target_version) else: - logger.warning('No message found in JSON response when analyzing layer ID %s', request['ID']) - return None + logger.warning('Got non-201 when analyzing layer ID %s: %s', request['ID'], jsonResponse) - api_version = jsonResponse['Version'] - if api_version < self._target_version: - logger.warning('An engine runs on version %d but the target version is %d') + return None - try: - _update_image(image, True, api_version) - logger.debug('Layer %s analyzed successfully', request['ID']) - except: - logger.exception('Failed to update image to be indexed') + # Verify that the version matches. + api_version = jsonResponse['Version'] + if api_version < self._target_version: + logger.warning('An engine runs on version %d but the target version is %d') - logger.debug('Loading vulnerabilities for layer %s', image['image_id']) + # Mark the image as analyzed. + logger.debug('Layer %s analyzed successfully; Loading vulnerabilities for layer', + image['image_id']) + _update_image(image, True, api_version) + + # Lookup the vulnerabilities for the image, now that it is analyzed. try: response = secscan_api.call('layers/%s/vulnerabilities', None, request['ID']) logger.debug('Got response %s for vulnerabilities for layer %s', response.status_code, image['image_id']) if response.status_code == 404: return None - except: + except (requests.exceptions.RequestException, ValueError): logger.exception('Failed to get vulnerability response for %s', image['image_id']) return None @@ -246,6 +240,7 @@ class SecurityWorker(Worker): with UseThenDisconnect(app.config): while True: + # Lookup the images to index. images = [] try: logger.debug('Looking up images to index') @@ -256,9 +251,10 @@ class SecurityWorker(Worker): if not images: logger.debug('No more images left to analyze') return - logger.debug('Found %d images to index', len(images)) + logger.debug('Found %d images to index', len(images)) for image in images: + # Analyze the image, retrieving the vulnerabilities (if any). sec_data = self._analyze_image(image) if sec_data is None: continue @@ -267,7 +263,7 @@ class SecurityWorker(Worker): continue # Dispatch events for any detected vulnerabilities - logger.debug('Got response vulnerabilities for layer %s: %s', image['image_id'], sec_data) + logger.debug('Got vulnerabilities for layer %s: %s', image['image_id'], sec_data) event = ExternalNotificationEvent.get(name='vulnerability_found') matching = (RepositoryTag .select(RepositoryTag, Repository) @@ -275,9 +271,11 @@ class SecurityWorker(Worker): .join(Repository) .join(RepositoryNotification) .where(RepositoryNotification.event == event, - RepositoryTag.image == image['image_id'])) + RepositoryTag.image == image['image_id'], + RepositoryTag.hidden == False, + RepositoryTag.lifetime_end_ts >> None)) - repository_map = defaultdict() + repository_map = defaultdict(list) for tag in matching: repository_map[tag.repository_id].append(tag)