WIP: Towards sec demo
This commit is contained in:
Joseph Schorr
Jimmy Zelinskie
parent
fb3d0fa27d
commit
407eaae137
6 changed files with 164 additions and 14 deletions
|
|
@ -1,22 +1,29 @@
|
|||
import logging
|
||||
import logging.config
|
||||
|
||||
import requests
|
||||
import features
|
||||
import time
|
||||
import os
|
||||
import random
|
||||
import json
|
||||
|
||||
from endpoints.notificationhelper import spawn_notification
|
||||
from collections import defaultdict
|
||||
from sys import exc_info
|
||||
from peewee import JOIN_LEFT_OUTER
|
||||
from app import app, storage, OVERRIDE_CONFIG_DIRECTORY
|
||||
from app import app, storage, OVERRIDE_CONFIG_DIRECTORY, sec_endpoint
|
||||
from workers.worker import Worker
|
||||
from data.database import Image, ImageStorage, ImageStorageLocation, ImageStoragePlacement, db_random_func, UseThenDisconnect
|
||||
from data.database import (Image, ImageStorage, ImageStorageLocation, ImageStoragePlacement,
|
||||
db_random_func, UseThenDisconnect, RepositoryTag, Repository,
|
||||
ExternalNotificationEvent, RepositoryNotification)
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
BATCH_SIZE = 20
|
||||
INDEXING_INTERVAL = 10
|
||||
API_METHOD_INSERT = '/layers'
|
||||
API_METHOD_VERSION = '/versions/engine'
|
||||
API_METHOD_INSERT = '/v1/layers'
|
||||
API_METHOD_VERSION = '/v1/versions/engine'
|
||||
|
||||
def _get_image_to_export(version):
|
||||
Parent = Image.alias()
|
||||
|
|
@ -25,14 +32,14 @@ def _get_image_to_export(version):
|
|||
|
||||
# Without parent
|
||||
candidates = (Image
|
||||
.select(Image.docker_image_id, ImageStorage.uuid, ImageStorage.checksum)
|
||||
.select(Image.id, Image.docker_image_id, ImageStorage.uuid, ImageStorage.checksum)
|
||||
.join(ImageStorage)
|
||||
.where(Image.security_indexed_engine < version, Image.parent >> None, ImageStorage.uploading == False, ImageStorage.checksum != '')
|
||||
.limit(BATCH_SIZE*10)
|
||||
.alias('candidates'))
|
||||
|
||||
images = (Image
|
||||
.select(candidates.c.docker_image_id, candidates.c.uuid, candidates.c.checksum)
|
||||
.select(candidates.c.id, candidates.c.docker_image_id, candidates.c.uuid, candidates.c.checksum)
|
||||
.distinct()
|
||||
.from_(candidates)
|
||||
.order_by(db_random_func())
|
||||
|
|
@ -40,11 +47,11 @@ def _get_image_to_export(version):
|
|||
.limit(BATCH_SIZE))
|
||||
|
||||
for image in images:
|
||||
rimages.append({'docker_image_id': image[0], 'storage_uuid': image[1], 'storage_checksum': image[2], 'parent_docker_image_id': None, 'parent_storage_uuid': None})
|
||||
rimages.append({'image_id': image[0], 'docker_image_id': image[1], 'storage_uuid': image[2], 'storage_checksum': image[3], 'parent_docker_image_id': None, 'parent_storage_uuid': None})
|
||||
|
||||
# With analyzed parent
|
||||
candidates = (Image
|
||||
.select(Image.docker_image_id, ImageStorage.uuid, ImageStorage.checksum, Parent.docker_image_id.alias('parent_docker_image_id'), ParentImageStorage.uuid.alias('parent_storage_uuid'))
|
||||
.select(Image.id, Image.docker_image_id, ImageStorage.uuid, ImageStorage.checksum, Parent.docker_image_id.alias('parent_docker_image_id'), ParentImageStorage.uuid.alias('parent_storage_uuid'))
|
||||
.join(Parent, on=(Image.parent == Parent.id))
|
||||
.join(ParentImageStorage, on=(ParentImageStorage.id == Parent.storage))
|
||||
.switch(Image)
|
||||
|
|
@ -54,7 +61,7 @@ def _get_image_to_export(version):
|
|||
.alias('candidates'))
|
||||
|
||||
images = (Image
|
||||
.select(candidates.c.docker_image_id, candidates.c.uuid, candidates.c.checksum, candidates.c.parent_docker_image_id, candidates.c.parent_storage_uuid)
|
||||
.select(candidates.c.id, candidates.c.docker_image_id, candidates.c.uuid, candidates.c.checksum, candidates.c.parent_docker_image_id, candidates.c.parent_storage_uuid)
|
||||
.distinct()
|
||||
.from_(candidates)
|
||||
.order_by(db_random_func())
|
||||
|
|
@ -62,7 +69,7 @@ def _get_image_to_export(version):
|
|||
.limit(BATCH_SIZE))
|
||||
|
||||
for image in images:
|
||||
rimages.append({'docker_image_id': image[0], 'storage_uuid': image[1], 'storage_checksum': image[2], 'parent_docker_image_id': None, 'parent_storage_uuid': None})
|
||||
rimages.append({'image_id': image[0], 'docker_image_id': image[1], 'storage_uuid': image[2], 'storage_checksum': image[3], 'parent_docker_image_id': image[4], 'parent_storage_uuid': image[5]})
|
||||
|
||||
# Re-shuffle, otherwise the images without parents will always be on the top
|
||||
random.shuffle(rimages)
|
||||
|
|
@ -134,15 +141,24 @@ class SecurityWorker(Worker):
|
|||
return True
|
||||
|
||||
def _index_images(self):
|
||||
logger.debug('Starting indexing')
|
||||
|
||||
with UseThenDisconnect(app.config):
|
||||
while True:
|
||||
logger.debug('Looking up images to index')
|
||||
|
||||
# Get images to analyze
|
||||
try:
|
||||
images = _get_image_to_export(self._target_version)
|
||||
if not images:
|
||||
logger.debug('No more image to analyze')
|
||||
return
|
||||
|
||||
except Image.DoesNotExist:
|
||||
logger.debug('No more image to analyze')
|
||||
return
|
||||
|
||||
logger.debug('Found images to index: %s', images)
|
||||
for img in images:
|
||||
# Get layer storage URL
|
||||
path = storage.image_layer_path(img['storage_uuid'])
|
||||
|
|
@ -191,6 +207,62 @@ class SecurityWorker(Worker):
|
|||
logger.warning('An engine runs on version %d but the target version is %d')
|
||||
_update_image(img, True, api_version)
|
||||
logger.info('Layer ID %s : analyzed successfully', request['ID'])
|
||||
|
||||
|
||||
# TODO(jschorr): Put this in a proper place, properly comment, unify with the
|
||||
# callback code, etc.
|
||||
try:
|
||||
logger.debug('Loading vulnerabilities for layer %s', img['image_id'])
|
||||
response = sec_endpoint.call_api('layers/%s/vulnerabilities', request['ID'])
|
||||
except requests.exceptions.Timeout:
|
||||
logger.debug('Timeout when calling Sec')
|
||||
continue
|
||||
except requests.exceptions.ConnectionError:
|
||||
logger.debug('Connection error when calling Sec')
|
||||
continue
|
||||
|
||||
logger.debug('Got response %s for vulnerabilities for layer %s', response.status_code, img['image_id'])
|
||||
|
||||
if response.status_code == 404:
|
||||
continue
|
||||
|
||||
sec_data = json.loads(response.text)
|
||||
logger.debug('Got response vulnerabilities for layer %s: %s', img['image_id'], sec_data)
|
||||
|
||||
if not sec_data['Vulnerabilities']:
|
||||
continue
|
||||
|
||||
event = ExternalNotificationEvent.get(name='vulnerability_found')
|
||||
matching = (RepositoryTag
|
||||
.select(RepositoryTag, Repository)
|
||||
.distinct()
|
||||
.where(RepositoryTag.image_id == img['id'])
|
||||
.join(Repository)
|
||||
.join(RepositoryNotification)
|
||||
.where(RepositoryNotification.event == event))
|
||||
|
||||
repository_map = defaultdict(list)
|
||||
|
||||
for tag in matching:
|
||||
repository_map[tag.repository_id].append(tag)
|
||||
|
||||
for repository_id in repository_map:
|
||||
tags = repository_map[repository_id]
|
||||
|
||||
for vuln in sec_data['Vulnerabilities']:
|
||||
event_data = {
|
||||
'tags': [tag.name for tag in tags],
|
||||
'vulnerability': {
|
||||
'id': vuln['ID'],
|
||||
'description': vuln['Description'],
|
||||
'link': vuln['Link'],
|
||||
'priority': vuln['Priority'],
|
||||
},
|
||||
}
|
||||
|
||||
spawn_notification(tags[0].repository, 'vulnerability_found', event_data)
|
||||
|
||||
|
||||
else:
|
||||
if 'Message' in jsonResponse:
|
||||
if 'OS and/or package manager are not supported' in jsonResponse['Message']:
|
||||
|
|
@ -206,13 +278,11 @@ class SecurityWorker(Worker):
|
|||
return
|
||||
|
||||
if __name__ == '__main__':
|
||||
logging.getLogger('requests').setLevel(logging.WARNING)
|
||||
logging.getLogger('apscheduler').setLevel(logging.CRITICAL)
|
||||
|
||||
if not features.SECURITY_SCANNER:
|
||||
logger.debug('Security scanner disabled; skipping')
|
||||
while True:
|
||||
time.sleep(100000)
|
||||
|
||||
logging.config.fileConfig('conf/logging_debug.conf', disable_existing_loggers=False)
|
||||
worker = SecurityWorker()
|
||||
worker.start()
|
||||
|
|
|
|||
Reference in a new issue