vbatts/quay
Archived
1
0
Fork 0
Code Issues Pull requests Releases Wiki Activity

WIP: Towards sec demo

Browse source
This commit is contained in:
Joseph Schorr 2015-10-27 17:38:48 -04:00 committed by Jimmy Zelinskie
parent fb3d0fa27d
commit 407eaae137
6 changed files with 164 additions and 14 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

13
data/model/image.py
View file

@ -12,6 +12,19 @@ from data.database import (Image, Repository, ImageStoragePlacement, Namespace,
logger = logging.getLogger(__name__)
def get_repository_images_recursive(docker_image_ids):
""" Returns a query matching the given docker image IDs, along with any which have the image IDs
as parents.
Note: This is a DB intensive operation and should be used sparingly.
"""
inner_images = Image.select('%/' + Image.id + '/%').where(Image.docker_image_id << docker_image_ids)
images = Image.select(Image.id).where(Image.docker_image_id << docker_image_ids)
recursive_images = Image.select(Image.id).where(Image.ancestors ** inner_images)
return recursive_images | images
def get_parent_images(namespace_name, repository_name, image_obj):
""" Returns a list of parent Image objects in chronilogical order. """
parents = image_obj.ancestors

10
data/model/tag.py
View file

@ -12,6 +12,16 @@ def _tag_alive(query, now_ts=None):
(RepositoryTag.lifetime_end_ts > now_ts))
def get_matching_tags(docker_image_ids, *args):
""" Returns a query pointing to all tags that contain the given image(s). """
return (RepositoryTag
.select(*args)
.distinct()
.join(Image)
.where(Image.id << image.get_repository_images_recursive(docker_image_ids),
RepositoryTag.lifetime_end_ts >> None))
def list_repository_tags(namespace_name, repository_name, include_hidden=False,
include_storage=False):
to_select = (RepositoryTag, Image)

55
endpoints/sec.py Normal file
View file

@ -0,0 +1,55 @@
import logging
from flask import request, make_response, Blueprint
from data import model
from data.database import RepositoryNotification, Repository, ExternalNotificationEvent, RepositoryTag, Image
from endpoints.notificationhelper import spawn_notification
from collections import defaultdict
logger = logging.getLogger(__name__)
sec = Blueprint('sec', __name__)
@sec.route('/notification', methods=['POST'])
def sec_notification():
data = request.get_json()
print data
# Find all tags that contain the layer(s) introducing the vulnerability.
# TODO: remove this check once fixed.
if not 'IntroducingLayersIDs' in data['Content']:
return make_response('Okay')
layer_ids = data['Content']['IntroducingLayersIDs']
tags = model.tag.get_matching_tags(layer_ids, RepositoryTag, Repository, Image)
# For any repository that has a notification setup, issue a notification.
event = ExternalNotificationEvent.get(name='vulnerability_found')
matching = (tags.switch(RepositoryTag)
.join(Repository)
.join(RepositoryNotification)
.where(RepositoryNotification.event == event))
repository_map = defaultdict(list)
for tag in matching:
repository_map[tag.repository_id].append(tag)
for repository_id in repository_map:
tags = repository_map[repository_id]
# TODO(jschorr): Pull out the other metadata once added.
event_data = {
'tags': [tag.name for tag in tags],
'vulnerability': {
'id': data['Name'],
'description': 'Some description',
'link': 'https://security-tracker.debian.org/tracker/CVE-FAKE-CVE',
'priority': 'Medium',
},
}
spawn_notification(tags[0].repository, 'vulnerability_found', event_data)
return make_response('Okay')

2
web.py
View file

@ -11,6 +11,7 @@ from endpoints.oauthlogin import oauthlogin
from endpoints.githubtrigger import githubtrigger
from endpoints.gitlabtrigger import gitlabtrigger
from endpoints.bitbuckettrigger import bitbuckettrigger
from endpoints.sec import sec
if os.environ.get('DEBUGLOG') == 'true':
logging.config.fileConfig('conf/logging_debug.conf', disable_existing_loggers=False)
@ -23,3 +24,4 @@ application.register_blueprint(bitbuckettrigger, url_prefix='/oauth1')
application.register_blueprint(api_bp, url_prefix='/api')
application.register_blueprint(webhooks, url_prefix='/webhooks')
application.register_blueprint(realtime, url_prefix='/realtime')
application.register_blueprint(sec, url_prefix='/sec')

96
workers/securityworker.py
View file

@ -1,22 +1,29 @@
import logging
import logging.config
import requests
import features
import time
import os
import random
import json
from endpoints.notificationhelper import spawn_notification
from collections import defaultdict
from sys import exc_info
from peewee import JOIN_LEFT_OUTER
from app import app, storage, OVERRIDE_CONFIG_DIRECTORY
from app import app, storage, OVERRIDE_CONFIG_DIRECTORY, sec_endpoint
from workers.worker import Worker
from data.database import Image, ImageStorage, ImageStorageLocation, ImageStoragePlacement, db_random_func, UseThenDisconnect
from data.database import (Image, ImageStorage, ImageStorageLocation, ImageStoragePlacement,
db_random_func, UseThenDisconnect, RepositoryTag, Repository,
ExternalNotificationEvent, RepositoryNotification)
logger = logging.getLogger(__name__)
BATCH_SIZE = 20
INDEXING_INTERVAL = 10
API_METHOD_INSERT = '/layers'
API_METHOD_VERSION = '/versions/engine'
API_METHOD_INSERT = '/v1/layers'
API_METHOD_VERSION = '/v1/versions/engine'
def _get_image_to_export(version):
Parent = Image.alias()
@ -25,14 +32,14 @@ def _get_image_to_export(version):
# Without parent
candidates = (Image
.select(Image.docker_image_id, ImageStorage.uuid, ImageStorage.checksum)
.select(Image.id, Image.docker_image_id, ImageStorage.uuid, ImageStorage.checksum)
.join(ImageStorage)
.where(Image.security_indexed_engine < version, Image.parent >> None, ImageStorage.uploading == False, ImageStorage.checksum != '')
.limit(BATCH_SIZE*10)
.alias('candidates'))
images = (Image
.select(candidates.c.docker_image_id, candidates.c.uuid, candidates.c.checksum)
.select(candidates.c.id, candidates.c.docker_image_id, candidates.c.uuid, candidates.c.checksum)
.distinct()
.from_(candidates)
.order_by(db_random_func())
@ -40,11 +47,11 @@ def _get_image_to_export(version):
.limit(BATCH_SIZE))
for image in images:
rimages.append({'docker_image_id': image[0], 'storage_uuid': image[1], 'storage_checksum': image[2], 'parent_docker_image_id': None, 'parent_storage_uuid': None})
rimages.append({'image_id': image[0], 'docker_image_id': image[1], 'storage_uuid': image[2], 'storage_checksum': image[3], 'parent_docker_image_id': None, 'parent_storage_uuid': None})
# With analyzed parent
candidates = (Image
.select(Image.docker_image_id, ImageStorage.uuid, ImageStorage.checksum, Parent.docker_image_id.alias('parent_docker_image_id'), ParentImageStorage.uuid.alias('parent_storage_uuid'))
.select(Image.id, Image.docker_image_id, ImageStorage.uuid, ImageStorage.checksum, Parent.docker_image_id.alias('parent_docker_image_id'), ParentImageStorage.uuid.alias('parent_storage_uuid'))
.join(Parent, on=(Image.parent == Parent.id))
.join(ParentImageStorage, on=(ParentImageStorage.id == Parent.storage))
.switch(Image)
@ -54,7 +61,7 @@ def _get_image_to_export(version):
.alias('candidates'))
images = (Image
.select(candidates.c.docker_image_id, candidates.c.uuid, candidates.c.checksum, candidates.c.parent_docker_image_id, candidates.c.parent_storage_uuid)
.select(candidates.c.id, candidates.c.docker_image_id, candidates.c.uuid, candidates.c.checksum, candidates.c.parent_docker_image_id, candidates.c.parent_storage_uuid)
.distinct()
.from_(candidates)
.order_by(db_random_func())
@ -62,7 +69,7 @@ def _get_image_to_export(version):
.limit(BATCH_SIZE))
for image in images:
rimages.append({'docker_image_id': image[0], 'storage_uuid': image[1], 'storage_checksum': image[2], 'parent_docker_image_id': None, 'parent_storage_uuid': None})
rimages.append({'image_id': image[0], 'docker_image_id': image[1], 'storage_uuid': image[2], 'storage_checksum': image[3], 'parent_docker_image_id': image[4], 'parent_storage_uuid': image[5]})
# Re-shuffle, otherwise the images without parents will always be on the top
random.shuffle(rimages)
@ -134,15 +141,24 @@ class SecurityWorker(Worker):
return True
def _index_images(self):
logger.debug('Starting indexing')
with UseThenDisconnect(app.config):
while True:
logger.debug('Looking up images to index')
# Get images to analyze
try:
images = _get_image_to_export(self._target_version)
if not images:
logger.debug('No more image to analyze')
return
except Image.DoesNotExist:
logger.debug('No more image to analyze')
return
logger.debug('Found images to index: %s', images)
for img in images:
# Get layer storage URL
path = storage.image_layer_path(img['storage_uuid'])
@ -191,6 +207,62 @@ class SecurityWorker(Worker):
logger.warning('An engine runs on version %d but the target version is %d')
_update_image(img, True, api_version)
logger.info('Layer ID %s : analyzed successfully', request['ID'])
# TODO(jschorr): Put this in a proper place, properly comment, unify with the
# callback code, etc.
try:
logger.debug('Loading vulnerabilities for layer %s', img['image_id'])
response = sec_endpoint.call_api('layers/%s/vulnerabilities', request['ID'])
except requests.exceptions.Timeout:
logger.debug('Timeout when calling Sec')
continue
except requests.exceptions.ConnectionError:
logger.debug('Connection error when calling Sec')
continue
logger.debug('Got response %s for vulnerabilities for layer %s', response.status_code, img['image_id'])
if response.status_code == 404:
continue
sec_data = json.loads(response.text)
logger.debug('Got response vulnerabilities for layer %s: %s', img['image_id'], sec_data)
if not sec_data['Vulnerabilities']:
continue
event = ExternalNotificationEvent.get(name='vulnerability_found')
matching = (RepositoryTag
.select(RepositoryTag, Repository)
.distinct()
.where(RepositoryTag.image_id == img['id'])
.join(Repository)
.join(RepositoryNotification)
.where(RepositoryNotification.event == event))
repository_map = defaultdict(list)
for tag in matching:
repository_map[tag.repository_id].append(tag)
for repository_id in repository_map:
tags = repository_map[repository_id]
for vuln in sec_data['Vulnerabilities']:
event_data = {
'tags': [tag.name for tag in tags],
'vulnerability': {
'id': vuln['ID'],
'description': vuln['Description'],
'link': vuln['Link'],
'priority': vuln['Priority'],
},
}
spawn_notification(tags[0].repository, 'vulnerability_found', event_data)
else:
if 'Message' in jsonResponse:
if 'OS and/or package manager are not supported' in jsonResponse['Message']:
@ -206,13 +278,11 @@ class SecurityWorker(Worker):
return
if __name__ == '__main__':
logging.getLogger('requests').setLevel(logging.WARNING)
logging.getLogger('apscheduler').setLevel(logging.CRITICAL)
if not features.SECURITY_SCANNER:
logger.debug('Security scanner disabled; skipping')
while True:
time.sleep(100000)
logging.config.fileConfig('conf/logging_debug.conf', disable_existing_loggers=False)
worker = SecurityWorker()
worker.start()

2
workers/worker.py
View file

@ -61,7 +61,7 @@ class Worker(object):
pass
def start(self):
logging.config.fileConfig('conf/logging.conf', disable_existing_loggers=False)
logging.config.fileConfig('conf/logging_debug.conf', disable_existing_loggers=False)
if not app.config.get('SETUP_COMPLETE', False):
logger.info('Product setup is not yet complete; skipping worker startup')