vbatts/quay
Archived
1
0
Fork 0
Code Issues Pull requests Releases Wiki Activity

Ready for demo

Browse source
This commit is contained in:
Joseph Schorr 2015-10-28 16:32:46 -04:00 committed by Jimmy Zelinskie
parent 75173d5573
commit b408cfd2cc
7 changed files with 94 additions and 80 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

5
data/model/image.py
View file

@ -1,7 +1,7 @@
import logging
import dateutil.parser
from peewee import JOIN_LEFT_OUTER, fn
from peewee import JOIN_LEFT_OUTER, fn, SQL
from datetime import datetime
from data.model import DataModelException, db_transaction, _basequery, storage
@ -18,7 +18,8 @@ def get_repository_images_recursive(docker_image_ids):
Note: This is a DB intensive operation and should be used sparingly.
"""
inner_images = Image.select('%/' + Image.id + '/%').where(Image.docker_image_id << docker_image_ids)
# TODO: test this on MySQL and Postgres
inner_images = Image.select(SQL('"%/" || id || "/%"')).where(Image.docker_image_id << docker_image_ids)
images = Image.select(Image.id).where(Image.docker_image_id << docker_image_ids)
recursive_images = Image.select(Image.id).where(Image.ancestors ** inner_images)

8
endpoints/api/sec.py
View file

@ -65,7 +65,8 @@ class RepositoryTagVulnerabilities(RepositoryParamResource):
'security_indexed': False
}
data = _call_security_api('layers/%s/vulnerabilities', tag_image.docker_image_id,
data = _call_security_api('layers/%s.%s/vulnerabilities', tag_image.docker_image_id,
tag_image.storage.uuid,
minimumPriority=args.minimumPriority)
return {
@ -79,7 +80,7 @@ class RepositoryTagVulnerabilities(RepositoryParamResource):
@path_param('repository', 'The full path of the repository. e.g. namespace/name')
@path_param('imageid', 'The image ID')
class RepositoryImagePackages(RepositoryParamResource):
""" Operations for listing the packages added/removed in an image. """
""" Operations for listing the packages in an image. """
@require_repo_read
@nickname('getRepoImagePackages')
@ -94,7 +95,8 @@ class RepositoryImagePackages(RepositoryParamResource):
'security_indexed': False
}
data = _call_security_api('layers/%s/packages/diff', repo_image.docker_image_id)
data = _call_security_api('layers/%s.%s/packages', repo_image.docker_image_id,
repo_image.storage.uuid)
return {
'security_indexed': True,

7
endpoints/sec.py
View file

@ -13,14 +13,17 @@ sec = Blueprint('sec', __name__)
@sec.route('/notification', methods=['POST'])
def sec_notification():
data = request.get_json()
print data
# Find all tags that contain the layer(s) introducing the vulnerability.
# TODO: remove this check once fixed.
if not 'IntroducingLayersIDs' in data['Content']:
return make_response('Okay')
layer_ids = data['Content']['IntroducingLayersIDs']
# TODO: fix this for the image_id.storage thing properly.
layer_ids = [full_id.split('.')[0] for full_id in data['Content']['IntroducingLayersIDs']]
if not layer_ids:
return make_response('Okay')
tags = model.tag.get_matching_tags(layer_ids, RepositoryTag, Repository, Image)
# For any repository that has a notification setup, issue a notification.

7
static/css/directives/repo-view/repo-panel-tags.css
View file

@ -94,8 +94,9 @@
}
.repo-panel-tags-element .vuln-description {
color: #ccc;
color: #aaa;
font-size: 10px;
white-space: normal;
}
.repo-panel-tags-element .fa-flag.None {
@ -110,6 +111,10 @@
color: red;
}
.repo-panel-tags-element .vuln-dropdown ul {
min-width: 400px;
}
@keyframes flickerAnimation { /* flame pulses */
0% { opacity:1; }
50% { opacity:0; }

2
static/directives/repo-view/repo-panel-tags.html
View file

@ -126,7 +126,7 @@
data-title="Image has no vulnerabilities"
bs-tooltip>
</i>
<div class="dropdown" style="text-align: left;"
<div class="dropdown vuln-dropdown" style="text-align: left;"
ng-if="getTagVulnerabilities(tag).indexed && getTagVulnerabilities(tag).hasVulnerabilities">
<i class="fa fa-flag"
data-title="Image has vulnerabilities"

6
static/partials/image-view.html
View file

@ -68,21 +68,21 @@
Please try again in a few minutes.
</div>
</div>
<div class="empty" ng-if="packages.security_indexed && !packages.data.InstalledPackages.length">
<div class="empty" ng-if="packages.security_indexed && !packages.data.Packages.length">
<div class="empty-primary-msg">This image contains no recognized packages</div>
<div class="empty-secondary-msg">
Quay currently indexes Debian, Red Hat and Ubuntu packages.
</div>
</div>
<table class="table" ng-if="packages.security_indexed && packages.data.InstalledPackages.length">
<table class="table" ng-if="packages.security_indexed && packages.data.Packages.length">
<thead>
<th>Package Name</th>
<th>Package Version</th>
<th>OS</th>
</thead>
<tr ng-repeat="package in packages.data.InstalledPackages | orderBy:'Name'">
<tr ng-repeat="package in packages.data.Packages | orderBy:'Name'">
<td>{{ package.Name }}</td>
<td>{{ package.Version }}</td>
<td>{{ package.OS }}</td>

139
workers/securityworker.py
View file

@ -97,13 +97,13 @@ def _update_image(image, indexed, version):
.where(Image.docker_image_id == image['docker_image_id'], ImageStorage.uuid == image['storage_uuid']))
updated_images = list()
for image in query:
updated_images.append(image.id)
for row in query:
updated_images.append(row.id)
query = (Image
(Image
.update(security_indexed=indexed, security_indexed_engine=version)
.where(Image.id << updated_images))
query.execute()
.where(Image.id << updated_images)
.execute())
class SecurityWorker(Worker):
def __init__(self):
@ -163,14 +163,17 @@ class SecurityWorker(Worker):
# Get layer storage URL
path = storage.image_layer_path(img['storage_uuid'])
locations = self._default_storage_locations
if not storage.exists(locations, path):
locations = _get_storage_locations(img['storage_uuid'])
if not storage.exists(locations, path):
logger.warning('Could not find a valid location to download layer %s', img['docker_image_id']+'.'+img['storage_uuid'])
# Mark as analyzed because that error is most likely to occur during the pre-process, with the database copy
# when images are actually removed on the real database (and therefore in S3)
_update_image(img, False, self._target_version)
continue
uri = storage.get_direct_download_url(locations, path)
if uri == None:
# Local storage hack
@ -200,69 +203,7 @@ class SecurityWorker(Worker):
logger.exception('An exception occurred when analyzing layer ID %s : the response is not valid JSON (%s)', request['ID'], httpResponse.text)
return
if httpResponse.status_code == 201:
# The layer has been successfully indexed
api_version = jsonResponse['Version']
if api_version < self._target_version:
logger.warning('An engine runs on version %d but the target version is %d')
_update_image(img, True, api_version)
logger.info('Layer ID %s : analyzed successfully', request['ID'])
# TODO(jschorr): Put this in a proper place, properly comment, unify with the
# callback code, etc.
try:
logger.debug('Loading vulnerabilities for layer %s', img['image_id'])
response = sec_endpoint.call_api('layers/%s/vulnerabilities', request['ID'])
except requests.exceptions.Timeout:
logger.debug('Timeout when calling Sec')
continue
except requests.exceptions.ConnectionError:
logger.debug('Connection error when calling Sec')
continue
logger.debug('Got response %s for vulnerabilities for layer %s', response.status_code, img['image_id'])
if response.status_code == 404:
continue
sec_data = json.loads(response.text)
logger.debug('Got response vulnerabilities for layer %s: %s', img['image_id'], sec_data)
if not sec_data['Vulnerabilities']:
continue
event = ExternalNotificationEvent.get(name='vulnerability_found')
matching = (RepositoryTag
.select(RepositoryTag, Repository)
.distinct()
.join(Repository)
.join(RepositoryNotification)
.where(RepositoryNotification.event == event,
RepositoryTag.image == img['image_id']))
repository_map = defaultdict(list)
for tag in matching:
repository_map[tag.repository_id].append(tag)
for repository_id in repository_map:
tags = repository_map[repository_id]
for vuln in sec_data['Vulnerabilities']:
event_data = {
'tags': [tag.name for tag in tags],
'vulnerability': {
'id': vuln['ID'],
'description': vuln['Description'],
'link': vuln['Link'],
'priority': vuln['Priority'],
},
}
spawn_notification(tags[0].repository, 'vulnerability_found', event_data)
else:
if httpResponse.status_code != 201:
if 'Message' in jsonResponse:
if 'OS and/or package manager are not supported' in jsonResponse['Message']:
# The current engine could not index this layer
@ -276,6 +217,68 @@ class SecurityWorker(Worker):
logger.exception('An exception occurred when analyzing layer ID %s : %d', request['ID'], httpResponse.status_code)
return
# The layer has been successfully indexed
api_version = jsonResponse['Version']
if api_version < self._target_version:
logger.warning('An engine runs on version %d but the target version is %d')
logger.debug('Layer %s analyzed successfully', request['ID'])
_update_image(img, True, api_version)
# TODO(jschorr): Put this in a proper place, properly comment, unify with the
# callback code, etc.
try:
logger.debug('Loading vulnerabilities for layer %s', img['image_id'])
response = sec_endpoint.call_api('layers/%s/vulnerabilities', request['ID'])
except requests.exceptions.Timeout:
logger.debug('Timeout when calling Sec')
continue
except requests.exceptions.ConnectionError:
logger.debug('Connection error when calling Sec')
continue
logger.debug('Got response %s for vulnerabilities for layer %s', response.status_code, img['image_id'])
if response.status_code == 404:
continue
sec_data = json.loads(response.text)
logger.debug('Got response vulnerabilities for layer %s: %s', img['image_id'], sec_data)
if not sec_data['Vulnerabilities']:
continue
event = ExternalNotificationEvent.get(name='vulnerability_found')
matching = (RepositoryTag
.select(RepositoryTag, Repository)
.distinct()
.join(Repository)
.join(RepositoryNotification)
.where(RepositoryNotification.event == event,
RepositoryTag.image == img['image_id']))
repository_map = defaultdict(list)
for tag in matching:
repository_map[tag.repository_id].append(tag)
for repository_id in repository_map:
tags = repository_map[repository_id]
for vuln in sec_data['Vulnerabilities']:
event_data = {
'tags': [tag.name for tag in tags],
'vulnerability': {
'id': vuln['ID'],
'description': vuln['Description'],
'link': vuln['Link'],
'priority': vuln['Priority'],
},
}
spawn_notification(tags[0].repository, 'vulnerability_found', event_data)
if __name__ == '__main__':
if not features.SECURITY_SCANNER:
logger.debug('Security scanner disabled; skipping')