vbatts/quay
Archived
1
0
Fork 0
Code Issues Pull requests Releases Wiki Activity

Implement setup tool support for Clair

Browse source 
Fixes #1387
This commit is contained in:
Joseph Schorr 2016-05-02 15:29:31 -04:00
parent 53ce4de6aa
commit 2cbdecb043
23 changed files with 584 additions and 116 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
app.py
View file

@ -194,7 +194,7 @@ dockerfile_build_queue = WorkQueue(app.config['DOCKERFILE_BUILD_QUEUE_NAME'], tf
reporter=MetricQueueReporter(metric_queue))
notification_queue = WorkQueue(app.config['NOTIFICATION_QUEUE_NAME'], tf)
secscan_notification_queue = WorkQueue(app.config['SECSCAN_NOTIFICATION_QUEUE_NAME'], tf)
secscan_api = SecurityScannerAPI(app.config, config_provider, storage)
secscan_api = SecurityScannerAPI(app.config, storage)
# Check for a key in config. If none found, generate a new signing key for Docker V2 manifests.
_v2_key_path = os.path.join(OVERRIDE_CONFIG_DIRECTORY, DOCKER_V2_SIGNINGKEY_FILENAME)

39
boot.py
View file

@ -5,8 +5,9 @@ from urlparse import urlunparse
from jinja2 import Template
from cachetools import lru_cache
import release
import release
import os.path
from app import app
from data.model.release import set_region_release
@ -37,49 +38,49 @@ def get_audience():
return urlunparse((scheme, hostname + ':' + port, '', '', '', ''))
def create_quay_service_key():
def setup_jwt_proxy():
"""
Creates a service key for quay to use in the jwtproxy
Creates a service key for quay to use in the jwtproxy and generates the JWT proxy configuration.
"""
if os.path.exists('conf/jwtproxy_conf.yaml'):
# Proxy is already setup.
return
# Generate the key for this Quay instance to use.
minutes_until_expiration = app.config.get('QUAY_SERVICE_KEY_EXPIRATION', 120)
expiration = datetime.now() + timedelta(minutes=minutes_until_expiration)
quay_key, key_id = generate_key('quay', get_audience(), expiration_date=expiration)
quay_key, quay_key_id = generate_key('quay', get_audience(), expiration_date=expiration)
with open('/conf/quay.kid', mode='w') as f:
with open('conf/quay.kid', mode='w') as f:
f.truncate(0)
f.write(key_id)
f.write(quay_key_id)
with open('/conf/quay.pem', mode='w') as f:
with open('conf/quay.pem', mode='w') as f:
f.truncate(0)
f.write(quay_key.exportKey())
return key_id
def create_jwtproxy_conf(quay_key_id):
"""
Generates the jwtproxy conf from the jinja template
"""
# Generate the JWT proxy configuration.
audience = get_audience()
registry = audience + '/keys'
security_issuer = app.config.get('SECURITY_SCANNER_ISSUER_NAME', 'security_scanner')
with open("/conf/jwtproxy_conf.yaml.jnj") as f:
with open("conf/jwtproxy_conf.yaml.jnj") as f:
template = Template(f.read())
rendered = template.render(
audience=audience,
registry=registry,
key_id=quay_key_id
key_id=quay_key_id,
security_issuer=security_issuer,
)
with open('/conf/jwtproxy_conf.yaml', 'w') as f:
with open('conf/jwtproxy_conf.yaml', 'w') as f:
f.write(rendered)
def main():
if app.config.get('SETUP_COMPLETE', False):
sync_database_with_config(app.config)
quay_key_id = create_quay_service_key()
create_jwtproxy_conf(quay_key_id)
setup_jwt_proxy()
# Record deploy
if release.REGION and release.GIT_HEAD:

2
conf/jwtproxy_conf.yaml.jnj
View file

@ -23,5 +23,5 @@ jwtproxy:
key_server:
type: keyregistry
options:
issuer: clair
issuer: {{ security_issuer }}
registry: {{ registry }}

31
config.py
View file

@ -282,18 +282,33 @@ class DefaultConfig(object):
# Security scanner
FEATURE_SECURITY_SCANNER = False
FEATURE_SECURITY_NOTIFICATIONS = False
SECURITY_SCANNER = {
'ENDPOINT': 'http://192.168.99.101:6060',
'ENGINE_VERSION_TARGET': 2,
'API_VERSION': 'v1',
'API_TIMEOUT_SECONDS': 10,
'API_TIMEOUT_POST_SECONDS': 480,
}
# The endpoint for the security scanner.
SECURITY_SCANNER_ENDPOINT = 'http://192.168.99.101:6060'
# If specified, the endpoint to be used for all POST calls to the security scanner.
SECURITY_SCANNER_ENDPOINT_BATCH = None
# The indexing engine version running inside the security scanner.
SECURITY_SCANNER_ENGINE_VERSION_TARGET = 2
# The version of the API to use for the security scanner.
SECURITY_SCANNER_API_VERSION = 'v1'
# API call timeout for the security scanner.
SECURITY_SCANNER_API_TIMEOUT_SECONDS = 10
# POST call timeout for the security scanner.
SECURITY_SCANNER_API_TIMEOUT_POST_SECONDS = 480
# The issuer name for the security scanner.
SECURITY_SCANNER_ISSUER_NAME = 'security_scanner'
# JWTProxy Settings
# The address (sans schema) to proxy outgoing requests through the jwtproxy
# to be signed
JWTPROXY_SIGNER = 'localhost:8080'
# The audience that jwtproxy should verify on incoming requests
# If None, will be calculated off of the SERVER_HOSTNAME (default)
JWTPROXY_AUDIENCE = None
@ -322,7 +337,9 @@ class DefaultConfig(object):
# The location of the private key generated for this instance
INSTANCE_SERVICE_KEY_LOCATION = 'conf/quay.pem'
# This instance's service key expiration in minutes
INSTANCE_SERVICE_KEY_EXPIRATION = 120
# Number of minutes between expiration refresh in minutes
INSTANCE_SERVICE_KEY_REFRESH = 60

1
endpoints/api/superuser.py
View file

@ -599,6 +599,7 @@ class SuperUserServiceKeyManagement(ApiResource):
return jsonify({
'kid': key.kid,
'name': body.get('name', ''),
'service': body['service'],
'public_key': private_key.publickey().exportKey('PEM'),
'private_key': private_key.exportKey('PEM'),
})

19
static/css/core-ui.css
View file

@ -545,6 +545,18 @@ a:focus {
margin-left: 10px;
}
.config-service-key-field-element {
position: relative;
}
.config-service-key-field-element .co-modify-link {
margin-left: 10px;
}
.config-service-key-field-element .fa-check {
margin-right: 4px;
}
.co-checkbox {
position: relative;
}
@ -1457,4 +1469,11 @@ a:focus {
padding-bottom: 10px;
}
.co-option-table .help-text {
margin-top: 4px;
margin-bottom: 10px;
font-size: 14px;
color: #aaa;
}

11
static/css/directives/ui/request-service-key-dialog.css Normal file
View file

@ -0,0 +1,11 @@
.request-service-key-dialog-element .co-option-table {
margin-top: 20px;
}
.request-service-key-dialog-element .key-display {
margin-top: 10px;
font-size: 12px;
font-family: Menlo,Monaco,Consolas,"Courier New",monospace;
background: white;
min-height: 500px;
}

29
static/directives/config/config-service-key-field.html Normal file
View file

@ -0,0 +1,29 @@
<div class="config-service-key-field-element">
<!-- Loading -->
<div class="cor-loader" ng-if="loading"></div>
<!-- Loading error -->
<div class="co-alert co-alert-warning" ng-if="loadError">
Could not load service keys
</div>
<!-- Key config -->
<div ng-show="!loading && !loadError">
<div ng-show="hasValidKey">
<i class="fa fa-check"></i>
Valid key for service <code>{{ serviceName }}</code> exists
</div>
<div ng-show="!hasValidKey">
No valid key found for service <code>{{ serviceName }}</code>
<a class="co-modify-link" ng-click="showRequestServiceKey()">Create Key</a>
</div>
</div>
<!-- Note: This field is a hidden text field that binds to a model that is set to non-empty
when there is a valid key. This allows us to use the existing Angular form validation
code.
-->
<input type="text" ng-model="hasValidKeyStr" ng-required="true" style="position: absolute; top: 0px; left: 0px; visibility: hidden; width: 0px; height: 0px;">
<div class="request-service-key-dialog" request-key-info="requestKeyInfo" key-created="handleKeyCreated(key)"></div>
</div>

47
static/directives/config/config-setup-tool.html
View file

@ -286,6 +286,53 @@
</div>
</div>
<!-- Security Scanner -->
<div class="co-panel">
<div class="co-panel-heading">
<i class="fa fa-bug"></i> Security Scanner
</div>
<div class="co-panel-body">
<div class="description">
<p>If enabled, all images pushed to Quay will be scanned via the external security scanning service, with vulnerability information available in the UI and API, as well
as async notification support.
</p>
</div>
<div class="co-checkbox">
<input id="ftsecurity" type="checkbox" ng-model="config.FEATURE_SECURITY_SCANNER">
<label for="ftsecurity">Enable Security Scanning</label>
</div>
<div class="co-alert co-alert-info" ng-if="config.FEATURE_SECURITY_SCANNER" style="margin-top: 20px;">
A scanner compliant with the Quay Security Scanning API must be running to use this feature. Documentation on running <a href="https://github.com/coreos/clair" ng-safenewtab>Clair</a> can be found at <a href="https://tectonic.com/quay-enterprise/docs/latest/clair.html" ng-safenewtab>Running Clair Security Scanner</a>.
</div>
<table class="config-table" ng-if="config.FEATURE_SECURITY_SCANNER">
<tr>
<td>Security Scanner Endpoint:</td>
<td>
<span class="config-string-field" binding="config.SECURITY_SCANNER_ENDPOINT"
placeholder="Security Scanner API endpoint (Example: http://myhost:6060)"
pattern="http(s)?://.+"></span>
<div class="help-text">
The HTTP URL at which the security scanner is running.
</div>
</td>
</tr>
<tr>
<td>Authentication Key:</td>
<td>
<span class="config-service-key-field" service-name="{{ config.SECURITY_SCANNER_ISSUER_NAME }}"></span>
<div class="help-text">
The security scanning service requires an authorized service key to speak to Quay. Once setup, the key
can be managed in the Service Keys panel under the Super User Admin Panel.
</div>
</td>
</tr>
</table>
</div>
</div>
<!-- ACI Conversion -->
<div class="co-panel">
<div class="co-panel-heading">

134
static/directives/request-service-key-dialog.html Normal file
View file

@ -0,0 +1,134 @@
<div class="request-service-key-dialog-element">
<!-- Modal message dialog -->
<div class="co-dialog modal fade">
<div class="modal-dialog">
<div class="modal-content">
<div class="modal-header">
<button type="button" class="close" data-dismiss="modal" aria-hidden="true" ng-show="!working">&times;</button>
<h4 class="modal-title">Create key for service {{ requestKeyInfo.service }}</h4>
</div>
<div class="modal-body" ng-show="working">
<div class="cor-loader"></div>
</div>
<div class="modal-body" ng-show="!working">
<!-- Step 0 -->
<div ng-show="step == 0">
<table class="co-option-table">
<tr>
<td><input type="radio" id="automaticKey" ng-model="requestKind" value="automatic"></td>
<td>
<label for="automaticKey">Have the service provide a key</label>
<div class="help-text">Recommended for <code>{{ requestKeyInfo.service }}</code> installations where the single instance is setup now.</div>
</td>
</tr>
<tr>
<td><input type="radio" id="presharedKey" ng-model="requestKind" value="preshared"></td>
<td>
<label for="presharedKey">Generate shared key</label>
<div class="help-text">Recommended for <code>{{ requestKeyInfo.service }}</code> installations where the instances are dynamically started.</div>
</td>
</tr>
</table>
</div>
<!-- Step 1 (automatic) -->
<div ng-show="step == 1 && requestKind == 'automatic'" style="text-align: center">
<div style="margin-top: 20px;">
Please start the <code>{{ requestKeyInfo.service }}</code> service now, configured for <a href="https://github.com/coreos/jwtproxy#autogenerated-private-key" ng-safenewtab>autogenerated private key</a>. The key approval process will continue automatically once the service connects to Quay.
</div>
<div style="margin-top: 20px;">
Waiting for service to connect
</div>
<div style="margin-top: 10px; margin-bottom: 20px;">
<div class="cor-loader-inline"></div>
</div>
</div>
<!-- Step 2 (automatic) -->
<div ng-show="step == 2 && requestKind == 'automatic'" style="text-align: center">
A key for service <code>{{ requestKeyInfo.service }}</code> has been automatically generated, approved and saved in the service's keystore.
</div>
<!-- Step 1 (generate) -->
<div ng-show="step == 1 && requestKind == 'preshared'">
<form name="createForm" ng-submit="createPresharedKey()">
<table class="co-form-table">
<tr>
<td><label for="create-key-name">Key Name:</label></td>
<td>
<input class="form-control" name="create-key-name" type="text" ng-model="preshared.name" placeholder="Friendly Key Name">
<span class="co-help-text">
A friendly name for the key for later reference.
</span>
</td>
</tr>
<tr>
<td><label for="create-key-expiration">Expiration date (optional):</label></td>
<td>
<span class="datetime-picker" datetime="preshared.expiration"></span>
<span class="co-help-text">
The date and time that the key expires. If left blank, the key will never expire.
</span>
</td>
</tr>
<tr>
<td><label for="create-key-notes">Approval Notes (optional):</label></td>
<td>
<div class="markdown-editor" content="preshared.notes"></div>
<span class="co-help-text">
Optional notes for additional human-readable information about why the key was created.
</span>
</td>
</tr>
</table>
</form>
</div>
<!-- Step 2 (generate) -->
<div ng-show="step == 2 && requestKind == 'preshared'">
<div class="co-alert co-alert-warning">
The following key has been generated for service <code>{{ requestKeyInfo.service }}</code>.
<br><br>
Please copy the key's ID and copy/download the key's private contents and place it in the directory with the service's configuration.
<br><br>
<strong>Once this dialog is closed this private key will not be accessible anywhere else!</strong>
</div>
<label>Key ID:</label>
<div class="copy-box" value="createdKey.kid"></div>
<label>Private Key (PEM):</label>
<textarea class="key-display form-control" onclick="this.focus();this.select()" readonly>{{ createdKey.private_key }}</textarea>
</div>
</div>
<div class="modal-footer" ng-show="!working">
<button type="button" class="btn btn-primary" ng-show="step == 1 && requestKind == 'preshared'"
ng-disabled="createForm.$invalid"
ng-click="createPresharedKey()">
Generate Key
</button>
<button type="button" class="btn btn-primary" ng-show="step == 0 && requestKind == 'preshared'"
ng-click="showGenerate()">
Continue
</button>
<button type="button" class="btn btn-primary" ng-show="step == 0 && requestKind == 'automatic'"
ng-click="startApproval()">
Start Approval
</button>
<button type="button" class="btn btn-primary" ng-click="downloadPrivateKey(createdKey)" ng-if="createdKey && isDownloadSupported()">
<i class="fa fa-download"></i> Download Private Key
</button>
<button type="button" class="btn btn-default" data-dismiss="modal" ng-show="step == 2">Close</button>
<button type="button" class="btn btn-default" data-dismiss="modal" ng-show="step != 2">Cancel</button>
</div>
</div><!-- /.modal-content -->
</div><!-- /.modal-dialog -->
</div><!-- /.modal -->
</div>
</div>

85
static/js/core-config-setup.js
View file

@ -61,6 +61,10 @@ angular.module("core-config-setup", ['angularFileUpload'])
{'id': 'gitlab-trigger', 'title': 'GitLab Build Triggers', 'condition': function(config) {
return config.FEATURE_GITLAB_BUILD;
}},
{'id': 'security-scanner', 'title': 'Quay Security Scanner', 'condition': function(config) {
return config.FEATURE_SECURITY_SCANNER;
}}
];
@ -1029,6 +1033,87 @@ angular.module("core-config-setup", ['angularFileUpload'])
return directiveDefinitionObject;
})
.directive('configServiceKeyField', function (ApiService) {
var directiveDefinitionObject = {
priority: 0,
templateUrl: '/static/directives/config/config-service-key-field.html',
replace: false,
transclude: false,
restrict: 'C',
scope: {
'serviceName': '@serviceName',
},
controller: function($scope, $element) {
$scope.foundKeys = [];
$scope.loading = false;
$scope.loadError = false;
$scope.hasValidKey = false;
$scope.hasValidKeyStr = null;
$scope.updateKeys = function() {
$scope.foundKeys = [];
$scope.loading = true;
ApiService.listServiceKeys().then(function(resp) {
$scope.loading = false;
$scope.loadError = false;
resp['keys'].forEach(function(key) {
if (key['service'] == $scope.serviceName) {
$scope.foundKeys.push(key);
}
});
$scope.hasValidKey = checkValidKey($scope.foundKeys);
$scope.hasValidKeyStr = $scope.hasValidKey ? 'true' : '';
}, function() {
$scope.loading = false;
$scope.loadError = true;
});
};
// Perform initial loading of the keys.
$scope.updateKeys();
$scope.isKeyExpired = function(key) {
if (key.expiration_date != null) {
var expiration_date = moment(key.expiration_date);
return moment().isAfter(expiration_date);
}
return false;
};
$scope.showRequestServiceKey = function() {
$scope.requestKeyInfo = {
'service': $scope.serviceName
};
};
$scope.handleKeyCreated = function() {
$scope.updateKeys();
};
var checkValidKey = function(keys) {
for (var i = 0; i < keys.length; ++i) {
var key = keys[i];
if (!key.approval) {
continue;
}
if ($scope.isKeyExpired(key)) {
continue;
}
return true;
}
return false;
};
}
};
return directiveDefinitionObject;
})
.directive('configStringField', function () {
var directiveDefinitionObject = {
priority: 0,

120
static/js/directives/ui/request-service-key-dialog.js Normal file
View file

@ -0,0 +1,120 @@
/**
* An element which displays a dialog for requesting or creating a service key.
*/
angular.module('quay').directive('requestServiceKeyDialog', function () {
var directiveDefinitionObject = {
priority: 0,
templateUrl: '/static/directives/request-service-key-dialog.html',
replace: false,
transclude: true,
restrict: 'C',
scope: {
'requestKeyInfo': '=requestKeyInfo',
'keyCreated': '&keyCreated'
},
controller: function($scope, $element, $timeout, ApiService) {
var handleNewKey = function(key) {
var data = {
'notes': 'Approved during setup of service ' + key.service
};
var params = {
'kid': key.kid
};
ApiService.approveServiceKey(data, params).then(function(resp) {
$scope.keyCreated({'key': key});
$scope.step = 2;
}, ApiService.errorDisplay('Could not approve service key'));
};
var checkKeys = function() {
var isShown = ($element.find('.modal').data('bs.modal') || {}).isShown;
if (!isShown) {
return;
}
// TODO: filter by service.
ApiService.listServiceKeys().then(function(resp) {
var keys = resp['keys'];
for (var i = 0; i < keys.length; ++i) {
var key = keys[i];
if (key.service == $scope.requestKeyInfo.service && !key.approval && key.rotation_duration) {
handleNewKey(key);
return;
}
}
$timeout(checkKeys, 1000);
}, ApiService.errorDisplay('Could not list service keys'));
};
$scope.show = function() {
$scope.working = false;
$scope.step = 0;
$scope.requestKind = null;
$scope.preshared = {
'name': $scope.requestKeyInfo.service + ' Service Key',
'notes': 'Created during setup for service `' + $scope.requestKeyInfo.service + '`'
};
$element.find('.modal').modal({});
};
$scope.hide = function() {
$scope.loading = false;
$element.find('.modal').modal('hide');
};
$scope.showGenerate = function() {
$scope.step = 1;
};
$scope.startApproval = function() {
$scope.step = 1;
checkKeys();
};
$scope.isDownloadSupported = function() {
var isSafari = /^((?!chrome).)*safari/i.test(navigator.userAgent);
if (isSafari) {
// Doesn't work properly in Safari, sadly.
return false;
}
try { return !!new Blob(); } catch(e) {}
return false;
};
$scope.downloadPrivateKey = function(key) {
var blob = new Blob([key.private_key]);
saveAs(blob, key.service + '.pem');
};
$scope.createPresharedKey = function() {
$scope.working = true;
var data = {
'name': $scope.preshared.name,
'service': $scope.requestKeyInfo.service,
'expiration': $scope.preshared.expiration || null,
'notes': $scope.preshared.notes
};
ApiService.createServiceKey(data).then(function(resp) {
$scope.working = false;
$scope.step = 2;
$scope.createdKey = resp;
$scope.keyCreated({'key': resp});
}, ApiService.errorDisplay('Could not create service key'));
};
$scope.$watch('requestKeyInfo', function(info) {
if (info && info.service) {
$scope.show();
}
});
}
};
return directiveDefinitionObject;
});

5
static/js/pages/setup.js
View file

@ -1,13 +1,12 @@
(function() {
/**
* The Setup page provides a nice GUI walkthrough experience for setting up the Enterprise
* Registry.
* The Setup page provides a nice GUI walkthrough experience for setting up Quay Enterprise.
*/
angular.module('quayPages').config(['pages', function(pages) {
pages.create('setup', 'setup.html', SetupCtrl,
{
'newLayout': true,
'title': 'Enterprise Registry Setup'
'title': 'Quay Enterprise Setup'
})
}]);

2
static/js/pages/superuser.js
View file

@ -1,6 +1,6 @@
(function() {
/**
* The superuser admin page provides a new management UI for the Enterprise Registry.
* The superuser admin page provides a new management UI for Quay Enterprise.
*/
angular.module('quayPages').config(['pages', function(pages) {
pages.create('superuser', 'super-user.html', SuperuserCtrl,

2
static/partials/setup.html
View file

@ -3,7 +3,7 @@
<div class="page-content" quay-show="Features.SUPER_USERS && currentStep == States.CONFIG">
<div class="cor-title">
<span class="cor-title-link"></span>
<span class="cor-title-content">Enterprise Registry Setup</span>
<span class="cor-title-content">Quay Enterprise Setup</span>
</div>
<div class="cor-tab-panel" style="padding: 20px;">

2
test/test_api_usage.py
View file

@ -3546,7 +3546,7 @@ class TestRepositoryImageSecurity(ApiTestCase):
# Mark the layer as indexed.
layer.security_indexed = True
layer.security_indexed_engine = app.config['SECURITY_SCANNER']['ENGINE_VERSION_TARGET']
layer.security_indexed_engine = app.config['SECURITY_SCANNER_ENGINE_VERSION_TARGET']
layer.save()
# Grab the security info again.

2
test/test_secscan.py
View file

@ -122,7 +122,7 @@ class TestSecurityScanner(unittest.TestCase):
self.ctx = app.test_request_context()
self.ctx.__enter__()
self.api = SecurityScannerAPI(app.config, config_provider, storage)
self.api = SecurityScannerAPI(app.config, storage)
def tearDown(self):
storage.put_content(['local_us'], 'supports_direct_download', 'false')

10
test/testconfig.py
View file

@ -61,9 +61,7 @@ class TestConfig(DefaultConfig):
FEATURE_SECURITY_SCANNER = True
FEATURE_SECURITY_NOTIFICATIONS = True
SECURITY_SCANNER = {
'ENDPOINT': 'http://mockclairservice/',
'API_VERSION': 'v1',
'ENGINE_VERSION_TARGET': 1,
'API_TIMEOUT_SECONDS': 1
}
SECURITY_SCANNER_ENDPOINT = 'http://mockclairservice/'
SECURITY_SCANNER_API_VERSION = 'v1'
SECURITY_SCANNER_ENGINE_VERSION_TARGET = 1
SECURITY_SCANNER_API_TIMEOUT_SECONDS = 1

20
util/config/configutil.py
View file

@ -30,10 +30,20 @@ def add_enterprise_config_defaults(config_obj, current_secret_key, hostname):
'signing-public.gpg')
config_obj['SIGNING_ENGINE'] = config_obj.get('SIGNING_ENGINE', 'gpg2')
# Default security scanner config.
config_obj['FEATURE_SECURITY_NOTIFICATIONS'] = config_obj.get(
'FEATURE_SECURITY_NOTIFICATIONS', True)
config_obj['FEATURE_SECURITY_SCANNER'] = config_obj.get(
'FEATURE_SECURITY_SCANNER', False)
config_obj['SECURITY_SCANNER_ISSUER_NAME'] = config_obj.get(
'SECURITY_SCANNER_ISSUER_NAME', 'security_scanner')
# Default mail setings.
config_obj['MAIL_USE_TLS'] = True
config_obj['MAIL_PORT'] = 587
config_obj['MAIL_DEFAULT_SENDER'] = 'support@quay.io'
config_obj['MAIL_USE_TLS'] = config_obj.get('MAIL_USE_TLS', True)
config_obj['MAIL_PORT'] = config_obj.get('MAIL_PORT', 587)
config_obj['MAIL_DEFAULT_SENDER'] = config_obj.get('MAIL_DEFAULT_SENDER', 'support@quay.io')
# Default auth type.
if not 'AUTHENTICATION_TYPE' in config_obj:
@ -60,5 +70,5 @@ def add_enterprise_config_defaults(config_obj, current_secret_key, hostname):
# Misc configuration.
config_obj['PREFERRED_URL_SCHEME'] = config_obj.get('PREFERRED_URL_SCHEME', 'http')
config_obj['ENTERPRISE_LOGO_URL'] = config_obj.get('ENTERPRISE_LOGO_URL',
'/static/img/quay-logo.png')
config_obj['ENTERPRISE_LOGO_URL'] = config_obj.get(
'ENTERPRISE_LOGO_URL', '/static/img/QuayEnterprise_horizontal_color.svg')

25
util/config/validator.py
View file

@ -1,10 +1,9 @@
import redis
import os
import json
import ldap
import peewee
import OpenSSL
import logging
import time
from StringIO import StringIO
from fnmatch import fnmatch
@ -14,12 +13,14 @@ from data.users.externalldap import LDAPConnection, LDAPUsers
from flask import Flask
from flask.ext.mail import Mail, Message
from data.database import validate_database_url, User
from data.database import validate_database_url
from storage import get_storage_driver
from auth.auth_context import get_authenticated_user
from util.config.oauth import GoogleOAuthConfig, GithubOAuthConfig, GitLabOAuthConfig
from bitbucket import BitBucket
from util.security.signing import SIGNING_ENGINES
from util.secscan.api import SecurityScannerAPI
from boot import setup_jwt_proxy
from app import app, config_provider, get_app_url, OVERRIDE_CONFIG_DIRECTORY
@ -424,6 +425,23 @@ def _validate_signer(config, _):
engine.detached_sign(StringIO('test string'))
def _validate_security_scanner(config, _):
""" Validates the configuration for talking to a Quay Security Scanner. """
# Generate a temporary Quay key to use for signing the outgoing requests.
setup_jwt_proxy()
# Wait a few seconds for the JWT proxy to startup.
time.sleep(2)
# Make a ping request to the security service.
client = app.config['HTTPCLIENT']
api = SecurityScannerAPI(config, None, client=client, skip_validation=True)
response = api.ping()
if response.status_code != 200:
message = 'Expected 200 status code, got %s: %s' % (response.status_code, response.text)
raise Exception('Could not ping security scanner: %s' % message)
_VALIDATORS = {
'database': _validate_database,
'redis': _validate_redis,
@ -439,4 +457,5 @@ _VALIDATORS = {
'jwt': _validate_jwt,
'keystone': _validate_keystone,
'signer': _validate_signer,
'security-scanner': _validate_security_scanner,
}

5
util/secscan/analyzer.py
View file

@ -17,10 +17,8 @@ logger = logging.getLogger(__name__)
class LayerAnalyzer(object):
""" Helper class to perform analysis of a layer via the security scanner. """
def __init__(self, config, api):
secscan_config = config.get('SECURITY_SCANNER')
self._api = api
self._target_version = secscan_config['ENGINE_VERSION_TARGET']
self._target_version = config.get('SECURITY_SCANNER_ENGINE_VERSION_TARGET', 2)
def analyze_recursively(self, layer):
@ -62,7 +60,6 @@ class LayerAnalyzer(object):
- The second one is set to False when another worker pre-empted the candidate's analysis
for us.
"""
# If the parent couldn't be analyzed with the target version or higher, we can't analyze
# this image. Mark it as failed with the current target version.
if (layer.parent_id and not layer.parent.security_indexed and

63
util/secscan/api.py
View file

@ -21,26 +21,23 @@ _API_METHOD_INSERT = 'layers'
_API_METHOD_GET_LAYER = 'layers/%s'
_API_METHOD_MARK_NOTIFICATION_READ = 'notifications/%s'
_API_METHOD_GET_NOTIFICATION = 'notifications/%s'
_API_METHOD_PING = 'metrics'
class SecurityScannerAPI(object):
""" Helper class for talking to the Security Scan service (Clair). """
def __init__(self, config, config_provider, storage):
self.config = config
self.config_provider = config_provider
def __init__(self, config, storage, client=None, skip_validation=False):
if not skip_validation:
config_validator = SecurityConfigValidator(config)
if not config_validator.valid():
logger.warning('Invalid config provided to SecurityScannerAPI')
return
self._config = config
self._client = client or config['HTTPCLIENT']
self._storage = storage
self._security_config = None
config_validator = SecurityConfigValidator(config, config_provider)
if not config_validator.valid():
logger.warning('Invalid config provided to SecurityScannerAPI')
return
self._default_storage_locations = config['DISTRIBUTED_STORAGE_PREFERENCE']
self._security_config = config.get('SECURITY_SCANNER')
self._target_version = self._security_config['ENGINE_VERSION_TARGET']
self._target_version = config.get('SECURITY_SCANNER_ENGINE_VERSION_TARGET', 2)
def _get_image_url(self, image):
@ -62,7 +59,7 @@ class SecurityScannerAPI(object):
if uri is None:
# Handle local storage.
local_storage_enabled = False
for storage_type, _ in self.config.get('DISTRIBUTED_STORAGE_CONFIG', {}).values():
for storage_type, _ in self._config.get('DISTRIBUTED_STORAGE_CONFIG', {}).values():
if storage_type == 'LocalStorage':
local_storage_enabled = True
@ -99,6 +96,23 @@ class SecurityScannerAPI(object):
return request
def ping(self):
""" Calls GET on the metrics endpoint of the security scanner to ensure it is running
and properly configured. Returns the HTTP response.
"""
try:
return self._call('GET', _API_METHOD_PING)
except requests.exceptions.Timeout:
logger.exception('Timeout when trying to connect to security scanner endpoint')
raise Exception('Timeout when trying to connect to security scanner endpoint')
except requests.exceptions.ConnectionError:
logger.exception('Connection error when trying to connect to security scanner endpoint')
raise Exception('Connection error when trying to connect to security scanner endpoint')
except (requests.exceptions.RequestException, ValueError):
logger.exception('Exception when trying to connect to security scanner endpoint')
raise Exception('Exception when trying to connect to security scanner endpoint')
def analyze_layer(self, layer):
""" Posts the given layer to the security scanner for analysis, blocking until complete.
Returns a tuple containing the analysis version (on success, None on failure) and
@ -122,6 +136,7 @@ class SecurityScannerAPI(object):
logger.exception('Failed to post layer data response for %s', layer.id)
return None, False
# Handle any errors from the security scanner.
if response.status_code != 201:
message = json_response.get('Error').get('Message', '')
@ -235,25 +250,23 @@ class SecurityScannerAPI(object):
This function disconnects from the database while awaiting a response
from the API server.
"""
security_config = self._security_config
if security_config is None:
if self._config is None:
raise Exception('Cannot call unconfigured security system')
client = self.config['HTTPCLIENT']
client = self._client
headers = {'Connection': 'close'}
timeout = security_config['API_TIMEOUT_SECONDS']
endpoint = security_config['ENDPOINT']
timeout = self._config.get('SECURITY_SCANNER_API_TIMEOUT_SECONDS', 10)
endpoint = self._config['SECURITY_SCANNER_ENDPOINT']
if method != 'GET':
timeout = security_config.get('API_BATCH_TIMEOUT_SECONDS', timeout)
endpoint = security_config.get('ENDPOINT_BATCH', endpoint)
timeout = self._config.get('SECURITY_SCANNER_API_BATCH_TIMEOUT_SECONDS', timeout)
endpoint = self._config.get('SECURITY_SCANNER_ENDPOINT_BATCH') or endpoint
api_url = urljoin(endpoint, '/' + security_config['API_VERSION']) + '/'
api_url = urljoin(endpoint, '/' + self._config.get('SECURITY_SCANNER_API_VERSION', 'v1')) + '/'
url = urljoin(api_url, relative_url)
signer_proxy_url = self.config.get('JWTPROXY_SIGNER', 'localhost:8080')
signer_proxy_url = self._config.get('JWTPROXY_SIGNER', 'localhost:8080')
with CloseForLongOperation(self.config):
with CloseForLongOperation(self._config):
logger.debug('%sing security URL %s', method.upper(), url)
return client.request(method, url, json=body, params=params, timeout=timeout,
verify='/conf/mitm.cert', headers=headers,

44
util/secscan/validator.py
View file

@ -6,55 +6,23 @@ logger = logging.getLogger(__name__)
class SecurityConfigValidator(object):
""" Helper class for validating the security scanner configuration. """
def __init__(self, config, config_provider):
self._config_provider = config_provider
def __init__(self, config):
if not features.SECURITY_SCANNER:
return
self._security_config = config['SECURITY_SCANNER']
if self._security_config is None:
return
self._certificate = self._get_filepath('CA_CERTIFICATE_FILENAME') or False
self._public_key = self._get_filepath('PUBLIC_KEY_FILENAME')
self._private_key = self._get_filepath('PRIVATE_KEY_FILENAME')
if self._public_key and self._private_key:
self._keys = (self._public_key, self._private_key)
else:
self._keys = None
def _get_filepath(self, key):
config = self._security_config
if key in config:
with self._config_provider.get_volume_file(config[key]) as f:
return f.name
return None
def cert(self):
return self._certificate
def keypair(self):
return self._keys
self._config = config
def valid(self):
if not features.SECURITY_SCANNER:
return False
if not self._security_config:
logger.debug('Missing SECURITY_SCANNER block in configuration')
if self._config.get('SECURITY_SCANNER_ENDPOINT') is None:
logger.debug('Missing SECURITY_SCANNER_ENDPOINT configuration')
return False
if not 'ENDPOINT' in self._security_config:
logger.debug('Missing ENDPOINT field in SECURITY_SCANNER configuration')
return False
endpoint = self._security_config['ENDPOINT'] or ''
endpoint = self._config.get('SECURITY_SCANNER_ENDPOINT')
if not endpoint.startswith('http://') and not endpoint.startswith('https://'):
logger.debug('ENDPOINT field in SECURITY_SCANNER configuration must start with http or https')
logger.debug('SECURITY_SCANNER_ENDPOINT configuration must start with http or https')
return False
return True