vbatts/quay
Archived
1
0
Fork 0
Code Issues Pull requests Releases Wiki Activity

Merge branch 'master' into ldapreferfix

Browse source
This commit is contained in:
Joseph Schorr 2015-05-27 15:15:47 -04:00
commit 386b1710ed
14 changed files with 403 additions and 40 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

30
endpoints/web.py
View file

@ -9,10 +9,11 @@ from health.healthcheck import get_healthchecker
from data import model
from data.model.oauth import DatabaseAuthorizationProvider
from app import app, billing as stripe, build_logs, avatar, signer
from app import app, billing as stripe, build_logs, avatar, signer, log_archive
from auth.auth import require_session_login, process_oauth
from auth.permissions import (AdministerOrganizationPermission, ReadRepositoryPermission,
SuperUserPermission, AdministerRepositoryPermission)
SuperUserPermission, AdministerRepositoryPermission,
ModifyRepositoryPermission)
from util.invoice import renderInvoiceToPdf
from util.seo import render_snapshot
@ -248,6 +249,31 @@ def robots():
return send_from_directory('static', 'robots.txt')
@web.route('/buildlogs/<build_uuid>', methods=['GET'])
@route_show_if(features.BUILD_SUPPORT)
@require_session_login
def buildlogs(build_uuid):
build = model.get_repository_build(build_uuid)
if not build:
abort(403)
repo = build.repository
if not ModifyRepositoryPermission(repo.namespace_user.username, repo.name).can():
abort(403)
# If the logs have been archived, just return a URL of the completed archive
if build.logs_archived:
return redirect(log_archive.get_file_url(build.uuid))
_, logs = build_logs.get_log_entries(build.uuid, 0)
response = jsonify({
'logs': [log for log in logs]
})
response.headers["Content-Disposition"] = "attachment;filename=" + build.uuid + ".json"
return response
@web.route('/receipt', methods=['GET'])
@route_show_if(features.BILLING)
@require_session_login

5
requirements-nover.txt
View file

@ -21,7 +21,6 @@ paramiko
xhtml2pdf
redis
hiredis
docker-py
flask-restful==0.2.12
jsonschema
git+https://github.com/NateFerrero/oauth2lib.git
@ -51,4 +50,6 @@ pygpgme
cachetools
mock
psutil
stringscore
stringscore
python-swiftclient
python-keystoneclient

83
requirements.txt
View file

@ -1,67 +1,90 @@
APScheduler==3.0.1
APScheduler==3.0.3
Babel==1.3
Flask==0.10.1
Flask-Login==0.2.11
Flask-Mail==0.9.1
Flask-Principal==0.4.0
Flask-RESTful==0.2.12
Jinja2==2.7.3
LogentriesLogger==0.2.1
Mako==1.0.0
Logentries==0.7
Mako==1.0.1
MarkupSafe==0.23
Pillow==2.7.0
PyMySQL==0.6.3
Pillow==2.8.1
PyMySQL==0.6.6
PyPDF2==1.24
PyYAML==3.11
SQLAlchemy==0.9.8
WebOb==1.4
Werkzeug==0.9.6
aiowsgi==0.3
alembic==0.7.4
SQLAlchemy==1.0.3
WebOb==1.4.1
Werkzeug==0.10.4
aiowsgi==0.5
alembic==0.7.5.post2
argparse==1.3.0
autobahn==0.9.3-3
backports.ssl-match-hostname==3.4.0.2
beautifulsoup4==4.3.2
blinker==1.3
boto==2.35.1
boto==2.38.0
cachetools==1.0.0
docker-py==0.7.1
ecdsa==0.11
certifi==2015.04.28
cffi==0.9.2
cryptography==0.8.2
ecdsa==0.13
enum34==1.0.4
funcparserlib==0.3.6
futures==2.2.0
gevent==1.0.1
gipc==0.5.0
greenlet==0.4.5
gunicorn==18.0
hiredis==0.1.5
html5lib==0.999
hiredis==0.2.0
html5lib==0.99999
iso8601==0.1.10
itsdangerous==0.24
jsonschema==2.4.0
marisa-trie==0.7
mixpanel-py==3.2.1
marisa-trie==0.7.2
mixpanel-py==4.0.2
mock==1.0.1
msgpack-python==0.4.6
netaddr==0.7.14
netifaces==0.10.4
oauthlib==0.7.2
oslo.config==1.11.0
oslo.i18n==1.6.0
oslo.serialization==1.5.0
oslo.utils==1.5.0
paramiko==1.15.2
peewee==2.4.7
pbr==0.11.0
peewee==2.6.0
prettytable==0.7.2
psutil==2.2.1
psycopg2==2.5.4
psycopg2==2.6
py-bcrypt==0.4
pyOpenSSL==0.15.1
pyasn1==0.1.7
pycparser==2.12
pycrypto==2.6.1
python-dateutil==2.4.0
pygpgme==0.3
python-dateutil==2.4.2
python-keystoneclient==1.4.0
python-ldap==2.4.19
python-magic==0.4.6
pygpgme==0.3
pytz==2014.10
pyOpenSSL==0.14
raven==5.1.1
python-swiftclient==2.4.0
pytz==2015.2
raven==5.3.0
redis==2.10.3
reportlab==2.7
requests==2.5.1
requests==2.6.2
requests-oauthlib==0.4.2
simplejson==3.7.1
six==1.9.0
stevedore==1.4.0
stringscore==0.1.0
stripe==1.20.1
stripe==1.22.2
trollius==1.0.4
tzlocal==1.1.2
urllib3==1.10.2
tzlocal==1.1.3
urllib3==1.10.3
waitress==0.8.9
websocket-client==0.23.0
websocket-client==0.30.0
wsgiref==0.1.2
xhtml2pdf==0.0.6
git+https://github.com/DevTable/aniso8601-fake.git
@ -73,4 +96,4 @@ git+https://github.com/DevTable/python-etcd.git
git+https://github.com/NateFerrero/oauth2lib.git
git+https://github.com/coreos/py-bitbucket.git
git+https://github.com/coreos/pyapi-gitlab.git
git+https://github.com/coreos/mockldap.git
git+https://github.com/coreos/mockldap.git

23
static/css/core-ui.css
View file

@ -388,6 +388,29 @@ a:focus {
width: 400px;
}
.config-map-field-element table {
margin-bottom: 10px;
}
.config-map-field-element .form-control-container {
border-top: 1px solid #eee;
padding-top: 10px;
}
.config-map-field-element .form-control-container select, .config-map-field-element .form-control-container input {
margin-bottom: 10px;
}
.config-map-field-element .empty {
color: #ccc;
margin-bottom: 10px;
display: block;
}
.config-map-field-element .item-title {
font-weight: bold;
}
.config-contact-field {
margin-bottom: 4px;
}

18
static/css/directives/ui/build-logs-view.css
View file

@ -157,6 +157,24 @@
transition: all 0.15s ease-in-out;
}
.build-logs-view .download-button i.fa {
margin-right: 10px;
}
.build-logs-view .download-button {
position: absolute;
top: 6px;
right: 124px;
z-index: 2;
transition: all 0.15s ease-in-out;
}
.build-logs-view .download-button:not(:hover) {
background: transparent;
border: 1px solid transparent;
color: #ddd;
}
.build-logs-view .copy-button:not(.zeroclipboard-is-hover) {
background: transparent;
border: 1px solid transparent;

6
static/directives/build-logs-view.html
View file

@ -3,6 +3,12 @@
<i class="fa fa-clipboard"></i>Copy Logs
</button>
<a id="downloadButton" class="btn btn-primary download-button"
ng-href="/buildlogs/{{ currentBuild.id }}"
target="_blank">
<i class="fa fa-download"></i>Download Logs
</a>
<span class="cor-loader" ng-if="!logEntries"></span>
<span class="no-logs" ng-if="!logEntries.length && currentBuild.phase == 'waiting'">

20
static/directives/config/config-map-field.html Normal file
View file

@ -0,0 +1,20 @@
<div class="config-map-field-element">
<table class="table" ng-show="hasValues(binding)">
<tr class="item" ng-repeat="(key, value) in binding">
<td class="item-title">{{ key }}</td>
<td class="item-value">{{ value }}</td>
<td class="item-delete">
<a href="javascript:void(0)" ng-click="removeKey(key)">Remove</a>
</td>
</tr>
</table>
<span class="empty" ng-if="!hasValues(binding)">No entries defined</span>
<form class="form-control-container" ng-submit="addEntry()">
Add Key-Value:
<select ng-model="newKey">
<option ng-repeat="key in keys" value="{{ key }}">{{ key }}</option>
</select>
<input type="text" class="form-control" placeholder="Value" ng-model="newValue">
<button class="btn btn-default" style="display: inline-block">Add Entry</button>
</form>
</div>

8
static/directives/config/config-setup-tool.html
View file

@ -189,6 +189,7 @@
<option value="S3Storage">Amazon S3</option>
<option value="GoogleCloudStorage">Google Cloud Storage</option>
<option value="RadosGWStorage">Ceph Object Gateway (RADOS)</option>
<option value="SwiftStorage">OpenStack Storage (Swift)</option>
</select>
</td>
</tr>
@ -197,10 +198,15 @@
<tr ng-repeat="field in STORAGE_CONFIG_FIELDS[config.DISTRIBUTED_STORAGE_CONFIG.local[0]]">
<td>{{ field.title }}:</td>
<td>
<span class="config-map-field"
binding="config.DISTRIBUTED_STORAGE_CONFIG.local[1][field.name]"
ng-if="field.kind == 'map'"
keys="field.keys"></span>
<span class="config-string-field"
binding="config.DISTRIBUTED_STORAGE_CONFIG.local[1][field.name]"
placeholder="{{ field.placeholder }}"
ng-if="field.kind == 'text'"></span>
ng-if="field.kind == 'text'"
is-optional="field.optional"></span>
<div class="co-checkbox" ng-if="field.kind == 'bool'">
<input id="dsc-{{ field.name }}" type="checkbox"
ng-model="config.DISTRIBUTED_STORAGE_CONFIG.local[1][field.name]">

2
static/directives/config/config-string-field.html
View file

@ -2,7 +2,7 @@
<form name="fieldform" novalidate>
<input type="text" class="form-control" placeholder="{{ placeholder || '' }}"
ng-model="binding" ng-trim="false" ng-minlength="1"
ng-pattern="getRegexp(pattern)" required>
ng-pattern="getRegexp(pattern)" ng-required="!isOptional">
<div class="alert alert-danger" ng-show="errorMessage">
{{ errorMessage }}
</div>

4
static/directives/repository-events-table.html
View file

@ -15,10 +15,10 @@
<div class="empty" ng-if="!notifications.length">
<div class="empty-primary-msg">No notifications have been setup for this repository.</div>
<div class="empty-secondary-msg hidden-xs" ng-if="repository.can_write">
<div class="empty-secondary-msg hidden-sm hidden-xs" ng-if="repository.can_write">
Click the "Create Notification" button above to add a new notification for a repository event.
</div>
<div class="empty-secondary-msg visible-xs" ng-if="repository.can_write">
<div class="empty-secondary-msg visible-sm visible-xs" ng-if="repository.can_write">
<a href="javascript:void(0)" ng-click="askCreateNotification()">Click here</a> to add a new notification for a repository event.
</div>
</div>

52
static/js/core-config-setup.js
View file

@ -78,6 +78,19 @@ angular.module("core-config-setup", ['angularFileUpload'])
{'name': 'secret_key', 'title': 'Secret Key', 'placeholder': 'secretkeyhere', 'kind': 'text'},
{'name': 'bucket_name', 'title': 'Bucket Name', 'placeholder': 'my-cool-bucket', 'kind': 'text'},
{'name': 'storage_path', 'title': 'Storage Directory', 'placeholder': '/path/inside/bucket', 'kind': 'text'}
],
'SwiftStorage': [
{'name': 'auth_url', 'title': 'Swift Auth URL', 'placeholder': '', 'kind': 'text'},
{'name': 'swift_container', 'title': 'Swift Container Name', 'placeholder': 'mycontainer', 'kind': 'text'},
{'name': 'storage_path', 'title': 'Storage Path', 'placeholder': '/path/inside/container', 'kind': 'text'},
{'name': 'swift_user', 'title': 'Username', 'placeholder': 'accesskeyhere', 'kind': 'text'},
{'name': 'swift_password', 'title': 'Password/Key', 'placeholder': 'secretkeyhere', 'kind': 'text'},
{'name': 'ca_cert_path', 'title': 'CA Cert Filename', 'placeholder': 'conf/stack/swift.cert', 'kind': 'text', 'optional': true},
{'name': 'os_options', 'title': 'OS Options', 'kind': 'map',
'keys': ['tenant_id', 'auth_token', 'service_type', 'endpoint_type', 'tenant_name', 'object_storage_url', 'region_name']}
]
};
@ -760,6 +773,42 @@ angular.module("core-config-setup", ['angularFileUpload'])
return directiveDefinitionObject;
})
.directive('configMapField', function () {
var directiveDefinitionObject = {
priority: 0,
templateUrl: '/static/directives/config/config-map-field.html',
replace: false,
transclude: false,
restrict: 'C',
scope: {
'binding': '=binding',
'keys': '=keys'
},
controller: function($scope, $element) {
$scope.newKey = null;
$scope.newValue = null;
$scope.hasValues = function(binding) {
return binding && Object.keys(binding).length;
};
$scope.removeKey = function(key) {
delete $scope.binding[key];
};
$scope.addEntry = function() {
if (!$scope.newKey || !$scope.newValue) { return; }
$scope.binding = $scope.binding || {};
$scope.binding[$scope.newKey] = $scope.newValue;
$scope.newKey = null;
$scope.newValue = null;
}
}
};
return directiveDefinitionObject;
})
.directive('configStringField', function () {
var directiveDefinitionObject = {
priority: 0,
@ -772,7 +821,8 @@ angular.module("core-config-setup", ['angularFileUpload'])
'placeholder': '@placeholder',
'pattern': '@pattern',
'defaultValue': '@defaultValue',
'validator': '&validator'
'validator': '&validator',
'isOptional': '=isOptional'
},
controller: function($scope, $element) {
$scope.getRegexp = function(pattern) {

2
static/js/pages/user-view.js
View file

@ -94,7 +94,7 @@
}, ApiService.errorDisplay('Could not generate token'));
};
UIService.showPasswordDialog('Enter your password to generated an encrypted version:', generateToken);
UIService.showPasswordDialog('Enter your password to generate an encrypted version:', generateToken);
};
$scope.changeEmail = function() {

2
storage/__init__.py
View file

@ -2,6 +2,7 @@ from storage.local import LocalStorage
from storage.cloud import S3Storage, GoogleCloudStorage, RadosGWStorage
from storage.fakestorage import FakeStorage
from storage.distributedstorage import DistributedStorage
from storage.swift import SwiftStorage
STORAGE_DRIVER_CLASSES = {
@ -9,6 +10,7 @@ STORAGE_DRIVER_CLASSES = {
'S3Storage': S3Storage,
'GoogleCloudStorage': GoogleCloudStorage,
'RadosGWStorage': RadosGWStorage,
'SwiftStorage': SwiftStorage,
}
def get_storage_driver(storage_params):

188
storage/swift.py Normal file
View file

@ -0,0 +1,188 @@
""" Swift storage driver. Based on: github.com/bacongobbler/docker-registry-driver-swift/ """
from swiftclient.client import Connection, ClientException
from storage.basestorage import BaseStorage
from random import SystemRandom
import string
import logging
logger = logging.getLogger(__name__)
class SwiftStorage(BaseStorage):
def __init__(self, swift_container, storage_path, auth_url, swift_user,
swift_password, auth_version=None, os_options=None, ca_cert_path=None):
self._swift_container = swift_container
self._storage_path = storage_path
self._auth_url = auth_url
self._ca_cert_path = ca_cert_path
self._swift_user = swift_user
self._swift_password = swift_password
self._auth_version = auth_version or 2
self._os_options = os_options or {}
self._initialized = False
self._swift_connection = None
def _initialize(self):
if self._initialized:
return
self._initialized = True
self._swift_connection = self._get_connection()
def _get_connection(self):
return Connection(
authurl=self._auth_url,
cacert=self._ca_cert_path,
user=self._swift_user,
key=self._swift_password,
auth_version=self._auth_version,
os_options=self._os_options)
def _get_relative_path(self, path):
if path.startswith(self._storage_path):
path = path[len(self._storage_path)]
if path.endswith('/'):
path = path[:-1]
return path
def _normalize_path(self, path=None):
path = self._storage_path + (path or '')
# Openstack does not like paths starting with '/' and we always normalize
# to remove trailing '/'
if path.startswith('/'):
path = path[1:]
if path.endswith('/'):
path = path[:-1]
return path
def _get_container(self, path):
self._initialize()
path = self._normalize_path(path)
if path and not path.endswith('/'):
path += '/'
try:
_, container = self._swift_connection.get_container(
container=self._swift_container,
prefix=path, delimiter='/')
return container
except:
logger.exception('Could not get container: %s', path)
raise IOError('Unknown path: %s' % path)
def _get_object(self, path, chunk_size=None):
self._initialize()
path = self._normalize_path(path)
try:
_, obj = self._swift_connection.get_object(self._swift_container, path,
resp_chunk_size=chunk_size)
return obj
except Exception:
logger.exception('Could not get object: %s', path)
raise IOError('Path %s not found' % path)
def _put_object(self, path, content, chunk=None, content_type=None, content_encoding=None):
self._initialize()
path = self._normalize_path(path)
headers = {}
if content_encoding is not None:
headers['Content-Encoding'] = content_encoding
try:
self._swift_connection.put_object(self._swift_container, path, content,
chunk_size=chunk, content_type=content_type,
headers=headers)
except ClientException:
# We re-raise client exception here so that validation of config during setup can see
# the client exception messages.
raise
except Exception:
logger.exception('Could not put object: %s', path)
raise IOError("Could not put content: %s" % path)
def _head_object(self, path):
self._initialize()
path = self._normalize_path(path)
try:
return self._swift_connection.head_object(self._swift_container, path)
except Exception:
logger.exception('Could not head object: %s', path)
return None
def get_direct_download_url(self, path, expires_in=60, requires_cors=False):
if requires_cors:
return None
# TODO(jschorr): This method is not strictly necessary but would result in faster operations
# when using this storage engine. However, the implementation (as seen in the link below)
# is not clean, so we punt on this for now.
# http://docs.openstack.org/juno/config-reference/content/object-storage-tempurl.html
return None
def get_content(self, path):
return self._get_object(path)
def put_content(self, path, content):
self._put_object(path, content)
def stream_read(self, path):
for data in self._get_object(path, self.buffer_size):
yield data
def stream_read_file(self, path):
raise NotImplementedError
def stream_write(self, path, fp, content_type=None, content_encoding=None):
self._put_object(path, fp, self.buffer_size, content_type=content_type,
content_encoding=content_encoding)
def list_directory(self, path=None):
container = self._get_container(path)
if not container:
raise OSError('Unknown path: %s' % path)
for entry in container:
param = None
if 'name' in entry:
param = 'name'
elif 'subdir' in entry:
param = 'subdir'
else:
continue
yield self._get_relative_path(entry[param])
def exists(self, path):
return bool(self._head_object(path))
def remove(self, path):
self._initialize()
path = self._normalize_path(path)
try:
self._swift_connection.delete_object(self._swift_container, path)
except Exception:
raise IOError('Cannot delete path: %s' % path)
def _random_checksum(self, count):
chars = string.ascii_uppercase + string.digits
return ''.join(SystemRandom().choice(chars) for _ in range(count))
def get_checksum(self, path):
headers = self._head_object(path)
if not headers:
raise IOError('Cannot lookup path: %s' % path)
return headers.get('etag', '')[1:-1][:7] or self._random_checksum(7)