Merge branch 'master' into tutorial

Conflicts:
	config.py
	static/js/app.js
	test/data/test.db
This commit is contained in:
yackob03 2014-02-13 14:35:20 -05:00
commit ade20952e2
38 changed files with 1140 additions and 224 deletions

View file

@ -1,5 +1,4 @@
import logging
import os
from app import app as application
from data.model import db as model_db
@ -21,10 +20,6 @@ from endpoints.realtime import realtime
logger = logging.getLogger(__name__)
if application.config.get('INCLUDE_TEST_ENDPOINTS', False):
logger.debug('Loading test endpoints.')
import endpoints.test
application.register_blueprint(web)
application.register_blueprint(index, url_prefix='/v1')
application.register_blueprint(tags, url_prefix='/v1')

View file

@ -1,5 +1,4 @@
import logging
import os
import logstash_formatter
from peewee import MySQLDatabase, SqliteDatabase
@ -12,6 +11,7 @@ from util import analytics
from test.teststorage import FakeStorage, FakeUserfiles
from test import analytics as fake_analytics
from test.testlogs import TestBuildLogs
class FlaskConfig(object):
@ -96,6 +96,11 @@ class UserEventConfig(object):
USER_EVENTS = UserEventBuilder('logs.quay.io')
class TestBuildLogs(object):
BUILDLOGS = TestBuildLogs('logs.quay.io', 'devtable', 'building',
'deadbeef-dead-beef-dead-beefdeadbeef')
class StripeTestConfig(object):
STRIPE_SECRET_KEY = 'sk_test_PEbmJCYrLXPW0VRLSnWUiZ7Y'
STRIPE_PUBLISHABLE_KEY = 'pk_test_uEDHANKm9CHCvVa2DLcipGRh'
@ -145,13 +150,13 @@ class BuildNodeConfig(object):
BUILD_NODE_PULL_TOKEN = 'F02O2E86CQLKZUQ0O81J8XDHQ6F0N1V36L9JTOEEK6GKKMT1GI8PTJQT4OU88Y6G'
def logs_init_builder(level=logging.DEBUG):
def logs_init_builder(level=logging.DEBUG,
formatter=logstash_formatter.LogstashFormatter()):
@staticmethod
def init_logs():
handler = logging.StreamHandler()
root_logger = logging.getLogger('')
root_logger.setLevel(level)
formatter = logstash_formatter.LogstashFormatter()
handler.setFormatter(formatter)
root_logger.addHandler(handler)
@ -164,17 +169,15 @@ class TestConfig(FlaskConfig, FakeStorage, EphemeralDB, FakeUserfiles,
LOGGING_CONFIG = logs_init_builder(logging.WARN)
POPULATE_DB_TEST_DATA = True
TESTING = True
INCLUDE_TEST_ENDPOINTS = True
class DebugConfig(FlaskConfig, MailConfig, LocalStorage, SQLiteDB,
StripeTestConfig, MixpanelTestConfig, GitHubTestConfig,
DigitalOceanConfig, BuildNodeConfig, S3Userfiles,
RedisBuildLogs, UserEventConfig):
LOGGING_CONFIG = logs_init_builder()
RedisBuildLogs, UserEventConfig, TestBuildLogs):
LOGGING_CONFIG = logs_init_builder(formatter=logging.Formatter())
SEND_FILE_MAX_AGE_DEFAULT = 0
POPULATE_DB_TEST_DATA = True
INCLUDE_TEST_ENDPOINTS = True
class LocalHostedConfig(FlaskConfig, MailConfig, S3Storage, RDSMySQL,

View file

@ -3,6 +3,10 @@ import json
class BuildLogs(object):
ERROR = 'error'
COMMAND = 'command'
PHASE = 'phase'
def __init__(self, redis_host):
self._redis = redis.StrictRedis(host=redis_host)
@ -17,24 +21,27 @@ class BuildLogs(object):
"""
return self._redis.rpush(self._logs_key(build_id), json.dumps(log_obj))
def append_log_message(self, build_id, log_message):
def append_log_message(self, build_id, log_message, log_type=None):
"""
Wraps the message in an envelope and push it to the end of the log entry
list and returns the new length of the list.
list and returns the index at which it was inserted.
"""
log_obj = {
'message': log_message
}
return self._redis.rpush(self._logs_key(build_id), json.dumps(log_obj))
def get_log_entries(self, build_id, start_index, end_index):
if log_type:
log_obj['type'] = log_type
return self._redis.rpush(self._logs_key(build_id), json.dumps(log_obj)) - 1
def get_log_entries(self, build_id, start_index):
"""
Returns a tuple of the current length of the list and an iterable of the
requested log entries. End index is inclusive.
requested log entries.
"""
llen = self._redis.llen(self._logs_key(build_id))
log_entries = self._redis.lrange(self._logs_key(build_id), start_index,
end_index)
log_entries = self._redis.lrange(self._logs_key(build_id), start_index, -1)
return (llen, (json.loads(entry) for entry in log_entries))
@staticmethod

View file

@ -215,6 +215,8 @@ class RepositoryBuild(BaseModel):
resource_key = CharField()
tag = CharField()
phase = CharField(default='waiting')
started = DateTimeField(default=datetime.now)
display_name = CharField()
class QueueItem(BaseModel):

View file

@ -1309,9 +1309,11 @@ def list_repository_builds(namespace_name, repository_name,
return fetched
def create_repository_build(repo, access_token, resource_key, tag):
def create_repository_build(repo, access_token, resource_key, tag,
display_name):
return RepositoryBuild.create(repository=repo, access_token=access_token,
resource_key=resource_key, tag=tag)
resource_key=resource_key, tag=tag,
display_name=display_name)
def create_webhook(repo, params_obj):

View file

@ -64,5 +64,5 @@ class WorkQueue(object):
image_diff_queue = WorkQueue('imagediff')
dockerfile_build_queue = WorkQueue('dockerfilebuild')
dockerfile_build_queue = WorkQueue('dockerfilebuild2')
webhook_queue = WorkQueue('webhook')

View file

@ -59,3 +59,9 @@ class UserRequestFiles(object):
full_key = os.path.join(self._prefix, file_id)
k = Key(self._bucket, full_key)
return k.generate_url(expires_in)
def get_file_checksum(self, file_id):
self._initialize_s3()
full_key = os.path.join(self._prefix, file_id)
k = self._bucket.lookup(full_key)
return k.etag[1:-1][:7]

View file

@ -70,7 +70,7 @@ def get_route_data():
routes = []
for rule in app.url_map.iter_rules():
if rule.endpoint.startswith('api.'):
endpoint_method = globals()[rule.endpoint[4:]] # Remove api.
endpoint_method = app.view_functions[rule.endpoint]
is_internal = '__internal_call' in dir(endpoint_method)
is_org_api = '__user_call' in dir(endpoint_method)
methods = list(rule.methods.difference(['HEAD', 'OPTIONS']))
@ -1154,6 +1154,8 @@ def build_status_view(build_obj):
return {
'id': build_obj.uuid,
'phase': build_obj.phase,
'started': build_obj.started,
'display_name': build_obj.display_name,
'status': status,
}
@ -1191,25 +1193,22 @@ def get_repo_build_status(namespace, repository, build_uuid):
def get_repo_build_logs(namespace, repository, build_uuid):
permission = ModifyRepositoryPermission(namespace, repository)
if permission.can():
response_obj = {}
build = model.get_repository_build(namespace, repository, build_uuid)
start = int(request.args.get('start', -1000))
end = int(request.args.get('end', -1))
count, logs = build_logs.get_log_entries(build.uuid, start, end)
start = int(request.args.get('start', 0))
if start < 0:
start = max(0, count + start)
count, logs = build_logs.get_log_entries(build.uuid, start)
if end < 0:
end = count + end
return jsonify({
response_obj.update({
'start': start,
'end': end,
'total': count,
'logs': [log for log in logs],
})
return jsonify(response_obj)
abort(403) # Permission denied
@ -1224,11 +1223,13 @@ def request_repo_build(namespace, repository):
repo = model.get_repository(namespace, repository)
token = model.create_access_token(repo, 'write')
display_name = user_files.get_file_checksum(dockerfile_id)
logger.debug('**********Md5: %s' % display_name)
host = urlparse.urlparse(request.url).netloc
tag = '%s/%s/%s' % (host, repo.namespace, repo.name)
build_request = model.create_repository_build(repo, token, dockerfile_id,
tag)
tag, display_name)
dockerfile_build_queue.put(json.dumps({
'build_uuid': build_request.uuid,
'namespace': namespace,

View file

@ -1,61 +0,0 @@
import math
from random import SystemRandom
from flask import jsonify
from app import app
def generate_image_completion(rand_func):
images = {}
for image_id in range(rand_func.randint(1, 11)):
total = int(math.pow(abs(rand_func.gauss(0, 1000)), 2))
current = rand_func.randint(0, total)
image_id = 'image_id_%s' % image_id
images[image_id] = {
'total': total,
'current': current,
}
return images
@app.route('/test/build/status', methods=['GET'])
def generate_random_build_status():
response = {
'id': 1,
'total_commands': None,
'current_command': None,
'push_completion': 0.0,
'status': None,
'message': None,
'image_completion': {},
}
random = SystemRandom()
phases = {
'waiting': {},
'starting': {
'total_commands': 7,
'current_command': 0,
},
'initializing': {},
'error': {
'message': 'Oops!'
},
'complete': {},
'building': {
'total_commands': 7,
'current_command': random.randint(1, 7),
},
'pushing': {
'total_commands': 7,
'current_command': 7,
'push_completion': random.random(),
'image_completion': generate_image_completion(random),
},
}
phase = random.choice(phases.keys())
response['status'] = phase
response.update(phases[phase])
return jsonify(response)

View file

@ -275,6 +275,13 @@ def populate_database():
'Empty repository which is building.',
False, [], (0, [], None))
token = model.create_access_token(building, 'write')
tag = 'ci.devtable.com:5000/%s/%s' % (building.namespace, building.name)
build = model.create_repository_build(building, token, '123-45-6789', tag,
'build-name')
build.uuid = 'deadbeef-dead-beef-dead-beefdeadbeef'
build.save()
org = model.create_organization('buynlarge', 'quay@devtable.com',
new_user_1)
org.stripe_id = TEST_STRIPE_ID
@ -298,19 +305,11 @@ def populate_database():
model.add_user_to_team(new_user_2, reader_team)
model.add_user_to_team(reader, reader_team)
token = model.create_access_token(building, 'write')
tag = 'ci.devtable.com:5000/%s/%s' % (building.namespace, building.name)
build = model.create_repository_build(building, token, '123-45-6789', tag)
build.build_node_id = 1
build.phase = 'building'
build.status_url = 'http://localhost:5000/test/build/status'
build.save()
__generate_repository(new_user_1, 'superwide', None, False, [],
[(10, [], 'latest2'),
(2, [], 'latest3'),
(2, [(1, [], 'latest11'), (2, [], 'latest12')], 'latest4'),
(2, [(1, [], 'latest11'), (2, [], 'latest12')],
'latest4'),
(2, [], 'latest5'),
(2, [], 'latest6'),
(2, [], 'latest7'),

View file

@ -21,4 +21,5 @@ xhtml2pdf
logstash_formatter
redis
hiredis
git+https://github.com/dotcloud/docker-py.git
git+https://github.com/dotcloud/docker-py.git
loremipsum

View file

@ -23,6 +23,7 @@ html5lib==1.0b3
itsdangerous==0.23
lockfile==0.9.1
logstash-formatter==0.5.8
loremipsum==1.0.2
marisa-trie==0.5.1
mixpanel-py==3.1.1
mock==1.0.1

View file

@ -507,35 +507,103 @@ i.toggle-icon:hover {
color: #428bca;
}
.status-boxes .popover {
margin-right: 20px;
.status-box a {
padding: 6px;
color: black;
}
.status-boxes .popover-content {
width: 260px;
.status-box a b {
margin-right: 10px;
}
.build-statuses {
.build-info {
margin: 4px;
padding: 4px;
margin-left: 6px;
margin-right: 6px;
border-bottom: 1px solid #eee;
}
.build-info.clickable:hover {
background: rgba(66, 139, 202, 0.2);
cursor: pointer;
border-radius: 4px;
}
.build-info:last-child {
border-bottom: 0px;
}
.phase-icon {
border-radius: 50%;
display: inline-block;
width: 12px;
height: 12px;
margin-right: 6px;
}
.active .build-tab-link .phase-icon {
box-shadow: 0px 0px 10px #FFFFFF, 0px 0px 10px #FFFFFF;
}
.build-status .phase-icon {
margin-top: 4px;
float: left;
}
.phase-icon.error {
background-color: red;
}
.phase-icon.waiting, .phase-icon.starting, .phase-icon.initializing {
background-color: #ddd;
}
.phase-icon.building {
background-color: #f0ad4e;
}
.phase-icon.pushing {
background-color: #5cb85c;
}
.phase-icon.complete {
background-color: #428bca;
}
.build-status {
display: inline-block;
}
.build-status-container {
padding: 4px;
margin-bottom: 10px;
border-bottom: 1px solid #eee;
width: 230px;
width: 350px;
}
.build-status-container .build-message {
display: block;
white-space: nowrap;
font-size: 12px;
font-size: 14px;
margin-bottom: 10px;
padding-bottom: 10px;
border-bottom: 1px solid #eee;
margin-left: 20px;
}
.build-status-container .progress {
height: 12px;
height: 10px;
margin: 0px;
margin-top: 10px;
width: 230px;
margin-left: 20px;
width: 310px;
}
.build-status-container .timing {
margin-left: 20px;
margin-top: 6px;
}
.build-status-container:last-child {
@ -1633,6 +1701,185 @@ p.editable:hover i {
padding-left: 44px;
}
.repo-build .build-id:before {
content: "Build ID: "
}
.repo-build .build-id {
float: right;
font-size: 12px;
color: #aaa;
padding: 10px;
}
.repo-build .build-pane .timing {
float: right;
}
.repo-build .build-tab-link {
white-space: nowrap;
}
.repo-build .build-pane .build-header {
padding-top: 10px;
border-bottom: 1px solid #eee;
padding-bottom: 10px;
}
.repo-build .build-pane .build-progress {
margin-top: 16px;
margin-bottom: 10px;
}
.repo-build .build-pane .build-progress .progress {
height: 14px;
margin-bottom: 0px;
}
.repo-build .build-pane .quay-spinner {
margin-top: 4px;
display: inline-block;
}
.repo-build .build-pane .build-logs {
background: #222;
color: white;
padding: 10px;
overflow: auto;
}
.repo-build .build-pane .build-logs .container-header {
padding: 2px;
}
.repo-build .build-pane .build-logs .container-logs {
margin: 4px;
padding-bottom: 4px;
}
.repo-build .build-pane .build-logs .command-title,
.repo-build .build-pane .build-logs .log-entry .message {
font-family: Consolas, "Lucida Console", Monaco, monospace;
font-size: 13px;
}
.repo-build .build-pane .build-logs .container-header {
cursor: pointer;
position: relative;
}
.repo-build .build-pane .build-logs .container-header i.fa.chevron {
color: #666;
margin-right: 4px;
width: 14px;
text-align: center;
position: absolute;
top: 6px;
left: 0px;
}
.repo-build .build-pane .build-logs .log-container.command {
margin-left: 42px;
}
.repo-build .build-pane .build-logs .container-header.building {
margin-bottom: 10px;
}
.repo-build .build-pane .build-logs .container-header.pushing {
margin-top: 10px;
}
.repo-build .build-log-error-element {
position: relative;
display: inline-block;
margin: 10px;
padding: 10px;
background: rgba(255, 0, 0, 0.17);
border-radius: 10px;
margin-left: 22px;
}
.repo-build .build-log-error-element i.fa {
color: red;
position: absolute;
top: 13px;
left: 11px;
}
.repo-build .build-log-error-element .error-message {
display: inline-block;
margin-left: 25px;
}
.repo-build .build-pane .build-logs .container-header .label {
padding-top: 4px;
text-align: right;
margin-right: 4px;
width: 86px;
display: inline-block;
border-right: 4px solid #aaa;
background-color: #444;
position: absolute;
top: 4px;
left: 24px;
}
.repo-build .build-pane .build-logs .container-header .container-content {
display: block;
padding-left: 20px;
}
.repo-build .build-pane .build-logs .container-header .container-content.build-log-command {
padding-left: 120px;
}
.label.FROM {
border-color: #5bc0de !important;
}
.label.CMD, .label.EXPOSE, .label.ENTRYPOINT {
border-color: #428bca !important;
}
.label.RUN, .label.ADD {
border-color: #5cb85c !important;
}
.label.ENV, .label.VOLUME, .label.USER, .label.WORKDIR {
border-color: #f0ad4e !important;
}
.label.MAINTAINER {
border-color: #aaa !important;
}
.repo-build .build-pane .build-logs .log-entry {
position: relative;
}
.repo-build .build-pane .build-logs .log-entry .message {
display: inline-block;
margin-left: 46px;
}
.repo-build .build-pane .build-logs .log-entry .id {
color: #aaa;
padding-right: 6px;
margin-right: 6px;
text-align: right;
font-size: 12px;
width: 40px;
position: absolute;
top: 4px;
left: 4px;
}
.repo-admin .right-info {
font-size: 11px;
margin-top: 10px;
@ -1676,16 +1923,6 @@ p.editable:hover i {
cursor: pointer;
}
.repo .build-info {
padding: 10px;
margin: 0px;
}
.repo .build-info .progress {
margin: 0px;
margin-top: 10px;
}
.repo .section {
display: block;
margin-bottom: 20px;
@ -2935,4 +3172,4 @@ pre.command:before {
.about-basic-text {
display: inline-block;
}
}

View file

@ -0,0 +1,6 @@
<span class="command" bindonce>
<span class="label" bo-class="getCommandKind(command.message)" bo-show="getCommandKind(command.message)"
bo-text="getCommandKind(command.message)">
</span>
<span class="command-title" bo-html="getCommandTitleHtml(command.message)"></span>
</span>

View file

@ -0,0 +1,4 @@
<span bindonce class="build-log-error-element">
<i class="fa fa-exclamation-triangle"></i>
<span class="error-message" bo-text="error.message"></span>
</span>

View file

@ -0,0 +1,4 @@
<span bindonce class="build-log-phase-element">
<span class="phase-icon" ng-class="phase.message"></span>
<span class="build-message" phase="phase.message"></span>
</span>

View file

@ -0,0 +1 @@
<span class="build-message-element">{{ getBuildMessage(phase) }}</span>

View file

@ -0,0 +1,6 @@
<div class="build-progress-element">
<div class="progress" ng-class="getPercentage(build) < 100 ? 'active progress-striped' : ''">
<div class="progress-bar" role="progressbar" aria-valuenow="{{ getPercentage(build) }}" aria-valuemin="0" aria-valuemax="100" style="{{ 'width: ' + getPercentage(build) + '%' }}">
</div>
</div>
</div>

View file

@ -1,8 +1,11 @@
<div id="build-status-container" class="build-status-container">
<span class="build-message">{{ getBuildMessage(build) }}</span>
<div class="progress" ng-class="getBuildProgress(build) < 100 ? 'active progress-striped' : ''" ng-show="getBuildProgress(build) >= 0">
<div class="progress-bar" role="progressbar" aria-valuenow="{{ getBuildProgress(build) }}" aria-valuemin="0" aria-valuemax="100" style="{{ 'width: ' + getBuildProgress(build) + '%' }}">
</div>
<div>
<span class="phase-icon" ng-class="build.phase"></span>
<span class="build-message" phase="build.phase"></span>
</div>
<div class="timing">
<i class="fa fa-clock-o"></i>
Started: <span am-time-ago="build.started || 0"></span>
</div>
<div class="build-progress" build="build"></div>
</div>

View file

@ -102,9 +102,8 @@ function getMarkedDown(string) {
return Markdown.getSanitizingConverter().makeHtml(string || '');
}
// Start the application code itself.
quayApp = angular.module('quay', ['ngRoute', 'chieffancypants.loadingBar', 'angular-tour', 'restangular', 'angularMoment', 'angulartics', /*'angulartics.google.analytics',*/ 'angulartics.mixpanel', '$strap.directives', 'ngCookies', 'ngSanitize', 'angular-md5'], function($provide, cfpLoadingBarProvider) {
cfpLoadingBarProvider.includeSpinner = false;
quayApp = angular.module('quay', ['ngRoute', 'chieffancypants.loadingBar', 'angular-tour', 'restangular', 'angularMoment', 'angulartics', /*'angulartics.google.analytics',*/ 'angulartics.mixpanel', '$strap.directives', 'ngCookies', 'ngSanitize', 'angular-md5', 'pasvaz.bindonce'], function($provide, cfpLoadingBarProvider) {
cfpLoadingBarProvider.includeSpinner = false;
$provide.factory('UtilService', ['$sanitize', function($sanitize) {
var utilService = {};
@ -151,7 +150,7 @@ quayApp = angular.module('quay', ['ngRoute', 'chieffancypants.loadingBar', 'angu
$provide.factory('ApiService', ['Restangular', function(Restangular) {
var apiService = {};
var getResource = function(path) {
var getResource = function(path, opt_background) {
var resource = {};
resource.url = path;
resource.withOptions = function(options) {
@ -169,6 +168,12 @@ quayApp = angular.module('quay', ['ngRoute', 'chieffancypants.loadingBar', 'angu
'hasError': false
};
if (opt_background) {
performer.withHttpConfig({
'ignoreLoadingBar': true
});
}
performer.get(options).then(function(resp) {
result.value = processor(resp);
result.loading = false;
@ -240,27 +245,33 @@ quayApp = angular.module('quay', ['ngRoute', 'chieffancypants.loadingBar', 'angu
var buildMethodsForEndpoint = function(endpoint) {
var method = endpoint.methods[0].toLowerCase();
var methodName = formatMethodName(endpoint['name']);
apiService[methodName] = function(opt_options, opt_parameters) {
return Restangular.one(buildUrl(endpoint['path'], opt_parameters))['custom' + method.toUpperCase()](opt_options);
apiService[methodName] = function(opt_options, opt_parameters, opt_background) {
var one = Restangular.one(buildUrl(endpoint['path'], opt_parameters));
if (opt_background) {
one.withHttpConfig({
'ignoreLoadingBar': true
});
}
return one['custom' + method.toUpperCase()](opt_options);
};
if (method == 'get') {
apiService[methodName + 'AsResource'] = function(opt_parameters) {
return getResource(buildUrl(endpoint['path'], opt_parameters));
apiService[methodName + 'AsResource'] = function(opt_parameters, opt_background) {
return getResource(buildUrl(endpoint['path'], opt_parameters), opt_background);
};
}
if (endpoint['user_method']) {
apiService[getGenericMethodName(endpoint['user_method'])] = function(orgname, opt_options, opt_parameters) {
apiService[getGenericMethodName(endpoint['user_method'])] = function(orgname, opt_options, opt_parameters, opt_background) {
if (orgname) {
if (orgname.name) {
orgname = orgname.name;
}
var params = jQuery.extend({'orgname' : orgname}, opt_parameters || {});
var params = jQuery.extend({'orgname' : orgname}, opt_parameters || {}, opt_background);
return apiService[methodName](opt_options, params);
} else {
return apiService[formatMethodName(endpoint['user_method'])](opt_options, opt_parameters);
return apiService[formatMethodName(endpoint['user_method'])](opt_options, opt_parameters, opt_background);
}
};
}
@ -779,6 +790,7 @@ quayApp = angular.module('quay', ['ngRoute', 'chieffancypants.loadingBar', 'angu
fixFooter: false}).
when('/repository/:namespace/:name/image/:image', {templateUrl: '/static/partials/image-view.html', controller: ImageViewCtrl, reloadOnSearch: false}).
when('/repository/:namespace/:name/admin', {templateUrl: '/static/partials/repo-admin.html', controller:RepoAdminCtrl, reloadOnSearch: false}).
when('/repository/:namespace/:name/build', {templateUrl: '/static/partials/repo-build.html', controller:RepoBuildCtrl, reloadOnSearch: false}).
when('/repository/', {title: 'Repositories', description: 'Public and private docker repositories list',
templateUrl: '/static/partials/repo-list.html', controller: RepoListCtrl}).
when('/user/', {title: 'Account Settings', description:'Account settings for Quay.io', templateUrl: '/static/partials/user-admin.html',
@ -2471,6 +2483,119 @@ quayApp.directive('namespaceSelector', function () {
});
quayApp.directive('buildLogPhase', function () {
var directiveDefinitionObject = {
priority: 0,
templateUrl: '/static/directives/build-log-phase.html',
replace: false,
transclude: false,
restrict: 'C',
scope: {
'phase': '=phase'
},
controller: function($scope, $element) {
}
};
return directiveDefinitionObject;
});
quayApp.directive('buildLogError', function () {
var directiveDefinitionObject = {
priority: 0,
templateUrl: '/static/directives/build-log-error.html',
replace: false,
transclude: false,
restrict: 'C',
scope: {
'error': '=error'
},
controller: function($scope, $element) {
}
};
return directiveDefinitionObject;
});
quayApp.directive('buildLogCommand', function () {
var directiveDefinitionObject = {
priority: 0,
templateUrl: '/static/directives/build-log-command.html',
replace: false,
transclude: false,
restrict: 'C',
scope: {
'command': '=command'
},
controller: function($scope, $element, $sanitize) {
var registryHandlers = {
'quay.io': function(pieces) {
var rnamespace = pieces[pieces.length - 2];
var rname = pieces[pieces.length - 1];
return '/repository/' + rnamespace + '/' + rname + '/';
},
'': function(pieces) {
var rnamespace = pieces.length == 1 ? '_' : pieces[0];
var rname = pieces[pieces.length - 1];
return 'https://index.docker.io/u/' + rnamespace + '/' + rname + '/';
}
};
var kindHandlers = {
'FROM': function(title) {
var pieces = title.split('/');
var registry = pieces.length < 3 ? '' : pieces[0];
if (!registryHandlers[registry]) {
return title;
}
return '<i class="fa fa-hdd-o"></i> <a href="' + registryHandlers[registry](pieces) + '">' + title + '</a>';
}
};
$scope.getCommandKind = function(fullTitle) {
var colon = fullTitle.indexOf(':');
var title = getTitleWithoutStep(fullTitle);
if (!title) {
return null;
}
var space = title.indexOf(' ');
return title.substring(0, space);
};
$scope.getCommandTitleHtml = function(fullTitle) {
var title = getTitleWithoutStep(fullTitle) || fullTitle;
var space = title.indexOf(' ');
if (space <= 0) {
return $sanitize(title);
}
var kind = $scope.getCommandKind(fullTitle);
var sanitized = $sanitize(title.substring(space + 1));
var handler = kindHandlers[kind || ''];
if (handler) {
return handler(sanitized);
} else {
return sanitized;
}
};
var getTitleWithoutStep = function(fullTitle) {
var colon = fullTitle.indexOf(':');
if (colon <= 0) {
return null;
}
return $.trim(fullTitle.substring(colon + 1));
};
}
};
return directiveDefinitionObject;
});
quayApp.directive('buildStatus', function () {
var directiveDefinitionObject = {
priority: 0,
@ -2482,55 +2607,85 @@ quayApp.directive('buildStatus', function () {
'build': '=build'
},
controller: function($scope, $element) {
$scope.getBuildProgress = function(buildInfo) {
switch (buildInfo.phase) {
case 'building':
return (buildInfo.status.current_command / buildInfo.status.total_commands) * 100;
break;
case 'pushing':
return buildInfo.status.push_completion * 100;
break;
}
};
return directiveDefinitionObject;
});
case 'complete':
return 100;
break;
case 'initializing':
case 'starting':
case 'waiting':
return 0;
break;
}
quayApp.directive('buildMessage', function () {
var directiveDefinitionObject = {
priority: 0,
templateUrl: '/static/directives/build-message.html',
replace: false,
transclude: false,
restrict: 'C',
scope: {
'phase': '=phase'
},
controller: function($scope, $element) {
$scope.getBuildMessage = function (phase) {
switch (phase) {
case 'starting':
case 'initializing':
return 'Starting Dockerfile build';
case 'waiting':
return 'Waiting for available build worker';
case 'building':
return 'Building image from Dockerfile';
case 'pushing':
return 'Pushing image built from Dockerfile';
return -1;
};
case 'complete':
return 'Dockerfile build completed and pushed';
case 'error':
return 'Dockerfile build failed';
}
};
}
};
return directiveDefinitionObject;
});
$scope.getBuildMessage = function(buildInfo) {
switch (buildInfo.phase) {
case 'initializing':
return 'Starting Dockerfile build';
break;
case 'starting':
case 'waiting':
case 'building':
return 'Building image from Dockerfile';
break;
quayApp.directive('buildProgress', function () {
var directiveDefinitionObject = {
priority: 0,
templateUrl: '/static/directives/build-progress.html',
replace: false,
transclude: false,
restrict: 'C',
scope: {
'build': '=build'
},
controller: function($scope, $element) {
$scope.getPercentage = function(buildInfo) {
switch (buildInfo.phase) {
case 'building':
return (buildInfo.status.current_command / buildInfo.status.total_commands) * 100;
break;
case 'pushing':
return buildInfo.status.push_completion * 100;
break;
case 'pushing':
return 'Pushing image built from Dockerfile';
break;
case 'complete':
return 100;
break;
case 'complete':
return 'Dockerfile build completed and pushed';
break;
case 'error':
return 'Dockerfile build failed.';
break;
}
};
case 'initializing':
case 'starting':
case 'waiting':
return 0;
break;
}
return -1;
};
}
};
return directiveDefinitionObject;
@ -2545,6 +2700,13 @@ quayApp.directive('ngBlur', function() {
};
});
quayApp.directive('ngVisible', function () {
return function (scope, element, attr) {
scope.$watch(attr.ngVisible, function (visible) {
element.css('visibility', visible ? 'visible' : 'hidden');
});
};
});
quayApp.run(['$location', '$rootScope', 'Restangular', 'UserService', 'PlanService', '$http', '$timeout',
function($location, $rootScope, Restangular, UserService, PlanService, $http, $timeout) {

View file

@ -326,6 +326,11 @@ function RepoCtrl($scope, $sanitize, Restangular, ImageMetadataService, ApiServi
$scope.getFormattedCommand = ImageMetadataService.getFormattedCommand;
$scope.showBuild = function(buildInfo) {
$location.path('/repository/' + namespace + '/' + name + '/build');
$location.search('current', buildInfo.id);
};
$scope.getTooltipCommand = function(image) {
var sanitized = ImageMetadataService.getEscapedFormattedCommand(image);
return '<span class=\'codetooltip\'>' + sanitized + '</span>';
@ -653,13 +658,11 @@ function RepoCtrl($scope, $sanitize, Restangular, ImageMetadataService, ApiServi
};
var getBuildInfo = function(repo) {
// Note: We use restangular manually here because we need to turn off the loading bar.
var buildInfo = Restangular.one('repository/' + repo.namespace + '/' + repo.name + '/build/');
buildInfo.withHttpConfig({
'ignoreLoadingBar': true
});
var params = {
'repository': repo.namespace + '/' + repo.name
};
buildInfo.get().then(function(resp) {
ApiService.getRepoBuilds(null, params, true).then(function(resp) {
var runningBuilds = [];
for (var i = 0; i < resp.builds.length; ++i) {
var build = resp.builds[i];
@ -745,6 +748,197 @@ function RepoCtrl($scope, $sanitize, Restangular, ImageMetadataService, ApiServi
loadViewInfo();
}
function RepoBuildCtrl($scope, Restangular, ApiService, $routeParams, $rootScope, $location, $interval, $sanitize) {
var namespace = $routeParams.namespace;
var name = $routeParams.name;
var pollTimerHandle = null;
$scope.$on('$destroy', function() {
stopPollTimer();
});
// Watch for changes to the current parameter.
$scope.$on('$routeUpdate', function(){
if ($location.search().current) {
$scope.setCurrentBuild($location.search().current, false);
}
});
$scope.builds = [];
$scope.polling = false;
$scope.adjustLogHeight = function() {
$('.build-logs').height($(window).height() - 365);
};
$scope.hasLogs = function(container) {
return ((container.logs && container.logs.length) || (container._logs && container._logs.length));
};
$scope.toggleLogs = function(container) {
if (container._logs) {
container.logs = container._logs;
container._logs = null;
} else {
container._logs = container.logs;
container.logs = null;
}
};
$scope.setCurrentBuild = function(buildId, opt_updateURL) {
// Find the build.
for (var i = 0; i < $scope.builds.length; ++i) {
if ($scope.builds[i].id == buildId) {
$scope.setCurrentBuildInternal($scope.builds[i], opt_updateURL);
return;
}
}
};
$scope.setCurrentBuildInternal = function(build, opt_updateURL) {
if (build == $scope.currentBuild) { return; }
stopPollTimer();
$scope.logEntries = null;
$scope.logStartIndex = null;
$scope.currentParentEntry = null;
$scope.currentBuild = build;
if (opt_updateURL) {
if (build) {
$location.search('current', build.id);
} else {
$location.search('current', null);
}
}
// Timeout needed to ensure the log element has been created
// before its height is adjusted.
setTimeout(function() {
$scope.adjustLogHeight();
}, 1);
// Load the first set of logs.
getBuildStatusAndLogs();
// If the build is currently processing, start the build timer.
checkPollTimer();
};
var checkPollTimer = function() {
var build = $scope.currentBuild;
if (!build) {
stopPollTimer();
return;
}
if (build['phase'] != 'complete' && build['phase'] != 'error') {
startPollTimer();
return true;
} else {
stopPollTimer();
return false;
}
};
var stopPollTimer = function() {
$interval.cancel(pollTimerHandle);
};
var startPollTimer = function() {
stopPollTimer();
pollTimerHandle = $interval(getBuildStatusAndLogs, 2000);
};
var processLogs = function(logs, startIndex) {
if (!$scope.logEntries) { $scope.logEntries = []; }
for (var i = 0; i < logs.length; ++i) {
var entry = logs[i];
var type = entry['type'] || 'entry';
if (type == 'command' || type == 'phase' || type == 'error') {
entry['_logs'] = [];
entry['index'] = startIndex + i;
$scope.logEntries.push(entry);
$scope.currentParentEntry = entry;
} else if ($scope.currentParentEntry) {
if ($scope.currentParentEntry['logs']) {
$scope.currentParentEntry['logs'].push(entry);
} else {
$scope.currentParentEntry['_logs'].push(entry);
}
}
}
};
var getBuildStatusAndLogs = function() {
if (!$scope.currentBuild || $scope.polling) { return; }
$scope.polling = true;
var params = {
'repository': namespace + '/' + name,
'build_uuid': $scope.currentBuild.id
};
ApiService.getRepoBuildStatus(null, params, true).then(function(resp) {
// Note: We use extend here rather than replacing as Angular is depending on the
// root build object to remain the same object.
$.extend(true, $scope.currentBuild, resp);
checkPollTimer();
// Load the updated logs for the build.
var options = {
'start': $scope.logStartIndex
};
ApiService.getRepoBuildLogsAsResource(params, true).withOptions(options).get(function(resp) {
processLogs(resp['logs'], resp['start']);
$scope.logStartIndex = resp['total'];
$scope.polling = false;
});
});
};
var fetchRepository = function() {
var params = {'repository': namespace + '/' + name};
$rootScope.title = 'Loading Repository...';
$scope.repository = ApiService.getRepoAsResource(params).get(function(repo) {
if (!repo.can_write) {
$rootScope.title = 'Unknown builds';
$scope.accessDenied = true;
return;
}
$rootScope.title = 'Repository Builds';
$scope.repo = repo;
getBuildInfo();
});
};
var getBuildInfo = function(repo) {
var params = {
'repository': namespace + '/' + name
};
ApiService.getRepoBuilds(null, params).then(function(resp) {
$scope.builds = resp.builds;
if ($location.search().current) {
$scope.setCurrentBuild($location.search().current, false);
} else if ($scope.builds.length > 0) {
$scope.setCurrentBuild($scope.builds[0].id, true);
}
});
};
fetchRepository();
}
function RepoAdminCtrl($scope, Restangular, ApiService, $routeParams, $rootScope) {
var namespace = $routeParams.namespace;
var name = $routeParams.name;
@ -1002,8 +1196,13 @@ function RepoAdminCtrl($scope, Restangular, ApiService, $routeParams, $rootScope
};
$scope.repository = ApiService.getRepoAsResource(params).get(function(repo) {
$scope.repo = repo;
if (!repo.can_admin) {
$rootScope.title = 'Forbidden';
$scope.accessDenied = true;
return;
}
$scope.repo = repo;
$rootScope.title = 'Settings - ' + namespace + '/' + name;
$rootScope.description = 'Administrator settings for ' + namespace + '/' + name +
': Permissions, webhooks and other settings';

1
static/lib/bindonce.min.js vendored Normal file

File diff suppressed because one or more lines are too long

View file

@ -1,5 +1,8 @@
<div class="container" ng-show="deleting"><div class="quay-spinner"></div></div>
<div class="resource-view" resource="repository" error-message="'No repository found'"></div>
<div class="container repo repo-admin" ng-show="accessDenied">
You do not have permission to view this page
</div>
<div class="container repo repo-admin" ng-show="repo && !deleting">
<div class="header row">
<a href="{{ '/repository/' + repo.namespace + '/' + repo.name }}" class="back"><i class="fa fa-chevron-left"></i></a>

View file

@ -0,0 +1,83 @@
<div class="resource-view" resource="repository" error-message="'No repository found'"></div>
<div class="container repo repo-build" ng-show="accessDenied">
You do not have permission to view this page
</div>
<div class="container repo repo-build" ng-show="repo">
<div class="header row">
<a href="{{ '/repository/' + repo.namespace + '/' + repo.name }}" class="back"><i class="fa fa-chevron-left"></i></a>
<h3>
<span class="repo-circle no-background" repo="repo"></span>
<span class="repo-breadcrumb" repo="repo"></span>
</h3>
</div>
<div class="row" ng-show="!builds.length">
There are no builds for this repository
</div>
<div class="row" ng-show="builds.length">
<!-- Side tabs -->
<div class="col-sm-2">
<ul class="nav nav-pills nav-stacked">
<li ng-class="currentBuild == build ? 'active' : ''" ng-repeat="build in builds">
<a class="build-tab-link" href="javascript:void(0)" ng-click="setCurrentBuild(build.id, true)">
<span class="phase-icon" ng-class="build.phase"></span>
<span>{{ build.display_name }}</span>
</a>
</li>
</ul>
</div>
<!-- Content -->
<div class="col-sm-10">
<div class="tab-content" onresize="adjustLogHeight()">
<div ng-repeat="build in builds" class="tab-pane build-pane" ng-class="currentBuild == build ? 'active' : ''">
<div class="build-header">
<div class="timing">
<i class="fa fa-clock-o"></i>
Started: <span am-time-ago="build.started || 0"></span>
</div>
<span class="phase-icon" ng-class="build.phase"></span>
<span class="build-message" phase="build.phase"></span>
<div class="build-progress" build="build"></div>
</div>
<div class="build-logs">
<div ng-show="!logEntries">
<span class="quay-spinner"></span>
</div>
<div class="log-container" ng-class="container.type" ng-repeat="container in logEntries">
<div class="container-header" ng-class="container.type == 'phase' ? container.message : ''"
ng-switch on="container.type" ng-click="toggleLogs(container)">
<i class="fa chevron"
ng-class="container.logs ? 'fa-chevron-down' : 'fa-chevron-right'" ng-show="hasLogs(container)"></i>
<div ng-switch-when="phase">
<span class="container-content build-log-phase" phase="container"></span>
</div>
<div ng-switch-when="error">
<span class="container-content build-log-error" error="container"></span>
</div>
<div ng-switch-when="command">
<span class="container-content build-log-command" command="container"></span>
</div>
</div>
<!-- Display the entries for the container -->
<div class="container-logs" ng-show="container.logs">
<div class="log-entry" bindonce ng-repeat="entry in container.logs">
<span class="id" bo-text="$index + container.index + 1"></span>
<span class="message" bo-text="entry.message"></span>
</div>
</div>
</div>
</div>
<div>
<span class="quay-spinner" ng-show="polling"></span>
<span class="build-id">{{ build.id }}</span>
</div>
</div>
</div>
</div>
</div>
</div>

View file

@ -38,13 +38,18 @@
<!-- Status boxes -->
<div class="status-boxes">
<div id="buildInfoBox" class="status-box" ng-show="repo.is_building"
bs-popover="'static/partials/build-status-item.html'" data-placement="bottom">
<span class="title">
<span class="quay-spinner"></span>
<b>Building Images</b>
</span>
<span class="count" ng-class="buildsInfo ? 'visible' : ''"><span>{{ buildsInfo ? buildsInfo.length : '-' }}</span></span>
<div id="buildInfoBox" class="status-box" ng-show="repo.is_building">
<div class="dropdown" data-placement="top">
<span class="count" ng-class="buildsInfo ? 'visible' : ''"><span>{{ buildsInfo ? buildsInfo.length : '-' }}</span></span>
<a href="javascript:void(0)" class="dropdown-toggle" data-toggle="dropdown">Building Dockerfile<span ng-show="buildsInfo.length > 1">s</span> <b class="caret"></b></a>
<ul class="dropdown-menu pull-right">
<li ng-repeat="buildInfo in buildsInfo">
<div class="build-info" ng-class="repo.can_write ? 'clickable' : ''" ng-click="showBuild(buildInfo)">
<span class="build-status" build="buildInfo"></span>
</div>
</li>
</ul>
</div>
</div>
</div>

View file

@ -38,14 +38,12 @@
<script src="//code.jquery.com/jquery.js"></script>
<script src="//netdna.bootstrapcdn.com/bootstrap/3.0.0/js/bootstrap.min.js"></script>
<script src="//cdnjs.cloudflare.com/ajax/libs/bootbox.js/4.0.0/bootbox.min.js"></script>
<script src="//ajax.googleapis.com/ajax/libs/angularjs/1.2.1/angular.min.js"></script>
<script src="//ajax.googleapis.com/ajax/libs/angularjs/1.2.1/angular-route.min.js"></script>
<script src="//ajax.googleapis.com/ajax/libs/angularjs/1.2.1/angular-sanitize.min.js"></script>
<script src="//cdn.jsdelivr.net/underscorejs/1.5.2/underscore-min.js"></script>
<script src="//cdn.jsdelivr.net/restangular/1.2.0/restangular.min.js"></script>
<script src="//cdn.jsdelivr.net/g/bootbox@4.1.0,underscorejs@1.5.2,restangular@1.2.0"></script>
<script src="static/lib/loading-bar.js"></script>
<script src="static/lib/angular-strap.min.js"></script>
@ -53,6 +51,7 @@
<script src="static/lib/angulartics-mixpanel.js"></script>
<script src="static/lib/angulartics-google-analytics.js"></script>
<script src="static/lib/angular-md5.js"></script>
<script src="static/lib/bindonce.min.js"></script>
<script src="static/lib/angular-moment.min.js"></script>
<script src="static/lib/angular-cookies.min.js"></script>

Binary file not shown.

View file

@ -856,7 +856,6 @@ class TestGetRepoBuilds(ApiTestCase):
assert 'id' in build
assert 'status' in build
assert 'message' in build
class TestRequearRepoBuild(ApiTestCase):

189
test/testlogs.py Normal file
View file

@ -0,0 +1,189 @@
import logging
from random import SystemRandom
from loremipsum import get_sentence
from functools import wraps
from copy import deepcopy
from data.buildlogs import BuildLogs
logger = logging.getLogger(__name__)
random = SystemRandom()
def maybe_advance_script(is_get_status=False):
def inner_advance(func):
@wraps(func)
def wrapper(self, *args, **kwargs):
advance_units = random.randint(1, 500)
logger.debug('Advancing script %s units', advance_units)
while advance_units > 0 and self.remaining_script:
units = self.remaining_script[0][0]
if advance_units > units:
advance_units -= units
self.advance_script(is_get_status)
else:
break
return func(self, *args, **kwargs)
return wrapper
return inner_advance
class TestBuildLogs(BuildLogs):
COMMAND_TYPES = ['FROM', 'MAINTAINER', 'RUN', 'CMD', 'EXPOSE', 'ENV', 'ADD',
'ENTRYPOINT', 'VOLUME', 'USER', 'WORKDIR']
STATUS_TEMPLATE = {
'total_commands': None,
'current_command': None,
'push_completion': 0.0,
'image_completion': {},
}
def __init__(self, redis_host, namespace, repository, test_build_id):
super(TestBuildLogs, self).__init__(redis_host)
self.namespace = namespace
self.repository = repository
self.test_build_id = test_build_id
self.remaining_script = self._generate_script()
logger.debug('Total script size: %s', len(self.remaining_script))
self._logs = []
self._status = {}
self._last_status = {}
def advance_script(self, is_get_status):
(_, log, status_wrapper) = self.remaining_script.pop(0)
if log is not None:
self._logs.append(log)
if status_wrapper is not None:
(phase, status) = status_wrapper
from data import model
build_obj = model.get_repository_build(self.namespace, self.repository,
self.test_build_id)
build_obj.phase = phase
build_obj.save()
self._status = status
if not is_get_status:
self._last_status = status
def _generate_script(self):
script = []
# generate the init phase
script.append(self._generate_phase(400, 'initializing'))
script.extend(self._generate_logs(random.randint(1, 3)))
# move to the building phase
script.append(self._generate_phase(400, 'building'))
total_commands = random.randint(5, 20)
for command_num in range(1, total_commands + 1):
command_weight = random.randint(50, 100)
script.append(self._generate_command(command_num, total_commands,
command_weight))
# we want 0 logs some percent of the time
num_logs = max(0, random.randint(-50, 400))
script.extend(self._generate_logs(num_logs))
# move to the pushing phase
script.append(self._generate_phase(400, 'pushing'))
script.extend(self._generate_push_statuses(total_commands))
# move to the error or complete phase
if random.randint(0, 1) == 0:
script.append(self._generate_phase(400, 'complete'))
else:
script.append(self._generate_phase(400, 'error'))
script.append((1, {'message': 'Something bad happened! Oh noes!',
'type': self.ERROR}, None))
return script
def _generate_phase(self, start_weight, phase_name):
return (start_weight, {'message': phase_name, 'type': self.PHASE},
(phase_name, deepcopy(self.STATUS_TEMPLATE)))
def _generate_command(self, command_num, total_commands, command_weight):
sentence = get_sentence()
command = random.choice(self.COMMAND_TYPES)
if command == 'FROM':
sentence = random.choice(['ubuntu', 'lopter/raring-base',
'quay.io/devtable/simple',
'quay.io/buynlarge/orgrepo',
'stackbrew/ubuntu:precise'])
msg = {
'message': 'Step %s: %s %s' % (command_num, command, sentence),
'type': self.COMMAND,
}
status = deepcopy(self.STATUS_TEMPLATE)
status['total_commands'] = total_commands
status['current_command'] = command_num
return (command_weight, msg, ('building', status))
@staticmethod
def _generate_logs(count):
return [(1, {'message': get_sentence()}, None) for _ in range(count)]
@staticmethod
def _compute_total_completion(statuses, total_images):
percentage_with_sizes = float(len(statuses.values()))/total_images
sent_bytes = sum([status[u'current'] for status in statuses.values()])
total_bytes = sum([status[u'total'] for status in statuses.values()])
return float(sent_bytes)/total_bytes*percentage_with_sizes
@staticmethod
def _generate_push_statuses(total_commands):
push_status_template = deepcopy(TestBuildLogs.STATUS_TEMPLATE)
push_status_template['current_command'] = total_commands
push_status_template['total_commands'] = total_commands
push_statuses = []
one_mb = 1 * 1024 * 1024
num_images = random.randint(2, 7)
sizes = [random.randint(one_mb, one_mb * 5) for _ in range(num_images)]
image_completion = {}
for image_num, image_size in enumerate(sizes):
image_id = 'image_id_%s' % image_num
image_completion[image_id] = {
'current': 0,
'total': image_size,
}
for i in range(one_mb, image_size, one_mb):
image_completion[image_id]['current'] = i
new_status = deepcopy(push_status_template)
new_status['image_completion'] = deepcopy(image_completion)
completion = TestBuildLogs._compute_total_completion(image_completion,
num_images)
new_status['push_completion'] = completion
push_statuses.append((250, None, ('pushing', new_status)))
return push_statuses
@maybe_advance_script()
def get_log_entries(self, build_id, start_index):
if build_id == self.test_build_id:
return (len(self._logs), self._logs[start_index:])
else:
return super(TestBuildLogs, self).get_log_entries(build_id, start_index)
@maybe_advance_script(True)
def get_status(self, build_id):
if build_id == self.test_build_id:
returnable_status = self._last_status
self._last_status = self._status
return returnable_status
else:
return super(TestBuildLogs, self).get_status(build_id)

View file

@ -35,3 +35,6 @@ class FakeUserfiles(object):
def get_file_url(self, file_id, expires_in=300):
return ('http://fake/url')
def get_file_checksum(self, file_id):
return 'abcdefg'

39
workers/README.md Normal file
View file

@ -0,0 +1,39 @@
to prepare a new build node host:
```
sudo apt-get update
sudo apt-get install -y git python-virtualenv python-dev phantomjs libjpeg8 libjpeg62-dev libfreetype6 libfreetype6-dev libevent-dev gdebi-core
```
check out the code, install the kernel, custom docker, nsexec, and reboot:
```
git clone https://bitbucket.org/yackob03/quay.git
cd quay
sudo gdebi --n binary_dependencies/builder/linux-headers-3.11.0-17_3.11.0-17.28_all.deb
sudo gdebi --n binary_dependencies/builder/linux-headers-3.11.0-17-generic_3.11.0-17.28_amd64.deb
sudo gdebi --n binary_dependencies/builder/linux-image-3.11.0-17-generic_3.11.0-17.28_amd64.deb
sudo gdebi --n binary_dependencies/builder/linux-image-extra-3.11.0-17-generic_3.11.0-17.28_amd64.deb
sudo gdebi --n binary_dependencies/builder/nsexec_1.22ubuntu1trusty1_amd64.deb
sudo gdebi --n binary_dependencies/builder/lxc-docker-0.8.0-tutum_0.8.0-tutum-20140212002736-afad5c0-dirty_amd64.deb
sudo chown -R 100000:100000 /var/lib/docker
sudo shutdown -r now
```
pull some base images if you want (optional)
```
sudo docker pull ubuntu
sudo docker pull stackbrew/ubuntu
sudo docker pull busybox
sudo docker pull lopter/raring-base
```
start the worker
```
cd quay
virtualenv --distribute venv
source venv/bin/activate
pip install -r requirements.txt
sudo STACK=prod venv/bin/python -m workers.dockerfilebuild -D
```

View file

@ -10,6 +10,7 @@ import shutil
from docker import Client, APIError
from tempfile import TemporaryFile, mkdtemp
from zipfile import ZipFile
from functools import partial
from data.queue import dockerfile_build_queue
from data import model
@ -53,9 +54,9 @@ class DockerfileBuildContext(object):
self._build_dir = build_context_dir
self._tag_name = tag_name
self._push_token = push_token
self._build_uuid = build_uuid
self._cl = Client(timeout=1200)
self._status = StatusWrapper(self._build_uuid)
self._cl = Client(timeout=1200, version='1.7')
self._status = StatusWrapper(build_uuid)
self._build_logger = partial(build_logs.append_log_message, build_uuid)
dockerfile_path = os.path.join(self._build_dir, "Dockerfile")
self._num_steps = DockerfileBuildContext.__count_steps(dockerfile_path)
@ -93,22 +94,25 @@ class DockerfileBuildContext(object):
with self._status as status:
status['total_commands'] = self._num_steps
logger.debug('Building to tag names: %s' % self._tag_name)
logger.debug('Building to tag named: %s' % self._tag_name)
build_status = self._cl.build(path=self._build_dir, tag=self._tag_name,
stream=True)
current_step = 0
built_image = None
for status in build_status:
logger.debug('Status: %s', str(status))
build_logs.append_log_message(self._build_uuid, str(status))
status_str = str(status.encode('utf-8'))
logger.debug('Status: %s', status_str)
step_increment = re.search(r'Step ([0-9]+) :', status)
if step_increment:
self._build_logger(status_str, build_logs.COMMAND)
current_step = int(step_increment.group(1))
logger.debug('Step now: %s/%s' % (current_step, self._num_steps))
with self._status as status:
status['current_command'] = current_step
continue
else:
self._build_logger(status_str)
complete = re.match(r'Successfully built ([a-z0-9]+)$', status)
if complete:
@ -189,7 +193,11 @@ class DockerfileBuildContext(object):
repos = set()
for image in self._cl.images():
images_to_remove.add(image['Id'])
repos.add(image['Repository'])
for tag in image['RepoTags']:
tag_repo = tag.split(':')[0]
if tag_repo != '<none>':
repos.add(tag_repo)
for repo in repos:
repo_url = 'https://index.docker.io/v1/repositories/%s/images' % repo
@ -254,10 +262,15 @@ class DockerfileBuildWorker(Worker):
tag_name = repository_build.tag
access_token = repository_build.access_token.code
start_msg = ('Starting job with resource url: %s tag: %s and token: %s' %
(resource_url, tag_name, access_token))
log_appender = partial(build_logs.append_log_message,
repository_build.uuid)
log_appender('initializing', build_logs.PHASE)
start_msg = ('Starting job with resource url: %s tag: %s' % (resource_url,
tag_name))
logger.debug(start_msg)
build_logs.append_log_message(repository_build.uuid, start_msg)
log_appender(start_msg)
docker_resource = requests.get(resource_url)
c_type = docker_resource.headers['content-type']
@ -265,41 +278,45 @@ class DockerfileBuildWorker(Worker):
filetype_msg = ('Request to build file of type: %s with tag: %s' %
(c_type, tag_name))
logger.info(filetype_msg)
build_logs.append_log_message(repository_build.uuid, filetype_msg)
log_appender(filetype_msg)
if c_type not in self._mime_processors:
raise RuntimeError('Invalid dockerfile content type: %s' % c_type)
build_dir = self._mime_processors[c_type](docker_resource)
uuid = repository_build.uuid
log_appender('building', build_logs.PHASE)
repository_build.phase = 'building'
repository_build.save()
try:
with DockerfileBuildContext(build_dir, tag_name, access_token,
repository_build.uuid) as build_ctxt:
with DockerfileBuildContext(build_dir, tag_name, access_token,
repository_build.uuid) as build_ctxt:
try:
built_image = build_ctxt.build()
if not built_image:
log_appender('error', build_logs.PHASE)
repository_build.phase = 'error'
repository_build.save()
build_logs.append_log_message(uuid, 'Unable to build dockerfile.')
log_appender('Unable to build dockerfile.', build_logs.ERROR)
return False
log_appender('pushing', build_logs.PHASE)
repository_build.phase = 'pushing'
repository_build.save()
build_ctxt.push(built_image)
log_appender('complete', build_logs.PHASE)
repository_build.phase = 'complete'
repository_build.save()
except Exception as exc:
logger.exception('Exception when processing request.')
repository_build.phase = 'error'
repository_build.save()
build_logs.append_log_message(uuid, exc.message)
return False
except Exception as exc:
log_appender('error', build_logs.PHASE)
logger.exception('Exception when processing request.')
repository_build.phase = 'error'
repository_build.save()
log_appender(str(exc), build_logs.ERROR)
return False
return True