vbatts/quay
Archived
1
0
Fork 0
Code Issues Pull requests Releases Wiki Activity

Merge branch 'bobthe'

Browse source 
Conflicts:
	test/data/test.db
This commit is contained in:
yackob03 2014-02-13 12:47:59 -05:00
commit 8f6cdabde3
41 changed files with 1519 additions and 744 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

5
application.py
View file

@ -1,5 +1,4 @@
import logging
import os
from app import app as application
from data.model import db as model_db
@ -20,10 +19,6 @@ from endpoints.webhooks import webhooks
logger = logging.getLogger(__name__)
if application.config.get('INCLUDE_TEST_ENDPOINTS', False):
logger.debug('Loading test endpoints.')
import endpoints.test
application.register_blueprint(web)
application.register_blueprint(index, url_prefix='/v1')
application.register_blueprint(tags, url_prefix='/v1')

BIN
binary_dependencies/builder/linux-headers-3.11.0-17-generic_3.11.0-17.28_amd64.deb Normal file
View file

Binary file not shown.

BIN
binary_dependencies/builder/linux-headers-3.11.0-17_3.11.0-17.28_all.deb Normal file
View file

Binary file not shown.

BIN
binary_dependencies/builder/linux-image-3.11.0-17-generic_3.11.0-17.28_amd64.deb Normal file
View file

Binary file not shown.

BIN
binary_dependencies/builder/linux-image-extra-3.11.0-17-generic_3.11.0-17.28_amd64.deb Normal file
View file

Binary file not shown.

BIN
binary_dependencies/builder/lxc-docker-0.8.0-tutum_0.8.0-tutum-20140212002736-afad5c0-dirty_amd64.deb Normal file
View file

Binary file not shown.

BIN
binary_dependencies/builder/nsexec_1.22ubuntu1trusty1_amd64.deb Normal file
View file

Binary file not shown.

26
buildserver/Dockerfile
View file

@ -1,26 +0,0 @@
FROM lopter/raring-base
MAINTAINER jake@devtable.com
RUN echo deb http://archive.ubuntu.com/ubuntu precise universe > /etc/apt/sources.list.d/universe.list
RUN apt-get update -qq
RUN apt-get install -qqy iptables ca-certificates lxc python-virtualenv git python-dev xz-utils aufs-tools
# This will use the latest public release. To use your own, comment it out...
ADD https://get.docker.io/builds/Linux/x86_64/docker-latest /usr/local/bin/docker
# ...then uncomment the following line, and copy your docker binary to current dir.
#ADD ./docker /usr/local/bin/docker
# Install the files
ADD ./startserver /usr/local/bin/startserver
ADD ./buildserver.py ./buildserver.py
ADD ./requirements.txt ./requirements.txt
RUN chmod +x /usr/local/bin/docker /usr/local/bin/startserver
RUN virtualenv --distribute venv
RUN venv/bin/pip install -r requirements.txt
VOLUME /var/lib/docker
EXPOSE 5002
CMD startserver

13
buildserver/Readme.md
View file

@ -1,13 +0,0 @@
To build:
```
sudo docker build -t quay.io/quay/buildserver .
sudo docker push quay.io/quay/buildserver
```
To run:
```
sudo docker pull quay.io/quay/buildserver
sudo docker run -d -privileged -lxc-conf="lxc.aa_profile=unconfined" quay.io/quay/buildserver
```

214
buildserver/buildserver.py
View file

@ -1,214 +0,0 @@
import docker
import logging
import shutil
import os
import re
import requests
import json
from flask import Flask, jsonify, abort, make_response
from zipfile import ZipFile
from tempfile import TemporaryFile, mkdtemp
from multiprocessing.pool import ThreadPool
from base64 import b64encode
BUFFER_SIZE = 8 * 1024
LOG_FORMAT = '%(asctime)-15s - %(levelname)s - %(pathname)s - ' + \
'%(funcName)s - %(message)s'
app = Flask(__name__)
logger = logging.getLogger(__name__)
def count_steps(dockerfile_path):
with open(dockerfile_path, 'r') as dockerfileobj:
steps = 0
for line in dockerfileobj.readlines():
stripped = line.strip()
if stripped and stripped[0] is not '#':
steps += 1
return steps
def prepare_zip(request_file):
build_dir = mkdtemp(prefix='docker-build-')
# Save the zip file to temp somewhere
with TemporaryFile() as zip_file:
zip_file.write(request_file.content)
to_extract = ZipFile(zip_file)
to_extract.extractall(build_dir)
return build_dir
def prepare_dockerfile(request_file):
build_dir = mkdtemp(prefix='docker-build-')
dockerfile_path = os.path.join(build_dir, "Dockerfile")
with open(dockerfile_path, 'w') as dockerfile:
dockerfile.write(request_file.content)
return build_dir
def total_completion(statuses, total_images):
percentage_with_sizes = float(len(statuses.values()))/total_images
sent_bytes = sum([status[u'current'] for status in statuses.values()])
total_bytes = sum([status[u'total'] for status in statuses.values()])
return float(sent_bytes)/total_bytes*percentage_with_sizes
def build_image(build_dir, tag_name, num_steps, result_object):
try:
logger.debug('Starting build.')
docker_cl = docker.Client(timeout=1200)
result_object['status'] = 'building'
build_status = docker_cl.build(path=build_dir, tag=tag_name, stream=True)
current_step = 0
built_image = None
for status in build_status:
# logger.debug('Status: %s', str(status))
step_increment = re.search(r'Step ([0-9]+) :', status)
if step_increment:
current_step = int(step_increment.group(1))
logger.debug('Step now: %s/%s' % (current_step, num_steps))
result_object['current_command'] = current_step
continue
complete = re.match(r'Successfully built ([a-z0-9]+)$', status)
if complete:
built_image = complete.group(1)
logger.debug('Final image ID is: %s' % built_image)
continue
shutil.rmtree(build_dir)
# Get the image count
if not built_image:
result_object['status'] = 'error'
result_object['message'] = 'Unable to build dockerfile.'
return
history = json.loads(docker_cl.history(built_image))
num_images = len(history)
result_object['total_images'] = num_images
result_object['status'] = 'pushing'
logger.debug('Pushing to tag name: %s' % tag_name)
resp = docker_cl.push(tag_name, stream=True)
for status_str in resp:
status = json.loads(status_str)
logger.debug('Status: %s', status_str)
if u'status' in status:
status_msg = status[u'status']
if status_msg == 'Pushing':
if u'progressDetail' in status and u'id' in status:
image_id = status[u'id']
detail = status[u'progressDetail']
if u'current' in detail and 'total' in detail:
images = result_object['image_completion']
images[image_id] = detail
result_object['push_completion'] = total_completion(images,
num_images)
elif u'errorDetail' in status:
result_object['status'] = 'error'
if u'message' in status[u'errorDetail']:
result_object['message'] = str(status[u'errorDetail'][u'message'])
return
result_object['status'] = 'complete'
except Exception as e:
logger.exception('Exception when processing request.')
result_object['status'] = 'error'
result_object['message'] = str(e.message)
MIME_PROCESSORS = {
'application/zip': prepare_zip,
'text/plain': prepare_dockerfile,
'application/octet-stream': prepare_dockerfile,
}
# If this format it should also be changed in the api method get_repo_builds
build = {
'total_commands': None,
'current_command': None,
'push_completion': 0.0,
'status': 'waiting',
'message': None,
'image_completion': {},
}
pool = ThreadPool(1)
@app.before_first_request
def start_build():
resource_url = os.environ['RESOURCE_URL']
tag_name = os.environ['TAG']
acccess_token = os.environ['TOKEN']
logger.debug('Starting job with resource url: %s tag: %s and token: %s' %
(resource_url, tag_name, acccess_token))
# Save the token
host = re.match(r'([a-z0-9.:]+)/.+/.+$', tag_name)
if host:
docker_endpoint = 'http://%s/v1/' % host.group(1)
dockercfg_path = os.path.join(os.environ.get('HOME', '.'), '.dockercfg')
token = b64encode('$token:%s' % acccess_token)
with open(dockercfg_path, 'w') as dockercfg:
payload = {
docker_endpoint: {
'auth': token,
'email': '',
}
}
dockercfg.write(json.dumps(payload))
else:
raise Exception('Invalid tag name: %s' % tag_name)
docker_resource = requests.get(resource_url)
c_type = docker_resource.headers['content-type']
logger.info('Request to build file of type: %s with tag: %s' %
(c_type, tag_name))
if c_type not in MIME_PROCESSORS:
raise Exception('Invalid dockerfile content type: %s' % c_type)
build_dir = MIME_PROCESSORS[c_type](docker_resource)
dockerfile_path = os.path.join(build_dir, "Dockerfile")
num_steps = count_steps(dockerfile_path)
logger.debug('Dockerfile had %s steps' % num_steps)
logger.info('Sending job to builder pool.')
build['total_commands'] = num_steps
pool.apply_async(build_image, [build_dir, tag_name, num_steps,
build])
@app.route('/build/', methods=['GET'])
def get_status():
if build:
return jsonify(build)
abort(404)
@app.route('/status/', methods=['GET'])
def health_check():
return make_response('Running')
if __name__ == '__main__':
logging.basicConfig(level=logging.DEBUG, format=LOG_FORMAT)
app.run(host='0.0.0.0', port=5002, threaded=True)

5
buildserver/requirements.txt
View file

@ -1,5 +0,0 @@
mock==1.0.1
requests==1.2.3
six==1.3.0
flask==0.10.1
-e git+git://github.com/DevTable/docker-py.git#egg=docker-py

48
buildserver/startserver
View file

@ -1,48 +0,0 @@
#!/bin/bash
# First, make sure that cgroups are mounted correctly.
CGROUP=/sys/fs/cgroup
[ -d $CGROUP ] ||
mkdir $CGROUP
mountpoint -q $CGROUP ||
mount -n -t tmpfs -o uid=0,gid=0,mode=0755 cgroup $CGROUP || {
echo "Could not make a tmpfs mount. Did you use -privileged?"
exit 1
}
# Mount the cgroup hierarchies exactly as they are in the parent system.
for SUBSYS in $(cut -d: -f2 /proc/1/cgroup)
do
[ -d $CGROUP/$SUBSYS ] || mkdir $CGROUP/$SUBSYS
mountpoint -q $CGROUP/$SUBSYS ||
mount -n -t cgroup -o $SUBSYS cgroup $CGROUP/$SUBSYS
done
# Note: as I write those lines, the LXC userland tools cannot setup
# a "sub-container" properly if the "devices" cgroup is not in its
# own hierarchy. Let's detect this and issue a warning.
grep -q :devices: /proc/1/cgroup ||
echo "WARNING: the 'devices' cgroup should be in its own hierarchy."
grep -qw devices /proc/1/cgroup ||
echo "WARNING: it looks like the 'devices' cgroup is not mounted."
# Now, close extraneous file descriptors.
pushd /proc/self/fd
for FD in *
do
case "$FD" in
# Keep stdin/stdout/stderr
[012])
;;
# Nuke everything else
*)
eval exec "$FD>&-"
;;
esac
done
popd
docker -d &
exec venv/bin/python buildserver.py

29
config.py
View file

@ -1,15 +1,16 @@
import logging
import os
import logstash_formatter
from peewee import MySQLDatabase, SqliteDatabase
from storage.s3 import S3Storage
from storage.local import LocalStorage
from data.userfiles import UserRequestFiles
from data.buildlogs import BuildLogs
from util import analytics
from test.teststorage import FakeStorage, FakeUserfiles
from test import analytics as fake_analytics
from test.testlogs import TestBuildLogs
class FlaskConfig(object):
@ -86,6 +87,15 @@ class S3Userfiles(AWSCredentials):
AWSCredentials.REGISTRY_S3_BUCKET)
class RedisBuildLogs(object):
BUILDLOGS = BuildLogs('logs.quay.io')
class TestBuildLogs(object):
BUILDLOGS = TestBuildLogs('logs.quay.io', 'devtable', 'building',
'deadbeef-dead-beef-dead-beefdeadbeef')
class StripeTestConfig(object):
STRIPE_SECRET_KEY = 'sk_test_PEbmJCYrLXPW0VRLSnWUiZ7Y'
STRIPE_PUBLISHABLE_KEY = 'pk_test_uEDHANKm9CHCvVa2DLcipGRh'
@ -135,13 +145,13 @@ class BuildNodeConfig(object):
BUILD_NODE_PULL_TOKEN = 'F02O2E86CQLKZUQ0O81J8XDHQ6F0N1V36L9JTOEEK6GKKMT1GI8PTJQT4OU88Y6G'
def logs_init_builder(level=logging.DEBUG):
def logs_init_builder(level=logging.DEBUG,
formatter=logstash_formatter.LogstashFormatter()):
@staticmethod
def init_logs():
handler = logging.StreamHandler()
root_logger = logging.getLogger('')
root_logger.setLevel(level)
formatter = logstash_formatter.LogstashFormatter()
handler.setFormatter(formatter)
root_logger.addHandler(handler)
@ -149,26 +159,25 @@ def logs_init_builder(level=logging.DEBUG):
class TestConfig(FlaskConfig, FakeStorage, EphemeralDB, FakeUserfiles,
FakeAnalytics, StripeTestConfig):
FakeAnalytics, StripeTestConfig, RedisBuildLogs):
LOGGING_CONFIG = logs_init_builder(logging.WARN)
POPULATE_DB_TEST_DATA = True
TESTING = True
INCLUDE_TEST_ENDPOINTS = True
class DebugConfig(FlaskConfig, MailConfig, LocalStorage, SQLiteDB,
StripeTestConfig, MixpanelTestConfig, GitHubTestConfig,
DigitalOceanConfig, BuildNodeConfig, S3Userfiles):
LOGGING_CONFIG = logs_init_builder()
DigitalOceanConfig, BuildNodeConfig, S3Userfiles,
TestBuildLogs):
LOGGING_CONFIG = logs_init_builder(formatter=logging.Formatter())
SEND_FILE_MAX_AGE_DEFAULT = 0
POPULATE_DB_TEST_DATA = True
INCLUDE_TEST_ENDPOINTS = True
class LocalHostedConfig(FlaskConfig, MailConfig, S3Storage, RDSMySQL,
StripeLiveConfig, MixpanelTestConfig,
GitHubProdConfig, DigitalOceanConfig,
BuildNodeConfig, S3Userfiles):
BuildNodeConfig, S3Userfiles, RedisBuildLogs):
LOGGING_CONFIG = logs_init_builder()
SEND_FILE_MAX_AGE_DEFAULT = 0
@ -176,7 +185,7 @@ class LocalHostedConfig(FlaskConfig, MailConfig, S3Storage, RDSMySQL,
class ProductionConfig(FlaskProdConfig, MailConfig, S3Storage, RDSMySQL,
StripeLiveConfig, MixpanelProdConfig,
GitHubProdConfig, DigitalOceanConfig, BuildNodeConfig,
S3Userfiles):
S3Userfiles, RedisBuildLogs):
LOGGING_CONFIG = logs_init_builder()
SEND_FILE_MAX_AGE_DEFAULT = 0

63
data/buildlogs.py Normal file
View file

@ -0,0 +1,63 @@
import redis
import json
class BuildLogs(object):
ERROR = 'error'
COMMAND = 'command'
PHASE = 'phase'
def __init__(self, redis_host):
self._redis = redis.StrictRedis(host=redis_host)
@staticmethod
def _logs_key(build_id):
return 'builds/%s/logs' % build_id
def append_log_entry(self, build_id, log_obj):
"""
Appends the serialized form of log_obj to the end of the log entry list
and returns the new length of the list.
"""
return self._redis.rpush(self._logs_key(build_id), json.dumps(log_obj))
def append_log_message(self, build_id, log_message, log_type=None):
"""
Wraps the message in an envelope and push it to the end of the log entry
list and returns the index at which it was inserted.
"""
log_obj = {
'message': log_message
}
if log_type:
log_obj['type'] = log_type
return self._redis.rpush(self._logs_key(build_id), json.dumps(log_obj)) - 1
def get_log_entries(self, build_id, start_index):
"""
Returns a tuple of the current length of the list and an iterable of the
requested log entries.
"""
llen = self._redis.llen(self._logs_key(build_id))
log_entries = self._redis.lrange(self._logs_key(build_id), start_index, -1)
return (llen, (json.loads(entry) for entry in log_entries))
@staticmethod
def _status_key(build_id):
return 'builds/%s/status' % build_id
def set_status(self, build_id, status_obj):
"""
Sets the status key for this build to json serialized form of the supplied
obj.
"""
self._redis.set(self._status_key(build_id), json.dumps(status_obj))
def get_status(self, build_id):
"""
Loads the status information for the specified build id.
"""
fetched = self._redis.get(self._status_key(build_id))
return json.loads(fetched) if fetched else None

14
data/database.py
View file

@ -1,5 +1,6 @@
import string
import logging
import uuid
from random import SystemRandom
from datetime import datetime
@ -20,6 +21,10 @@ def random_string_generator(length=16):
return random_string
def uuid_generator():
return str(uuid.uuid4())
class BaseModel(Model):
class Meta:
database = db
@ -125,7 +130,7 @@ class RepositoryPermission(BaseModel):
class PermissionPrototype(BaseModel):
org = ForeignKeyField(User, index=True, related_name='orgpermissionproto')
uuid = CharField()
uuid = CharField(default=uuid_generator)
activating_user = ForeignKeyField(User, index=True, null=True,
related_name='userpermissionproto')
delegate_user = ForeignKeyField(User, related_name='receivingpermission',
@ -204,13 +209,14 @@ class RepositoryTag(BaseModel):
class RepositoryBuild(BaseModel):
repository = ForeignKeyField(Repository)
uuid = CharField(default=uuid_generator, index=True)
repository = ForeignKeyField(Repository, index=True)
access_token = ForeignKeyField(AccessToken)
resource_key = CharField()
tag = CharField()
build_node_id = IntegerField(null=True)
phase = CharField(default='waiting')
status_url = CharField(null=True)
started = DateTimeField(default=datetime.now)
display_name = CharField()
class QueueItem(BaseModel):

26
data/model.py
View file

@ -4,9 +4,7 @@ import datetime
import dateutil.parser
import operator
import json
import uuid
from datetime import timedelta
from database import *
from util.validation import *
@ -731,8 +729,7 @@ def update_prototype_permission(org, uid, role_name):
def add_prototype_permission(org, role_name, activating_user,
delegate_user=None, delegate_team=None):
new_role = Role.get(Role.name == role_name)
uid = str(uuid.uuid4())
return PermissionPrototype.create(org=org, uuid=uid, role=new_role,
return PermissionPrototype.create(org=org, role=new_role,
activating_user=activating_user,
delegate_user=delegate_user, delegate_team=delegate_team)
@ -1287,13 +1284,18 @@ def load_token_data(code):
raise InvalidTokenException('Invalid delegate token code: %s' % code)
def get_repository_build(request_dbid):
try:
return RepositoryBuild.get(RepositoryBuild.id == request_dbid)
except RepositoryBuild.DoesNotExist:
msg = 'Unable to locate a build by id: %s' % request_dbid
def get_repository_build(namespace_name, repository_name, build_uuid):
joined = RepositoryBuild.select().join(Repository)
fetched = list(joined.where(Repository.name == repository_name,
Repository.namespace == namespace_name,
RepositoryBuild.uuid == build_uuid))
if not fetched:
msg = 'Unable to locate a build by id: %s' % build_uuid
raise InvalidRepositoryBuildException(msg)
return fetched[0]
def list_repository_builds(namespace_name, repository_name,
include_inactive=True):
@ -1307,9 +1309,11 @@ def list_repository_builds(namespace_name, repository_name,
return fetched
def create_repository_build(repo, access_token, resource_key, tag):
def create_repository_build(repo, access_token, resource_key, tag,
display_name):
return RepositoryBuild.create(repository=repo, access_token=access_token,
resource_key=resource_key, tag=tag)
resource_key=resource_key, tag=tag,
display_name=display_name)
def create_webhook(repo, params_obj):

2
data/queue.py
View file

@ -64,5 +64,5 @@ class WorkQueue(object):
image_diff_queue = WorkQueue('imagediff')
dockerfile_build_queue = WorkQueue('dockerfilebuild')
dockerfile_build_queue = WorkQueue('dockerfilebuild2')
webhook_queue = WorkQueue('webhook')

6
data/userfiles.py
View file

@ -59,3 +59,9 @@ class UserRequestFiles(object):
full_key = os.path.join(self._prefix, file_id)
k = Key(self._bucket, full_key)
return k.generate_url(expires_in)
def get_file_checksum(self, file_id):
self._initialize_s3()
full_key = os.path.join(self._prefix, file_id)
k = self._bucket.lookup(full_key)
return k.etag[1:-1][:7]

92
endpoints/api.py
View file

@ -31,6 +31,7 @@ from datetime import datetime, timedelta
store = app.config['STORAGE']
user_files = app.config['USERFILES']
build_logs = app.config['BUILDLOGS']
logger = logging.getLogger(__name__)
route_data = None
@ -69,7 +70,7 @@ def get_route_data():
routes = []
for rule in app.url_map.iter_rules():
if rule.endpoint.startswith('api.'):
endpoint_method = globals()[rule.endpoint[4:]] # Remove api.
endpoint_method = app.view_functions[rule.endpoint]
is_internal = '__internal_call' in dir(endpoint_method)
is_org_api = '__user_call' in dir(endpoint_method)
methods = list(rule.methods.difference(['HEAD', 'OPTIONS']))
@ -1148,40 +1149,69 @@ def get_repo(namespace, repository):
abort(403) # Permission denied
def build_status_view(build_obj):
status = build_logs.get_status(build_obj.uuid)
return {
'id': build_obj.uuid,
'phase': build_obj.phase,
'started': build_obj.started,
'display_name': build_obj.display_name,
'status': status,
}
@api.route('/repository/<path:repository>/build/', methods=['GET'])
@parse_repository_name
def get_repo_builds(namespace, repository):
permission = ReadRepositoryPermission(namespace, repository)
is_public = model.repository_is_public(namespace, repository)
if permission.can() or is_public:
def build_view(build_obj):
# TODO(jake): Filter these logs if the current user can only *read* the repo.
if build_obj.status_url:
# Delegate the status to the build node
node_status = requests.get(build_obj.status_url).json()
node_status['id'] = build_obj.id
return node_status
# If there was no status url, do the best we can
# The format of this block should mirror that of the buildserver.
return {
'id': build_obj.id,
'total_commands': None,
'current_command': None,
'push_completion': 0.0,
'status': build_obj.phase,
'message': None,
'image_completion': {},
}
builds = model.list_repository_builds(namespace, repository)
return jsonify({
'builds': [build_view(build) for build in builds]
'builds': [build_status_view(build) for build in builds]
})
abort(403) # Permission denied
@api.route('/repository/<path:repository>/build/<build_uuid>/status',
methods=['GET'])
@parse_repository_name
def get_repo_build_status(namespace, repository, build_uuid):
permission = ReadRepositoryPermission(namespace, repository)
is_public = model.repository_is_public(namespace, repository)
if permission.can() or is_public:
build = model.get_repository_build(namespace, repository, build_uuid)
return jsonify(build_status_view(build))
abort(403) # Permission denied
@api.route('/repository/<path:repository>/build/<build_uuid>/logs',
methods=['GET'])
@parse_repository_name
def get_repo_build_logs(namespace, repository, build_uuid):
permission = ModifyRepositoryPermission(namespace, repository)
if permission.can():
response_obj = {}
build = model.get_repository_build(namespace, repository, build_uuid)
start = int(request.args.get('start', 0))
count, logs = build_logs.get_log_entries(build.uuid, start)
response_obj.update({
'start': start,
'total': count,
'logs': [log for log in logs],
})
return jsonify(response_obj)
abort(403) # Permission denied
@api.route('/repository/<path:repository>/build/', methods=['POST'])
@api_login_required
@parse_repository_name
@ -1193,20 +1223,28 @@ def request_repo_build(namespace, repository):
repo = model.get_repository(namespace, repository)
token = model.create_access_token(repo, 'write')
display_name = user_files.get_file_checksum(dockerfile_id)
logger.debug('**********Md5: %s' % display_name)
host = urlparse.urlparse(request.url).netloc
tag = '%s/%s/%s' % (host, repo.namespace, repo.name)
build_request = model.create_repository_build(repo, token, dockerfile_id,
tag)
dockerfile_build_queue.put(json.dumps({'build_id': build_request.id}))
tag, display_name)
dockerfile_build_queue.put(json.dumps({
'build_uuid': build_request.uuid,
'namespace': namespace,
'repository': repository,
}))
log_action('build_dockerfile', namespace,
{'repo': repository, 'namespace': namespace,
'fileid': dockerfile_id}, repo=repo)
resp = jsonify({
'started': True
})
resp = jsonify(build_status_view(build_request))
repo_string = '%s/%s' % (namespace, repository)
resp.headers['Location'] = url_for('api.get_repo_build_status',
repository=repo_string,
build_uuid=build_request.uuid)
resp.status_code = 201
return resp

61
endpoints/test.py
View file

@ -1,61 +0,0 @@
import math
from random import SystemRandom
from flask import jsonify
from app import app
def generate_image_completion(rand_func):
images = {}
for image_id in range(rand_func.randint(1, 11)):
total = int(math.pow(abs(rand_func.gauss(0, 1000)), 2))
current = rand_func.randint(0, total)
image_id = 'image_id_%s' % image_id
images[image_id] = {
'total': total,
'current': current,
}
return images
@app.route('/test/build/status', methods=['GET'])
def generate_random_build_status():
response = {
'id': 1,
'total_commands': None,
'current_command': None,
'push_completion': 0.0,
'status': None,
'message': None,
'image_completion': {},
}
random = SystemRandom()
phases = {
'waiting': {},
'starting': {
'total_commands': 7,
'current_command': 0,
},
'initializing': {},
'error': {
'message': 'Oops!'
},
'complete': {},
'building': {
'total_commands': 7,
'current_command': random.randint(1, 7),
},
'pushing': {
'total_commands': 7,
'current_command': 7,
'push_completion': random.random(),
'image_completion': generate_image_completion(random),
},
}
phase = random.choice(phases.keys())
response['status'] = phase
response.update(phases[phase])
return jsonify(response)

19
initdb.py
View file

@ -275,6 +275,13 @@ def populate_database():
'Empty repository which is building.',
False, [], (0, [], None))
token = model.create_access_token(building, 'write')
tag = 'ci.devtable.com:5000/%s/%s' % (building.namespace, building.name)
build = model.create_repository_build(building, token, '123-45-6789', tag,
'build-name')
build.uuid = 'deadbeef-dead-beef-dead-beefdeadbeef'
build.save()
org = model.create_organization('buynlarge', 'quay@devtable.com',
new_user_1)
org.stripe_id = TEST_STRIPE_ID
@ -298,19 +305,11 @@ def populate_database():
model.add_user_to_team(new_user_2, reader_team)
model.add_user_to_team(reader, reader_team)
token = model.create_access_token(building, 'write')
tag = 'ci.devtable.com:5000/%s/%s' % (building.namespace, building.name)
build = model.create_repository_build(building, token, '123-45-6789', tag)
build.build_node_id = 1
build.phase = 'building'
build.status_url = 'http://localhost:5000/test/build/status'
build.save()
__generate_repository(new_user_1, 'superwide', None, False, [],
[(10, [], 'latest2'),
(2, [], 'latest3'),
(2, [(1, [], 'latest11'), (2, [], 'latest12')], 'latest4'),
(2, [(1, [], 'latest11'), (2, [], 'latest12')],
'latest4'),
(2, [], 'latest5'),
(2, [], 'latest6'),
(2, [], 'latest7'),

4
requirements-nover.txt
View file

@ -19,3 +19,7 @@ paramiko
python-digitalocean
xhtml2pdf
logstash_formatter
redis
hiredis
git+https://github.com/dotcloud/docker-py.git
loremipsum

6
requirements.txt
View file

@ -13,16 +13,20 @@ beautifulsoup4==4.3.2
blinker==1.3
boto==2.24.0
distribute==0.6.34
git+https://github.com/dotcloud/docker-py.git
ecdsa==0.10
gevent==1.0
greenlet==0.4.2
gunicorn==18.0
hiredis==0.1.2
html5lib==1.0b3
itsdangerous==0.23
lockfile==0.9.1
logstash-formatter==0.5.8
loremipsum==1.0.2
marisa-trie==0.5.1
mixpanel-py==3.1.1
mock==1.0.1
paramiko==1.12.1
peewee==2.2.0
py-bcrypt==0.4
@ -31,9 +35,11 @@ pycrypto==2.6.1
python-daemon==1.6
python-dateutil==2.2
python-digitalocean==0.6
redis==2.9.1
reportlab==2.7
requests==2.2.1
six==1.5.2
stripe==1.12.0
websocket-client==0.11.0
wsgiref==0.1.2
xhtml2pdf==0.0.5

275
static/css/quay.css
View file

@ -507,35 +507,103 @@ i.toggle-icon:hover {
color: #428bca;
}
.status-boxes .popover {
margin-right: 20px;
.status-box a {
padding: 6px;
color: black;
}
.status-boxes .popover-content {
width: 260px;
.status-box a b {
margin-right: 10px;
}
.build-statuses {
.build-info {
margin: 4px;
padding: 4px;
margin-left: 6px;
margin-right: 6px;
border-bottom: 1px solid #eee;
}
.build-info.clickable:hover {
background: rgba(66, 139, 202, 0.2);
cursor: pointer;
border-radius: 4px;
}
.build-info:last-child {
border-bottom: 0px;
}
.phase-icon {
border-radius: 50%;
display: inline-block;
width: 12px;
height: 12px;
margin-right: 6px;
}
.active .build-tab-link .phase-icon {
box-shadow: 0px 0px 10px #FFFFFF, 0px 0px 10px #FFFFFF;
}
.build-status .phase-icon {
margin-top: 4px;
float: left;
}
.phase-icon.error {
background-color: red;
}
.phase-icon.waiting, .phase-icon.starting, .phase-icon.initializing {
background-color: #ddd;
}
.phase-icon.building {
background-color: #f0ad4e;
}
.phase-icon.pushing {
background-color: #5cb85c;
}
.phase-icon.complete {
background-color: #428bca;
}
.build-status {
display: inline-block;
}
.build-status-container {
padding: 4px;
margin-bottom: 10px;
border-bottom: 1px solid #eee;
width: 230px;
width: 350px;
}
.build-status-container .build-message {
display: block;
white-space: nowrap;
font-size: 12px;
font-size: 14px;
margin-bottom: 10px;
padding-bottom: 10px;
border-bottom: 1px solid #eee;
margin-left: 20px;
}
.build-status-container .progress {
height: 12px;
height: 10px;
margin: 0px;
margin-top: 10px;
width: 230px;
margin-left: 20px;
width: 310px;
}
.build-status-container .timing {
margin-left: 20px;
margin-top: 6px;
}
.build-status-container:last-child {
@ -1633,6 +1701,185 @@ p.editable:hover i {
padding-left: 44px;
}
.repo-build .build-id:before {
content: "Build ID: "
}
.repo-build .build-id {
float: right;
font-size: 12px;
color: #aaa;
padding: 10px;
}
.repo-build .build-pane .timing {
float: right;
}
.repo-build .build-tab-link {
white-space: nowrap;
}
.repo-build .build-pane .build-header {
padding-top: 10px;
border-bottom: 1px solid #eee;
padding-bottom: 10px;
}
.repo-build .build-pane .build-progress {
margin-top: 16px;
margin-bottom: 10px;
}
.repo-build .build-pane .build-progress .progress {
height: 14px;
margin-bottom: 0px;
}
.repo-build .build-pane .quay-spinner {
margin-top: 4px;
display: inline-block;
}
.repo-build .build-pane .build-logs {
background: #222;
color: white;
padding: 10px;
overflow: auto;
}
.repo-build .build-pane .build-logs .container-header {
padding: 2px;
}
.repo-build .build-pane .build-logs .container-logs {
margin: 4px;
padding-bottom: 4px;
}
.repo-build .build-pane .build-logs .command-title,
.repo-build .build-pane .build-logs .log-entry .message {
font-family: Consolas, "Lucida Console", Monaco, monospace;
font-size: 13px;
}
.repo-build .build-pane .build-logs .container-header {
cursor: pointer;
position: relative;
}
.repo-build .build-pane .build-logs .container-header i.fa.chevron {
color: #666;
margin-right: 4px;
width: 14px;
text-align: center;
position: absolute;
top: 6px;
left: 0px;
}
.repo-build .build-pane .build-logs .log-container.command {
margin-left: 42px;
}
.repo-build .build-pane .build-logs .container-header.building {
margin-bottom: 10px;
}
.repo-build .build-pane .build-logs .container-header.pushing {
margin-top: 10px;
}
.repo-build .build-log-error-element {
position: relative;
display: inline-block;
margin: 10px;
padding: 10px;
background: rgba(255, 0, 0, 0.17);
border-radius: 10px;
margin-left: 22px;
}
.repo-build .build-log-error-element i.fa {
color: red;
position: absolute;
top: 13px;
left: 11px;
}
.repo-build .build-log-error-element .error-message {
display: inline-block;
margin-left: 25px;
}
.repo-build .build-pane .build-logs .container-header .label {
padding-top: 4px;
text-align: right;
margin-right: 4px;
width: 86px;
display: inline-block;
border-right: 4px solid #aaa;
background-color: #444;
position: absolute;
top: 4px;
left: 24px;
}
.repo-build .build-pane .build-logs .container-header .container-content {
display: block;
padding-left: 20px;
}
.repo-build .build-pane .build-logs .container-header .container-content.build-log-command {
padding-left: 120px;
}
.label.FROM {
border-color: #5bc0de !important;
}
.label.CMD, .label.EXPOSE, .label.ENTRYPOINT {
border-color: #428bca !important;
}
.label.RUN, .label.ADD {
border-color: #5cb85c !important;
}
.label.ENV, .label.VOLUME, .label.USER, .label.WORKDIR {
border-color: #f0ad4e !important;
}
.label.MAINTAINER {
border-color: #aaa !important;
}
.repo-build .build-pane .build-logs .log-entry {
position: relative;
}
.repo-build .build-pane .build-logs .log-entry .message {
display: inline-block;
margin-left: 46px;
}
.repo-build .build-pane .build-logs .log-entry .id {
color: #aaa;
padding-right: 6px;
margin-right: 6px;
text-align: right;
font-size: 12px;
width: 40px;
position: absolute;
top: 4px;
left: 4px;
}
.repo-admin .right-info {
font-size: 11px;
margin-top: 10px;
@ -1676,16 +1923,6 @@ p.editable:hover i {
cursor: pointer;
}
.repo .build-info {
padding: 10px;
margin: 0px;
}
.repo .build-info .progress {
margin: 0px;
margin-top: 10px;
}
.repo .section {
display: block;
margin-bottom: 20px;

6
static/directives/build-log-command.html Normal file
View file

@ -0,0 +1,6 @@
<span class="command" bindonce>
<span class="label" bo-class="getCommandKind(command.message)" bo-show="getCommandKind(command.message)"
bo-text="getCommandKind(command.message)">
</span>
<span class="command-title" bo-html="getCommandTitleHtml(command.message)"></span>
</span>

4
static/directives/build-log-error.html Normal file
View file

@ -0,0 +1,4 @@
<span bindonce class="build-log-error-element">
<i class="fa fa-exclamation-triangle"></i>
<span class="error-message" bo-text="error.message"></span>
</span>

4
static/directives/build-log-phase.html Normal file
View file

@ -0,0 +1,4 @@
<span bindonce class="build-log-phase-element">
<span class="phase-icon" ng-class="phase.message"></span>
<span class="build-message" phase="phase.message"></span>
</span>

1
static/directives/build-message.html Normal file
View file

@ -0,0 +1 @@
<span class="build-message-element">{{ getBuildMessage(phase) }}</span>

6
static/directives/build-progress.html Normal file
View file

@ -0,0 +1,6 @@
<div class="build-progress-element">
<div class="progress" ng-class="getPercentage(build) < 100 ? 'active progress-striped' : ''">
<div class="progress-bar" role="progressbar" aria-valuenow="{{ getPercentage(build) }}" aria-valuemin="0" aria-valuemax="100" style="{{ 'width: ' + getPercentage(build) + '%' }}">
</div>
</div>
</div>

13
static/directives/build-status.html
View file

@ -1,8 +1,11 @@
<div id="build-status-container" class="build-status-container">
<span class="build-message">{{ getBuildMessage(build) }}</span>
<div class="progress" ng-class="getBuildProgress(build) < 100 ? 'active progress-striped' : ''" ng-show="getBuildProgress(build) >= 0">
<div class="progress-bar" role="progressbar" aria-valuenow="{{ getBuildProgress(build) }}" aria-valuemin="0" aria-valuemax="100" style="{{ 'width: ' + getBuildProgress(build) + '%' }}">
</div>
<div>
<span class="phase-icon" ng-class="build.phase"></span>
<span class="build-message" phase="build.phase"></span>
</div>
<div class="timing">
<i class="fa fa-clock-o"></i>
Started: <span am-time-ago="build.started || 0"></span>
</div>
<div class="build-progress" build="build"></div>
</div>

266
static/js/app.js
View file

@ -103,7 +103,7 @@ function getMarkedDown(string) {
}
// Start the application code itself.
quayApp = angular.module('quay', ['ngRoute', 'chieffancypants.loadingBar', 'restangular', 'angularMoment', 'angulartics', /*'angulartics.google.analytics',*/ 'angulartics.mixpanel', '$strap.directives', 'ngCookies', 'ngSanitize', 'angular-md5'], function($provide, cfpLoadingBarProvider) {
quayApp = angular.module('quay', ['ngRoute', 'chieffancypants.loadingBar', 'restangular', 'angularMoment', 'angulartics', /*'angulartics.google.analytics',*/ 'angulartics.mixpanel', '$strap.directives', 'ngCookies', 'ngSanitize', 'angular-md5', 'pasvaz.bindonce'], function($provide, cfpLoadingBarProvider) {
cfpLoadingBarProvider.includeSpinner = false;
$provide.factory('UtilService', ['$sanitize', function($sanitize) {
@ -151,7 +151,7 @@ quayApp = angular.module('quay', ['ngRoute', 'chieffancypants.loadingBar', 'rest
$provide.factory('ApiService', ['Restangular', function(Restangular) {
var apiService = {};
var getResource = function(path) {
var getResource = function(path, opt_background) {
var resource = {};
resource.url = path;
resource.withOptions = function(options) {
@ -169,6 +169,12 @@ quayApp = angular.module('quay', ['ngRoute', 'chieffancypants.loadingBar', 'rest
'hasError': false
};
if (opt_background) {
performer.withHttpConfig({
'ignoreLoadingBar': true
});
}
performer.get(options).then(function(resp) {
result.value = processor(resp);
result.loading = false;
@ -240,27 +246,33 @@ quayApp = angular.module('quay', ['ngRoute', 'chieffancypants.loadingBar', 'rest
var buildMethodsForEndpoint = function(endpoint) {
var method = endpoint.methods[0].toLowerCase();
var methodName = formatMethodName(endpoint['name']);
apiService[methodName] = function(opt_options, opt_parameters) {
return Restangular.one(buildUrl(endpoint['path'], opt_parameters))['custom' + method.toUpperCase()](opt_options);
apiService[methodName] = function(opt_options, opt_parameters, opt_background) {
var one = Restangular.one(buildUrl(endpoint['path'], opt_parameters));
if (opt_background) {
one.withHttpConfig({
'ignoreLoadingBar': true
});
}
return one['custom' + method.toUpperCase()](opt_options);
};
if (method == 'get') {
apiService[methodName + 'AsResource'] = function(opt_parameters) {
return getResource(buildUrl(endpoint['path'], opt_parameters));
apiService[methodName + 'AsResource'] = function(opt_parameters, opt_background) {
return getResource(buildUrl(endpoint['path'], opt_parameters), opt_background);
};
}
if (endpoint['user_method']) {
apiService[getGenericMethodName(endpoint['user_method'])] = function(orgname, opt_options, opt_parameters) {
apiService[getGenericMethodName(endpoint['user_method'])] = function(orgname, opt_options, opt_parameters, opt_background) {
if (orgname) {
if (orgname.name) {
orgname = orgname.name;
}
var params = jQuery.extend({'orgname' : orgname}, opt_parameters || {});
var params = jQuery.extend({'orgname' : orgname}, opt_parameters || {}, opt_background);
return apiService[methodName](opt_options, params);
} else {
return apiService[formatMethodName(endpoint['user_method'])](opt_options, opt_parameters);
return apiService[formatMethodName(endpoint['user_method'])](opt_options, opt_parameters, opt_background);
}
};
}
@ -779,6 +791,7 @@ quayApp = angular.module('quay', ['ngRoute', 'chieffancypants.loadingBar', 'rest
fixFooter: false}).
when('/repository/:namespace/:name/image/:image', {templateUrl: '/static/partials/image-view.html', controller: ImageViewCtrl, reloadOnSearch: false}).
when('/repository/:namespace/:name/admin', {templateUrl: '/static/partials/repo-admin.html', controller:RepoAdminCtrl, reloadOnSearch: false}).
when('/repository/:namespace/:name/build', {templateUrl: '/static/partials/repo-build.html', controller:RepoBuildCtrl, reloadOnSearch: false}).
when('/repository/', {title: 'Repositories', description: 'Public and private docker repositories list',
templateUrl: '/static/partials/repo-list.html', controller: RepoListCtrl}).
when('/user/', {title: 'Account Settings', description:'Account settings for Quay.io', templateUrl: '/static/partials/user-admin.html',
@ -2467,6 +2480,119 @@ quayApp.directive('namespaceSelector', function () {
});
quayApp.directive('buildLogPhase', function () {
var directiveDefinitionObject = {
priority: 0,
templateUrl: '/static/directives/build-log-phase.html',
replace: false,
transclude: false,
restrict: 'C',
scope: {
'phase': '=phase'
},
controller: function($scope, $element) {
}
};
return directiveDefinitionObject;
});
quayApp.directive('buildLogError', function () {
var directiveDefinitionObject = {
priority: 0,
templateUrl: '/static/directives/build-log-error.html',
replace: false,
transclude: false,
restrict: 'C',
scope: {
'error': '=error'
},
controller: function($scope, $element) {
}
};
return directiveDefinitionObject;
});
quayApp.directive('buildLogCommand', function () {
var directiveDefinitionObject = {
priority: 0,
templateUrl: '/static/directives/build-log-command.html',
replace: false,
transclude: false,
restrict: 'C',
scope: {
'command': '=command'
},
controller: function($scope, $element, $sanitize) {
var registryHandlers = {
'quay.io': function(pieces) {
var rnamespace = pieces[pieces.length - 2];
var rname = pieces[pieces.length - 1];
return '/repository/' + rnamespace + '/' + rname + '/';
},
'': function(pieces) {
var rnamespace = pieces.length == 1 ? '_' : pieces[0];
var rname = pieces[pieces.length - 1];
return 'https://index.docker.io/u/' + rnamespace + '/' + rname + '/';
}
};
var kindHandlers = {
'FROM': function(title) {
var pieces = title.split('/');
var registry = pieces.length < 3 ? '' : pieces[0];
if (!registryHandlers[registry]) {
return title;
}
return '<i class="fa fa-hdd-o"></i> <a href="' + registryHandlers[registry](pieces) + '">' + title + '</a>';
}
};
$scope.getCommandKind = function(fullTitle) {
var colon = fullTitle.indexOf(':');
var title = getTitleWithoutStep(fullTitle);
if (!title) {
return null;
}
var space = title.indexOf(' ');
return title.substring(0, space);
};
$scope.getCommandTitleHtml = function(fullTitle) {
var title = getTitleWithoutStep(fullTitle) || fullTitle;
var space = title.indexOf(' ');
if (space <= 0) {
return $sanitize(title);
}
var kind = $scope.getCommandKind(fullTitle);
var sanitized = $sanitize(title.substring(space + 1));
var handler = kindHandlers[kind || ''];
if (handler) {
return handler(sanitized);
} else {
return sanitized;
}
};
var getTitleWithoutStep = function(fullTitle) {
var colon = fullTitle.indexOf(':');
if (colon <= 0) {
return null;
}
return $.trim(fullTitle.substring(colon + 1));
};
}
};
return directiveDefinitionObject;
});
quayApp.directive('buildStatus', function () {
var directiveDefinitionObject = {
priority: 0,
@ -2478,55 +2604,85 @@ quayApp.directive('buildStatus', function () {
'build': '=build'
},
controller: function($scope, $element) {
$scope.getBuildProgress = function(buildInfo) {
switch (buildInfo.status) {
case 'building':
return (buildInfo.current_command / buildInfo.total_commands) * 100;
break;
case 'pushing':
return buildInfo.push_completion * 100;
break;
}
};
return directiveDefinitionObject;
});
case 'complete':
return 100;
break;
case 'initializing':
case 'starting':
case 'waiting':
return 0;
break;
}
quayApp.directive('buildMessage', function () {
var directiveDefinitionObject = {
priority: 0,
templateUrl: '/static/directives/build-message.html',
replace: false,
transclude: false,
restrict: 'C',
scope: {
'phase': '=phase'
},
controller: function($scope, $element) {
$scope.getBuildMessage = function (phase) {
switch (phase) {
case 'starting':
case 'initializing':
return 'Starting Dockerfile build';
case 'waiting':
return 'Waiting for available build worker';
case 'building':
return 'Building image from Dockerfile';
case 'pushing':
return 'Pushing image built from Dockerfile';
return -1;
};
case 'complete':
return 'Dockerfile build completed and pushed';
case 'error':
return 'Dockerfile build failed';
}
};
}
};
return directiveDefinitionObject;
});
$scope.getBuildMessage = function(buildInfo) {
switch (buildInfo.status) {
case 'initializing':
return 'Starting Dockerfile build';
break;
case 'starting':
case 'waiting':
case 'building':
return 'Building image from Dockerfile';
break;
quayApp.directive('buildProgress', function () {
var directiveDefinitionObject = {
priority: 0,
templateUrl: '/static/directives/build-progress.html',
replace: false,
transclude: false,
restrict: 'C',
scope: {
'build': '=build'
},
controller: function($scope, $element) {
$scope.getPercentage = function(buildInfo) {
switch (buildInfo.phase) {
case 'building':
return (buildInfo.status.current_command / buildInfo.status.total_commands) * 100;
break;
case 'pushing':
return buildInfo.status.push_completion * 100;
break;
case 'pushing':
return 'Pushing image built from Dockerfile';
break;
case 'complete':
return 100;
break;
case 'complete':
return 'Dockerfile build completed and pushed';
break;
case 'error':
return 'Dockerfile build failed: ' + buildInfo.message;
break;
}
};
case 'initializing':
case 'starting':
case 'waiting':
return 0;
break;
}
return -1;
};
}
};
return directiveDefinitionObject;
@ -2541,6 +2697,14 @@ quayApp.directive('ngBlur', function() {
};
});
quayApp.directive('ngVisible', function () {
return function (scope, element, attr) {
scope.$watch(attr.ngVisible, function (visible) {
element.css('visibility', visible ? 'visible' : 'hidden');
});
};
});
quayApp.run(['$location', '$rootScope', 'Restangular', 'UserService', 'PlanService', '$http', '$timeout',
function($location, $rootScope, Restangular, UserService, PlanService, $http, $timeout) {

213
static/js/controllers.js
View file

@ -185,6 +185,11 @@ function RepoCtrl($scope, $sanitize, Restangular, ImageMetadataService, ApiServi
$scope.getFormattedCommand = ImageMetadataService.getFormattedCommand;
$scope.showBuild = function(buildInfo) {
$location.path('/repository/' + namespace + '/' + name + '/build');
$location.search('current', buildInfo.id);
};
$scope.getTooltipCommand = function(image) {
var sanitized = ImageMetadataService.getEscapedFormattedCommand(image);
return '<span class=\'codetooltip\'>' + sanitized + '</span>';
@ -512,13 +517,11 @@ function RepoCtrl($scope, $sanitize, Restangular, ImageMetadataService, ApiServi
};
var getBuildInfo = function(repo) {
// Note: We use restangular manually here because we need to turn off the loading bar.
var buildInfo = Restangular.one('repository/' + repo.namespace + '/' + repo.name + '/build/');
buildInfo.withHttpConfig({
'ignoreLoadingBar': true
});
var params = {
'repository': repo.namespace + '/' + repo.name
};
buildInfo.get().then(function(resp) {
ApiService.getRepoBuilds(null, params, true).then(function(resp) {
var runningBuilds = [];
for (var i = 0; i < resp.builds.length; ++i) {
var build = resp.builds[i];
@ -604,6 +607,197 @@ function RepoCtrl($scope, $sanitize, Restangular, ImageMetadataService, ApiServi
loadViewInfo();
}
function RepoBuildCtrl($scope, Restangular, ApiService, $routeParams, $rootScope, $location, $interval, $sanitize) {
var namespace = $routeParams.namespace;
var name = $routeParams.name;
var pollTimerHandle = null;
$scope.$on('$destroy', function() {
stopPollTimer();
});
// Watch for changes to the current parameter.
$scope.$on('$routeUpdate', function(){
if ($location.search().current) {
$scope.setCurrentBuild($location.search().current, false);
}
});
$scope.builds = [];
$scope.polling = false;
$scope.adjustLogHeight = function() {
$('.build-logs').height($(window).height() - 365);
};
$scope.hasLogs = function(container) {
return ((container.logs && container.logs.length) || (container._logs && container._logs.length));
};
$scope.toggleLogs = function(container) {
if (container._logs) {
container.logs = container._logs;
container._logs = null;
} else {
container._logs = container.logs;
container.logs = null;
}
};
$scope.setCurrentBuild = function(buildId, opt_updateURL) {
// Find the build.
for (var i = 0; i < $scope.builds.length; ++i) {
if ($scope.builds[i].id == buildId) {
$scope.setCurrentBuildInternal($scope.builds[i], opt_updateURL);
return;
}
}
};
$scope.setCurrentBuildInternal = function(build, opt_updateURL) {
if (build == $scope.currentBuild) { return; }
stopPollTimer();
$scope.logEntries = null;
$scope.logStartIndex = null;
$scope.currentParentEntry = null;
$scope.currentBuild = build;
if (opt_updateURL) {
if (build) {
$location.search('current', build.id);
} else {
$location.search('current', null);
}
}
// Timeout needed to ensure the log element has been created
// before its height is adjusted.
setTimeout(function() {
$scope.adjustLogHeight();
}, 1);
// Load the first set of logs.
getBuildStatusAndLogs();
// If the build is currently processing, start the build timer.
checkPollTimer();
};
var checkPollTimer = function() {
var build = $scope.currentBuild;
if (!build) {
stopPollTimer();
return;
}
if (build['phase'] != 'complete' && build['phase'] != 'error') {
startPollTimer();
return true;
} else {
stopPollTimer();
return false;
}
};
var stopPollTimer = function() {
$interval.cancel(pollTimerHandle);
};
var startPollTimer = function() {
stopPollTimer();
pollTimerHandle = $interval(getBuildStatusAndLogs, 2000);
};
var processLogs = function(logs, startIndex) {
if (!$scope.logEntries) { $scope.logEntries = []; }
for (var i = 0; i < logs.length; ++i) {
var entry = logs[i];
var type = entry['type'] || 'entry';
if (type == 'command' || type == 'phase' || type == 'error') {
entry['_logs'] = [];
entry['index'] = startIndex + i;
$scope.logEntries.push(entry);
$scope.currentParentEntry = entry;
} else if ($scope.currentParentEntry) {
if ($scope.currentParentEntry['logs']) {
$scope.currentParentEntry['logs'].push(entry);
} else {
$scope.currentParentEntry['_logs'].push(entry);
}
}
}
};
var getBuildStatusAndLogs = function() {
if (!$scope.currentBuild || $scope.polling) { return; }
$scope.polling = true;
var params = {
'repository': namespace + '/' + name,
'build_uuid': $scope.currentBuild.id
};
ApiService.getRepoBuildStatus(null, params, true).then(function(resp) {
// Note: We use extend here rather than replacing as Angular is depending on the
// root build object to remain the same object.
$.extend(true, $scope.currentBuild, resp);
checkPollTimer();
// Load the updated logs for the build.
var options = {
'start': $scope.logStartIndex
};
ApiService.getRepoBuildLogsAsResource(params, true).withOptions(options).get(function(resp) {
processLogs(resp['logs'], resp['start']);
$scope.logStartIndex = resp['total'];
$scope.polling = false;
});
});
};
var fetchRepository = function() {
var params = {'repository': namespace + '/' + name};
$rootScope.title = 'Loading Repository...';
$scope.repository = ApiService.getRepoAsResource(params).get(function(repo) {
if (!repo.can_write) {
$rootScope.title = 'Unknown builds';
$scope.accessDenied = true;
return;
}
$rootScope.title = 'Repository Builds';
$scope.repo = repo;
getBuildInfo();
});
};
var getBuildInfo = function(repo) {
var params = {
'repository': namespace + '/' + name
};
ApiService.getRepoBuilds(null, params).then(function(resp) {
$scope.builds = resp.builds;
if ($location.search().current) {
$scope.setCurrentBuild($location.search().current, false);
} else if ($scope.builds.length > 0) {
$scope.setCurrentBuild($scope.builds[0].id, true);
}
});
};
fetchRepository();
}
function RepoAdminCtrl($scope, Restangular, ApiService, $routeParams, $rootScope) {
var namespace = $routeParams.namespace;
var name = $routeParams.name;
@ -854,8 +1048,13 @@ function RepoAdminCtrl($scope, Restangular, ApiService, $routeParams, $rootScope
};
$scope.repository = ApiService.getRepoAsResource(params).get(function(repo) {
$scope.repo = repo;
if (!repo.can_admin) {
$rootScope.title = 'Forbidden';
$scope.accessDenied = true;
return;
}
$scope.repo = repo;
$rootScope.title = 'Settings - ' + namespace + '/' + name;
$rootScope.description = 'Administrator settings for ' + namespace + '/' + name +
': Permissions, webhooks and other settings';

1
static/lib/bindonce.min.js vendored Normal file
View file

File diff suppressed because one or more lines are too long

3
static/partials/repo-admin.html
View file

@ -1,5 +1,8 @@
<div class="container" ng-show="deleting"><div class="quay-spinner"></div></div>
<div class="resource-view" resource="repository" error-message="'No repository found'"></div>
<div class="container repo repo-admin" ng-show="accessDenied">
You do not have permission to view this page
</div>
<div class="container repo repo-admin" ng-show="repo && !deleting">
<div class="header row">
<a href="{{ '/repository/' + repo.namespace + '/' + repo.name }}" class="back"><i class="fa fa-chevron-left"></i></a>

83
static/partials/repo-build.html Normal file
View file

@ -0,0 +1,83 @@
<div class="resource-view" resource="repository" error-message="'No repository found'"></div>
<div class="container repo repo-build" ng-show="accessDenied">
You do not have permission to view this page
</div>
<div class="container repo repo-build" ng-show="repo">
<div class="header row">
<a href="{{ '/repository/' + repo.namespace + '/' + repo.name }}" class="back"><i class="fa fa-chevron-left"></i></a>
<h3>
<span class="repo-circle no-background" repo="repo"></span>
<span class="repo-breadcrumb" repo="repo"></span>
</h3>
</div>
<div class="row" ng-show="!builds.length">
There are no builds for this repository
</div>
<div class="row" ng-show="builds.length">
<!-- Side tabs -->
<div class="col-sm-2">
<ul class="nav nav-pills nav-stacked">
<li ng-class="currentBuild == build ? 'active' : ''" ng-repeat="build in builds">
<a class="build-tab-link" href="javascript:void(0)" ng-click="setCurrentBuild(build.id, true)">
<span class="phase-icon" ng-class="build.phase"></span>
<span>{{ build.display_name }}</span>
</a>
</li>
</ul>
</div>
<!-- Content -->
<div class="col-sm-10">
<div class="tab-content" onresize="adjustLogHeight()">
<div ng-repeat="build in builds" class="tab-pane build-pane" ng-class="currentBuild == build ? 'active' : ''">
<div class="build-header">
<div class="timing">
<i class="fa fa-clock-o"></i>
Started: <span am-time-ago="build.started || 0"></span>
</div>
<span class="phase-icon" ng-class="build.phase"></span>
<span class="build-message" phase="build.phase"></span>
<div class="build-progress" build="build"></div>
</div>
<div class="build-logs">
<div ng-show="!logEntries">
<span class="quay-spinner"></span>
</div>
<div class="log-container" ng-class="container.type" ng-repeat="container in logEntries">
<div class="container-header" ng-class="container.type == 'phase' ? container.message : ''"
ng-switch on="container.type" ng-click="toggleLogs(container)">
<i class="fa chevron"
ng-class="container.logs ? 'fa-chevron-down' : 'fa-chevron-right'" ng-show="hasLogs(container)"></i>
<div ng-switch-when="phase">
<span class="container-content build-log-phase" phase="container"></span>
</div>
<div ng-switch-when="error">
<span class="container-content build-log-error" error="container"></span>
</div>
<div ng-switch-when="command">
<span class="container-content build-log-command" command="container"></span>
</div>
</div>
<!-- Display the entries for the container -->
<div class="container-logs" ng-show="container.logs">
<div class="log-entry" bindonce ng-repeat="entry in container.logs">
<span class="id" bo-text="$index + container.index + 1"></span>
<span class="message" bo-text="entry.message"></span>
</div>
</div>
</div>
</div>
<div>
<span class="quay-spinner" ng-show="polling"></span>
<span class="build-id">{{ build.id }}</span>
</div>
</div>
</div>
</div>
</div>
</div>

19
static/partials/view-repo.html
View file

@ -38,13 +38,18 @@
<!-- Status boxes -->
<div class="status-boxes">
<div id="buildInfoBox" class="status-box" ng-show="repo.is_building"
bs-popover="'static/partials/build-status-item.html'" data-placement="bottom">
<span class="title">
<span class="quay-spinner"></span>
<b>Building Images</b>
</span>
<span class="count" ng-class="buildsInfo ? 'visible' : ''"><span>{{ buildsInfo ? buildsInfo.length : '-' }}</span></span>
<div id="buildInfoBox" class="status-box" ng-show="repo.is_building">
<div class="dropdown" data-placement="top">
<span class="count" ng-class="buildsInfo ? 'visible' : ''"><span>{{ buildsInfo ? buildsInfo.length : '-' }}</span></span>
<a href="javascript:void(0)" class="dropdown-toggle" data-toggle="dropdown">Building Dockerfile<span ng-show="buildsInfo.length > 1">s</span> <b class="caret"></b></a>
<ul class="dropdown-menu pull-right">
<li ng-repeat="buildInfo in buildsInfo">
<div class="build-info" ng-class="repo.can_write ? 'clickable' : ''" ng-click="showBuild(buildInfo)">
<span class="build-status" build="buildInfo"></span>
</div>
</li>
</ul>
</div>
</div>
</div>

1
templates/base.html
View file

@ -51,6 +51,7 @@
<script src="static/lib/angulartics-mixpanel.js"></script>
<script src="static/lib/angulartics-google-analytics.js"></script>
<script src="static/lib/angular-md5.js"></script>
<script src="static/lib/bindonce.min.js"></script>
<script src="static/lib/angular-moment.min.js"></script>
<script src="static/lib/angular-cookies.min.js"></script>

BIN
test/data/test.db
View file

Binary file not shown.

189
test/testlogs.py Normal file
View file

@ -0,0 +1,189 @@
import logging
from random import SystemRandom
from loremipsum import get_sentence
from functools import wraps
from copy import deepcopy
from data.buildlogs import BuildLogs
logger = logging.getLogger(__name__)
random = SystemRandom()
def maybe_advance_script(is_get_status=False):
def inner_advance(func):
@wraps(func)
def wrapper(self, *args, **kwargs):
advance_units = random.randint(1, 500)
logger.debug('Advancing script %s units', advance_units)
while advance_units > 0 and self.remaining_script:
units = self.remaining_script[0][0]
if advance_units > units:
advance_units -= units
self.advance_script(is_get_status)
else:
break
return func(self, *args, **kwargs)
return wrapper
return inner_advance
class TestBuildLogs(BuildLogs):
COMMAND_TYPES = ['FROM', 'MAINTAINER', 'RUN', 'CMD', 'EXPOSE', 'ENV', 'ADD',
'ENTRYPOINT', 'VOLUME', 'USER', 'WORKDIR']
STATUS_TEMPLATE = {
'total_commands': None,
'current_command': None,
'push_completion': 0.0,
'image_completion': {},
}
def __init__(self, redis_host, namespace, repository, test_build_id):
super(TestBuildLogs, self).__init__(redis_host)
self.namespace = namespace
self.repository = repository
self.test_build_id = test_build_id
self.remaining_script = self._generate_script()
logger.debug('Total script size: %s', len(self.remaining_script))
self._logs = []
self._status = {}
self._last_status = {}
def advance_script(self, is_get_status):
(_, log, status_wrapper) = self.remaining_script.pop(0)
if log is not None:
self._logs.append(log)
if status_wrapper is not None:
(phase, status) = status_wrapper
from data import model
build_obj = model.get_repository_build(self.namespace, self.repository,
self.test_build_id)
build_obj.phase = phase
build_obj.save()
self._status = status
if not is_get_status:
self._last_status = status
def _generate_script(self):
script = []
# generate the init phase
script.append(self._generate_phase(400, 'initializing'))
script.extend(self._generate_logs(random.randint(1, 3)))
# move to the building phase
script.append(self._generate_phase(400, 'building'))
total_commands = random.randint(5, 20)
for command_num in range(1, total_commands + 1):
command_weight = random.randint(50, 100)
script.append(self._generate_command(command_num, total_commands,
command_weight))
# we want 0 logs some percent of the time
num_logs = max(0, random.randint(-50, 400))
script.extend(self._generate_logs(num_logs))
# move to the pushing phase
script.append(self._generate_phase(400, 'pushing'))
script.extend(self._generate_push_statuses(total_commands))
# move to the error or complete phase
if random.randint(0, 1) == 0:
script.append(self._generate_phase(400, 'complete'))
else:
script.append(self._generate_phase(400, 'error'))
script.append((1, {'message': 'Something bad happened! Oh noes!',
'type': self.ERROR}, None))
return script
def _generate_phase(self, start_weight, phase_name):
return (start_weight, {'message': phase_name, 'type': self.PHASE},
(phase_name, deepcopy(self.STATUS_TEMPLATE)))
def _generate_command(self, command_num, total_commands, command_weight):
sentence = get_sentence()
command = random.choice(self.COMMAND_TYPES)
if command == 'FROM':
sentence = random.choice(['ubuntu', 'lopter/raring-base',
'quay.io/devtable/simple',
'quay.io/buynlarge/orgrepo',
'stackbrew/ubuntu:precise'])
msg = {
'message': 'Step %s: %s %s' % (command_num, command, sentence),
'type': self.COMMAND,
}
status = deepcopy(self.STATUS_TEMPLATE)
status['total_commands'] = total_commands
status['current_command'] = command_num
return (command_weight, msg, ('building', status))
@staticmethod
def _generate_logs(count):
return [(1, {'message': get_sentence()}, None) for _ in range(count)]
@staticmethod
def _compute_total_completion(statuses, total_images):
percentage_with_sizes = float(len(statuses.values()))/total_images
sent_bytes = sum([status[u'current'] for status in statuses.values()])
total_bytes = sum([status[u'total'] for status in statuses.values()])
return float(sent_bytes)/total_bytes*percentage_with_sizes
@staticmethod
def _generate_push_statuses(total_commands):
push_status_template = deepcopy(TestBuildLogs.STATUS_TEMPLATE)
push_status_template['current_command'] = total_commands
push_status_template['total_commands'] = total_commands
push_statuses = []
one_mb = 1 * 1024 * 1024
num_images = random.randint(2, 7)
sizes = [random.randint(one_mb, one_mb * 5) for _ in range(num_images)]
image_completion = {}
for image_num, image_size in enumerate(sizes):
image_id = 'image_id_%s' % image_num
image_completion[image_id] = {
'current': 0,
'total': image_size,
}
for i in range(one_mb, image_size, one_mb):
image_completion[image_id]['current'] = i
new_status = deepcopy(push_status_template)
new_status['image_completion'] = deepcopy(image_completion)
completion = TestBuildLogs._compute_total_completion(image_completion,
num_images)
new_status['push_completion'] = completion
push_statuses.append((250, None, ('pushing', new_status)))
return push_statuses
@maybe_advance_script()
def get_log_entries(self, build_id, start_index):
if build_id == self.test_build_id:
return (len(self._logs), self._logs[start_index:])
else:
return super(TestBuildLogs, self).get_log_entries(build_id, start_index)
@maybe_advance_script(True)
def get_status(self, build_id):
if build_id == self.test_build_id:
returnable_status = self._last_status
self._last_status = self._status
return returnable_status
else:
return super(TestBuildLogs, self).get_status(build_id)

39
workers/README.md Normal file
View file

@ -0,0 +1,39 @@
to prepare a new build node host:
```
sudo apt-get update
sudo apt-get install -y git python-virtualenv python-dev phantomjs libjpeg8 libjpeg62-dev libfreetype6 libfreetype6-dev libevent-dev gdebi-core
```
check out the code, install the kernel, custom docker, nsexec, and reboot:
```
git clone https://bitbucket.org/yackob03/quay.git
cd quay
sudo gdebi --n binary_dependencies/builder/linux-headers-3.11.0-17_3.11.0-17.28_all.deb
sudo gdebi --n binary_dependencies/builder/linux-headers-3.11.0-17-generic_3.11.0-17.28_amd64.deb
sudo gdebi --n binary_dependencies/builder/linux-image-3.11.0-17-generic_3.11.0-17.28_amd64.deb
sudo gdebi --n binary_dependencies/builder/linux-image-extra-3.11.0-17-generic_3.11.0-17.28_amd64.deb
sudo gdebi --n binary_dependencies/builder/nsexec_1.22ubuntu1trusty1_amd64.deb
sudo gdebi --n binary_dependencies/builder/lxc-docker-0.8.0-tutum_0.8.0-tutum-20140212002736-afad5c0-dirty_amd64.deb
sudo chown -R 100000:100000 /var/lib/docker
sudo shutdown -r now
```
pull some base images if you want (optional)
```
sudo docker pull ubuntu
sudo docker pull stackbrew/ubuntu
sudo docker pull busybox
sudo docker pull lopter/raring-base
```
start the worker
```
cd quay
virtualenv --distribute venv
source venv/bin/activate
pip install -r requirements.txt
sudo STACK=prod venv/bin/python -m workers.dockerfilebuild -D
```

507
workers/dockerfilebuild.py
View file

@ -1,20 +1,20 @@
import logging
import json
import daemon
import time
import argparse
import digitalocean
import requests
import os
import requests
import re
import json
import shutil
from apscheduler.scheduler import Scheduler
from multiprocessing.pool import ThreadPool
from base64 import b64encode
from requests.exceptions import ConnectionError
from docker import Client, APIError
from tempfile import TemporaryFile, mkdtemp
from zipfile import ZipFile
from functools import partial
from data.queue import dockerfile_build_queue
from data import model
from data.database import db as db_connection
from workers.worker import Worker
from app import app
@ -26,234 +26,300 @@ formatter = logging.Formatter(FORMAT)
logger = logging.getLogger(__name__)
BUILD_SERVER_CMD = ('docker run -d -p 5002:5002 ' +
'-lxc-conf="lxc.aa_profile=unconfined" ' +
'-privileged -e \'RESOURCE_URL=%s\' -e \'TAG=%s\' ' +
'-e \'TOKEN=%s\' quay.io/quay/buildserver')
user_files = app.config['USERFILES']
build_logs = app.config['BUILDLOGS']
def retry_command(to_call, args=[], kwargs={}, retries=5, period=5):
try:
return to_call(*args, **kwargs)
except Exception as ex:
if retries:
logger.debug('Retrying command after %ss' % period)
time.sleep(period)
return retry_command(to_call, args, kwargs, retries-1, period)
raise ex
def get_status(url):
return retry_command(requests.get, [url]).json()['status']
def babysit_builder(request):
""" Spin up a build node and ask it to build our job. Retryable errors
should return False, while fatal errors should return True.
"""
try:
logger.debug('Starting work item: %s' % request)
repository_build = model.get_repository_build(request['build_id'])
logger.debug('Request details: %s' % repository_build)
# Initialize digital ocean API
do_client_id = app.config['DO_CLIENT_ID']
do_api_key = app.config['DO_CLIENT_SECRET']
manager = digitalocean.Manager(client_id=do_client_id, api_key=do_api_key)
# check if there is already a DO node for this build, if so clean it up
old_id = repository_build.build_node_id
if old_id:
logger.debug('Cleaning up old DO node: %s' % old_id)
old_droplet = digitalocean.Droplet(id=old_id, client_id=do_client_id,
api_key=do_api_key)
retry_command(old_droplet.destroy)
# Pick the region for the new droplet
allowed_regions = app.config['DO_ALLOWED_REGIONS']
regions = retry_command(manager.get_all_regions)
available_regions = {region.id for region in regions}
regions = available_regions.intersection(allowed_regions)
if not regions:
logger.error('No droplets in our allowed regtions, available: %s' %
available_regions)
return False
# start the DO node
name = 'dockerfile-build-%s' % repository_build.id
logger.debug('Starting DO node: %s' % name)
droplet = digitalocean.Droplet(client_id=do_client_id,
api_key=do_api_key,
name=name,
region_id=regions.pop(),
image_id=app.config['DO_DOCKER_IMAGE'],
size_id=66, # 512MB,
backup_active=False)
retry_command(droplet.create, [],
{'ssh_key_ids': [app.config['DO_SSH_KEY_ID']]})
repository_build.build_node_id = droplet.id
repository_build.phase = 'starting'
repository_build.save()
logger.debug('Waiting for DO node to be available.')
startup = retry_command(droplet.get_events)[0]
while not startup.percentage or int(startup.percentage) != 100:
logger.debug('Droplet startup percentage: %s' % startup.percentage)
time.sleep(5)
retry_command(startup.load)
retry_command(droplet.load)
logger.debug('Droplet started at ip address: %s' % droplet.ip_address)
# connect to it with ssh
repository_build.phase = 'initializing'
repository_build.save()
# We wait until here to import paramiko because otherwise it doesn't work
# under the daemon context.
import paramiko
ssh_client = paramiko.SSHClient()
ssh_client.set_missing_host_key_policy(paramiko.AutoAddPolicy())
logger.debug('Connecting to droplet through ssh at ip: %s' %
droplet.ip_address)
retry_command(ssh_client.connect, [droplet.ip_address, 22, 'root'],
{'look_for_keys': False, 'timeout': 10.0,
'key_filename': app.config['DO_SSH_PRIVATE_KEY_FILENAME']})
# Load the node with the pull token
token = app.config['BUILD_NODE_PULL_TOKEN']
basicauth = b64encode('%s:%s' % ('$token', token))
auth_object = {
'https://quay.io/v1/': {
'auth': basicauth,
'email': '',
},
class StatusWrapper(object):
def __init__(self, build_uuid):
self._uuid = build_uuid
self._status = {
'total_commands': None,
'current_command': None,
'push_completion': 0.0,
'image_completion': {},
}
create_auth_cmd = 'echo \'%s\' > .dockercfg' % json.dumps(auth_object)
ssh_client.exec_command(create_auth_cmd)
self.__exit__(None, None, None)
# Pull and run the buildserver
pull_cmd = 'docker pull quay.io/quay/buildserver'
_, stdout, _ = ssh_client.exec_command(pull_cmd)
pull_status = stdout.channel.recv_exit_status()
def __enter__(self):
return self._status
if pull_status != 0:
logger.error('Pull command failed for host: %s' % droplet.ip_address)
return False
else:
logger.debug('Pull status was: %s' % pull_status)
def __exit__(self, exc_type, value, traceback):
build_logs.set_status(self._uuid, self._status)
# Remove the credentials we used to pull so crafty users cant steal them
remove_auth_cmd = 'rm .dockercfg'
ssh_client.exec_command(remove_auth_cmd)
# Prepare the signed resource url the build node can fetch the job from
user_files = app.config['USERFILES']
class DockerfileBuildContext(object):
def __init__(self, build_context_dir, tag_name, push_token, build_uuid):
self._build_dir = build_context_dir
self._tag_name = tag_name
self._push_token = push_token
self._cl = Client(timeout=1200, version='1.7')
self._status = StatusWrapper(build_uuid)
self._build_logger = partial(build_logs.append_log_message, build_uuid)
dockerfile_path = os.path.join(self._build_dir, "Dockerfile")
self._num_steps = DockerfileBuildContext.__count_steps(dockerfile_path)
logger.debug('Will build and push to tag named: %s' % self._tag_name)
def __enter__(self):
return self
def __exit__(self, exc_type, value, traceback):
self.__cleanup()
shutil.rmtree(self._build_dir)
@staticmethod
def __count_steps(dockerfile_path):
with open(dockerfile_path, 'r') as dockerfileobj:
steps = 0
for line in dockerfileobj.readlines():
stripped = line.strip()
if stripped and stripped[0] is not '#':
steps += 1
return steps
@staticmethod
def __total_completion(statuses, total_images):
percentage_with_sizes = float(len(statuses.values()))/total_images
sent_bytes = sum([status[u'current'] for status in statuses.values()])
total_bytes = sum([status[u'total'] for status in statuses.values()])
return float(sent_bytes)/total_bytes*percentage_with_sizes
def build(self):
logger.debug('Starting build.')
with self._status as status:
status['total_commands'] = self._num_steps
logger.debug('Building to tag named: %s' % self._tag_name)
build_status = self._cl.build(path=self._build_dir, tag=self._tag_name,
stream=True)
current_step = 0
built_image = None
for status in build_status:
status_str = str(status.encode('utf-8'))
logger.debug('Status: %s', status_str)
step_increment = re.search(r'Step ([0-9]+) :', status)
if step_increment:
self._build_logger(status_str, build_logs.COMMAND)
current_step = int(step_increment.group(1))
logger.debug('Step now: %s/%s' % (current_step, self._num_steps))
with self._status as status:
status['current_command'] = current_step
continue
else:
self._build_logger(status_str)
complete = re.match(r'Successfully built ([a-z0-9]+)$', status)
if complete:
built_image = complete.group(1)
logger.debug('Final image ID is: %s' % built_image)
continue
# Get the image count
if not built_image:
return
return built_image
def push(self, built_image):
# Login to the registry
host = re.match(r'([a-z0-9.:]+)/.+/.+$', self._tag_name)
if not host:
raise RuntimeError('Invalid tag name: %s' % self._tag_name)
for protocol in ['https', 'http']:
registry_endpoint = '%s://%s/v1/' % (protocol, host.group(1))
logger.debug('Attempting login to registry: %s' % registry_endpoint)
try:
self._cl.login('$token', self._push_token, registry=registry_endpoint)
break
except APIError:
pass # Probably the wrong protocol
history = json.loads(self._cl.history(built_image))
num_images = len(history)
with self._status as status:
status['total_images'] = num_images
logger.debug('Pushing to tag name: %s' % self._tag_name)
resp = self._cl.push(self._tag_name, stream=True)
for status_str in resp:
status = json.loads(status_str)
logger.debug('Status: %s', status_str)
if u'status' in status:
status_msg = status[u'status']
if status_msg == 'Pushing':
if u'progressDetail' in status and u'id' in status:
image_id = status[u'id']
detail = status[u'progressDetail']
if u'current' in detail and 'total' in detail:
with self._status as status:
images = status['image_completion']
images[image_id] = detail
status['push_completion'] = \
DockerfileBuildContext.__total_completion(images, num_images)
elif u'errorDetail' in status:
message = 'Error pushing image.'
if u'message' in status[u'errorDetail']:
message = str(status[u'errorDetail'][u'message'])
raise RuntimeError(message)
def __cleanup(self):
# First clean up any containers that might be holding the images
for running in self._cl.containers(quiet=True):
logger.debug('Killing container: %s' % running['Id'])
self._cl.kill(running['Id'])
# Next, remove all of the containers (which should all now be killed)
for container in self._cl.containers(all=True, quiet=True):
logger.debug('Removing container: %s' % container['Id'])
self._cl.remove_container(container['Id'])
# Iterate all of the images and remove the ones that the public registry
# doesn't know about, this should preserve base images.
images_to_remove = set()
repos = set()
for image in self._cl.images():
images_to_remove.add(image['Id'])
for tag in image['RepoTags']:
tag_repo = tag.split(':')[0]
if tag_repo != '<none>':
repos.add(tag_repo)
for repo in repos:
repo_url = 'https://index.docker.io/v1/repositories/%s/images' % repo
repo_info = requests.get(repo_url)
if repo_info.status_code / 100 == 2:
for repo_image in repo_info.json():
if repo_image['id'] in images_to_remove:
logger.debug('Image was deemed public: %s' % repo_image['id'])
images_to_remove.remove(repo_image['id'])
for to_remove in images_to_remove:
logger.debug('Removing private image: %s' % to_remove)
try:
self._cl.remove_image(to_remove)
except APIError:
# Sometimes an upstream image removed this one
pass
# Verify that our images were actually removed
for image in self._cl.images():
if image['Id'] in images_to_remove:
raise RuntimeError('Image was not removed: %s' % image['Id'])
class DockerfileBuildWorker(Worker):
def __init__(self, *vargs, **kwargs):
super(DockerfileBuildWorker, self).__init__(*vargs, **kwargs)
self._mime_processors = {
'application/zip': DockerfileBuildWorker.__prepare_zip,
'text/plain': DockerfileBuildWorker.__prepare_dockerfile,
'application/octet-stream': DockerfileBuildWorker.__prepare_dockerfile,
}
@staticmethod
def __prepare_zip(request_file):
build_dir = mkdtemp(prefix='docker-build-')
# Save the zip file to temp somewhere
with TemporaryFile() as zip_file:
zip_file.write(request_file.content)
to_extract = ZipFile(zip_file)
to_extract.extractall(build_dir)
return build_dir
@staticmethod
def __prepare_dockerfile(request_file):
build_dir = mkdtemp(prefix='docker-build-')
dockerfile_path = os.path.join(build_dir, "Dockerfile")
with open(dockerfile_path, 'w') as dockerfile:
dockerfile.write(request_file.content)
return build_dir
def process_queue_item(self, job_details):
repository_build = model.get_repository_build(job_details['namespace'],
job_details['repository'],
job_details['build_uuid'])
resource_url = user_files.get_file_url(repository_build.resource_key)
tag_name = repository_build.tag
access_token = repository_build.access_token.code
# Start the build server
start_cmd = BUILD_SERVER_CMD % (resource_url, repository_build.tag,
repository_build.access_token.code)
logger.debug('Sending build server request with command: %s' % start_cmd)
ssh_client.exec_command(start_cmd)
log_appender = partial(build_logs.append_log_message,
repository_build.uuid)
status_endpoint = 'http://%s:5002/build/' % droplet.ip_address
# wait for the server to be ready
logger.debug('Waiting for buildserver to be ready')
retry_command(requests.get, [status_endpoint])
log_appender('initializing', build_logs.PHASE)
# wait for the job to be complete
start_msg = ('Starting job with resource url: %s tag: %s' % (resource_url,
tag_name))
logger.debug(start_msg)
log_appender(start_msg)
docker_resource = requests.get(resource_url)
c_type = docker_resource.headers['content-type']
filetype_msg = ('Request to build file of type: %s with tag: %s' %
(c_type, tag_name))
logger.info(filetype_msg)
log_appender(filetype_msg)
if c_type not in self._mime_processors:
raise RuntimeError('Invalid dockerfile content type: %s' % c_type)
build_dir = self._mime_processors[c_type](docker_resource)
log_appender('building', build_logs.PHASE)
repository_build.phase = 'building'
repository_build.status_url = status_endpoint
repository_build.save()
logger.debug('Waiting for job to be complete')
status = get_status(status_endpoint)
while status != 'error' and status != 'complete':
logger.debug('Job status is: %s' % status)
time.sleep(5)
status = get_status(status_endpoint)
with DockerfileBuildContext(build_dir, tag_name, access_token,
repository_build.uuid) as build_ctxt:
try:
built_image = build_ctxt.build()
logger.debug('Job complete with status: %s' % status)
if status == 'error':
error_message = requests.get(status_endpoint).json()['message']
logger.warning('Job error: %s' % error_message)
repository_build.phase = 'error'
else:
repository_build.phase = 'complete'
if not built_image:
log_appender('error', build_logs.PHASE)
repository_build.phase = 'error'
repository_build.save()
log_appender('Unable to build dockerfile.', build_logs.ERROR)
return False
# clean up the DO node
logger.debug('Cleaning up DO node.')
retry_command(droplet.destroy)
log_appender('pushing', build_logs.PHASE)
repository_build.phase = 'pushing'
repository_build.save()
repository_build.status_url = None
repository_build.build_node_id = None
repository_build.save()
build_ctxt.push(built_image)
log_appender('complete', build_logs.PHASE)
repository_build.phase = 'complete'
repository_build.save()
except Exception as exc:
log_appender('error', build_logs.PHASE)
logger.exception('Exception when processing request.')
repository_build.phase = 'error'
repository_build.save()
log_appender(str(exc), build_logs.ERROR)
return False
return True
except Exception as outer_ex:
# We don't really know what these are, but they are probably retryable
logger.exception('Exception processing job: %s' % outer_ex.message)
return False
finally:
if not db_connection.is_closed():
logger.debug('Closing thread db connection.')
db_connection.close()
def process_work_items(pool):
logger.debug('Getting work item from queue.')
item = dockerfile_build_queue.get(processing_time=60*60) # allow 1 hr
while item:
logger.debug('Queue gave us some work: %s' % item.body)
request = json.loads(item.body)
def build_callback(item):
local_item = item
def complete_callback(completed):
if completed:
logger.debug('Queue item completed successfully, will be removed.')
dockerfile_build_queue.complete(local_item)
else:
# We have a retryable error, add the job back to the queue
logger.debug('Queue item incomplete, will be retryed.')
dockerfile_build_queue.incomplete(local_item)
return complete_callback
logger.debug('Sending work item to thread pool: %s' % pool)
pool.apply_async(babysit_builder, [request],
callback=build_callback(item))
item = dockerfile_build_queue.get()
logger.debug('No more work.')
if not db_connection.is_closed():
logger.debug('Closing thread db connection.')
db_connection.close()
def start_worker():
pool = ThreadPool(3)
logger.debug('Scheduling worker.')
sched = Scheduler()
sched.start()
sched.add_interval_job(process_work_items, args=[pool], seconds=30)
while True:
time.sleep(60 * 60 * 24) # sleep one day, basically forever
desc = 'Worker daemon to monitor dockerfile build'
parser = argparse.ArgumentParser(description=desc)
@ -264,16 +330,17 @@ parser.add_argument('--log', default='dockerfilebuild.log',
args = parser.parse_args()
worker = DockerfileBuildWorker(dockerfile_build_queue)
if args.D:
handler = logging.FileHandler(args.log)
handler.setFormatter(formatter)
root_logger.addHandler(handler)
with daemon.DaemonContext(files_preserve=[handler.stream],
working_directory=os.getcwd()):
start_worker()
with daemon.DaemonContext(files_preserve=[handler.stream]):
worker.start()
else:
handler = logging.StreamHandler()
handler.setFormatter(formatter)
root_logger.addHandler(handler)
start_worker()
worker.start()