From cd28ae6b1aecb974713c841667bb0b00acfbdf39 Mon Sep 17 00:00:00 2001 From: yackob03 Date: Thu, 24 Oct 2013 00:05:58 -0400 Subject: [PATCH 01/57] First stab at a buildserver that will puppet docker to build dockerfiles remotely. --- buildserver/Readme.md | 1 + buildserver/buildserver.py | 115 +++++++++++++++++++++++++++++ buildserver/requirements-nover.txt | 2 + buildserver/test.html | 8 ++ 4 files changed, 126 insertions(+) create mode 100644 buildserver/Readme.md create mode 100644 buildserver/buildserver.py create mode 100644 buildserver/requirements-nover.txt create mode 100644 buildserver/test.html diff --git a/buildserver/Readme.md b/buildserver/Readme.md new file mode 100644 index 000000000..8cc88d12d --- /dev/null +++ b/buildserver/Readme.md @@ -0,0 +1 @@ +This must be run on Python version 2.7.4 or later (Ubuntu 13.04) for security reasons. \ No newline at end of file diff --git a/buildserver/buildserver.py b/buildserver/buildserver.py new file mode 100644 index 000000000..f9eba8416 --- /dev/null +++ b/buildserver/buildserver.py @@ -0,0 +1,115 @@ +import docker +import logging +import shutil +import os +import re + +from flask import Flask, request, send_file, make_response, jsonify +from zipfile import ZipFile +from tempfile import TemporaryFile, mkdtemp + + +BUFFER_SIZE = 8 * 1024 +LOG_FORMAT = '%(asctime)-15s - %(levelname)s - %(pathname)s - ' + \ + '%(funcName)s - %(message)s' + +app = Flask(__name__) +logger = logging.getLogger(__name__) + + +def count_steps(dockerfileobj): + steps = 0 + for line in dockerfileobj.readlines(): + stripped = line.strip() + if stripped and stripped[0] is not '#': + steps += 1 + return steps + +@app.route('/') +def index(): + return send_file('test.html') + + +@app.route('/start/zip/', methods=['POST']) +def start_build(tag_name): + docker_zip = request.files['dockerfile'] + + build_dir = mkdtemp(prefix='docker-build-') + + # Save the zip file to temp somewhere + with TemporaryFile() as zip_file: + docker_zip.save(zip_file) + to_extract = ZipFile(zip_file) + to_extract.extractall(build_dir) + + docker_cl = docker.Client(version='1.5') + + build_status = docker_cl.build(path=build_dir, tag=tag_name) + + dockerfile_path = os.path.join(build_dir, "Dockerfile") + with open(dockerfile_path, 'r') as dockerfileobj: + num_steps = count_steps(dockerfileobj) + logger.debug('Dockerfile had %s steps' % num_steps) + + current_step = 0 + built_image = None + for status in build_status: + step_increment = re.search(r'Step ([0-9]+) :', status) + if step_increment: + current_step = int(step_increment.group(1)) + logger.debug('Step now: %s/%s' % (current_step, num_steps)) + continue + + complete = re.match(r'Successfully built ([a-z0-9]+)', status) + if complete: + built_image = complete.group(1) + logger.debug('Final image ID is: %s' % built_image) + continue + + shutil.rmtree(build_dir) + + # Get the image count + if not built_image: + abort(500) + + history = docker_cl.history(built_image) + num_images = len(history) + + logger.debug('Pushing to tag name: %s' % tag_name) + resp = docker_cl.push(tag_name) + + current_image = 0 + image_progress = 0 + for status in resp: + if u'status' in status: + status_msg = status[u'status'] + + next_image = r'(Pushing:|Image) [a-z0-9]+( already pushed, skipping)?' + match = re.match(next_image, status_msg) + if match: + current_image += 1 + image_progress = 0 + logger.debug('Now pushing image %s/%s' % (current_image, num_images)) + continue + + if status_msg == u'Pushing' and u'progress' in status: + percent = r'\(([0-9]+)%\)' + match = re.search(percent, status[u'progress']) + if match: + image_progress = int(match.group(1)) + continue + + return jsonify({ + 'images_pushed': num_images, + 'commands_run': num_steps, + }) + + +@app.route('/status') +def get_status(): + return 'building' + + +if __name__ == '__main__': + logging.basicConfig(level=logging.DEBUG, format=LOG_FORMAT) + app.run(host='0.0.0.0', port=5002, debug=True) \ No newline at end of file diff --git a/buildserver/requirements-nover.txt b/buildserver/requirements-nover.txt new file mode 100644 index 000000000..7a7ebd121 --- /dev/null +++ b/buildserver/requirements-nover.txt @@ -0,0 +1,2 @@ +flask +-e git+git://github.com/dotcloud/docker-py.git#egg=docker-py \ No newline at end of file diff --git a/buildserver/test.html b/buildserver/test.html new file mode 100644 index 000000000..472f4dd6d --- /dev/null +++ b/buildserver/test.html @@ -0,0 +1,8 @@ + + +
+ + +
+ + \ No newline at end of file From 5d5d027cc0610e6cff0a10461dd30e7cab337dc4 Mon Sep 17 00:00:00 2001 From: yackob03 Date: Thu, 24 Oct 2013 01:29:34 -0400 Subject: [PATCH 02/57] Move the source dockerfile processing to a modular system. Move actual build to a thread. Wire up the status API. Fix a bug with the way current_image was computed. Add untested support for raw dockerfile upload. --- buildserver/buildserver.py | 202 ++++++++++++++++++++--------- buildserver/requirements-nover.txt | 2 +- buildserver/test.html | 3 +- 3 files changed, 143 insertions(+), 64 deletions(-) diff --git a/buildserver/buildserver.py b/buildserver/buildserver.py index f9eba8416..6ebdd5c9b 100644 --- a/buildserver/buildserver.py +++ b/buildserver/buildserver.py @@ -4,9 +4,11 @@ import shutil import os import re -from flask import Flask, request, send_file, make_response, jsonify +from flask import Flask, request, send_file, jsonify, redirect, url_for, abort from zipfile import ZipFile from tempfile import TemporaryFile, mkdtemp +from uuid import uuid4 +from multiprocessing.pool import ThreadPool BUFFER_SIZE = 8 * 1024 @@ -17,97 +19,173 @@ app = Flask(__name__) logger = logging.getLogger(__name__) -def count_steps(dockerfileobj): - steps = 0 - for line in dockerfileobj.readlines(): - stripped = line.strip() - if stripped and stripped[0] is not '#': - steps += 1 - return steps - @app.route('/') def index(): return send_file('test.html') -@app.route('/start/zip/', methods=['POST']) -def start_build(tag_name): - docker_zip = request.files['dockerfile'] +def count_steps(dockerfile_path): + with open(dockerfile_path, 'r') as dockerfileobj: + steps = 0 + for line in dockerfileobj.readlines(): + stripped = line.strip() + if stripped and stripped[0] is not '#': + steps += 1 + return steps + +def prepare_zip(request_file): build_dir = mkdtemp(prefix='docker-build-') # Save the zip file to temp somewhere with TemporaryFile() as zip_file: - docker_zip.save(zip_file) + request_file.save(zip_file) to_extract = ZipFile(zip_file) to_extract.extractall(build_dir) - docker_cl = docker.Client(version='1.5') + return build_dir - build_status = docker_cl.build(path=build_dir, tag=tag_name) +def prepare_dockerfile(request_file): + build_dir = mkdtemp(prefix='docker-build-') dockerfile_path = os.path.join(build_dir, "Dockerfile") - with open(dockerfile_path, 'r') as dockerfileobj: - num_steps = count_steps(dockerfileobj) - logger.debug('Dockerfile had %s steps' % num_steps) + request_file.save(dockerfile_path) - current_step = 0 - built_image = None - for status in build_status: - step_increment = re.search(r'Step ([0-9]+) :', status) - if step_increment: - current_step = int(step_increment.group(1)) - logger.debug('Step now: %s/%s' % (current_step, num_steps)) - continue + return build_dir - complete = re.match(r'Successfully built ([a-z0-9]+)', status) - if complete: - built_image = complete.group(1) - logger.debug('Final image ID is: %s' % built_image) - continue - shutil.rmtree(build_dir) +MIME_PROCESSORS = { + 'application/zip': prepare_zip, + 'text/plain': prepare_dockerfile, +} - # Get the image count - if not built_image: - abort(500) - history = docker_cl.history(built_image) - num_images = len(history) +builds = {} +pool = ThreadPool(1) - logger.debug('Pushing to tag name: %s' % tag_name) - resp = docker_cl.push(tag_name) - current_image = 0 - image_progress = 0 - for status in resp: - if u'status' in status: - status_msg = status[u'status'] +def build_image(build_dir, tag_name, num_steps, result_object): + try: + logger.debug('Does this show up?') + docker_cl = docker.Client(version='1.5') + result_object['status'] = 'building' + build_status = docker_cl.build(path=build_dir, tag=tag_name) - next_image = r'(Pushing:|Image) [a-z0-9]+( already pushed, skipping)?' - match = re.match(next_image, status_msg) - if match: - current_image += 1 - image_progress = 0 - logger.debug('Now pushing image %s/%s' % (current_image, num_images)) + current_step = 0 + built_image = None + for status in build_status: + step_increment = re.search(r'Step ([0-9]+) :', status) + if step_increment: + current_step = int(step_increment.group(1)) + logger.debug('Step now: %s/%s' % (current_step, num_steps)) + result_object['current_command'] = current_step continue - if status_msg == u'Pushing' and u'progress' in status: - percent = r'\(([0-9]+)%\)' - match = re.search(percent, status[u'progress']) + complete = re.match(r'Successfully built ([a-z0-9]+)$', status) + if complete: + built_image = complete.group(1) + logger.debug('Final image ID is: %s' % built_image) + continue + + shutil.rmtree(build_dir) + + # Get the image count + if not built_image: + result_object['status'] = 'error' + result_object['message'] = 'Unable to build dockerfile.' + return + + history = docker_cl.history(built_image) + num_images = len(history) + result_object['total_images'] = num_images + + result_object['status'] = 'pushing' + logger.debug('Pushing to tag name: %s' % tag_name) + resp = docker_cl.push(tag_name) + + current_image = 0 + image_progress = 0 + for status in resp: + logger.debug(status) + + if u'status' in status: + status_msg = status[u'status'] + + next_image = r'(Pushing|Image) [a-z0-9]+( already pushed, skipping)?$' + match = re.match(next_image, status_msg) if match: - image_progress = int(match.group(1)) - continue + current_image += 1 + image_progress = 0 + logger.debug('Now pushing image %s/%s' % + (current_image, num_images)) - return jsonify({ - 'images_pushed': num_images, - 'commands_run': num_steps, - }) + elif status_msg == u'Pushing' and u'progress' in status: + percent = r'\(([0-9]+)%\)' + match = re.search(percent, status[u'progress']) + if match: + image_progress = int(match.group(1)) + + result_object['current_image'] = current_image + result_object['image_completion_percent'] = image_progress + + elif u'errorDetail' in status: + result_object['status'] = 'error' + if u'message' in status[u'errorDetail']: + result_object['message'] = status[u'errorDetail'][u'message'] + return + + result_object['status'] = 'complete' + except Exception as e: + logger.exception('Exception when processing request.') + result_object['status'] = 'error' + result_object['message'] = e.message -@app.route('/status') -def get_status(): - return 'building' +@app.route('/build', methods=['POST']) +def start_build(): + docker_input = request.files['dockerfile'] + c_type = docker_input.content_type + tag_name = request.values['tag'] + + logger.info('Request to build file of type: %s with tag: %s' % + (c_type, tag_name)) + + if c_type not in MIME_PROCESSORS: + logger.error('Invalid dockerfile content type: %s' % c_type) + abort(400) + + build_dir = MIME_PROCESSORS[c_type](docker_input) + + dockerfile_path = os.path.join(build_dir, "Dockerfile") + num_steps = count_steps(dockerfile_path) + logger.debug('Dockerfile had %s steps' % num_steps) + + job_id = str(uuid4()) + logger.info('Sending job to builder pool: %s' % job_id) + + result_object = { + 'id': job_id, + 'total_commands': num_steps, + 'total_images': None, + 'current_command': 0, + 'current_image': 0, + 'image_completion_percent': 0, + 'status': 'waiting', + 'message': None, + } + builds[job_id] = result_object + pool.apply_async(build_image, [build_dir, tag_name, num_steps, + result_object]) + + return redirect(url_for('get_status', job_id=job_id)) + + +@app.route('/status/') +def get_status(job_id): + if job_id not in builds: + abort(400) + + return jsonify(builds[job_id]) if __name__ == '__main__': diff --git a/buildserver/requirements-nover.txt b/buildserver/requirements-nover.txt index 7a7ebd121..e5ec39f3c 100644 --- a/buildserver/requirements-nover.txt +++ b/buildserver/requirements-nover.txt @@ -1,2 +1,2 @@ flask --e git+git://github.com/dotcloud/docker-py.git#egg=docker-py \ No newline at end of file +-e git+git://github.com/DevTable/docker-py.git#egg=docker-py diff --git a/buildserver/test.html b/buildserver/test.html index 472f4dd6d..0137b766b 100644 --- a/buildserver/test.html +++ b/buildserver/test.html @@ -1,7 +1,8 @@ -
+ +
From 9dc9e0c940316161b218a532e704125246127d50 Mon Sep 17 00:00:00 2001 From: yackob03 Date: Thu, 24 Oct 2013 14:57:05 -0400 Subject: [PATCH 03/57] Add a docker-in-docker container that will run the build server. --- buildserver/Dockerfile | 27 +++++++++++++++++++++ buildserver/Readme.md | 14 ++++++++++- buildserver/buildserver.py | 35 +++++++++++++++------------ buildserver/startserver | 48 ++++++++++++++++++++++++++++++++++++++ buildserver/test.html | 2 +- 5 files changed, 109 insertions(+), 17 deletions(-) create mode 100644 buildserver/Dockerfile create mode 100644 buildserver/startserver diff --git a/buildserver/Dockerfile b/buildserver/Dockerfile new file mode 100644 index 000000000..c58e9f0ad --- /dev/null +++ b/buildserver/Dockerfile @@ -0,0 +1,27 @@ +FROM lopter/raring-base +MAINTAINER jake@devtable.com + +RUN echo deb http://archive.ubuntu.com/ubuntu precise universe > /etc/apt/sources.list.d/universe.list +RUN apt-get update -qq +RUN apt-get install -qqy iptables ca-certificates lxc python-virtualenv git python-dev + +# This will use the latest public release. To use your own, comment it out... +ADD https://get.docker.io/builds/Linux/x86_64/docker-latest /usr/local/bin/docker +# ...then uncomment the following line, and copy your docker binary to current dir. +#ADD ./docker /usr/local/bin/docker + +# Install the files +ADD ./startserver /usr/local/bin/startserver +ADD ./buildserver.py ./buildserver.py +ADD ./test.html ./test.html +ADD ./requirements-nover.txt ./requirements-nover.txt + +RUN chmod +x /usr/local/bin/docker /usr/local/bin/startserver + +RUN virtualenv --distribute venv +RUN venv/bin/pip install -r requirements-nover.txt + +VOLUME /var/lib/docker + +EXPOSE 5002:5002 +CMD startserver \ No newline at end of file diff --git a/buildserver/Readme.md b/buildserver/Readme.md index 8cc88d12d..6b36df9ee 100644 --- a/buildserver/Readme.md +++ b/buildserver/Readme.md @@ -1 +1,13 @@ -This must be run on Python version 2.7.4 or later (Ubuntu 13.04) for security reasons. \ No newline at end of file +To build: + +``` +sudo docker build -t quay.io/quay/buildserver . +sudo docker push quay.io/quay/buildserver +``` + +To run: + +``` +sudo docker pull quay.io/quay/buildserver +sudo docker run -d -privileged -lxc-conf="aa_profile=unconfined" quay.io/quay/buildserver +``` \ No newline at end of file diff --git a/buildserver/buildserver.py b/buildserver/buildserver.py index 6ebdd5c9b..94bf2dbe2 100644 --- a/buildserver/buildserver.py +++ b/buildserver/buildserver.py @@ -54,16 +54,6 @@ def prepare_dockerfile(request_file): return build_dir -MIME_PROCESSORS = { - 'application/zip': prepare_zip, - 'text/plain': prepare_dockerfile, -} - - -builds = {} -pool = ThreadPool(1) - - def build_image(build_dir, tag_name, num_steps, result_object): try: logger.debug('Does this show up?') @@ -106,11 +96,8 @@ def build_image(build_dir, tag_name, num_steps, result_object): current_image = 0 image_progress = 0 for status in resp: - logger.debug(status) - if u'status' in status: status_msg = status[u'status'] - next_image = r'(Pushing|Image) [a-z0-9]+( already pushed, skipping)?$' match = re.match(next_image, status_msg) if match: @@ -141,7 +128,18 @@ def build_image(build_dir, tag_name, num_steps, result_object): result_object['message'] = e.message -@app.route('/build', methods=['POST']) +MIME_PROCESSORS = { + 'application/zip': prepare_zip, + 'text/plain': prepare_dockerfile, + 'application/octet-stream': prepare_dockerfile, +} + + +builds = {} +pool = ThreadPool(1) + + +@app.route('/build/', methods=['POST']) def start_build(): docker_input = request.files['dockerfile'] c_type = docker_input.content_type @@ -180,7 +178,7 @@ def start_build(): return redirect(url_for('get_status', job_id=job_id)) -@app.route('/status/') +@app.route('/build/') def get_status(job_id): if job_id not in builds: abort(400) @@ -188,6 +186,13 @@ def get_status(job_id): return jsonify(builds[job_id]) +@app.route('/build/', methods=['GET']) +def get_all_status(): + return jsonify({ + 'builds': builds, + }) + + if __name__ == '__main__': logging.basicConfig(level=logging.DEBUG, format=LOG_FORMAT) app.run(host='0.0.0.0', port=5002, debug=True) \ No newline at end of file diff --git a/buildserver/startserver b/buildserver/startserver new file mode 100644 index 000000000..4947a618a --- /dev/null +++ b/buildserver/startserver @@ -0,0 +1,48 @@ +#!/bin/bash + +# First, make sure that cgroups are mounted correctly. +CGROUP=/sys/fs/cgroup + +[ -d $CGROUP ] || + mkdir $CGROUP + +mountpoint -q $CGROUP || + mount -n -t tmpfs -o uid=0,gid=0,mode=0755 cgroup $CGROUP || { + echo "Could not make a tmpfs mount. Did you use -privileged?" + exit 1 + } + +# Mount the cgroup hierarchies exactly as they are in the parent system. +for SUBSYS in $(cut -d: -f2 /proc/1/cgroup) +do + [ -d $CGROUP/$SUBSYS ] || mkdir $CGROUP/$SUBSYS + mountpoint -q $CGROUP/$SUBSYS || + mount -n -t cgroup -o $SUBSYS cgroup $CGROUP/$SUBSYS +done + +# Note: as I write those lines, the LXC userland tools cannot setup +# a "sub-container" properly if the "devices" cgroup is not in its +# own hierarchy. Let's detect this and issue a warning. +grep -q :devices: /proc/1/cgroup || + echo "WARNING: the 'devices' cgroup should be in its own hierarchy." +grep -qw devices /proc/1/cgroup || + echo "WARNING: it looks like the 'devices' cgroup is not mounted." + +# Now, close extraneous file descriptors. +pushd /proc/self/fd +for FD in * +do + case "$FD" in + # Keep stdin/stdout/stderr + [012]) + ;; + # Nuke everything else + *) + eval exec "$FD>&-" + ;; + esac +done +popd + +docker -d & +exec venv/bin/python buildserver.py 2> buildserver.log \ No newline at end of file diff --git a/buildserver/test.html b/buildserver/test.html index 0137b766b..9a08eb994 100644 --- a/buildserver/test.html +++ b/buildserver/test.html @@ -1,6 +1,6 @@ -
+ From 847e91b696956be2a593d5ef9ec9810672a4505f Mon Sep 17 00:00:00 2001 From: yackob03 Date: Thu, 24 Oct 2013 16:37:03 -0400 Subject: [PATCH 04/57] Add the first part of the build worker that runs on quay. --- buildserver/buildserver.py | 2 +- certs/digital_ocean | 27 ++++++++ certs/digital_ocean.pub | 1 + config.py | 13 +++- data/database.py | 9 ++- data/model.py | 12 ++++ data/queue.py | 1 + test/data/test.db | Bin 75776 -> 77824 bytes workers/diffsworker.py | 6 -- workers/dockerfilebuild.py | 136 +++++++++++++++++++++++++++++++++++++ 10 files changed, 196 insertions(+), 11 deletions(-) create mode 100644 certs/digital_ocean create mode 100644 certs/digital_ocean.pub create mode 100644 workers/dockerfilebuild.py diff --git a/buildserver/buildserver.py b/buildserver/buildserver.py index 94bf2dbe2..b6de17da5 100644 --- a/buildserver/buildserver.py +++ b/buildserver/buildserver.py @@ -195,4 +195,4 @@ def get_all_status(): if __name__ == '__main__': logging.basicConfig(level=logging.DEBUG, format=LOG_FORMAT) - app.run(host='0.0.0.0', port=5002, debug=True) \ No newline at end of file + app.run(host='0.0.0.0', port=5002, debug=True) diff --git a/certs/digital_ocean b/certs/digital_ocean new file mode 100644 index 000000000..700f2b586 --- /dev/null +++ b/certs/digital_ocean @@ -0,0 +1,27 @@ +-----BEGIN RSA PRIVATE KEY----- +MIIEowIBAAKCAQEAwjlIK0HodmDNrZAmaALtr9RLriRSeeLh76gV8KHmjRweeT7v +dmhKeGP1nOAs17caZkcwsW0tiDbCeIv2MisV405sScjPOxFivWpY8tL72sgVuOAl +ReZauOGZ4M1ZcSa/YbT7tnFCIayYE9pde4ih5LmYZqKsBsaNq3ErcMnAzqG77D95 +8swuVwhz/INioBWwe4FjO76/0DqS357hT5yHDWthJD6UUH12VajPKBtXEvGNUtNL +vdq+drm9omt2y0seMn47fZXiNIulLv7ojsWKwtRMTsGcjnv6VMZAVAuX11u4cJd+ +oPTbDl0D+02B7XYcxABqdMZcOc1/7VTUlFFd4wIDAQABAoIBAAs4V+z3z8AW84rV +SwKzOJvxvbV/r6wO6VJ4+Vt/XtxEBZanhhnnCHZP//5iDPUhRMsnza5SSlEWKMHi +BAT97DPHcgYJLb+Rz4x1ulG80oPfDzIw8LZLCm6nycXs1v/sZx3z4J63iER9vgNX +mBLs371g42b6esmhasm+re3EGflV0LeY1IX0MY40pqGndmW8Fly1QH179TrMzVUJ +btu3i2JrwWmKk5zO5YGm0SYY5QQGCdjPj6SL+idDniAefEvbjJYz2qOaPOF3wj/7 +r8dAnmyaP10Q3JojT01Et5ltMfr0oF2/pic9tWYGrgn/aIuoXUXj0SF3Pfgrb/4L +Et1kzFECgYEA8Tb/9bYzQgtaQTQfzFU/KnsIKKnrxh73rZwnIxG59WvN0Ws41Byf +rv8fEbXWU8Yj0drxRSud9fADr99lZGWFxle8rSW5+qqoUxG8n/fkktzHxyPE/9Mh +pZW7un7a5/glKgUpHpjaOCZj9rhdF1AwdUXLSo1sFc7VBsKvKiKJAT0CgYEAziDt +A9h5lOgiLGf1xdBq3qmLIlARz7fivAcZ5acSGN5k6MFFxjHNqhcXRusqs7g+hvCN +eRupdwfgSdLwrTfvxuY4pCcddfYIZO3uUZYs/glvYRtIxaP2kMBkZTs9KzI02Bjv +zT3NPReR/46SqW0zvYTlRFSY7VZ0eRED/5xnjZ8CgYAZdlrSjyceA6DFXUE2CpGe +ZFpaIIW45i/y7ZbcBtUAaR7SymS3T0Yz7M5UykMTmMjTMC9jw9Tqzyk0eXp0fJsA +cuaByIe3RCh8jFTC9iH0tsWH6eizsI/OsN2eNCHbdsBFjUHn7u6qGrNWqeN5wIc8 ++d8ZwY/1RV4LVqWy5u5baQKBgHLFvJMWluQFuPl2zU9etBLU3ma1pKU/I11EqvPH +afk044UCEKLBml1pzAkt6jH1lcM2798OOvbPCOCyNlaMvdLG36TvLqU+3/+qx7bf +4p90i3LLaWK64BBLP9tp9640n13vzJ5AGiY5GI7uSNVTu6p789hvLlOAfwvmII7T +/IjLAoGBAO6iU8i6pAOaKa7+/uExXx6xwk3vqQtovxByo1/m7NpyUtT+ElDSq+t9 +7f+3TzzPB6ggdMl8d+PSyHR3o7KjVPgOSe7zld7eePhUrLjwZ4lh5ohcvhvYfaRL +0EgRTaTb+zLtCAvJS/ilNnJoIcxUmD8u5uSXpY7vAleSOiQTJRTh +-----END RSA PRIVATE KEY----- diff --git a/certs/digital_ocean.pub b/certs/digital_ocean.pub new file mode 100644 index 000000000..95db83601 --- /dev/null +++ b/certs/digital_ocean.pub @@ -0,0 +1 @@ +ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDCOUgrQeh2YM2tkCZoAu2v1EuuJFJ54uHvqBXwoeaNHB55Pu92aEp4Y/Wc4CzXtxpmRzCxbS2INsJ4i/YyKxXjTmxJyM87EWK9aljy0vvayBW44CVF5lq44ZngzVlxJr9htPu2cUIhrJgT2l17iKHkuZhmoqwGxo2rcStwycDOobvsP3nyzC5XCHP8g2KgFbB7gWM7vr/QOpLfnuFPnIcNa2EkPpRQfXZVqM8oG1cS8Y1S00u92r52ub2ia3bLSx4yfjt9leI0i6Uu/uiOxYrC1ExOwZyOe/pUxkBUC5fXW7hwl36g9NsOXQP7TYHtdhzEAGp0xlw5zX/tVNSUUV3j jake@coreserver diff --git a/config.py b/config.py index 3ae57769a..71297a00b 100644 --- a/config.py +++ b/config.py @@ -85,8 +85,15 @@ class GitHubProdConfig(GitHubTestConfig): GITHUB_CLIENT_SECRET = 'f89d8bb28ea3bd4e1c68808500d185a816be53b1' +class DigitalOceanConfig(): + DO_CLIENT_ID = 'LJ44y2wwYj1MD0BRxS6qHA' + DO_CLIENT_SECRET = 'b9357a6f6ff45a33bb03f6dbbad135f9' + DO_SSH_KEY_ID = '46986' + + class DebugConfig(FlaskConfig, MailConfig, LocalStorage, SQLiteDB, - StripeTestConfig, MixpanelTestConfig, GitHubTestConfig): + StripeTestConfig, MixpanelTestConfig, GitHubTestConfig, + DigitalOceanConfig): REGISTRY_SERVER = 'localhost:5000' LOGGING_CONFIG = { 'level': logging.DEBUG, @@ -98,7 +105,7 @@ class DebugConfig(FlaskConfig, MailConfig, LocalStorage, SQLiteDB, class LocalHostedConfig(FlaskConfig, MailConfig, S3Storage, RDSMySQL, StripeLiveConfig, MixpanelTestConfig, - GitHubProdConfig): + GitHubProdConfig, DigitalOceanConfig): REGISTRY_SERVER = 'localhost:5000' LOGGING_CONFIG = { 'level': logging.DEBUG, @@ -109,7 +116,7 @@ class LocalHostedConfig(FlaskConfig, MailConfig, S3Storage, RDSMySQL, class ProductionConfig(FlaskConfig, MailConfig, S3Storage, RDSMySQL, StripeLiveConfig, MixpanelProdConfig, - GitHubProdConfig): + GitHubProdConfig, DigitalOceanConfig): REGISTRY_SERVER = 'quay.io' LOGGING_CONFIG = { 'stream': sys.stderr, diff --git a/data/database.py b/data/database.py index 0f3aef888..38bc2d2b7 100644 --- a/data/database.py +++ b/data/database.py @@ -150,6 +150,13 @@ class RepositoryTag(BaseModel): ) +class RepositoryBuild(BaseModel): + digitalocean_build_node_id = IntegerField(null=True) + phase = CharField(default='waiting') + status_url = CharField(null=True) + repository = ForeignKeyField(Repository) + + class QueueItem(BaseModel): queue_name = CharField(index=True) body = TextField() @@ -162,7 +169,7 @@ def initialize_db(): create_model_tables([User, Repository, Image, AccessToken, Role, RepositoryPermission, Visibility, RepositoryTag, EmailConfirmation, FederatedLogin, LoginService, - QueueItem]) + QueueItem, RepositoryBuild]) Role.create(name='admin') Role.create(name='write') Role.create(name='read') diff --git a/data/model.py b/data/model.py index 15827cca4..182ee2a21 100644 --- a/data/model.py +++ b/data/model.py @@ -30,6 +30,10 @@ class InvalidTokenException(DataModelException): pass +class InvalidRepositoryBuildException(DataModelException): + pass + + def create_user(username, password, email): if not validate_email(email): raise InvalidEmailAddressException('Invalid email address: %s' % email) @@ -548,3 +552,11 @@ def load_token_data(code): return fetched[0] else: raise InvalidTokenException('Invalid delegate token code: %s' % code) + + +def get_repository_build(request_dbid): + try: + return RepositoryBuild.get(RepositoryBuild == request_dbid) + except RepositoryBuild.DoesNotExist: + msg = 'Unable to locate a build by id: %s' % request_dbid + raise InvalidRepositoryBuildException(msg) diff --git a/data/queue.py b/data/queue.py index 0e1383aa2..3038b8389 100644 --- a/data/queue.py +++ b/data/queue.py @@ -56,3 +56,4 @@ class WorkQueue(object): image_diff_queue = WorkQueue('imagediff') +dockerfile_build_queue = WorkQueue('dockerfilebuild') diff --git a/test/data/test.db b/test/data/test.db index 96e878b95e9124c1618e41e662406d431a5af964..83688fab4d7c9acbd3b2a6402aa7b0b0cc274e8b 100644 GIT binary patch delta 925 zcmZ9KT})D87{||Z&Ou!w&1QqL@<@JwfXG*B^dS&Jf>|dJy>NzdG!n%SN~|rhF1oBe z7l~PGdC|v}aB6c|&CRxMnsm{+s;#=MRX5dIt@XmHHQ(+1?fKZA_y4@lGyR;Me#2g> zEZ?Gq3XrQ)c`m1NGuNCaSkg#t$EF42~F`y#sc8;AqR~ zn9(;F+}ZQ-@CZ^m>0ZfH|uIubZ}5Jl#E0!s!9`Xs6+X zNobw)@Vu`#;TuX?Tus5Qv6eyq$O(Qb86OHXb+`IM{-YzEpGVJhwj)|RV>$N;JiN`9Eb=pVLUn!jz?q3fkZeGTIbd}c&E+h=lCnow6 zv51oEw($NE)CcaJYs4Q4YPW7MkyfpPpLp8pfF@p+A&wpW3G9|vBt;RF6 zmH0Sai_5cm*hwOLpdnum+tOC3!Lw;QX#V)6cr9IqpVGRUmr8EVOJ!wb=nSk_6o1S~fDY8- zjZ78jaV66%%`cUVd5~W!uPD8pVUBp(h}~f&a5odUj0EjE0=t^PR!h)UO|UP&se88& z?5QAV-AK@~fuLDJ69SVXqz*7SqV@?RacG%7K*Lw%mCq6=0%i7IDGLWh>C#ggocQ8_ z2lgYk>;WrA2zQH1%Wk$J-*yqzOTjm~kNySI)OV_v%*)B7w(D>(*9yD-_5Lt-fJw1h ze6?l<3;D$^I-$YPnjXx=*@la2YC5FC@0l8CB8^wzVzvzTXE{*gv8)*kg*tb$-DDyl T)`1aK;;RLAMJlq-e+GU7y+<8n delta 697 zcmXxgOH5Ni6adh7XI^cBrKSXJYe7EpD}~a}E2V-;3*}Q9D}vMzQCcVuY^8k4r);Fb zwUOL~u^1La7r3L(g)uIO5TIbJi7s?OqO3G=;m(*4;t+K*b24-9By;D^f8pof^AB|@ z9mjDVw*C*pVwIHzs}IU$EF`D<#|dF=Y@9-cbU>puwgLBqIBW6Dgx!&_f7&eiY+lox z&p6s8v<_VNS_5I%q)V8J3_FJ=n}c?vzs}cV@?)l6gUeQ}%@`W&w%-xMq1h|KOqZ>x zW5!Zj6S7UmBGzj`qu?DDqr<(f_K01)YKk~}$2)skhvL!?>shbDz;On+2j7WEJ}f%5 zQp@}*A(aKFdMu!BL5sH@@27Hkg~UHOl;>o#vSpl~WAhutNgnx%D&9>K%0kQ}i|}?* z2W4nZYtWQB$xd>7mefNL%QZm7zjzMTF6p5d`HQX?xB*tI_{xIQoLBBo$ZZm-&W6Pu`}EjMsqcz zQO~$s%D7a-Xvk;OYZ(`E8TM0*+AM}mMwmb|OwL6!l8WD?GG`U-12i8MKl+&sCsf79 z-{)bOgJt+d`p6Nx^#F*!6j!AJsKT^lCxd19J6!@sJh$tFa_rv~poabGG(O+$VLgDo zGB9D@f(9G+HYNVGoQGCPvA#{92|vE+fQwkT(E$#OFzck38*O~*Ew9L;B*>8<`AqxS zlifr)rI)K6<&aR1j1GooaXo!TIT0F*PL9N)6Y*F;G_z*s;k&e+CiqG`+!4USB0ieP XH{i`32UuB21=yZe;nt3I< Date: Thu, 24 Oct 2013 16:39:25 -0400 Subject: [PATCH 05/57] Revert some fabric changes to the build worker. --- workers/dockerfilebuild.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/workers/dockerfilebuild.py b/workers/dockerfilebuild.py index 60f145c21..35fdb5720 100644 --- a/workers/dockerfilebuild.py +++ b/workers/dockerfilebuild.py @@ -7,7 +7,6 @@ import digitalocean from apscheduler.scheduler import Scheduler from multiprocessing.pool import ThreadPool -from fabric.api import env from data.queue import dockerfile_build_queue from data import model @@ -61,7 +60,6 @@ def babysit_builder(request): repository_build.phase = 'initializing' repository_build.save() - env.host_string = 'root@%s' % droplet.ip_address # tell it to pull and run the buildserver From b4d916bcc55720bbfae4b0c13d6c83baabc7ce2f Mon Sep 17 00:00:00 2001 From: yackob03 Date: Thu, 24 Oct 2013 17:41:37 -0400 Subject: [PATCH 06/57] Update to Font Awesome 4. --- static/css/quay.css | 21 +++++++++---------- static/directives/repo-circle.html | 4 ++-- static/js/controllers.js | 4 ++-- static/js/graphing.js | 10 ++++----- static/lib/browser-chrome.js | 2 +- static/lib/pagedown/Markdown.Editor.js | 26 ++++++++++++------------ static/partials/header.html | 6 +++--- static/partials/image-view.html | 10 ++++----- static/partials/landing.html | 14 ++++++------- static/partials/repo-admin.html | 22 ++++++++++---------- static/partials/repo-list.html | 2 +- static/partials/signin.html | 4 ++-- static/partials/user-admin.html | 6 +++--- static/partials/view-repo.html | 28 +++++++++++++------------- templates/base.html | 2 +- 15 files changed, 80 insertions(+), 81 deletions(-) diff --git a/static/css/quay.css b/static/css/quay.css index fd8e2f109..ecf162013 100644 --- a/static/css/quay.css +++ b/static/css/quay.css @@ -19,8 +19,7 @@ width: 40px; } -.repo-circle .icon-lock { - font-size: 50%; +.repo-circle .fa-lock { position: absolute; bottom: -6px; right: 0px; @@ -31,11 +30,11 @@ text-align: center; height: 20px; line-height: 21px; - font-size: 12px; + font-size: 16px; } -.repo-circle.no-background .icon-lock { - bottom: -4px; +.repo-circle.no-background .fa-lock { + bottom: 2px; right: -6px; color: #444; } @@ -660,7 +659,7 @@ p.editable:hover i { } .repo-listing i { - font-size: 1.5em; + font-size: 2em; color: #999; display: inline-block; margin-right: 6px; @@ -751,15 +750,15 @@ p.editable:hover i { margin-right: 10px; } -.repo .changes-container i.icon-plus-sign-alt { +.repo .changes-container i.fa-plus-square { color: rgb(73, 209, 73); } -.repo .changes-container i.icon-minus-sign-alt { +.repo .changes-container i.fa-minus-square { color: rgb(209, 73, 73); } -.repo .changes-container i.icon-edit-sign { +.repo .changes-container i.fa-pencil-square { color: rgb(73, 100, 209); } @@ -875,11 +874,11 @@ p.editable:hover i { width: 580px; } -.repo-admin .repo-access-state .state-icon i.icon-lock { +.repo-admin .repo-access-state .state-icon i.fa-lock { background: rgb(253, 191, 191); } -.repo-admin .repo-access-state .state-icon i.icon-unlock-alt { +.repo-admin .repo-access-state .state-icon i.fa-unlock-alt { background: rgb(170, 236, 170); } diff --git a/static/directives/repo-circle.html b/static/directives/repo-circle.html index 62986d4bf..8c03accc8 100644 --- a/static/directives/repo-circle.html +++ b/static/directives/repo-circle.html @@ -1,2 +1,2 @@ - - + + diff --git a/static/js/controllers.js b/static/js/controllers.js index f624c10b7..53edd400d 100644 --- a/static/js/controllers.js +++ b/static/js/controllers.js @@ -85,7 +85,7 @@ function HeaderCtrl($scope, $location, UserService, Restangular) { }, template: function (datum) { template = '
'; - template += '' + template += '' template += '' + datum.repo.namespace +'/' + datum.repo.name + '' if (datum.repo.description) { template += '' + getFirstTextLine(datum.repo.description) + '' @@ -485,7 +485,7 @@ function RepoAdminCtrl($scope, Restangular, $routeParams, $rootScope) { }, template: function (datum) { template = '
'; - template += '' + template += '' template += '' + datum.username + '' template += '
' return template; diff --git a/static/js/graphing.js b/static/js/graphing.js index 1066c0594..c38ee273e 100644 --- a/static/js/graphing.js +++ b/static/js/graphing.js @@ -1039,17 +1039,17 @@ ImageFileChangeTree.prototype.update_ = function(source) { node.select('.node-icon') .html(function(d) { if (!d.kind) { - var folder = d._children ? 'icon-folder-close' : 'icon-folder-open'; + var folder = d._children ? 'fa fa-folder' : 'fa fa-folder-open'; return ''; } var icon = { - 'added': 'plus-sign-alt', - 'removed': 'minus-sign-alt', - 'changed': 'edit-sign' + 'added': 'plus-square', + 'removed': 'minus-square', + 'changed': 'pencil-square' }; - return ''; + return ''; }); // Transition exiting nodes to the parent's new position. diff --git a/static/lib/browser-chrome.js b/static/lib/browser-chrome.js index deda3b11f..e0a129227 100644 --- a/static/lib/browser-chrome.js +++ b/static/lib/browser-chrome.js @@ -1,5 +1,5 @@ (function(browserchrome, $) { - var htmlTemplate = '
Tab Title
http://google.com/
' + var htmlTemplate = '
Tab Title
http://google.com/
' browserchrome.update = function() { $('[data-screenshot-url]').each(function(index, element) { diff --git a/static/lib/pagedown/Markdown.Editor.js b/static/lib/pagedown/Markdown.Editor.js index 6bf690ba9..30e1a7b79 100755 --- a/static/lib/pagedown/Markdown.Editor.js +++ b/static/lib/pagedown/Markdown.Editor.js @@ -1359,42 +1359,42 @@ } group1 = makeGroup(1); - buttons.bold = makeButton("wmd-bold-button", "Bold - Ctrl+B", "icon-bold", bindCommand("doBold"), group1); - buttons.italic = makeButton("wmd-italic-button", "Italic - Ctrl+I", "icon-italic", bindCommand("doItalic"), group1); + buttons.bold = makeButton("wmd-bold-button", "Bold - Ctrl+B", "fa fa-bold", bindCommand("doBold"), group1); + buttons.italic = makeButton("wmd-italic-button", "Italic - Ctrl+I", "fa fa-italic", bindCommand("doItalic"), group1); group2 = makeGroup(2); /* - buttons.link = makeButton("wmd-link-button", "Link - Ctrl+L", "icon-link", bindCommand(function (chunk, postProcessing) { + buttons.link = makeButton("wmd-link-button", "Link - Ctrl+L", "fa fa-link", bindCommand(function (chunk, postProcessing) { return this.doLinkOrImage(chunk, postProcessing, false); }), group2); */ - buttons.quote = makeButton("wmd-quote-button", "Blockquote - Ctrl+Q", "icon-quote-left", bindCommand("doBlockquote"), group2); - buttons.code = makeButton("wmd-code-button", "Code Sample - Ctrl+K", "icon-code", bindCommand("doCode"), group2); + buttons.quote = makeButton("wmd-quote-button", "Blockquote - Ctrl+Q", "fa fa-quote-left", bindCommand("doBlockquote"), group2); + buttons.code = makeButton("wmd-code-button", "Code Sample - Ctrl+K", "fa fa-code", bindCommand("doCode"), group2); /* - buttons.image = makeButton("wmd-image-button", "Image - Ctrl+G", "icon-picture", bindCommand(function (chunk, postProcessing) { + buttons.image = makeButton("wmd-image-button", "Image - Ctrl+G", "fa fa-picture", bindCommand(function (chunk, postProcessing) { return this.doLinkOrImage(chunk, postProcessing, true); }), group2); */ group3 = makeGroup(3); - buttons.olist = makeButton("wmd-olist-button", "Numbered List - Ctrl+O", "icon-list", bindCommand(function (chunk, postProcessing) { + buttons.olist = makeButton("wmd-olist-button", "Numbered List - Ctrl+O", "fa fa-list", bindCommand(function (chunk, postProcessing) { this.doList(chunk, postProcessing, true); }), group3); - buttons.ulist = makeButton("wmd-ulist-button", "Bulleted List - Ctrl+U", "icon-list-ul", bindCommand(function (chunk, postProcessing) { + buttons.ulist = makeButton("wmd-ulist-button", "Bulleted List - Ctrl+U", "fa fa-list-ul", bindCommand(function (chunk, postProcessing) { this.doList(chunk, postProcessing, false); }), group3); - buttons.heading = makeButton("wmd-heading-button", "Heading - Ctrl+H", "icon-tasks", bindCommand("doHeading"), group3); - buttons.hr = makeButton("wmd-hr-button", "Horizontal Rule - Ctrl+R", "icon-minus", bindCommand("doHorizontalRule"), group3); + buttons.heading = makeButton("wmd-heading-button", "Heading - Ctrl+H", "fa fa-tasks", bindCommand("doHeading"), group3); + buttons.hr = makeButton("wmd-hr-button", "Horizontal Rule - Ctrl+R", "fa fa-minus", bindCommand("doHorizontalRule"), group3); group4 = makeGroup(4); - buttons.undo = makeButton("wmd-undo-button", "Undo - Ctrl+Z", "icon-undo", null, group4); + buttons.undo = makeButton("wmd-undo-button", "Undo - Ctrl+Z", "fa fa-undo", null, group4); buttons.undo.execute = function (manager) { if (manager) manager.undo(); }; var redoTitle = /win/.test(nav.platform.toLowerCase()) ? "Redo - Ctrl+Y" : "Redo - Ctrl+Shift+Z"; // mac and other non-Windows platforms - buttons.redo = makeButton("wmd-redo-button", redoTitle, "icon-share-alt", null, group4); + buttons.redo = makeButton("wmd-redo-button", redoTitle, "fa fa-share", null, group4); buttons.redo.execute = function (manager) { if (manager) manager.redo(); }; if (helpOptions) { @@ -1402,7 +1402,7 @@ group5.className = group5.className + " pull-right"; var helpButton = document.createElement("button"); var helpButtonImage = document.createElement("i"); - helpButtonImage.className = "icon-question-sign"; + helpButtonImage.className = "fa fa-question-sign"; helpButton.appendChild(helpButtonImage); helpButton.className = "btn"; helpButton.id = "wmd-help-button" + postfix; diff --git a/static/partials/header.html b/static/partials/header.html index 321e08612..2590c0a4b 100644 --- a/static/partials/header.html +++ b/static/partials/header.html @@ -2,9 +2,9 @@
diff --git a/static/partials/image-view.html b/static/partials/image-view.html index 99523f61e..5e4477116 100644 --- a/static/partials/image-view.html +++ b/static/partials/image-view.html @@ -3,14 +3,14 @@
- +
- +

- + {{repo.namespace}} / {{repo.name}} @@ -31,7 +31,7 @@
- +

@@ -75,7 +75,7 @@ No matching changes
- + {{folder}}/{{getFilename(change.file)}} diff --git a/static/partials/landing.html b/static/partials/landing.html index 890566750..f425552e5 100644 --- a/static/partials/landing.html +++ b/static/partials/landing.html @@ -10,7 +10,7 @@
- +

Your Top Repositories

@@ -44,10 +44,10 @@ @@ -68,7 +68,7 @@
- + Secure Store your private docker containers where only you and your team @@ -77,7 +77,7 @@
- + Shareable Have to share a repository? No problem! Share with anyone you choose @@ -85,7 +85,7 @@
- + Cloud Hosted Accessible from anywhere, anytime @@ -97,7 +97,7 @@
-
+
Take a tour of Quay
diff --git a/static/partials/repo-admin.html b/static/partials/repo-admin.html index b7ebbce1e..f8c2a0428 100644 --- a/static/partials/repo-admin.html +++ b/static/partials/repo-admin.html @@ -1,5 +1,5 @@
- +
@@ -8,7 +8,7 @@
- +

{{repo.namespace}} / {{repo.name}}

@@ -18,7 +18,7 @@
User Access Permissions - +
@@ -33,7 +33,7 @@ - + {{username}} @@ -46,7 +46,7 @@ - + @@ -64,7 +64,7 @@
Access Token Permissions - +
@@ -79,7 +79,7 @@ - + {{ token.friendlyName }} @@ -91,7 +91,7 @@ - + @@ -114,7 +114,7 @@
Repository Settings
-
+
This repository is currently private. Only users on the above access list may view and interact with it. @@ -124,7 +124,7 @@
-
+
This repository is currently public and is visible to all users, and may be pulled by all users. @@ -171,7 +171,7 @@
-

{{ shownToken.friendlyName }}

+

{{ shownToken.friendlyName }}

The docker username is $token and the password is the token. You may use any value for email.
diff --git a/static/partials/repo-list.html b/static/partials/repo-list.html index a1292e6f2..602c08265 100644 --- a/static/partials/repo-list.html +++ b/static/partials/repo-list.html @@ -1,5 +1,5 @@
- +
diff --git a/static/partials/signin.html b/static/partials/signin.html index da203d1c8..71821ec0a 100644 --- a/static/partials/signin.html +++ b/static/partials/signin.html @@ -18,11 +18,11 @@ - Sign In with GitHub + Sign In with GitHub
Invalid username or password.
diff --git a/static/partials/user-admin.html b/static/partials/user-admin.html index 16df0fe00..8408863fb 100644 --- a/static/partials/user-admin.html +++ b/static/partials/user-admin.html @@ -1,6 +1,6 @@
- +
@@ -18,7 +18,7 @@
{{ plan.title }} - + Subscribed
@@ -59,7 +59,7 @@
- +
diff --git a/static/partials/view-repo.html b/static/partials/view-repo.html index fc0587ad4..671b06fba 100644 --- a/static/partials/view-repo.html +++ b/static/partials/view-repo.html @@ -3,7 +3,7 @@
- +
@@ -16,7 +16,7 @@ - + @@ -29,7 +29,7 @@
- +
@@ -44,7 +44,7 @@

- +

@@ -62,7 +62,7 @@
- {{getTagCount(repo)}} + {{getTagCount(repo)}} {{currentTag.name}}
  • @@ -76,7 +76,7 @@
    - +
    @@ -90,7 +90,7 @@
    - {{imageHistory.length}} + {{imageHistory.length}} {{currentImage.id.substr(0, 12)}}
    • @@ -117,22 +117,22 @@
      - +
      - + {{currentImageChanges.added.length}} - + {{currentImageChanges.removed.length}} - + {{currentImageChanges.changed.length}}
      @@ -140,15 +140,15 @@
      - + {{file}}
      - + {{file}}
      - + {{file}}
      diff --git a/templates/base.html b/templates/base.html index 50cb9b482..99c58d690 100644 --- a/templates/base.html +++ b/templates/base.html @@ -12,7 +12,7 @@ - + From 70fd9afb2bc9722d48bc8087a79b71b2cbf1eb53 Mon Sep 17 00:00:00 2001 From: Joseph Schorr Date: Thu, 24 Oct 2013 17:41:55 -0400 Subject: [PATCH 07/57] Start on new repo page --- endpoints/web.py | 5 ++ static/css/quay.css | 52 +++++++++++++++ static/directives/repo-circle.html | 2 +- static/js/app.js | 1 + static/js/controllers.js | 38 +++++++++++ static/partials/header.html | 6 +- static/partials/new-repo.html | 103 +++++++++++++++++++++++++++++ 7 files changed, 204 insertions(+), 3 deletions(-) create mode 100644 static/partials/new-repo.html diff --git a/endpoints/web.py b/endpoints/web.py index 3e6b2fc4d..ddfbca3d6 100644 --- a/endpoints/web.py +++ b/endpoints/web.py @@ -67,6 +67,11 @@ def signin(): return index('') +@app.route('/new/') +def new(): + return index('') + + @app.route('/repository/') def repository(): return index('') diff --git a/static/css/quay.css b/static/css/quay.css index fd8e2f109..aaad9aa5f 100644 --- a/static/css/quay.css +++ b/static/css/quay.css @@ -75,6 +75,57 @@ text-decoration: none !important; } +.new-repo .new-header { + font-size: 22px; +} + +.new-repo .new-header .repo-circle { + margin-right: 14px; +} + +.new-repo .new-header .name-container { + display: inline-block; + width: 300px; +} + +.new-repo .description { + margin-left: 10px; + margin-top: 10px; +} + +.new-repo .section { + padding-bottom: 20px; + border-bottom: 1px solid #eee; + margin-bottom: 16px; +} + +.new-repo .repo-option { + margin: 6px; + margin-top: 16px; +} + +.new-repo .repo-option i { + font-size: 18px; + padding-left: 10px; + padding-right: 10px; + width: 42px; + display: inline-block; + text-align: center; +} + +.new-repo .option-description { + display: inline-block; + vertical-align: top; +} + +.new-repo .option-description label { + display: block; +} + +.new-repo .cbox { + margin: 10px; +} + .user-guide h3 { margin-bottom: 20px; } @@ -1051,6 +1102,7 @@ p.editable:hover i { border: 1px solid #eee; margin-top: 10px; padding: 10px; + min-height: 50px; } /* Overrides for typeahead to work with bootstrap 3. */ diff --git a/static/directives/repo-circle.html b/static/directives/repo-circle.html index 62986d4bf..76ebd2cfd 100644 --- a/static/directives/repo-circle.html +++ b/static/directives/repo-circle.html @@ -1,2 +1,2 @@ - + diff --git a/static/js/app.js b/static/js/app.js index 73161f17e..568c19637 100644 --- a/static/js/app.js +++ b/static/js/app.js @@ -155,6 +155,7 @@ quayApp = angular.module('quay', ['restangular', 'angularMoment', 'angulartics', when('/guide/', {title: 'User Guide', templateUrl: '/static/partials/guide.html', controller: GuideCtrl}). when('/plans/', {title: 'Plans and Pricing', templateUrl: '/static/partials/plans.html', controller: PlansCtrl}). when('/signin/', {title: 'Signin', templateUrl: '/static/partials/signin.html', controller: SigninCtrl}). + when('/new/', {title: 'Create new repository', templateUrl: '/static/partials/new-repo.html', controller: NewRepoCtrl}). when('/v1/', {title: 'Activation information', templateUrl: '/static/partials/v1-page.html', controller: V1Ctrl}). diff --git a/static/js/controllers.js b/static/js/controllers.js index f624c10b7..72610a23f 100644 --- a/static/js/controllers.js +++ b/static/js/controllers.js @@ -893,4 +893,42 @@ function V1Ctrl($scope, UserService) { $scope.browseRepos = function() { document.location = '/repository/'; }; +} + +function NewRepoCtrl($scope, UserService) { + $scope.repo = { + 'is_public': 1, + 'description': '', + 'initialize': false + }; + + $scope.$watch( function () { return UserService.currentUser(); }, function (currentUser) { + $scope.user = currentUser; + + if ($scope.user.anonymous) { + document.location = '/signin/'; + } + }, true); + + $scope.editDescription = function() { + if (!$scope.markdownDescriptionEditor) { + var converter = Markdown.getSanitizingConverter(); + var editor = new Markdown.Editor(converter, '-description'); + editor.run(); + $scope.markdownDescriptionEditor = editor; + } + + $('#wmd-input-description')[0].value = $scope.repo.description; + $('#editModal').modal({}); + }; + + $scope.getMarkedDown = function(string) { + if (!string) { return ''; } + return getMarkedDown(string); + }; + + $scope.saveDescription = function() { + $('#editModal').modal('hide'); + $scope.repo.description = $('#wmd-input-description')[0].value; + }; } \ No newline at end of file diff --git a/static/partials/header.html b/static/partials/header.html index 321e08612..8e3f288dc 100644 --- a/static/partials/header.html +++ b/static/partials/header.html @@ -27,9 +27,11 @@
      -
    • - + + + +
    • {{ user.username }} diff --git a/static/partials/new-repo.html b/static/partials/new-repo.html new file mode 100644 index 000000000..71159bbc4 --- /dev/null +++ b/static/partials/new-repo.html @@ -0,0 +1,103 @@ +
      +
      + + +
      +
      +
      +
      +
      + + {{user.username}} / +
      +
      + +
      + Description:
      +

      + + +

      +
      +
      +
      + + +
      +
      +
      +
      +
      + + + +
      + + Anyone can see and pull from this repository. You choose who can push. +
      +
      +
      + + + +
      + + You choose who can see, pull and push from/to this repository. +
      +
      +
      +
      +
      + + +
      +
      +
      +
      + +
      + Dockerfile + Automatically populate your repository with a new image constructed from a Dockerfile +
      + +
      + Initialize +
      +
      +
      +
      + +
      +
      +
      + +
      +
      + +
      +
      + + + + From 1d51618c9d840da932c4bb8864e07798a019bdcd Mon Sep 17 00:00:00 2001 From: Joseph Schorr Date: Thu, 24 Oct 2013 17:51:09 -0400 Subject: [PATCH 08/57] Fix repo circle positioning by changing to absolute sizes everywhere --- static/css/quay.css | 18 +++++++++++++----- static/directives/repo-circle.html | 2 +- 2 files changed, 14 insertions(+), 6 deletions(-) diff --git a/static/css/quay.css b/static/css/quay.css index eabc24d86..8783e18b1 100644 --- a/static/css/quay.css +++ b/static/css/quay.css @@ -15,14 +15,23 @@ .repo-circle.no-background { background: transparent; + position: relative; height: 40px; width: 40px; } +.repo-circle .fa-hdd { + font-size: 36px; +} + +.repo-circle.no-background .fa-hdd { + font-size: 30px; +} + .repo-circle .fa-lock { position: absolute; - bottom: -6px; - right: 0px; + bottom: -2px; + right: -4px; background: rgb(253, 191, 191); width: 20px; display: inline-block; @@ -30,11 +39,11 @@ text-align: center; height: 20px; line-height: 21px; - font-size: 16px; + font-size: 16px !important; } .repo-circle.no-background .fa-lock { - bottom: 2px; + bottom: -2px; right: -6px; color: #444; } @@ -710,7 +719,6 @@ p.editable:hover i { } .repo-listing i { - font-size: 2em; color: #999; display: inline-block; margin-right: 6px; diff --git a/static/directives/repo-circle.html b/static/directives/repo-circle.html index 8c03accc8..49d60ce56 100644 --- a/static/directives/repo-circle.html +++ b/static/directives/repo-circle.html @@ -1,2 +1,2 @@ - + From 2afb8c85b19c291e6c2260c99c9bff710058516d Mon Sep 17 00:00:00 2001 From: Joseph Schorr Date: Thu, 24 Oct 2013 18:13:24 -0400 Subject: [PATCH 09/57] Get the new repo page ready for wiring --- static/css/quay.css | 30 ++++++++++++++++++++++++++++++ static/partials/header.html | 4 ++-- static/partials/new-repo.html | 14 +++++++++----- 3 files changed, 41 insertions(+), 7 deletions(-) diff --git a/static/css/quay.css b/static/css/quay.css index 8783e18b1..f1082c8dc 100644 --- a/static/css/quay.css +++ b/static/css/quay.css @@ -2,6 +2,18 @@ font-family: 'Droid Sans', sans-serif; } +.user-tools .user-tool { + font-size: 24px; + margin-top: 14px; + color: black; +} + +.user-tools i.user-tool:hover { + cursor: pointer; + color: #428bca; +} + + .repo-circle { position: relative; background: #eee; @@ -134,6 +146,24 @@ margin: 10px; } +.new-repo .initialize-repo { + margin: 10px; + margin-top: 16px; + margin-left: 20px; + padding: 10px; + border: 1px dashed #ccc; +} + +.new-repo .initialize-repo .init-description { + color: #444; + font-size: 12px; + text-align: center; +} + +.new-repo .initialize-repo .file-drop { + margin: 10px; +} + .user-guide h3 { margin-bottom: 20px; } diff --git a/static/partials/header.html b/static/partials/header.html index 1ff5bd1b6..5ddca4e88 100644 --- a/static/partials/header.html +++ b/static/partials/header.html @@ -27,8 +27,8 @@
    - - + +
  • diff --git a/static/partials/new-repo.html b/static/partials/new-repo.html index 71159bbc4..bc6c8becb 100644 --- a/static/partials/new-repo.html +++ b/static/partials/new-repo.html @@ -16,7 +16,7 @@ Description:

    - +

@@ -29,7 +29,7 @@
- +
@@ -38,7 +38,7 @@
- +
@@ -60,8 +60,12 @@ Automatically populate your repository with a new image constructed from a Dockerfile
-
- Initialize +
+
+ Upload a Dockerfile or a zip file containing a Dockerfile in the root directory +
+ +
From 9b9a29c31052ef098ede69328356e3b6b1810e40 Mon Sep 17 00:00:00 2001 From: yackob03 Date: Fri, 25 Oct 2013 01:14:38 -0400 Subject: [PATCH 10/57] Flesh out the create API and wire everything up together. Next up, testing. --- buildserver/buildserver.py | 16 ++++--- buildserver/requirements-nover.txt | 1 + config.py | 8 +++- data/database.py | 3 +- data/model.py | 8 +++- data/userfiles.py | 34 +++++++++++++++ endpoints/api.py | 30 ++++++++++++- requirements-nover.txt | 3 +- test/data/test.db | Bin 77824 -> 77824 bytes workers/dockerfilebuild.py | 68 ++++++++++++++++++++++++++++- 10 files changed, 156 insertions(+), 15 deletions(-) create mode 100644 data/userfiles.py diff --git a/buildserver/buildserver.py b/buildserver/buildserver.py index b6de17da5..87dc53878 100644 --- a/buildserver/buildserver.py +++ b/buildserver/buildserver.py @@ -3,6 +3,7 @@ import logging import shutil import os import re +import requests from flask import Flask, request, send_file, jsonify, redirect, url_for, abort from zipfile import ZipFile @@ -39,7 +40,7 @@ def prepare_zip(request_file): # Save the zip file to temp somewhere with TemporaryFile() as zip_file: - request_file.save(zip_file) + zip_file.write(request_file.content) to_extract = ZipFile(zip_file) to_extract.extractall(build_dir) @@ -49,7 +50,8 @@ def prepare_zip(request_file): def prepare_dockerfile(request_file): build_dir = mkdtemp(prefix='docker-build-') dockerfile_path = os.path.join(build_dir, "Dockerfile") - request_file.save(dockerfile_path) + with open(dockerfile_path, 'w') as dockerfile: + dockerfile.write(request_file.content) return build_dir @@ -141,10 +143,12 @@ pool = ThreadPool(1) @app.route('/build/', methods=['POST']) def start_build(): - docker_input = request.files['dockerfile'] - c_type = docker_input.content_type + resource_url = request.values['resource_url'] tag_name = request.values['tag'] + download_resource = requests.get(resource_url) + download_resource.get() + logger.info('Request to build file of type: %s with tag: %s' % (c_type, tag_name)) @@ -175,7 +179,9 @@ def start_build(): pool.apply_async(build_image, [build_dir, tag_name, num_steps, result_object]) - return redirect(url_for('get_status', job_id=job_id)) + resp = make_response('Created', 201) + resp.headers['Location'] = url_for('get_status', job_id=job_id) + return resp @app.route('/build/') diff --git a/buildserver/requirements-nover.txt b/buildserver/requirements-nover.txt index e5ec39f3c..0aba1d6c9 100644 --- a/buildserver/requirements-nover.txt +++ b/buildserver/requirements-nover.txt @@ -1,2 +1,3 @@ flask +requests -e git+git://github.com/DevTable/docker-py.git#egg=docker-py diff --git a/config.py b/config.py index 71297a00b..1aab5b5fe 100644 --- a/config.py +++ b/config.py @@ -42,10 +42,13 @@ class RDSMySQL(object): DB_DRIVER = MySQLDatabase -class S3Storage(object): +class AWSCredentials(object): AWS_ACCESS_KEY = 'AKIAJWZWUIS24TWSMWRA' AWS_SECRET_KEY = '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' REGISTRY_S3_BUCKET = 'quay-registry' + + +class S3Storage(AWSCredentials): STORAGE_KIND = 's3' @@ -89,11 +92,12 @@ class DigitalOceanConfig(): DO_CLIENT_ID = 'LJ44y2wwYj1MD0BRxS6qHA' DO_CLIENT_SECRET = 'b9357a6f6ff45a33bb03f6dbbad135f9' DO_SSH_KEY_ID = '46986' + DO_SSH_PRIVATE_KEY_FILENAME = 'certs/digital_ocean' class DebugConfig(FlaskConfig, MailConfig, LocalStorage, SQLiteDB, StripeTestConfig, MixpanelTestConfig, GitHubTestConfig, - DigitalOceanConfig): + DigitalOceanConfig, AWSCredentials): REGISTRY_SERVER = 'localhost:5000' LOGGING_CONFIG = { 'level': logging.DEBUG, diff --git a/data/database.py b/data/database.py index 38bc2d2b7..0c979a2c2 100644 --- a/data/database.py +++ b/data/database.py @@ -151,10 +151,11 @@ class RepositoryTag(BaseModel): class RepositoryBuild(BaseModel): + repository = ForeignKeyField(Repository) + resource_key = CharField() digitalocean_build_node_id = IntegerField(null=True) phase = CharField(default='waiting') status_url = CharField(null=True) - repository = ForeignKeyField(Repository) class QueueItem(BaseModel): diff --git a/data/model.py b/data/model.py index 182ee2a21..4320ca31a 100644 --- a/data/model.py +++ b/data/model.py @@ -287,8 +287,8 @@ def set_repository_visibility(repo, visibility): repo.save() -def create_repository(namespace, name, owner): - private = Visibility.get(name='private') +def create_repository(namespace, name, owner, visibility='private'): + private = Visibility.get(name=visibility) repo = Repository.create(namespace=namespace, name=name, visibility=private) admin = Role.get(name='admin') @@ -560,3 +560,7 @@ def get_repository_build(request_dbid): except RepositoryBuild.DoesNotExist: msg = 'Unable to locate a build by id: %s' % request_dbid raise InvalidRepositoryBuildException(msg) + + +def create_repository_build(repo, resource_key): + return RepositoryBuild.create(repository=repo, resource_key=resource_key) diff --git a/data/userfiles.py b/data/userfiles.py new file mode 100644 index 000000000..7c017090f --- /dev/null +++ b/data/userfiles.py @@ -0,0 +1,34 @@ +import boto +import os + +from boto.s3.key import Key +from uuid import uuid4 + + +class S3FileWriteException(Exception): + pass + + +class UserRequestFiles(object): + def __init__(self, s3_access_key, s3_secret_key, bucket_name): + self._s3_conn = boto.s3.connection.S3Connection(s3_access_key, + s3_secret_key, + is_secure=False) + self._bucket = self._s3_conn.get_bucket(bucket_name) + self._prefix = 'userfiles' + + def store_file(self, flask_file): + file_id = str(uuid4()) + full_key = os.path.join(self._prefix, file_id) + k = Key(full_key) + bytes_written = k.set_contents_from_file(flask_file) + + if bytes_written == 0: + raise S3FileWriteException('Unable to write file to S3') + + return file_id + + def get_file_url(self, file_id, expires_in=300): + full_key = os.path.join(self._prefix, file_id) + k = Key(full_key) + return k.generate_url(expires_in) diff --git a/endpoints/api.py b/endpoints/api.py index ab69b71b9..9d30ef141 100644 --- a/endpoints/api.py +++ b/endpoints/api.py @@ -2,7 +2,7 @@ import logging import stripe import re -from flask import request, make_response, jsonify, abort +from flask import request, make_response, jsonify, abort, url_for from flask.ext.login import login_required, current_user, logout_user from flask.ext.principal import identity_changed, AnonymousIdentity from functools import wraps @@ -11,6 +11,8 @@ from collections import defaultdict import storage from data import model +from data.userfiles import UserRequestFiles +from data.queue import dockerfile_build_queue from app import app from util.email import send_confirmation_email, send_recovery_email from util.names import parse_repository_name @@ -170,10 +172,34 @@ def get_matching_users(prefix): }) +user_files = UserRequestFiles(app.config['AWS_ACCESS_KEY'], + app.config['AWS_SECRET_KEY'], + app.config['REGISTRY_S3_BUCKET']) + + @app.route('/api/repository/', methods=['POST']) @api_login_required def create_repo_api(): - pass + namespace_name = request.values['namespace'] + repository_name = request.values['repository'] + visibility = request.values['visibility'] + + owner = current_user.db_user() + repo = model.create_repository(namespace_name, repository_name, owner, + visibility) + + if request.values['initialize']: + logger.debug('User requested repository initialization.') + dockerfile_source = request.files['initializedata'] + dockerfile_id = user_files.store_file(dockerfile_source) + + build_request = model.create_repository_build(repo, dockerfile_id) + dockerfile_build_queue.put(json.dumps({'request_id': build_request.id})) + + resp = make_response('Created', 201) + resp.headers['Location'] = url_for('get_repo_api', namespace=namespace_name, + repository=repository_name) + return resp @app.route('/api/find/repository', methods=['GET']) diff --git a/requirements-nover.txt b/requirements-nover.txt index fe43012cd..a12714454 100644 --- a/requirements-nover.txt +++ b/requirements-nover.txt @@ -14,4 +14,5 @@ mixpanel-py beautifulsoup4 marisa-trie apscheduler -python-daemon \ No newline at end of file +python-daemon +paramiko \ No newline at end of file diff --git a/test/data/test.db b/test/data/test.db index 83688fab4d7c9acbd3b2a6402aa7b0b0cc274e8b..48de7babe0ffc6e2660afbbce0d78e75f59a6781 100644 GIT binary patch delta 192 zcmV;x06+hL-~@o+1h7#vHFQX6a7i{tM`c54Y)VsQZ)ZqVMM8CHS6NSFE>}}CQdLe) zM{#aUVMbYSV`f8AR(g75Ra2ACI~bFmGZ{5#XLn3xG&F8=X!0?4yCfVTn( uH6I5TD=PtnA(QxmXp=&-LJV?cb8mHWV`X1!WqFg3W*U=VgI%-#g8{5h1w=vs delta 183 zcmV;o07(CU-~@o+1h7#vHF$JTXf9!EGDTTpLPA$lICpX~LrpF*GI>&EHA72Sa!6-P zFF1KvF-v$tQhH8yR4*@8Z&#DgI~bFmGZ{5YR#cV diff --git a/workers/dockerfilebuild.py b/workers/dockerfilebuild.py index 35fdb5720..e153354b1 100644 --- a/workers/dockerfilebuild.py +++ b/workers/dockerfilebuild.py @@ -4,11 +4,13 @@ import daemon import time import argparse import digitalocean +import requests from apscheduler.scheduler import Scheduler from multiprocessing.pool import ThreadPool from data.queue import dockerfile_build_queue +from data.userfiles import UserRequestFiles from data import model from app import app @@ -22,6 +24,21 @@ formatter = logging.Formatter(FORMAT) logger = logging.getLogger(__name__) +def try_connection(url, retries=5, period=5): + try: + return requests.get(url) + except ConnectionError as ex: + if retries: + logger.debug('Retrying connection to url: %s after %ss' % (url, period)) + time.sleep(period) + return try_connection(url, retries-1, period) + raise ex + + +def get_status(url): + return requests.get(url).json()['status'] + + def babysit_builder(request): manager = digitalocean.Manager(client_id=app.config['DO_CLIENT_ID'], api_key=app.config['DO_CLIENT_SECRET']) @@ -60,16 +77,62 @@ def babysit_builder(request): repository_build.phase = 'initializing' repository_build.save() + ssh_client = paramiko.SSHClient() + ssh_client.set_missing_host_key_policy(paramiko.AutoAddPolicy()) + ssh_client.connect(self._container_ip, self._config.sshd_port, "root", + look_for_keys=False, + key_filename=app.config['DO_SSH_PRIVATE_KEY_FILENAME']) - # tell it to pull and run the buildserver + # Pull and run the buildserver + pull_cmd = 'docker pull quay.io/quay/buildserver' + _, stdout, _ = ssh_client.exec_command(pull_cmd) + + start_cmd = 'sudo docker run -d -privileged quay.io/quay/buildserver' + _, stdout, _ = ssh_client.exec_command(start_cmd) # wait for the server to be ready + logger.debug('Waiting for buildserver to be ready') + build_endpoint = 'http://%s:5002/build/' % droplet.ip_address + try: + try_connection() + except ConnectionError: + #TODO cleanup + pass # send it the job + logger.debug('Sending build server request') + + user_files = UserRequestFiles(app.config['AWS_ACCESS_KEY'], + app.config['AWS_SECRET_KEY'], + app.config['REGISTRY_S3_BUCKET']) + + repo = repository_build.repository + payload = { + 'tag': 'quay.io/%s/%s' % (repo.namespace, repo.name), + 'resource_url': user_files.get_file_url(repository_build.resource_key), + } + start_build = requests.post(build_endpoint, data=payload) # wait for the job to be complete + status_url = start_build.headers['Location'] + + logger.debug('Waiting for job to be complete') + status = get_status(status_url) + while status != 'error' and status != 'completed': + logger.debug('Job status is: %s' % status) + time.sleep(5) + status = get_status(status_url) + + logger.debug('Job complete with status: %s' % status) + if status == 'error': + repository_build.phase = 'error' + else: + repository_build.phase = 'completed' + repository_build.save() # clean up the DO node + logger.debug('Cleaning up DO node.') + droplet.destroy() return True @@ -91,7 +154,8 @@ def process_work_items(pool): dockerfile_build_queue.complete(local_item) return complete_callback - pool.apply_async(babysit_builder, [request], callback=build_callback(item)) + pool.apply_async(babysit_builder, [request], + callback=build_callback(item)) item = dockerfile_build_queue.get() From ffc33e454c84cd4a4c30d89a6b0aa42fe41f9acf Mon Sep 17 00:00:00 2001 From: yackob03 Date: Fri, 25 Oct 2013 15:13:11 -0400 Subject: [PATCH 11/57] Fixes and refinements to the dockerfile build system. --- README.md | 1 + buildserver/buildserver.py | 6 +++--- data/database.py | 2 +- data/model.py | 7 +++++++ endpoints/api.py | 41 +++++++++++++++++++++++++++++++++++--- workers/dockerfilebuild.py | 11 +++++++--- 6 files changed, 58 insertions(+), 10 deletions(-) diff --git a/README.md b/README.md index 01ca6b799..2f608b81b 100644 --- a/README.md +++ b/README.md @@ -30,6 +30,7 @@ start the workers: ``` STACK=prod python -m workers.diffsworker -D +STACK=prod python -m workers.dockerfilebuild -D ``` bouncing the servers: diff --git a/buildserver/buildserver.py b/buildserver/buildserver.py index 87dc53878..1f5c7f0d8 100644 --- a/buildserver/buildserver.py +++ b/buildserver/buildserver.py @@ -169,9 +169,9 @@ def start_build(): 'id': job_id, 'total_commands': num_steps, 'total_images': None, - 'current_command': 0, - 'current_image': 0, - 'image_completion_percent': 0, + 'current_command': None, + 'current_image': None, + 'image_completion_percent': None, 'status': 'waiting', 'message': None, } diff --git a/data/database.py b/data/database.py index 0c979a2c2..7b5168ff5 100644 --- a/data/database.py +++ b/data/database.py @@ -153,7 +153,7 @@ class RepositoryTag(BaseModel): class RepositoryBuild(BaseModel): repository = ForeignKeyField(Repository) resource_key = CharField() - digitalocean_build_node_id = IntegerField(null=True) + build_node_id = IntegerField(null=True) phase = CharField(default='waiting') status_url = CharField(null=True) diff --git a/data/model.py b/data/model.py index 4320ca31a..7888543a9 100644 --- a/data/model.py +++ b/data/model.py @@ -562,5 +562,12 @@ def get_repository_build(request_dbid): raise InvalidRepositoryBuildException(msg) +def list_repository_builds(namespace_name, repository_name): + joined = RepositoryBuild.select().join(Repository) + fetched = list(joined.where(Repository.name == repository_name, + Repository.namespace == namespace_name)) + return fetched + + def create_repository_build(repo, resource_key): return RepositoryBuild.create(repository=repo, resource_key=resource_key) diff --git a/endpoints/api.py b/endpoints/api.py index 9d30ef141..a8b3c59c9 100644 --- a/endpoints/api.py +++ b/endpoints/api.py @@ -1,6 +1,7 @@ import logging import stripe import re +import requests from flask import request, make_response, jsonify, abort, url_for from flask.ext.login import login_required, current_user, logout_user @@ -180,11 +181,12 @@ user_files = UserRequestFiles(app.config['AWS_ACCESS_KEY'], @app.route('/api/repository/', methods=['POST']) @api_login_required def create_repo_api(): - namespace_name = request.values['namespace'] + owner = current_user.db_user() + + namespace_name = owner.username repository_name = request.values['repository'] visibility = request.values['visibility'] - owner = current_user.db_user() repo = model.create_repository(namespace_name, repository_name, owner, visibility) @@ -349,6 +351,8 @@ def get_repo_api(namespace, repository): tag_dict = {tag.name: tag_view(tag) for tag in tags} can_write = ModifyRepositoryPermission(namespace, repository).can() can_admin = AdministerRepositoryPermission(namespace, repository).can() + builds = model.list_repository_builds(namespace, repository) + return jsonify({ 'namespace': namespace, 'name': repository, @@ -356,13 +360,44 @@ def get_repo_api(namespace, repository): 'tags': tag_dict, 'can_write': can_write, 'can_admin': can_admin, - 'is_public': is_public + 'is_public': is_public, + 'is_building': len(builds) > 0, }) abort(404) # Not fount abort(403) # Permission denied +@app.route('/api/repository//build/', methods=['GET']) +@parse_repository_name +def get_repo_builds(namespace, repository): + permission = AdministerRepositoryPermission(namespace, repository) + if permission.can(): + def build_view(build_obj): + if build_obj.status_url: + # Delegate the status to the build node + node_status = requests.get(build_obj.status_url).json() + node_status['id'] = build_obj.id + return node_status + + # If there was no status url, do the best we can + return { + 'id': build_obj.id, + 'total_commands': None, + 'total_images': None, + 'current_command': None, + 'current_image': None, + 'image_completion_percent': None, + 'status': build_obj.phase, + 'message': None, + } + + builds = model.list_repository_builds(namespace, repository) + return jsonify({ + 'builds': [build_view(build) for build in builds] + }) + + def role_view(repo_perm_obj): return { 'role': repo_perm_obj.role.name diff --git a/workers/dockerfilebuild.py b/workers/dockerfilebuild.py index e153354b1..4fbc48cac 100644 --- a/workers/dockerfilebuild.py +++ b/workers/dockerfilebuild.py @@ -45,7 +45,7 @@ def babysit_builder(request): repository_build = model.get_repository_build(request['build_id']) # check if there is already a DO node for this build job, if so clean it up - old_id = repository_build.digitalocean_build_node_id + old_id = repository_build.build_node_id if old_id old_droplet = digitalocean.Droplet(old_id) old_droplet.destroy() @@ -60,7 +60,7 @@ def babysit_builder(request): size_id=66, # 512MB, backup_active=False) droplet.create(ssh_key_ids=[app.config['DO_SSH_KEY_ID']]) - repository_build.digitalocean_build_node_id = droplet.id + repository_build.build_node_id = droplet.id repository_build.phase = 'starting' repository_build.save() @@ -115,6 +115,8 @@ def babysit_builder(request): # wait for the job to be complete status_url = start_build.headers['Location'] + repository_build.status_url = status_url + repository_build.save() logger.debug('Waiting for job to be complete') status = get_status(status_url) @@ -128,12 +130,15 @@ def babysit_builder(request): repository_build.phase = 'error' else: repository_build.phase = 'completed' - repository_build.save() # clean up the DO node logger.debug('Cleaning up DO node.') droplet.destroy() + repository_build.status_url = None + repository_build.build_node_id = None; + repository_build.save() + return True From 684ce83058f35bf19b0f2dd6b3df3c45f658398b Mon Sep 17 00:00:00 2001 From: yackob03 Date: Fri, 25 Oct 2013 15:47:34 -0400 Subject: [PATCH 12/57] Add a test api to test repositories which are currenlty building. --- application.py | 13 +++++++++++-- config.py | 1 + endpoints/api.py | 2 ++ initdb.py | 15 ++++++++++++--- test/data/test.db | Bin 77824 -> 77824 bytes workers/dockerfilebuild.py | 5 +++-- 6 files changed, 29 insertions(+), 7 deletions(-) diff --git a/application.py b/application.py index d8d29a177..2244902c1 100644 --- a/application.py +++ b/application.py @@ -2,16 +2,25 @@ import logging from app import app as application + +logging.basicConfig(**application.config['LOGGING_CONFIG']) + + import endpoints.index import endpoints.api import endpoints.web import endpoints.tags import endpoints.registry + +logger = logging.getLogger(__name__) + +if application.config.get('INCLUDE_TEST_ENDPOINTS', False): + logger.debug('Loading test endpoints.') + import endpoints.test + # Remove this for prod config application.debug = True -logging.basicConfig(**application.config['LOGGING_CONFIG']) - if __name__ == '__main__': application.run(port=5000, debug=True, host='0.0.0.0') diff --git a/config.py b/config.py index 1aab5b5fe..a51e2700f 100644 --- a/config.py +++ b/config.py @@ -105,6 +105,7 @@ class DebugConfig(FlaskConfig, MailConfig, LocalStorage, SQLiteDB, } SEND_FILE_MAX_AGE_DEFAULT = 0 POPULATE_DB_TEST_DATA = True + INCLUDE_TEST_ENDPOINTS = True class LocalHostedConfig(FlaskConfig, MailConfig, S3Storage, RDSMySQL, diff --git a/endpoints/api.py b/endpoints/api.py index a8b3c59c9..d56cd12f4 100644 --- a/endpoints/api.py +++ b/endpoints/api.py @@ -397,6 +397,8 @@ def get_repo_builds(namespace, repository): 'builds': [build_view(build) for build in builds] }) + abort(403) # Permissions denied + def role_view(repo_perm_obj): return { diff --git a/initdb.py b/initdb.py index 562a29e16..79519ee7f 100644 --- a/initdb.py +++ b/initdb.py @@ -5,6 +5,7 @@ import os import hashlib from datetime import datetime, timedelta +from flask import url_for import storage @@ -95,6 +96,8 @@ def __generate_repository(user, name, description, is_public, permissions, create_subtree(repo, structure, None) + return repo + if __name__ == '__main__': initialize_db() @@ -141,6 +144,12 @@ if __name__ == '__main__': 'Shared repository, another user can write.', False, [(new_user_2, 'write')], (5, [], 'latest')) - __generate_repository(new_user_1, 'empty', - 'Empty repository with no images or tags.', False, - [], (0, [], None)) \ No newline at end of file + building = __generate_repository(new_user_1, 'building', + 'Empty repository which is building.', + False, [], (0, [], None)) + + build = model.create_repository_build(building, '123-45-6789') + build.build_node_id = 1 + build.phase = 'building' + build.status_url = 'http://localhost:5000/test/build/status' + build.save() diff --git a/test/data/test.db b/test/data/test.db index 48de7babe0ffc6e2660afbbce0d78e75f59a6781..32075f2bdf9f616e443e970241a748215068c41c 100644 GIT binary patch delta 414 zcmZp8z|!!5Wr8%L{6raNM){2iAB{}?@;&p6f`cm@4YC6Cv$70I%G1*_^@>dj^aG0Y za}D!?!YiGeeLT`UbCO-NlOszqLd(4LCZD&Km^{x|(zGNu-^eW^%ET$Dz`w}ZO}{Wy z-!0PL$R{bsG&HHqI6E)U)hNs@&DkW%z$M5dE6df7wG|s1gRCrLa7t=fNn%n?YEo%tPD*B8I+JT|K}n@TQEEYcab`(= zQKdq8MrLw`LT0f7Ov&cYMZ&CX4Ghc;%r`dkRFp7_in0KWLN{si)apVO9$f|oUB*Vn z*G!pAS0_43Z&qkn$;7E4&n~X5&Dg>|`BKw`&Fh;(dDs}4_b@Q;*(}JiiP_JIk)452 znNv2J&CtkL*ThuU%-q5fW_w0SNr9EVeolUJVopYWafy|wfq{X3NosM4K3Iu_i!7M%j%CAB{{)JTePB&D`BnoilU1f>ZO;JwjcbN;AVF{ZsV9 zgN=hi{rubu^L!HBA`6n!oP)xus!~FOC!e>Lm^{x|(ljHz+$Yt&_>C`n8&-u$^pn3b)Wf%yh=^Jbok5@t~b1_mJ(pm`|9ZJt_P$ikz* zz@Wfb!g!AZ;@aAbEsB%xH(lUO$xP2INzBPlPEE{<-@Lv#lxH!E M1jlBU8Goh&0HniiZU6uP diff --git a/workers/dockerfilebuild.py b/workers/dockerfilebuild.py index 4fbc48cac..d04f98ced 100644 --- a/workers/dockerfilebuild.py +++ b/workers/dockerfilebuild.py @@ -115,12 +115,13 @@ def babysit_builder(request): # wait for the job to be complete status_url = start_build.headers['Location'] + repository_build.phase = 'building' repository_build.status_url = status_url repository_build.save() logger.debug('Waiting for job to be complete') status = get_status(status_url) - while status != 'error' and status != 'completed': + while status != 'error' and status != 'complete': logger.debug('Job status is: %s' % status) time.sleep(5) status = get_status(status_url) @@ -129,7 +130,7 @@ def babysit_builder(request): if status == 'error': repository_build.phase = 'error' else: - repository_build.phase = 'completed' + repository_build.phase = 'complete' # clean up the DO node logger.debug('Cleaning up DO node.') From c5ff08e57d41c122fc80da95616f9b3af0bf68c3 Mon Sep 17 00:00:00 2001 From: yackob03 Date: Fri, 25 Oct 2013 18:14:35 -0400 Subject: [PATCH 13/57] Update the buildserver to use the new post format. --- buildserver/Dockerfile | 5 ++-- buildserver/buildserver.py | 37 ++++++++++++++++++++++-------- buildserver/requirements-nover.txt | 3 --- buildserver/requirements.txt | 5 ++++ buildserver/test.html | 9 -------- 5 files changed, 34 insertions(+), 25 deletions(-) delete mode 100644 buildserver/requirements-nover.txt create mode 100644 buildserver/requirements.txt delete mode 100644 buildserver/test.html diff --git a/buildserver/Dockerfile b/buildserver/Dockerfile index c58e9f0ad..eb69c1eb2 100644 --- a/buildserver/Dockerfile +++ b/buildserver/Dockerfile @@ -13,13 +13,12 @@ ADD https://get.docker.io/builds/Linux/x86_64/docker-latest /usr/local/bin/docke # Install the files ADD ./startserver /usr/local/bin/startserver ADD ./buildserver.py ./buildserver.py -ADD ./test.html ./test.html -ADD ./requirements-nover.txt ./requirements-nover.txt +ADD ./requirements.txt ./requirements.txt RUN chmod +x /usr/local/bin/docker /usr/local/bin/startserver RUN virtualenv --distribute venv -RUN venv/bin/pip install -r requirements-nover.txt +RUN venv/bin/pip install -r requirements.txt VOLUME /var/lib/docker diff --git a/buildserver/buildserver.py b/buildserver/buildserver.py index 1f5c7f0d8..5e65a2392 100644 --- a/buildserver/buildserver.py +++ b/buildserver/buildserver.py @@ -4,12 +4,14 @@ import shutil import os import re import requests +import json -from flask import Flask, request, send_file, jsonify, redirect, url_for, abort +from flask import Flask, request, jsonify, url_for, abort, make_response from zipfile import ZipFile from tempfile import TemporaryFile, mkdtemp from uuid import uuid4 from multiprocessing.pool import ThreadPool +from base64 import b64encode BUFFER_SIZE = 8 * 1024 @@ -20,11 +22,6 @@ app = Flask(__name__) logger = logging.getLogger(__name__) -@app.route('/') -def index(): - return send_file('test.html') - - def count_steps(dockerfile_path): with open(dockerfile_path, 'r') as dockerfileobj: steps = 0 @@ -58,7 +55,7 @@ def prepare_dockerfile(request_file): def build_image(build_dir, tag_name, num_steps, result_object): try: - logger.debug('Does this show up?') + logger.debug('Starting build.') docker_cl = docker.Client(version='1.5') result_object['status'] = 'building' build_status = docker_cl.build(path=build_dir, tag=tag_name) @@ -145,9 +142,29 @@ pool = ThreadPool(1) def start_build(): resource_url = request.values['resource_url'] tag_name = request.values['tag'] + acccess_token = request.values['token'] - download_resource = requests.get(resource_url) - download_resource.get() + # Save the token + host = re.match(r'([a-z0-9.:]+)/.+/.+$', tag_name) + if host: + docker_endpoint = 'http://%s/v1/' % host.group(0) + dockercfg_path = os.path.join(os.environ.get('HOME', '.'), '.dockercfg') + token = b64encode('$token:%s' % acccess_token) + with open(dockercfg_path, 'w') as dockercfg: + payload = { + docker_endpoint: { + 'auth': token, + 'email': '', + } + } + dockercfg.write(json.dumps(payload)) + + else: + logger.warning('Invalid tag name: %s' % tag_name) + abort(400) + + docker_resource = requests.get(resource_url) + c_type = docker_resource.headers['content-type'] logger.info('Request to build file of type: %s with tag: %s' % (c_type, tag_name)) @@ -156,7 +173,7 @@ def start_build(): logger.error('Invalid dockerfile content type: %s' % c_type) abort(400) - build_dir = MIME_PROCESSORS[c_type](docker_input) + build_dir = MIME_PROCESSORS[c_type](docker_resource) dockerfile_path = os.path.join(build_dir, "Dockerfile") num_steps = count_steps(dockerfile_path) diff --git a/buildserver/requirements-nover.txt b/buildserver/requirements-nover.txt deleted file mode 100644 index 0aba1d6c9..000000000 --- a/buildserver/requirements-nover.txt +++ /dev/null @@ -1,3 +0,0 @@ -flask -requests --e git+git://github.com/DevTable/docker-py.git#egg=docker-py diff --git a/buildserver/requirements.txt b/buildserver/requirements.txt new file mode 100644 index 000000000..3093eeb14 --- /dev/null +++ b/buildserver/requirements.txt @@ -0,0 +1,5 @@ +mock==1.0.1 +requests==1.2.3 +six==1.3.0 +flask==0.10.1 +-e git+git://github.com/DevTable/docker-py.git#egg=docker-py \ No newline at end of file diff --git a/buildserver/test.html b/buildserver/test.html deleted file mode 100644 index 9a08eb994..000000000 --- a/buildserver/test.html +++ /dev/null @@ -1,9 +0,0 @@ - - -
- - - -
- - \ No newline at end of file From 78d2d6cad07a81e1c3b87b28b4fd5e392dfad1c7 Mon Sep 17 00:00:00 2001 From: yackob03 Date: Fri, 25 Oct 2013 18:17:43 -0400 Subject: [PATCH 14/57] Add a test endpoint that allows us to manually create builds. Add access tokens to the build data. --- data/database.py | 2 ++ data/model.py | 5 ++-- endpoints/api.py | 35 ++++++++++++++++++------- endpoints/test.py | 53 ++++++++++++++++++++++++++++++++++++++ initdb.py | 5 +++- templates/testbuild.html | 8 ++++++ workers/dockerfilebuild.py | 5 ++-- 7 files changed, 99 insertions(+), 14 deletions(-) create mode 100644 endpoints/test.py create mode 100644 templates/testbuild.html diff --git a/data/database.py b/data/database.py index 7b5168ff5..31581d443 100644 --- a/data/database.py +++ b/data/database.py @@ -152,7 +152,9 @@ class RepositoryTag(BaseModel): class RepositoryBuild(BaseModel): repository = ForeignKeyField(Repository) + access_token = ForeignKeyField(AccessToken) resource_key = CharField() + tag = CharField() build_node_id = IntegerField(null=True) phase = CharField(default='waiting') status_url = CharField(null=True) diff --git a/data/model.py b/data/model.py index 7888543a9..f796c3572 100644 --- a/data/model.py +++ b/data/model.py @@ -569,5 +569,6 @@ def list_repository_builds(namespace_name, repository_name): return fetched -def create_repository_build(repo, resource_key): - return RepositoryBuild.create(repository=repo, resource_key=resource_key) +def create_repository_build(repo, access_token, resource_key, tag): + return RepositoryBuild.create(repository=repo, access_token=access_token, + resource_key=resource_key, tag=tag) diff --git a/endpoints/api.py b/endpoints/api.py index d56cd12f4..630130c98 100644 --- a/endpoints/api.py +++ b/endpoints/api.py @@ -2,6 +2,8 @@ import logging import stripe import re import requests +import urlparse +import json from flask import request, make_response, jsonify, abort, url_for from flask.ext.login import login_required, current_user, logout_user @@ -190,14 +192,6 @@ def create_repo_api(): repo = model.create_repository(namespace_name, repository_name, owner, visibility) - if request.values['initialize']: - logger.debug('User requested repository initialization.') - dockerfile_source = request.files['initializedata'] - dockerfile_id = user_files.store_file(dockerfile_source) - - build_request = model.create_repository_build(repo, dockerfile_id) - dockerfile_build_queue.put(json.dumps({'request_id': build_request.id})) - resp = make_response('Created', 201) resp.headers['Location'] = url_for('get_repo_api', namespace=namespace_name, repository=repository_name) @@ -371,7 +365,7 @@ def get_repo_api(namespace, repository): @app.route('/api/repository//build/', methods=['GET']) @parse_repository_name def get_repo_builds(namespace, repository): - permission = AdministerRepositoryPermission(namespace, repository) + permission = ModifyRepositoryPermission(namespace, repository) if permission.can(): def build_view(build_obj): if build_obj.status_url: @@ -400,6 +394,29 @@ def get_repo_builds(namespace, repository): abort(403) # Permissions denied +@app.route('/api/repository//build/', methods=['POST']) +@parse_repository_name +def request_repo_build(namespace, repository): + permission = ModifyRepositoryPermission(namespace, repository) + if permission.can(): + logger.debug('User requested repository initialization.') + dockerfile_source = request.files['initializedata'] + dockerfile_id = user_files.store_file(dockerfile_source) + + repo = model.get_repository(namespace, repository) + token = model.create_access_token(repo, 'write') + + host = urlparse.urlparse(request.url).netloc + tag = '%s/%s/%s' % (host, repo.namespace, repo.name) + build_request = model.create_repository_build(repo, token, dockerfile_id, + tag) + dockerfile_build_queue.put(json.dumps({'request_id': build_request.id})) + + return make_response('Created', 201) + + abort(403) # Permissions denied + + def role_view(repo_perm_obj): return { 'role': repo_perm_obj.role.name diff --git a/endpoints/test.py b/endpoints/test.py new file mode 100644 index 000000000..230ae2f70 --- /dev/null +++ b/endpoints/test.py @@ -0,0 +1,53 @@ +from random import SystemRandom +from flask import jsonify, send_file +from app import app + + +@app.route('/test/build/status', methods=['GET']) +def generate_random_build_status(): + response = { + 'id': 1, + 'total_commands': None, + 'total_images': None, + 'current_command': None, + 'current_image': None, + 'image_completion_percent': None, + 'status': None, + 'message': None, + } + + random = SystemRandom() + phases = { + 'waiting': {}, + 'starting': { + 'total_commands': 7, + 'current_command': 0, + }, + 'initializing': {}, + 'error': { + 'message': 'Oops!' + }, + 'complete': {}, + 'building': { + 'total_commands': 7, + 'current_command': random.randint(1, 7), + }, + 'pushing': { + 'total_commands': 7, + 'current_command': 7, + 'total_images': 11, + 'current_image': random.randint(1, 11), + 'image_completion_percent': random.randint(0, 100), + }, + } + + phase = random.choice(phases.keys()) + response['status'] = phase + response.update(phases[phase]) + + return jsonify(response) + + +@app.route('/test/build', methods=['GET']) +def create_build(): + return send_file('templates/testbuild.html') diff --git a/initdb.py b/initdb.py index 79519ee7f..c5afbdc93 100644 --- a/initdb.py +++ b/initdb.py @@ -148,7 +148,10 @@ if __name__ == '__main__': 'Empty repository which is building.', False, [], (0, [], None)) - build = model.create_repository_build(building, '123-45-6789') + token = model.create_access_token(building, 'write') + tag = 'ci.devtable.com:5000/%s/%s' % (building.namespace, building.name) + build = model.create_repository_build(building, token, '123-45-6789', tag) + build.build_node_id = 1 build.phase = 'building' build.status_url = 'http://localhost:5000/test/build/status' diff --git a/templates/testbuild.html b/templates/testbuild.html new file mode 100644 index 000000000..be740c804 --- /dev/null +++ b/templates/testbuild.html @@ -0,0 +1,8 @@ + + +
+ + +
+ + \ No newline at end of file diff --git a/workers/dockerfilebuild.py b/workers/dockerfilebuild.py index d04f98ced..cbf8718d9 100644 --- a/workers/dockerfilebuild.py +++ b/workers/dockerfilebuild.py @@ -108,8 +108,9 @@ def babysit_builder(request): repo = repository_build.repository payload = { - 'tag': 'quay.io/%s/%s' % (repo.namespace, repo.name), + 'tag': repository_build.tag, 'resource_url': user_files.get_file_url(repository_build.resource_key), + 'token': repository_build.access_token.code, } start_build = requests.post(build_endpoint, data=payload) @@ -146,7 +147,7 @@ def babysit_builder(request): def process_work_items(pool): logger.debug('Getting work item from queue.') - item = dockerfile_build_queue.get() + item = dockerfile_build_queue.get(processing_time=60*60) # allow 1 hr while item: logger.debug('Queue gave us some work: %s' % item.body) From 1ff67f688df1fef4ac43079dba7af8f3208c79f4 Mon Sep 17 00:00:00 2001 From: yackob03 Date: Fri, 25 Oct 2013 18:18:06 -0400 Subject: [PATCH 15/57] Fix the content-type for S3 uploads of user data. --- data/userfiles.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/data/userfiles.py b/data/userfiles.py index 7c017090f..c63d8fe53 100644 --- a/data/userfiles.py +++ b/data/userfiles.py @@ -1,10 +1,14 @@ import boto import os +import logging from boto.s3.key import Key from uuid import uuid4 +logger = logging.getLogger(__name__) + + class S3FileWriteException(Exception): pass @@ -20,7 +24,9 @@ class UserRequestFiles(object): def store_file(self, flask_file): file_id = str(uuid4()) full_key = os.path.join(self._prefix, file_id) - k = Key(full_key) + k = Key(self._bucket, full_key) + logger.debug('Setting s3 content type to: %s' % flask_file.content_type) + k.set_metadata('Content-Type', flask_file.content_type) bytes_written = k.set_contents_from_file(flask_file) if bytes_written == 0: @@ -30,5 +36,5 @@ class UserRequestFiles(object): def get_file_url(self, file_id, expires_in=300): full_key = os.path.join(self._prefix, file_id) - k = Key(full_key) + k = Key(self._bucket, full_key) return k.generate_url(expires_in) From 660a931d6b506f08b6846a20686f32c5656f3792 Mon Sep 17 00:00:00 2001 From: yackob03 Date: Fri, 25 Oct 2013 18:46:56 -0400 Subject: [PATCH 16/57] Docker apparently sometimes uses xz as a compression format. --- buildserver/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/buildserver/Dockerfile b/buildserver/Dockerfile index eb69c1eb2..2b90e36a6 100644 --- a/buildserver/Dockerfile +++ b/buildserver/Dockerfile @@ -3,7 +3,7 @@ MAINTAINER jake@devtable.com RUN echo deb http://archive.ubuntu.com/ubuntu precise universe > /etc/apt/sources.list.d/universe.list RUN apt-get update -qq -RUN apt-get install -qqy iptables ca-certificates lxc python-virtualenv git python-dev +RUN apt-get install -qqy iptables ca-certificates lxc python-virtualenv git python-dev xz-utils # This will use the latest public release. To use your own, comment it out... ADD https://get.docker.io/builds/Linux/x86_64/docker-latest /usr/local/bin/docker From 89df3d10be0cfe61c2b12884d28cb3aaaa5ff8af Mon Sep 17 00:00:00 2001 From: yackob03 Date: Fri, 25 Oct 2013 20:08:02 -0400 Subject: [PATCH 17/57] Add dependencies for docker in docker. Fix the aufs startup command. Fix a bug in the buildserver. --- buildserver/Dockerfile | 2 +- buildserver/Readme.md | 2 +- buildserver/buildserver.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/buildserver/Dockerfile b/buildserver/Dockerfile index 2b90e36a6..0a190a7cd 100644 --- a/buildserver/Dockerfile +++ b/buildserver/Dockerfile @@ -3,7 +3,7 @@ MAINTAINER jake@devtable.com RUN echo deb http://archive.ubuntu.com/ubuntu precise universe > /etc/apt/sources.list.d/universe.list RUN apt-get update -qq -RUN apt-get install -qqy iptables ca-certificates lxc python-virtualenv git python-dev xz-utils +RUN apt-get install -qqy iptables ca-certificates lxc python-virtualenv git python-dev xz-utils aufs-tools # This will use the latest public release. To use your own, comment it out... ADD https://get.docker.io/builds/Linux/x86_64/docker-latest /usr/local/bin/docker diff --git a/buildserver/Readme.md b/buildserver/Readme.md index 6b36df9ee..8dc0dff56 100644 --- a/buildserver/Readme.md +++ b/buildserver/Readme.md @@ -9,5 +9,5 @@ To run: ``` sudo docker pull quay.io/quay/buildserver -sudo docker run -d -privileged -lxc-conf="aa_profile=unconfined" quay.io/quay/buildserver +sudo docker run -d -privileged -lxc-conf="lxc.aa_profile=unconfined" quay.io/quay/buildserver ``` \ No newline at end of file diff --git a/buildserver/buildserver.py b/buildserver/buildserver.py index 5e65a2392..2ac90e9fc 100644 --- a/buildserver/buildserver.py +++ b/buildserver/buildserver.py @@ -147,7 +147,7 @@ def start_build(): # Save the token host = re.match(r'([a-z0-9.:]+)/.+/.+$', tag_name) if host: - docker_endpoint = 'http://%s/v1/' % host.group(0) + docker_endpoint = 'http://%s/v1/' % host.group(1) dockercfg_path = os.path.join(os.environ.get('HOME', '.'), '.dockercfg') token = b64encode('$token:%s' % acccess_token) with open(dockercfg_path, 'w') as dockercfg: From dc7a62db67f71ed4dff918dda2ccb67c585aa0c6 Mon Sep 17 00:00:00 2001 From: yackob03 Date: Fri, 25 Oct 2013 20:08:44 -0400 Subject: [PATCH 18/57] Add the test db with the random test endpoint. --- test/data/test.db | Bin 77824 -> 79872 bytes 1 file changed, 0 insertions(+), 0 deletions(-) diff --git a/test/data/test.db b/test/data/test.db index 32075f2bdf9f616e443e970241a748215068c41c..77546d9b6e1109922e319cb38ca255e0026e7e62 100644 GIT binary patch delta 1268 zcmbu8Urbw79LMju=RzrzvEsIZ$bgPfN@;uB+rO|O6k2X6w5u)uG8bs+g>~!b9PI`q zW^~mTA9UtNB7Y1IjV8L7EcE(doDYjLNZBGW`hbZ>-HSeuI3Ls)$>wRZ;T|xU9O5RMtGUR%Otk9@KpgxDJoTtx{#C~ z_KuG5FZJ{}gN}eT5Oi7x27Ucr8_)9&&!8{J`GfXeOJK;#xg4&6eg|>R`CSgT$3JTG zoZ|UnBWvKa23Bh{*BjY3*4AdQ=&Uxgh2!XQqMJb@6dIvF!bk8TnHHK@&2%DByYK;B zT1pHE-=gp>64^R-_AsgP{~c3ik;yJT!exa35b=u_$?0$4ljn$TBt14Ntvy|AFmhUx zS!=P{>=RL4M7W#?g=2zlB7ULG%(AThiA5hyMq`oa^b~vz`qqLZiX>EtZqgm}H;)xP zHKa@HBnpiNz3zb@uicP^CMJZLnXyFtyf6*^w6bj9us?q_ZN!Pt6kMNt9&V=naCNC2 zslbq{f~BSB;8gY?)TE{GXsMzjRoNg>cr+;09-WQ|vzLU6@tJ5Merc`{-PqG!#-fpD zmYefOR{!TtWxdY}73sr=o;g9OzSg1y`LYSA!L|GcxzWd|+N1EED3|?)B}6mEQfcPh zlFIz86j8$z>A#{m`UU!DkTXuWx6%k>D>YEInja zYIGdNR~<-Okj`eyU}LonryNbNvEGMtAkWTKsU!-80==dvWGQ+fG$j(;G1btQlw~>I?67;AEx{nh$@-Muzmf*!|f{sH3?G*%V z`NFgp(S!(Z6H+d`O^WCXT;y!UXAl@3D&Kifj*8{V{VHFh&>H%SK0`kwyWT|fa5H=; z)*%*hqLq%Q;qP1{GQ!cVF4O|!TPDO2Rfplm))3MFy4!?I(0uoMk@-x*Ade`!{Uwch z;oHwWs0-@vc+g3R5uOk~xx+KrdyK3U(_xAZ)8FB7;x0%ZNVZB8b`?q~3grQ}b4TD< pt_8lzsd0+Y!rrC{S;+1BaH>`ZFK^nBmH4j%H_5@qCYRY6`4?D$YLEZ` delta 849 zcmX}pUr19?90&08yXS7JHCL9l#TwJn%|Dyh{jp`MoYSqTBmacj3M1#gTxwR!ih`W= z5Ito-rw>ttJ+zG=^Ac1D`45T;EU2E+MK3)hWG{sn)zQcc_j51joO?On^X*;Wdq45l zHOV?ch>zX>FH^tKs?!nEXM={9zgq1GQAM`t&r(fu2Ix8mg0je+=*o+~Ak**Wx zX^T*>|3q1r<-mb+#reT1AsEaLoo#Dv&pU1F7OGAOo#w8Z`tz<5pSQKFqse`|$sama z+f$y0GY&Q0wr(j1b)K+zjvcbO0^OCTN<6~JTEXM5wD+!B?x24Y0T4Fnt zUs~f02HnM;^LgUDRqegYZ{?K|Vj{!@Vff0mbMO0$HR8c=l#};m>13+hT(}}b<3JoW z5j`G>RF9_c3MtSrLdRH`QJU%k8G-Fj zpyV9fYubRxd?!ALI6#ld2{pcmY{GA`4X7Vf)4rT^{4i*PEUX$(%J7;ViK8@zXb~e5n2csG2jLWblmJJzrG3J8Y)m67*N!Wc^iq3=_ z|HQPzch(HwS16Y^Mq!G;6nB;0V@VzX4ZAW?7cU1RR_3JQc$*xZv5j~$?p5n$DwPVZ zs6vf_j+XY$# Date: Sat, 26 Oct 2013 16:03:11 -0400 Subject: [PATCH 19/57] Work in progress: Add the UI for the build status and start on the file drop stuff --- application.py | 2 +- endpoints/api.py | 14 ++-- static/css/quay.css | 101 +++++++++++++++++++++++-- static/directives/build-status.html | 8 ++ static/js/app.js | 69 ++++++++++++++++- static/js/controllers.js | 41 +++++++++- static/lib/angular-strap.min.js | 8 ++ static/partials/build-status-item.html | 5 ++ static/partials/new-repo.html | 26 ++++++- static/partials/repo-admin.html | 2 +- static/partials/view-repo.html | 32 +++++--- templates/base.html | 1 + test/data/test.db | Bin 79872 -> 79872 bytes 13 files changed, 278 insertions(+), 31 deletions(-) create mode 100644 static/directives/build-status.html create mode 100644 static/lib/angular-strap.min.js create mode 100644 static/partials/build-status-item.html diff --git a/application.py b/application.py index 2244902c1..d3feefcb0 100644 --- a/application.py +++ b/application.py @@ -23,4 +23,4 @@ if application.config.get('INCLUDE_TEST_ENDPOINTS', False): application.debug = True if __name__ == '__main__': - application.run(port=5000, debug=True, host='0.0.0.0') + application.run(port=5000, debug=True, threaded=True, host='0.0.0.0') diff --git a/endpoints/api.py b/endpoints/api.py index 630130c98..5ba2272a0 100644 --- a/endpoints/api.py +++ b/endpoints/api.py @@ -180,22 +180,22 @@ user_files = UserRequestFiles(app.config['AWS_ACCESS_KEY'], app.config['REGISTRY_S3_BUCKET']) -@app.route('/api/repository/', methods=['POST']) +@app.route('/api/repository', methods=['POST']) @api_login_required def create_repo_api(): owner = current_user.db_user() namespace_name = owner.username - repository_name = request.values['repository'] - visibility = request.values['visibility'] + repository_name = request.get_json()['repository'] + visibility = request.get_json()['visibility'] repo = model.create_repository(namespace_name, repository_name, owner, visibility) - resp = make_response('Created', 201) - resp.headers['Location'] = url_for('get_repo_api', namespace=namespace_name, - repository=repository_name) - return resp + return jsonify({ + 'namespace': namespace_name, + 'name': repository_name + }) @app.route('/api/find/repository', methods=['GET']) diff --git a/static/css/quay.css b/static/css/quay.css index f1082c8dc..15fd496c3 100644 --- a/static/css/quay.css +++ b/static/css/quay.css @@ -13,6 +13,30 @@ color: #428bca; } +.build-statuses { +} + +.build-status-container { + padding: 4px; + margin-bottom: 10px; + border-bottom: 1px solid #eee; +} + +.build-status-container .build-message { + display: block; + white-space: nowrap; +} + +.build-status-container .progress { + height: 12px; + margin: 0px; + margin-top: 10px; +} + +.build-status-container:last-child { + margin-bottom: 0px; + border-bottom: 0px solid white; +} .repo-circle { position: relative; @@ -579,6 +603,7 @@ p.editable:hover i { } .repo .description { + margin-top: 10px; margin-bottom: 40px; } @@ -608,22 +633,70 @@ p.editable:hover i { display: inline-block; } +.repo .status-boxes { + float: right; + margin-bottom: 20px; +} + +.repo .status-boxes .status-box { + cursor: pointer; + display: inline-block; + border: 1px solid #eee; + border-radius: 4px; +} + +.repo .status-boxes .status-box .title { + padding: 4px; + display: inline-block; + padding-left: 10px; + padding-right: 10px; +} + +.repo .status-boxes .status-box .title i { + margin-right: 6px; +} + +.repo .status-boxes .status-box .count { + display: inline-block; + background-image: linear-gradient(to bottom,#e8e8e8 0,#f5f5f5 100%); + padding: 4px; + padding-left: 10px; + padding-right: 10px; + font-weight: bold; + + transform: scaleX(0); + -webkit-transform: scaleX(0); + -moz-transform: scaleX(0); + + transition: transform 500ms ease-in-out; + -webkit-transition: -webkit-transform 500ms ease-in-out; + -moz-transition: -moz-transform 500ms ease-in-out; +} + +.repo .status-boxes .status-box .count.visible { + transform: scaleX(1); + -webkit-transform: scaleX(1); + -moz-transform: scaleX(1); +} + .repo .pull-command { float: right; display: inline-block; - font-size: 1.2em; + font-size: 0.8em; position: relative; - margin-right: 10px; + margin-top: 30px; + margin-right: 26px; } -.repo .pull-command .pull-container { +.repo .pull-container { display: inline-block; width: 300px; + margin-left: 10px; margin-right: 10px; vertical-align: middle; } -.repo .pull-command input { +.repo .pull-container input { cursor: default; background: white; color: #666; @@ -788,8 +861,22 @@ p.editable:hover i { cursor: pointer; } -.repo .description p { - margin-bottom: 6px; + +.repo .build-info { + padding: 10px; + margin: 0px; +} + +.repo .build-info .progress { + margin: 0px; + margin-top: 10px; +} + +.repo .section { + display: block; + margin-bottom: 20px; + padding-bottom: 20px; + border-bottom: 1px solid #eee; } .repo .description p:last-child { @@ -967,7 +1054,7 @@ p.editable:hover i { background: rgb(253, 191, 191); } -.repo-admin .repo-access-state .state-icon i.fa-unlock-alt { +.repo-admin .repo-access-state .state-icon i.fa-unlock { background: rgb(170, 236, 170); } diff --git a/static/directives/build-status.html b/static/directives/build-status.html new file mode 100644 index 000000000..e1c1e2ce6 --- /dev/null +++ b/static/directives/build-status.html @@ -0,0 +1,8 @@ +
+ {{ getBuildMessage(build) }} +
+
+
+
+ +
diff --git a/static/js/app.js b/static/js/app.js index 568c19637..ae555c2d4 100644 --- a/static/js/app.js +++ b/static/js/app.js @@ -1,5 +1,5 @@ // Start the application code itself. -quayApp = angular.module('quay', ['restangular', 'angularMoment', 'angulartics', 'angulartics.mixpanel'], function($provide) { +quayApp = angular.module('quay', ['restangular', 'angularMoment', 'angulartics', 'angulartics.mixpanel', '$strap.directives'], function($provide) { $provide.factory('UserService', ['Restangular', function(Restangular) { var userResponse = { verified: false, @@ -177,7 +177,72 @@ quayApp.directive('repoCircle', function () { 'repo': '=repo' }, controller: function($scope, $element) { - window.console.log($scope); + } + }; + return directiveDefinitionObject; +}); + + +quayApp.directive('buildStatus', function () { + var directiveDefinitionObject = { + priority: 0, + templateUrl: '/static/directives/build-status.html', + replace: false, + transclude: false, + restrict: 'C', + scope: { + 'build': '=build' + }, + controller: function($scope, $element) { + $scope.getBuildProgress = function(buildInfo) { + switch (buildInfo.status) { + case 'building': + return (buildInfo.current_command / buildInfo.total_commands) * 100; + break; + + case 'pushing': + return (buildInfo.current_image / buildInfo.total_images) * 100; + break; + + case 'complete': + return 100; + break; + + case 'initializing': + case 'starting': + case 'waiting': + return 0; + break; + } + + return -1; + }; + + $scope.getBuildMessage = function(buildInfo) { + switch (buildInfo.status) { + case 'initializing': + return 'Starting Dockerfile build'; + break; + + case 'starting': + case 'waiting': + case 'building': + return 'Building image from Dockerfile'; + break; + + case 'pushing': + return 'Pushing image built from Dockerfile'; + break; + + case 'complete': + return 'Dockerfile build completed and pushed'; + break; + + case 'error': + return 'Dockerfile build failed: ' + buildInfo.message; + break; + } + }; } }; return directiveDefinitionObject; diff --git a/static/js/controllers.js b/static/js/controllers.js index e867ac75d..984982c52 100644 --- a/static/js/controllers.js +++ b/static/js/controllers.js @@ -339,6 +339,19 @@ function RepoCtrl($scope, Restangular, $routeParams, $rootScope, $location, $tim } }); + var startBuildInfoTimer = function(repo) { + setInterval(function() { + $scope.$apply(function() { getBuildInfo(repo); }); + }, 5000); + }; + + var getBuildInfo = function(repo) { + var buildInfo = Restangular.one('repository/' + repo.namespace + '/' + repo.name + '/build/'); + buildInfo.get().then(function(resp) { + $scope.buildsInfo = resp.builds; + }); + }; + var listImages = function() { if ($scope.imageHistory) { return; } @@ -443,6 +456,10 @@ function RepoCtrl($scope, Restangular, $routeParams, $rootScope, $location, $tim $('#copyClipboard').clipboardCopy(); $scope.loading = false; + + if (repo.is_building) { + getBuildInfo(repo); + } }, function() { $scope.repo = null; $scope.loading = false; @@ -895,7 +912,7 @@ function V1Ctrl($scope, UserService) { }; } -function NewRepoCtrl($scope, UserService) { +function NewRepoCtrl($scope, $location, UserService, Restangular) { $scope.repo = { 'is_public': 1, 'description': '', @@ -906,7 +923,7 @@ function NewRepoCtrl($scope, UserService) { $scope.user = currentUser; if ($scope.user.anonymous) { - document.location = '/signin/'; + $location.path('/signin/'); } }, true); @@ -931,4 +948,24 @@ function NewRepoCtrl($scope, UserService) { $('#editModal').modal('hide'); $scope.repo.description = $('#wmd-input-description')[0].value; }; + + $scope.createNewRepo = function() { + $scope.creating = true; + var repo = $scope.repo; + var data = { + 'repository': repo.name, + 'visibility': repo.is_public == '1' ? 'public' : 'private' + }; + + var createPost = Restangular.one('repository'); + createPost.customPOST(data).then(function(created) { + // Repository created. Start the upload process if applicable. + + // Otherwise, redirect to the repo page. + $location.path('/repository/' + created.namespace + '/' + created.name); + }, function() { + $('#cannotcreateModal').modal(); + $scope.creating = false; + }); + }; } \ No newline at end of file diff --git a/static/lib/angular-strap.min.js b/static/lib/angular-strap.min.js new file mode 100644 index 000000000..7bf8788fb --- /dev/null +++ b/static/lib/angular-strap.min.js @@ -0,0 +1,8 @@ +/** + * AngularStrap - Twitter Bootstrap directives for AngularJS + * @version v0.7.5 - 2013-07-21 + * @link http://mgcrea.github.com/angular-strap + * @author Olivier Louvignes + * @license MIT License, http://www.opensource.org/licenses/MIT + */ +angular.module("$strap.config",[]).value("$strapConfig",{}),angular.module("$strap.filters",["$strap.config"]),angular.module("$strap.directives",["$strap.config"]),angular.module("$strap",["$strap.filters","$strap.directives","$strap.config"]),angular.module("$strap.directives").directive("bsAlert",["$parse","$timeout","$compile",function(t,e,n){return{restrict:"A",link:function(a,i,o){var r=t(o.bsAlert),s=(r.assign,r(a)),l=function(t){e(function(){i.alert("close")},1*t)};o.bsAlert?a.$watch(o.bsAlert,function(t,e){s=t,i.html((t.title?""+t.title+" ":"")+t.content||""),t.closed&&i.hide(),n(i.contents())(a),(t.type||e.type)&&(e.type&&i.removeClass("alert-"+e.type),t.type&&i.addClass("alert-"+t.type)),angular.isDefined(t.closeAfter)?l(t.closeAfter):o.closeAfter&&l(o.closeAfter),(angular.isUndefined(o.closeButton)||"0"!==o.closeButton&&"false"!==o.closeButton)&&i.prepend('')},!0):((angular.isUndefined(o.closeButton)||"0"!==o.closeButton&&"false"!==o.closeButton)&&i.prepend(''),o.closeAfter&&l(o.closeAfter)),i.addClass("alert").alert(),i.hasClass("fade")&&(i.removeClass("in"),setTimeout(function(){i.addClass("in")}));var u=o.ngRepeat&&o.ngRepeat.split(" in ").pop();i.on("close",function(t){var e;u?(t.preventDefault(),i.removeClass("in"),e=function(){i.trigger("closed"),a.$parent&&a.$parent.$apply(function(){for(var t=u.split("."),e=a.$parent,n=0;t.length>n;++n)e&&(e=e[t[n]]);e&&e.splice(a.$index,1)})},$.support.transition&&i.hasClass("fade")?i.on($.support.transition.end,e):e()):s&&(t.preventDefault(),i.removeClass("in"),e=function(){i.trigger("closed"),a.$apply(function(){s.closed=!0})},$.support.transition&&i.hasClass("fade")?i.on($.support.transition.end,e):e())})}}}]),angular.module("$strap.directives").directive("bsButton",["$parse","$timeout",function(t){return{restrict:"A",require:"?ngModel",link:function(e,n,a,i){if(i){n.parent('[data-toggle="buttons-checkbox"], [data-toggle="buttons-radio"]').length||n.attr("data-toggle","button");var o=!!e.$eval(a.ngModel);o&&n.addClass("active"),e.$watch(a.ngModel,function(t,e){var a=!!t,i=!!e;a!==i?$.fn.button.Constructor.prototype.toggle.call(r):a&&!o&&n.addClass("active")})}n.hasClass("btn")||n.on("click.button.data-api",function(){n.button("toggle")}),n.button();var r=n.data("button");r.toggle=function(){if(!i)return $.fn.button.Constructor.prototype.toggle.call(this);var a=n.parent('[data-toggle="buttons-radio"]');a.length?(n.siblings("[ng-model]").each(function(n,a){t($(a).attr("ng-model")).assign(e,!1)}),e.$digest(),i.$modelValue||(i.$setViewValue(!i.$modelValue),e.$digest())):e.$apply(function(){i.$setViewValue(!i.$modelValue)})}}}}]).directive("bsButtonsCheckbox",["$parse",function(){return{restrict:"A",require:"?ngModel",compile:function(t){t.attr("data-toggle","buttons-checkbox").find("a, button").each(function(t,e){$(e).attr("bs-button","")})}}}]).directive("bsButtonsRadio",["$timeout",function(t){return{restrict:"A",require:"?ngModel",compile:function(e,n){return e.attr("data-toggle","buttons-radio"),n.ngModel||e.find("a, button").each(function(t,e){$(e).attr("bs-button","")}),function(e,n,a,i){i&&(t(function(){n.find("[value]").button().filter('[value="'+i.$viewValue+'"]').addClass("active")}),n.on("click.button.data-api",function(t){e.$apply(function(){i.$setViewValue($(t.target).closest("button").attr("value"))})}),e.$watch(a.ngModel,function(t,i){if(t!==i){var o=n.find('[value="'+e.$eval(a.ngModel)+'"]');o.length&&o.button("toggle")}}))}}}}]),angular.module("$strap.directives").directive("bsButtonSelect",["$parse","$timeout",function(t){return{restrict:"A",require:"?ngModel",link:function(e,n,a,i){var o=t(a.bsButtonSelect);o.assign,i&&(n.text(e.$eval(a.ngModel)),e.$watch(a.ngModel,function(t){n.text(t)}));var r,s,l,u;n.bind("click",function(){r=o(e),s=i?e.$eval(a.ngModel):n.text(),l=r.indexOf(s),u=l>r.length-2?r[0]:r[l+1],e.$apply(function(){n.text(u),i&&i.$setViewValue(u)})})}}}]),angular.module("$strap.directives").directive("bsDatepicker",["$timeout","$strapConfig",function(t,e){var n=/(iP(a|o)d|iPhone)/g.test(navigator.userAgent),a=function a(t){return t=t||"en",{"/":"[\\/]","-":"[-]",".":"[.]"," ":"[\\s]",dd:"(?:(?:[0-2]?[0-9]{1})|(?:[3][01]{1}))",d:"(?:(?:[0-2]?[0-9]{1})|(?:[3][01]{1}))",mm:"(?:[0]?[1-9]|[1][012])",m:"(?:[0]?[1-9]|[1][012])",DD:"(?:"+$.fn.datepicker.dates[t].days.join("|")+")",D:"(?:"+$.fn.datepicker.dates[t].daysShort.join("|")+")",MM:"(?:"+$.fn.datepicker.dates[t].months.join("|")+")",M:"(?:"+$.fn.datepicker.dates[t].monthsShort.join("|")+")",yyyy:"(?:(?:[1]{1}[0-9]{1}[0-9]{1}[0-9]{1})|(?:[2]{1}[0-9]{3}))(?![[0-9]])",yy:"(?:(?:[0-9]{1}[0-9]{1}))(?![[0-9]])"}},i=function i(t,e){var n,i=t,o=a(e);return n=0,angular.forEach(o,function(t,e){i=i.split(e).join("${"+n+"}"),n++}),n=0,angular.forEach(o,function(t){i=i.split("${"+n+"}").join(t),n++}),RegExp("^"+i+"$",["i"])};return{restrict:"A",require:"?ngModel",link:function(t,a,o,r){var s=angular.extend({autoclose:!0},e.datepicker||{}),l=o.dateType||s.type||"date";angular.forEach(["format","weekStart","calendarWeeks","startDate","endDate","daysOfWeekDisabled","autoclose","startView","minViewMode","todayBtn","todayHighlight","keyboardNavigation","language","forceParse"],function(t){angular.isDefined(o[t])&&(s[t]=o[t])});var u=s.language||"en",c=o.dateFormat||s.format||$.fn.datepicker.dates[u]&&$.fn.datepicker.dates[u].format||"mm/dd/yyyy",d=n?"yyyy-mm-dd":c,p=i(d,u);r&&(r.$formatters.unshift(function(t){return"date"===l&&angular.isString(t)&&t?$.fn.datepicker.DPGlobal.parseDate(t,$.fn.datepicker.DPGlobal.parseFormat(c),u):t}),r.$parsers.unshift(function(t){return t?"date"===l&&angular.isDate(t)?(r.$setValidity("date",!0),t):angular.isString(t)&&p.test(t)?(r.$setValidity("date",!0),n?new Date(t):"string"===l?t:$.fn.datepicker.DPGlobal.parseDate(t,$.fn.datepicker.DPGlobal.parseFormat(d),u)):(r.$setValidity("date",!1),void 0):(r.$setValidity("date",!0),null)}),r.$render=function(){if(n){var t=r.$viewValue?$.fn.datepicker.DPGlobal.formatDate(r.$viewValue,$.fn.datepicker.DPGlobal.parseFormat(d),u):"";return a.val(t),t}return r.$viewValue||a.val(""),a.datepicker("update",r.$viewValue)}),n?a.prop("type","date").css("-webkit-appearance","textfield"):(r&&a.on("changeDate",function(e){t.$apply(function(){r.$setViewValue("string"===l?a.val():e.date)})}),a.datepicker(angular.extend(s,{format:d,language:u})),t.$on("$destroy",function(){var t=a.data("datepicker");t&&(t.picker.remove(),a.data("datepicker",null))}),o.$observe("startDate",function(t){a.datepicker("setStartDate",t)}),o.$observe("endDate",function(t){a.datepicker("setEndDate",t)}));var f=a.siblings('[data-toggle="datepicker"]');f.length&&f.on("click",function(){a.prop("disabled")||a.trigger("focus")})}}}]),angular.module("$strap.directives").directive("bsDropdown",["$parse","$compile","$timeout",function(t,e,n){var a=function(t,e){return e||(e=['"]),angular.forEach(t,function(t,n){if(t.divider)return e.splice(n+1,0,'
  • ');var i=""+'"+(t.text||"")+"";t.submenu&&t.submenu.length&&(i+=a(t.submenu).join("\n")),i+="",e.splice(n+1,0,i)}),e};return{restrict:"EA",scope:!0,link:function(i,o,r){var s=t(r.bsDropdown),l=s(i);n(function(){!angular.isArray(l);var t=angular.element(a(l).join(""));t.insertAfter(o),e(o.next("ul.dropdown-menu"))(i)}),o.addClass("dropdown-toggle").attr("data-toggle","dropdown")}}}]),angular.module("$strap.directives").factory("$modal",["$rootScope","$compile","$http","$timeout","$q","$templateCache","$strapConfig",function(t,e,n,a,i,o,r){var s=function s(s){function l(s){var l=angular.extend({show:!0},r.modal,s),u=l.scope?l.scope:t.$new(),c=l.template;return i.when(o.get(c)||n.get(c,{cache:!0}).then(function(t){return t.data})).then(function(t){var n=c.replace(".html","").replace(/[\/|\.|:]/g,"-")+"-"+u.$id,i=$('
    ').attr("id",n).addClass("fade").html(t);return l.modalClass&&i.addClass(l.modalClass),$("body").append(i),a(function(){e(i)(u)}),u.$modal=function(t){i.modal(t)},angular.forEach(["show","hide"],function(t){u[t]=function(){i.modal(t)}}),u.dismiss=u.hide,angular.forEach(["show","shown","hide","hidden"],function(t){i.on(t,function(e){u.$emit("modal-"+t,e)})}),i.on("shown",function(){$("input[autofocus], textarea[autofocus]",i).first().trigger("focus")}),i.on("hidden",function(){l.persist||u.$destroy()}),u.$on("$destroy",function(){i.remove()}),i.modal(l),i})}return new l(s)};return s}]).directive("bsModal",["$q","$modal",function(t,e){return{restrict:"A",scope:!0,link:function(n,a,i){var o={template:n.$eval(i.bsModal),persist:!0,show:!1,scope:n};angular.forEach(["modalClass","backdrop","keyboard"],function(t){angular.isDefined(i[t])&&(o[t]=i[t])}),t.when(e(o)).then(function(t){a.attr("data-target","#"+t.attr("id")).attr("data-toggle","modal")})}}}]),angular.module("$strap.directives").directive("bsNavbar",["$location",function(t){return{restrict:"A",link:function(e,n){e.$watch(function(){return t.path()},function(t){$("li[data-match-route]",n).each(function(e,n){var a=angular.element(n),i=a.attr("data-match-route"),o=RegExp("^"+i+"$",["i"]);o.test(t)?a.addClass("active").find(".collapse.in").collapse("hide"):a.removeClass("active")})})}}}]),angular.module("$strap.directives").directive("bsPopover",["$parse","$compile","$http","$timeout","$q","$templateCache",function(t,e,n,a,i,o){return $("body").on("keyup",function(t){27===t.keyCode&&$(".popover.in").each(function(){$(this).popover("hide")})}),{restrict:"A",scope:!0,link:function(r,s,l){var u=t(l.bsPopover),c=(u.assign,u(r)),d={};angular.isObject(c)&&(d=c),i.when(d.content||o.get(c)||n.get(c,{cache:!0})).then(function(t){angular.isObject(t)&&(t=t.data),l.unique&&s.on("show",function(){$(".popover.in").each(function(){var t=$(this),e=t.data("bs.popover");e&&!e.$element.is(s)&&t.popover("hide")})}),l.hide&&r.$watch(l.hide,function(t,e){t?n.hide():t!==e&&n.show()}),l.show&&r.$watch(l.show,function(t,e){t?a(function(){n.show()}):t!==e&&n.hide()}),s.popover(angular.extend({},d,{content:t,html:!0}));var n=s.data("bs.popover");n.hasContent=function(){return this.getTitle()||t},n.getPosition=function(){var t=$.fn.popover.Constructor.prototype.getPosition.apply(this,arguments);return e(this.$tip)(r),r.$digest(),this.$tip.data("bs.popover",this),t},r.$popover=function(t){n(t)},angular.forEach(["show","hide"],function(t){r[t]=function(){n[t]()}}),r.dismiss=r.hide,angular.forEach(["show","shown","hide","hidden"],function(t){s.on(t,function(e){r.$emit("popover-"+t,e)})})})}}}]),angular.module("$strap.directives").directive("bsSelect",["$timeout",function(t){return{restrict:"A",require:"?ngModel",link:function(e,n,a,i){var o=e.$eval(a.bsSelect)||{};t(function(){n.selectpicker(o),n.next().removeClass("ng-scope")}),i&&e.$watch(a.ngModel,function(t,e){angular.equals(t,e)||n.selectpicker("refresh")})}}}]),angular.module("$strap.directives").directive("bsTabs",["$parse","$compile","$timeout",function(t,e,n){var a='
    ';return{restrict:"A",require:"?ngModel",priority:0,scope:!0,template:a,replace:!0,transclude:!0,compile:function(){return function(e,a,i,o){var r=t(i.bsTabs);r.assign,r(e),e.panes=[];var s,l,u,c=a.find("ul.nav-tabs"),d=a.find("div.tab-content"),p=0;n(function(){d.find("[data-title], [data-tab]").each(function(t){var n=angular.element(this);s="tab-"+e.$id+"-"+t,l=n.data("title")||n.data("tab"),u=!u&&n.hasClass("active"),n.attr("id",s).addClass("tab-pane"),i.fade&&n.addClass("fade"),e.panes.push({id:s,title:l,content:this.innerHTML,active:u})}),e.panes.length&&!u&&(d.find(".tab-pane:first-child").addClass("active"+(i.fade?" in":"")),e.panes[0].active=!0)}),o&&(a.on("show",function(t){var n=$(t.target);e.$apply(function(){o.$setViewValue(n.data("index"))})}),e.$watch(i.ngModel,function(t){angular.isUndefined(t)||(p=t,setTimeout(function(){var e=$(c[0].querySelectorAll("li")[1*t]);e.hasClass("active")||e.children("a").tab("show")}))}))}}}}]),angular.module("$strap.directives").directive("bsTimepicker",["$timeout","$strapConfig",function(t,e){var n="((?:(?:[0-1][0-9])|(?:[2][0-3])|(?:[0-9])):(?:[0-5][0-9])(?::[0-5][0-9])?(?:\\s?(?:am|AM|pm|PM))?)";return{restrict:"A",require:"?ngModel",link:function(a,i,o,r){if(r){i.on("changeTime.timepicker",function(){t(function(){r.$setViewValue(i.val())})});var s=RegExp("^"+n+"$",["i"]);r.$parsers.unshift(function(t){return!t||s.test(t)?(r.$setValidity("time",!0),t):(r.$setValidity("time",!1),void 0)})}i.attr("data-toggle","timepicker"),i.parent().addClass("bootstrap-timepicker"),i.timepicker(e.timepicker||{});var l=i.data("timepicker"),u=i.siblings('[data-toggle="timepicker"]');u.length&&u.on("click",$.proxy(l.showWidget,l))}}}]),angular.module("$strap.directives").directive("bsTooltip",["$parse","$compile",function(t){return{restrict:"A",scope:!0,link:function(e,n,a){var i=t(a.bsTooltip),o=(i.assign,i(e));e.$watch(a.bsTooltip,function(t,e){t!==e&&(o=t)}),a.unique&&n.on("show",function(){$(".tooltip.in").each(function(){var t=$(this),e=t.data("tooltip");e&&!e.$element.is(n)&&t.tooltip("hide")})}),n.tooltip({title:function(){return angular.isFunction(o)?o.apply(null,arguments):o},html:!0});var r=n.data("tooltip");r.show=function(){var t=$.fn.tooltip.Constructor.prototype.show.apply(this,arguments);return this.tip().data("tooltip",this),t},e._tooltip=function(t){n.tooltip(t)},e.hide=function(){n.tooltip("hide")},e.show=function(){n.tooltip("show")},e.dismiss=e.hide}}}]),angular.module("$strap.directives").directive("bsTypeahead",["$parse",function(t){return{restrict:"A",require:"?ngModel",link:function(e,n,a,i){var o=t(a.bsTypeahead),r=(o.assign,o(e));e.$watch(a.bsTypeahead,function(t,e){t!==e&&(r=t)}),n.attr("data-provide","typeahead"),n.typeahead({source:function(){return angular.isFunction(r)?r.apply(null,arguments):r},minLength:a.minLength||1,items:a.items,updater:function(t){return i&&e.$apply(function(){i.$setViewValue(t)}),e.$emit("typeahead-updated",t),t}});var s=n.data("typeahead");s.lookup=function(){var t;return this.query=this.$element.val()||"",this.query.length +
    +
    +
    +
    diff --git a/static/partials/new-repo.html b/static/partials/new-repo.html index bc6c8becb..fb4ff2507 100644 --- a/static/partials/new-repo.html +++ b/static/partials/new-repo.html @@ -1,4 +1,8 @@ -
    +
    + +
    + +
    @@ -74,7 +78,7 @@
    - +
    @@ -105,3 +109,21 @@
    + + + diff --git a/static/partials/repo-admin.html b/static/partials/repo-admin.html index f8c2a0428..cda0c8470 100644 --- a/static/partials/repo-admin.html +++ b/static/partials/repo-admin.html @@ -124,7 +124,7 @@
    -
    +
    This repository is currently public and is visible to all users, and may be pulled by all users. diff --git a/static/partials/view-repo.html b/static/partials/view-repo.html index 671b06fba..a10c609a3 100644 --- a/static/partials/view-repo.html +++ b/static/partials/view-repo.html @@ -20,11 +20,10 @@ - + -
    - Get this repository: - + + + + + From be035637f618ab5281e89b845c929c4625bfd118 Mon Sep 17 00:00:00 2001 From: yackob03 Date: Sat, 26 Oct 2013 17:41:29 -0400 Subject: [PATCH 21/57] Require API login for the build APIs. --- endpoints/api.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/endpoints/api.py b/endpoints/api.py index 630130c98..82c2bd807 100644 --- a/endpoints/api.py +++ b/endpoints/api.py @@ -363,6 +363,7 @@ def get_repo_api(namespace, repository): @app.route('/api/repository//build/', methods=['GET']) +@api_login_required @parse_repository_name def get_repo_builds(namespace, repository): permission = ModifyRepositoryPermission(namespace, repository) @@ -395,6 +396,7 @@ def get_repo_builds(namespace, repository): @app.route('/api/repository//build/', methods=['POST']) +@api_login_required @parse_repository_name def request_repo_build(namespace, repository): permission = ModifyRepositoryPermission(namespace, repository) From b1a5b9c343941e0887c6b69dac7302fe6646f010 Mon Sep 17 00:00:00 2001 From: yackob03 Date: Sat, 26 Oct 2013 18:37:53 -0400 Subject: [PATCH 22/57] Fix CORS s3 upload. --- data/userfiles.py | 16 ++++++---------- endpoints/api.py | 4 ++-- static/js/controllers.js | 20 ++++++++++++-------- 3 files changed, 20 insertions(+), 20 deletions(-) diff --git a/data/userfiles.py b/data/userfiles.py index 996e31cf0..43f75073e 100644 --- a/data/userfiles.py +++ b/data/userfiles.py @@ -29,17 +29,13 @@ class UserRequestFiles(object): self._secret_key = s3_secret_key self._prefix = 'userfiles' - def prepare_for_drop(self, mimeType): + def prepare_for_drop(self, mime_type): """ Returns a signed URL to upload a file to our bucket. """ - file_id = str(self._prefix + '/' + str(uuid4())) - - expires = str(int(time.time() + 300)) - signingString = "PUT\n\n" + mimeType + "\n" + expires + "\n/" + self._bucket_name + "/" + file_id; - - hmac_signer = hmac.new(self._secret_key, signingString, sha) - signature = base64.b64encode(hmac_signer.digest()) - - url = "http://s3.amazonaws.com/" + self._bucket_name + "/" + file_id + "?AWSAccessKeyId=" + self._access_key + "&Expires=" + expires + "&Signature=" + urllib.quote(signature); + logger.debug('Requested upload url with content type: %s' % mime_type) + file_id = str(uuid4()) + full_key = os.path.join(self._prefix, file_id) + k = Key(self._bucket, full_key) + url = k.generate_url(300, 'PUT', headers={'Content-Type': mime_type}) return (url, file_id) def store_file(self, flask_file): diff --git a/endpoints/api.py b/endpoints/api.py index 40f3a584e..94e0daced 100644 --- a/endpoints/api.py +++ b/endpoints/api.py @@ -398,8 +398,8 @@ def get_repo_builds(namespace, repository): @app.route('/api/filedrop/', methods=['POST']) def get_filedrop_url(): - mimeType = request.get_json()['mimeType'] - (url, file_id) = user_files.prepare_for_drop(mimeType) + mime_type = request.get_json()['mimeType'] + (url, file_id) = user_files.prepare_for_drop(mime_type) return jsonify({ 'url': url, 'file_id': file_id diff --git a/static/js/controllers.js b/static/js/controllers.js index 1e5493ec3..86cf05fa5 100644 --- a/static/js/controllers.js +++ b/static/js/controllers.js @@ -912,7 +912,7 @@ function V1Ctrl($scope, UserService) { }; } -function NewRepoCtrl($scope, $location, UserService, Restangular) { +function NewRepoCtrl($scope, $location, $http, UserService, Restangular) { $scope.repo = { 'is_public': 1, 'description': '', @@ -938,16 +938,20 @@ function NewRepoCtrl($scope, $location, UserService, Restangular) { var conductUpload = function(repo, file, url, fileId, mimeType) { var request = new XMLHttpRequest(); request.open('PUT', url, true); - request.overrideMimeType(mimeType); + request.setRequestHeader('Content-Type', mimeType); request.onprogress = function(e) { - var percentLoaded; - if (e.lengthComputable) { - $scope.upload_progress = (e.loaded / e.total) * 100; - } + $scope.apply(function() { + var percentLoaded; + if (e.lengthComputable) { + $scope.upload_progress = (e.loaded / e.total) * 100; + } + }); }; request.onerror = function() { - $('#couldnotbuildModal').modal(); - $location.path('/repository/' + repo.namespace + '/' + repo.name); + $scope.apply(function() { + $('#couldnotbuildModal').modal(); + $location.path('/repository/' + repo.namespace + '/' + repo.name); + }); }; request.onreadystatechange = function() { var state = request.readyState; From 0ff19d5401fbd990555bf9027027c808690778e5 Mon Sep 17 00:00:00 2001 From: yackob03 Date: Sun, 27 Oct 2013 16:00:44 -0400 Subject: [PATCH 23/57] Filter the inactive builds when computing the is_building flag. --- data/model.py | 11 ++++++++--- endpoints/api.py | 5 +++-- 2 files changed, 11 insertions(+), 5 deletions(-) diff --git a/data/model.py b/data/model.py index f796c3572..8c56da380 100644 --- a/data/model.py +++ b/data/model.py @@ -562,10 +562,15 @@ def get_repository_build(request_dbid): raise InvalidRepositoryBuildException(msg) -def list_repository_builds(namespace_name, repository_name): +def list_repository_builds(namespace_name, repository_name, + include_inactive=True): joined = RepositoryBuild.select().join(Repository) - fetched = list(joined.where(Repository.name == repository_name, - Repository.namespace == namespace_name)) + filtered = joined + if not include_inactive: + filtered = filtered.where(RepositoryBuild.phase != 'error', + RepositoryBuild.phase != 'complete') + fetched = list(filtered.where(Repository.name == repository_name, + Repository.namespace == namespace_name)) return fetched diff --git a/endpoints/api.py b/endpoints/api.py index 94e0daced..0d34b6a25 100644 --- a/endpoints/api.py +++ b/endpoints/api.py @@ -345,7 +345,8 @@ def get_repo_api(namespace, repository): tag_dict = {tag.name: tag_view(tag) for tag in tags} can_write = ModifyRepositoryPermission(namespace, repository).can() can_admin = AdministerRepositoryPermission(namespace, repository).can() - builds = model.list_repository_builds(namespace, repository) + active_builds = model.list_repository_builds(namespace, repository, + include_inactive=False) return jsonify({ 'namespace': namespace, @@ -355,7 +356,7 @@ def get_repo_api(namespace, repository): 'can_write': can_write, 'can_admin': can_admin, 'is_public': is_public, - 'is_building': len(builds) > 0, + 'is_building': len(active_builds) > 0, }) abort(404) # Not fount From 3026f83c2ce1b60a05e6b80af2212b731047e45b Mon Sep 17 00:00:00 2001 From: yackob03 Date: Sun, 27 Oct 2013 16:01:14 -0400 Subject: [PATCH 24/57] Remove the test build starter as it no longer works. --- endpoints/test.py | 5 ----- templates/testbuild.html | 8 -------- 2 files changed, 13 deletions(-) delete mode 100644 templates/testbuild.html diff --git a/endpoints/test.py b/endpoints/test.py index 230ae2f70..ff5e42c37 100644 --- a/endpoints/test.py +++ b/endpoints/test.py @@ -46,8 +46,3 @@ def generate_random_build_status(): response.update(phases[phase]) return jsonify(response) - - -@app.route('/test/build', methods=['GET']) -def create_build(): - return send_file('templates/testbuild.html') diff --git a/templates/testbuild.html b/templates/testbuild.html deleted file mode 100644 index be740c804..000000000 --- a/templates/testbuild.html +++ /dev/null @@ -1,8 +0,0 @@ - - - - - - - - \ No newline at end of file From d7f51fb7642fa3b1479492e077c03116692e02ba Mon Sep 17 00:00:00 2001 From: yackob03 Date: Sun, 27 Oct 2013 19:06:20 -0400 Subject: [PATCH 25/57] Bug fixes and associated changes involved with spinning up build nodes and sending jobs to them. --- config.py | 14 ++- data/model.py | 2 +- data/userfiles.py | 5 +- endpoints/api.py | 2 +- workers/dockerfilebuild.py | 248 +++++++++++++++++++++++-------------- 5 files changed, 169 insertions(+), 102 deletions(-) diff --git a/config.py b/config.py index a51e2700f..3647c886e 100644 --- a/config.py +++ b/config.py @@ -88,16 +88,21 @@ class GitHubProdConfig(GitHubTestConfig): GITHUB_CLIENT_SECRET = 'f89d8bb28ea3bd4e1c68808500d185a816be53b1' -class DigitalOceanConfig(): +class DigitalOceanConfig(object): DO_CLIENT_ID = 'LJ44y2wwYj1MD0BRxS6qHA' DO_CLIENT_SECRET = 'b9357a6f6ff45a33bb03f6dbbad135f9' DO_SSH_KEY_ID = '46986' DO_SSH_PRIVATE_KEY_FILENAME = 'certs/digital_ocean' + DO_ALLOWED_REGIONS = {1, 4} + + +class BuildNodeConfig(object): + BUILD_NODE_PULL_TOKEN = 'F02O2E86CQLKZUQ0O81J8XDHQ6F0N1V36L9JTOEEK6GKKMT1GI8PTJQT4OU88Y6G' class DebugConfig(FlaskConfig, MailConfig, LocalStorage, SQLiteDB, StripeTestConfig, MixpanelTestConfig, GitHubTestConfig, - DigitalOceanConfig, AWSCredentials): + DigitalOceanConfig, AWSCredentials, BuildNodeConfig): REGISTRY_SERVER = 'localhost:5000' LOGGING_CONFIG = { 'level': logging.DEBUG, @@ -110,7 +115,8 @@ class DebugConfig(FlaskConfig, MailConfig, LocalStorage, SQLiteDB, class LocalHostedConfig(FlaskConfig, MailConfig, S3Storage, RDSMySQL, StripeLiveConfig, MixpanelTestConfig, - GitHubProdConfig, DigitalOceanConfig): + GitHubProdConfig, DigitalOceanConfig, + BuildNodeConfig): REGISTRY_SERVER = 'localhost:5000' LOGGING_CONFIG = { 'level': logging.DEBUG, @@ -121,7 +127,7 @@ class LocalHostedConfig(FlaskConfig, MailConfig, S3Storage, RDSMySQL, class ProductionConfig(FlaskConfig, MailConfig, S3Storage, RDSMySQL, StripeLiveConfig, MixpanelProdConfig, - GitHubProdConfig, DigitalOceanConfig): + GitHubProdConfig, DigitalOceanConfig, BuildNodeConfig): REGISTRY_SERVER = 'quay.io' LOGGING_CONFIG = { 'stream': sys.stderr, diff --git a/data/model.py b/data/model.py index 8c56da380..a46ca1912 100644 --- a/data/model.py +++ b/data/model.py @@ -556,7 +556,7 @@ def load_token_data(code): def get_repository_build(request_dbid): try: - return RepositoryBuild.get(RepositoryBuild == request_dbid) + return RepositoryBuild.get(RepositoryBuild.id == request_dbid) except RepositoryBuild.DoesNotExist: msg = 'Unable to locate a build by id: %s' % request_dbid raise InvalidRepositoryBuildException(msg) diff --git a/data/userfiles.py b/data/userfiles.py index 43f75073e..b8ddd0d90 100644 --- a/data/userfiles.py +++ b/data/userfiles.py @@ -20,9 +20,8 @@ class S3FileWriteException(Exception): class UserRequestFiles(object): def __init__(self, s3_access_key, s3_secret_key, bucket_name): - self._s3_conn = boto.s3.connection.S3Connection(s3_access_key, - s3_secret_key, - is_secure=False) + self._s3_conn = boto.connect_s3(s3_access_key, s3_secret_key, + is_secure=False) self._bucket_name = bucket_name self._bucket = self._s3_conn.get_bucket(bucket_name) self._access_key = s3_access_key diff --git a/endpoints/api.py b/endpoints/api.py index 0d34b6a25..c5592d8b8 100644 --- a/endpoints/api.py +++ b/endpoints/api.py @@ -423,7 +423,7 @@ def request_repo_build(namespace, repository): tag = '%s/%s/%s' % (host, repo.namespace, repo.name) build_request = model.create_repository_build(repo, token, dockerfile_id, tag) - dockerfile_build_queue.put(json.dumps({'request_id': build_request.id})) + dockerfile_build_queue.put(json.dumps({'build_id': build_request.id})) return jsonify({ 'started': True diff --git a/workers/dockerfilebuild.py b/workers/dockerfilebuild.py index cbf8718d9..061de0c03 100644 --- a/workers/dockerfilebuild.py +++ b/workers/dockerfilebuild.py @@ -5,9 +5,12 @@ import time import argparse import digitalocean import requests +import paramiko from apscheduler.scheduler import Scheduler from multiprocessing.pool import ThreadPool +from base64 import b64encode +from requests.exceptions import ConnectionError from data.queue import dockerfile_build_queue from data.userfiles import UserRequestFiles @@ -35,113 +38,169 @@ def try_connection(url, retries=5, period=5): raise ex +def try_connect_ssh(client, ip_addr, port, user, key_filename, retries=5, + period=5): + try: + client.connect(ip_addr, port, user, look_for_keys=False, + key_filename=key_filename) + except Exception as ex: + if retries: + logger.debug('Retrying connection to ssh ip: %s:%s after %ss' % + (ip_addr, port, period)) + time.sleep(period) + return try_connect_ssh(client, ip_addr, port, user, key_filename, + retries-1, period) + raise ex + + def get_status(url): return requests.get(url).json()['status'] def babysit_builder(request): - manager = digitalocean.Manager(client_id=app.config['DO_CLIENT_ID'], - api_key=app.config['DO_CLIENT_SECRET']) - repository_build = model.get_repository_build(request['build_id']) - - # check if there is already a DO node for this build job, if so clean it up - old_id = repository_build.build_node_id - if old_id - old_droplet = digitalocean.Droplet(old_id) - old_droplet.destroy() - - # start the DO node - name = 'dockerfile-build-%s' % repository_build.id - droplet = digitalocean.Droplet(client_id=app.config['DO_CLIENT_ID'], - api_key=app.config['DO_CLIENT_SECRET'], - name=name, - region_id=1, # New York, - image_id=1004145, # Docker on 13.04 - size_id=66, # 512MB, - backup_active=False) - droplet.create(ssh_key_ids=[app.config['DO_SSH_KEY_ID']]) - repository_build.build_node_id = droplet.id - repository_build.phase = 'starting' - repository_build.save() - - startup = droplet.get_events()[0] - while int(startup.percentage) != 100: - logger.debug('Droplet startup percentage: %s' % startup.percentage) - time.sleep(5) - startup.load() - - droplet.load() - logger.debug('Droplet started at ip address: %s' % droplet.ip_address) - - # connect to it with ssh - repository_build.phase = 'initializing' - repository_build.save() - - ssh_client = paramiko.SSHClient() - ssh_client.set_missing_host_key_policy(paramiko.AutoAddPolicy()) - ssh_client.connect(self._container_ip, self._config.sshd_port, "root", - look_for_keys=False, - key_filename=app.config['DO_SSH_PRIVATE_KEY_FILENAME']) - - # Pull and run the buildserver - pull_cmd = 'docker pull quay.io/quay/buildserver' - _, stdout, _ = ssh_client.exec_command(pull_cmd) - - start_cmd = 'sudo docker run -d -privileged quay.io/quay/buildserver' - _, stdout, _ = ssh_client.exec_command(start_cmd) - - # wait for the server to be ready - logger.debug('Waiting for buildserver to be ready') - build_endpoint = 'http://%s:5002/build/' % droplet.ip_address try: - try_connection() - except ConnectionError: - #TODO cleanup - pass - # send it the job - logger.debug('Sending build server request') + logger.debug('Starting work item: %s' % request) + repository_build = model.get_repository_build(request['build_id']) + logger.debug('Request details: %s' % repository_build) - user_files = UserRequestFiles(app.config['AWS_ACCESS_KEY'], - app.config['AWS_SECRET_KEY'], - app.config['REGISTRY_S3_BUCKET']) + # Initialize digital ocean API + do_client_id = app.config['DO_CLIENT_ID'] + do_api_key = app.config['DO_CLIENT_SECRET'] + manager = digitalocean.Manager(client_id=do_client_id, api_key=do_api_key) - repo = repository_build.repository - payload = { - 'tag': repository_build.tag, - 'resource_url': user_files.get_file_url(repository_build.resource_key), - 'token': repository_build.access_token.code, - } - start_build = requests.post(build_endpoint, data=payload) + # check if there is already a DO node for this build, if so clean it up + old_id = repository_build.build_node_id + if old_id: + logger.debug('Cleaning up old DO node: %s' % old_id) + old_droplet = digitalocean.Droplet(id=old_id, client_id=do_client_id, + api_key=do_api_key) + old_droplet.destroy() - # wait for the job to be complete - status_url = start_build.headers['Location'] - repository_build.phase = 'building' - repository_build.status_url = status_url - repository_build.save() + # Pick the region for the new droplet + allowed_regions = app.config['DO_ALLOWED_REGIONS'] + available_regions = {region.id for region in manager.get_all_regions()} + regions = available_regions.intersection(allowed_regions) + if not regions: + logger.error('No droplets in our allowed regtions, available: %s' % + available_regions) + return False - logger.debug('Waiting for job to be complete') - status = get_status(status_url) - while status != 'error' and status != 'complete': - logger.debug('Job status is: %s' % status) - time.sleep(5) + # start the DO node + name = 'dockerfile-build-%s' % repository_build.id + logger.debug('Starting DO node: %s' % name) + droplet = digitalocean.Droplet(client_id=do_client_id, + api_key=do_api_key, + name=name, + region_id=regions.pop(), + image_id=1004145, # Docker on 13.04 + size_id=66, # 512MB, + backup_active=False) + droplet.create(ssh_key_ids=[app.config['DO_SSH_KEY_ID']]) + repository_build.build_node_id = droplet.id + repository_build.phase = 'starting' + repository_build.save() + + startup = droplet.get_events()[0] + startup.load() + while not startup.percentage or int(startup.percentage) != 100: + logger.debug('Droplet startup percentage: %s' % startup.percentage) + time.sleep(5) + startup.load() + + droplet.load() + logger.debug('Droplet started at ip address: %s' % droplet.ip_address) + + # connect to it with ssh + repository_build.phase = 'initializing' + repository_build.save() + + ssh_client = paramiko.SSHClient() + ssh_client.set_missing_host_key_policy(paramiko.AutoAddPolicy()) + try_connect_ssh(ssh_client, droplet.ip_address, 22, 'root', + key_filename=app.config['DO_SSH_PRIVATE_KEY_FILENAME']) + + # Load the node with the pull token + token = app.config['BUILD_NODE_PULL_TOKEN'] + basicauth = b64encode('%s:%s' % ('$token', token)) + auth_object = { + 'https://quay.io/v1/': { + 'auth': basicauth, + 'email': '', + }, + } + + create_auth_cmd = 'echo \'%s\' > .dockercfg' % json.dumps(auth_object) + ssh_client.exec_command(create_auth_cmd) + + # Pull and run the buildserver + pull_cmd = 'docker pull quay.io/quay/buildserver' + _, stdout, _ = ssh_client.exec_command(pull_cmd) + pull_status = stdout.channel.recv_exit_status() + + if pull_status != 0: + logger.error('Pull command failed for host: %s' % droplet.ip_address) + else: + logger.debug('Pull status was: %s' % pull_status) + + start_cmd = 'docker run -d -privileged -lxc-conf="lxc.aa_profile=unconfined" quay.io/quay/buildserver' + ssh_client.exec_command(start_cmd) + + # wait for the server to be ready + logger.debug('Waiting for buildserver to be ready') + build_endpoint = 'http://%s:5002/build/' % droplet.ip_address + try: + try_connection(build_endpoint) + except ConnectionError: + #TODO cleanup + pass + + # send it the job + logger.debug('Sending build server request') + + user_files = UserRequestFiles(app.config['AWS_ACCESS_KEY'], + app.config['AWS_SECRET_KEY'], + app.config['REGISTRY_S3_BUCKET']) + + repo = repository_build.repository + payload = { + 'tag': repository_build.tag, + 'resource_url': user_files.get_file_url(repository_build.resource_key), + 'token': repository_build.access_token.code, + } + start_build = requests.post(build_endpoint, data=payload) + + # wait for the job to be complete + status_url = start_build.headers['Location'] + repository_build.phase = 'building' + repository_build.status_url = status_url + repository_build.save() + + logger.debug('Waiting for job to be complete') status = get_status(status_url) + while status != 'error' and status != 'complete': + logger.debug('Job status is: %s' % status) + time.sleep(5) + status = get_status(status_url) - logger.debug('Job complete with status: %s' % status) - if status == 'error': - repository_build.phase = 'error' - else: - repository_build.phase = 'complete' + logger.debug('Job complete with status: %s' % status) + if status == 'error': + repository_build.phase = 'error' + else: + repository_build.phase = 'complete' - # clean up the DO node - logger.debug('Cleaning up DO node.') - droplet.destroy() + # clean up the DO node + logger.debug('Cleaning up DO node.') + # droplet.destroy() - repository_build.status_url = None - repository_build.build_node_id = None; - repository_build.save() + repository_build.status_url = None + repository_build.build_node_id = None; + repository_build.save() - return True + return True + + except Exception as outer_ex: + logger.exception('Exception processing job: %s' % outer_ex.message) def process_work_items(pool): @@ -161,6 +220,7 @@ def process_work_items(pool): dockerfile_build_queue.complete(local_item) return complete_callback + logger.debug('Sending work item to thread pool: %s' % pool) pool.apply_async(babysit_builder, [request], callback=build_callback(item)) @@ -171,12 +231,14 @@ def process_work_items(pool): def start_worker(): pool = ThreadPool(3) - logger.debug("Scheduling worker.") + logger.debug('Scheduling worker.') sched = Scheduler() sched.start() - sched.add_interval_job(process_work_items, args=[pool], seconds=30) + # sched.add_interval_job(process_work_items, args=[pool], seconds=30) + + process_work_items(pool) while True: time.sleep(60 * 60 * 24) # sleep one day, basically forever From 8cb177128b6c32f10b9a29068c572b9eef0f3c02 Mon Sep 17 00:00:00 2001 From: Joseph Schorr Date: Mon, 28 Oct 2013 13:09:22 -0400 Subject: [PATCH 26/57] Fixes to the new repo and build status: - make the new repo icon gray - have the build status properly center (this is a hack, unfortunately) - have the build status update when open --- static/css/quay.css | 13 ++++++++++++- static/directives/build-status.html | 2 +- static/js/controllers.js | 24 ++++++++++++++++-------- 3 files changed, 29 insertions(+), 10 deletions(-) diff --git a/static/css/quay.css b/static/css/quay.css index 15fd496c3..ed0c3b390 100644 --- a/static/css/quay.css +++ b/static/css/quay.css @@ -5,7 +5,7 @@ .user-tools .user-tool { font-size: 24px; margin-top: 14px; - color: black; + color: #aaa; } .user-tools i.user-tool:hover { @@ -13,6 +13,14 @@ color: #428bca; } +.status-boxes .popover { + margin-right: 20px; +} + +.status-boxes .popover-content { + width: 260px; +} + .build-statuses { } @@ -20,17 +28,20 @@ padding: 4px; margin-bottom: 10px; border-bottom: 1px solid #eee; + width: 230px; } .build-status-container .build-message { display: block; white-space: nowrap; + font-size: 12px; } .build-status-container .progress { height: 12px; margin: 0px; margin-top: 10px; + width: 230px; } .build-status-container:last-child { diff --git a/static/directives/build-status.html b/static/directives/build-status.html index e1c1e2ce6..8c27dba53 100644 --- a/static/directives/build-status.html +++ b/static/directives/build-status.html @@ -1,4 +1,4 @@ -
    +
    {{ getBuildMessage(build) }}
    diff --git a/static/js/controllers.js b/static/js/controllers.js index 86cf05fa5..64042558c 100644 --- a/static/js/controllers.js +++ b/static/js/controllers.js @@ -340,9 +340,13 @@ function RepoCtrl($scope, Restangular, $routeParams, $rootScope, $location, $tim }); var startBuildInfoTimer = function(repo) { + getBuildInfo(repo); setInterval(function() { - $scope.$apply(function() { getBuildInfo(repo); }); - }, 5000); + var container = document.getElementById('build-status-container'); + if (container != null && container.offsetWidth > 0) { + $scope.$apply(function() { getBuildInfo(repo); }); + } + }, 2000); }; var getBuildInfo = function(repo) { @@ -458,7 +462,7 @@ function RepoCtrl($scope, Restangular, $routeParams, $rootScope, $location, $tim $scope.loading = false; if (repo.is_building) { - getBuildInfo(repo); + startBuildInfoTimer(repo); } }, function() { $scope.repo = null; @@ -919,6 +923,12 @@ function NewRepoCtrl($scope, $location, $http, UserService, Restangular) { 'initialize': false }; + $('#couldnotbuildModal').on('hidden.bs.modal', function() { + $scope.$apply(function() { + $location.path('/repository/' + $scope.created.namespace + '/' + $scope.created.name); + }); + }); + var startBuild = function(repo, fileId) { $scope.building = true; @@ -931,7 +941,6 @@ function NewRepoCtrl($scope, $location, $http, UserService, Restangular) { $location.path('/repository/' + repo.namespace + '/' + repo.name); }, function() { $('#couldnotbuildModal').modal(); - $location.path('/repository/' + repo.namespace + '/' + repo.name); }); }; @@ -940,7 +949,7 @@ function NewRepoCtrl($scope, $location, $http, UserService, Restangular) { request.open('PUT', url, true); request.setRequestHeader('Content-Type', mimeType); request.onprogress = function(e) { - $scope.apply(function() { + $scope.$apply(function() { var percentLoaded; if (e.lengthComputable) { $scope.upload_progress = (e.loaded / e.total) * 100; @@ -948,9 +957,8 @@ function NewRepoCtrl($scope, $location, $http, UserService, Restangular) { }); }; request.onerror = function() { - $scope.apply(function() { + $scope.$apply(function() { $('#couldnotbuildModal').modal(); - $location.path('/repository/' + repo.namespace + '/' + repo.name); }); }; request.onreadystatechange = function() { @@ -984,7 +992,6 @@ function NewRepoCtrl($scope, $location, $http, UserService, Restangular) { conductUpload(repo, file, resp.url, resp.file_id, mimeType); }, function() { $('#couldnotbuildModal').modal(); - $location.path('/repository/' + repo.namespace + '/' + repo.name); }); }; @@ -1035,6 +1042,7 @@ function NewRepoCtrl($scope, $location, $http, UserService, Restangular) { var createPost = Restangular.one('repository'); createPost.customPOST(data).then(function(created) { $scope.creating = false; + $scope.created = created; // Repository created. Start the upload process if applicable. if ($scope.repo.initialize) { From 5175364e620f29eec16c63b2b6f88cba125e0300 Mon Sep 17 00:00:00 2001 From: Joseph Schorr Date: Mon, 28 Oct 2013 13:22:18 -0400 Subject: [PATCH 27/57] Add a create repo button in a few places --- static/js/controllers.js | 12 ++++++++---- static/partials/landing.html | 3 ++- static/partials/repo-list.html | 7 +++++++ 3 files changed, 17 insertions(+), 5 deletions(-) diff --git a/static/js/controllers.js b/static/js/controllers.js index 64042558c..334311918 100644 --- a/static/js/controllers.js +++ b/static/js/controllers.js @@ -204,7 +204,7 @@ function RepoListCtrl($scope, Restangular, UserService) { }); } -function LandingCtrl($scope, $timeout, Restangular, UserService, KeyService) { +function LandingCtrl($scope, $timeout, $location, Restangular, UserService, KeyService) { $('.form-signup').popover(); $scope.$watch( function () { return UserService.currentUser(); }, function (currentUser) { @@ -230,7 +230,11 @@ function LandingCtrl($scope, $timeout, Restangular, UserService, KeyService) { }; $scope.browseRepos = function() { - document.location = '/repository/'; + $location.path('/repository/'); + }; + + $scope.createRepo = function() { + $location.path('/new/'); }; $scope.register = function() { @@ -906,13 +910,13 @@ function ImageViewCtrl($scope, $routeParams, $rootScope, Restangular) { }); } -function V1Ctrl($scope, UserService) { +function V1Ctrl($scope, $location, UserService) { $scope.$watch( function () { return UserService.currentUser(); }, function (currentUser) { $scope.user = currentUser; }, true); $scope.browseRepos = function() { - document.location = '/repository/'; + $location.path('/repository/'); }; } diff --git a/static/partials/landing.html b/static/partials/landing.html index f425552e5..b53b4504f 100644 --- a/static/partials/landing.html +++ b/static/partials/landing.html @@ -61,7 +61,8 @@
    Welcome {{ user.username }}!
    - + +
    diff --git a/static/partials/repo-list.html b/static/partials/repo-list.html index 602c08265..5edd390d0 100644 --- a/static/partials/repo-list.html +++ b/static/partials/repo-list.html @@ -4,6 +4,13 @@
    + + + +

    Your Repositories

    From 59543a2e516153f69fc43d63494c5a9c64fef400 Mon Sep 17 00:00:00 2001 From: Joseph Schorr Date: Mon, 28 Oct 2013 13:31:43 -0400 Subject: [PATCH 28/57] Add the image push percent to the progress bar --- static/js/app.js | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/static/js/app.js b/static/js/app.js index ae555c2d4..70b8970e0 100644 --- a/static/js/app.js +++ b/static/js/app.js @@ -201,7 +201,9 @@ quayApp.directive('buildStatus', function () { break; case 'pushing': - return (buildInfo.current_image / buildInfo.total_images) * 100; + var imagePercentDecimal = (buildInfo.image_completion_percent / 100); + var portion = 1 / buildInfo.total_images; + return (buildInfo.current_image / buildInfo.total_images) * 100 + (portion * imagePercentDecimal); break; case 'complete': From bb2446c45c142488bfe640cc6a41ef03f4a2b01c Mon Sep 17 00:00:00 2001 From: Joseph Schorr Date: Mon, 28 Oct 2013 13:33:19 -0400 Subject: [PATCH 29/57] Actually, there is a simpler way to do this --- static/js/app.js | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/static/js/app.js b/static/js/app.js index 70b8970e0..69b3399aa 100644 --- a/static/js/app.js +++ b/static/js/app.js @@ -202,8 +202,7 @@ quayApp.directive('buildStatus', function () { case 'pushing': var imagePercentDecimal = (buildInfo.image_completion_percent / 100); - var portion = 1 / buildInfo.total_images; - return (buildInfo.current_image / buildInfo.total_images) * 100 + (portion * imagePercentDecimal); + return ((buildInfo.current_image + imagePercentDecimal) / buildInfo.total_images) * 100; break; case 'complete': From 1c7398e9209632bd940e32fda363318020539b57 Mon Sep 17 00:00:00 2001 From: yackob03 Date: Mon, 28 Oct 2013 14:49:23 -0400 Subject: [PATCH 30/57] Allow build node logs to flow to docker. Clean up the pull key before executing user code. Actually clean up the DO node when we are done. Run the status server threaded just in case. --- buildserver/buildserver.py | 2 +- buildserver/startserver | 2 +- workers/dockerfilebuild.py | 17 ++++++++++++----- 3 files changed, 14 insertions(+), 7 deletions(-) diff --git a/buildserver/buildserver.py b/buildserver/buildserver.py index 2ac90e9fc..0c9a6f081 100644 --- a/buildserver/buildserver.py +++ b/buildserver/buildserver.py @@ -218,4 +218,4 @@ def get_all_status(): if __name__ == '__main__': logging.basicConfig(level=logging.DEBUG, format=LOG_FORMAT) - app.run(host='0.0.0.0', port=5002, debug=True) + app.run(host='0.0.0.0', port=5002, debug=True, threaded=True) diff --git a/buildserver/startserver b/buildserver/startserver index 4947a618a..9eb2e1114 100644 --- a/buildserver/startserver +++ b/buildserver/startserver @@ -45,4 +45,4 @@ done popd docker -d & -exec venv/bin/python buildserver.py 2> buildserver.log \ No newline at end of file +exec venv/bin/python buildserver.py \ No newline at end of file diff --git a/workers/dockerfilebuild.py b/workers/dockerfilebuild.py index 061de0c03..1d4a7a828 100644 --- a/workers/dockerfilebuild.py +++ b/workers/dockerfilebuild.py @@ -58,8 +58,10 @@ def get_status(url): def babysit_builder(request): + """ Spin up a build node and ask it to build our job. Retryable errors + should return False, while fatal errors should return True. + """ try: - logger.debug('Starting work item: %s' % request) repository_build = model.get_repository_build(request['build_id']) logger.debug('Request details: %s' % repository_build) @@ -143,6 +145,11 @@ def babysit_builder(request): else: logger.debug('Pull status was: %s' % pull_status) + # Remove the credentials we used to pull so crafty users cant steal them + remove_auth_cmd = 'rm .dockercfg' + ssh_client.exec_command(remove_auth_cmd) + + # Start the build server start_cmd = 'docker run -d -privileged -lxc-conf="lxc.aa_profile=unconfined" quay.io/quay/buildserver' ssh_client.exec_command(start_cmd) @@ -191,7 +198,7 @@ def babysit_builder(request): # clean up the DO node logger.debug('Cleaning up DO node.') - # droplet.destroy() + droplet.destroy() repository_build.status_url = None repository_build.build_node_id = None; @@ -200,7 +207,9 @@ def babysit_builder(request): return True except Exception as outer_ex: + # We don't really know what these are, but they are probably retryable logger.exception('Exception processing job: %s' % outer_ex.message) + return False def process_work_items(pool): @@ -236,9 +245,7 @@ def start_worker(): sched = Scheduler() sched.start() - # sched.add_interval_job(process_work_items, args=[pool], seconds=30) - - process_work_items(pool) + sched.add_interval_job(process_work_items, args=[pool], seconds=30) while True: time.sleep(60 * 60 * 24) # sleep one day, basically forever From 1ee21318d4bcbb1296156bca45eef5d76e8b1d66 Mon Sep 17 00:00:00 2001 From: Joseph Schorr Date: Mon, 28 Oct 2013 17:08:26 -0400 Subject: [PATCH 31/57] Have the private option in the new repo form alert the user about needing to purchase a plan and let them do so --- static/css/quay.css | 10 +++++ static/js/app.js | 49 +++++++++++++++++++-- static/js/controllers.js | 80 +++++++++++++++++++++-------------- static/partials/new-repo.html | 30 ++++++++++++- 4 files changed, 133 insertions(+), 36 deletions(-) diff --git a/static/css/quay.css b/static/css/quay.css index ed0c3b390..47791564c 100644 --- a/static/css/quay.css +++ b/static/css/quay.css @@ -130,6 +130,16 @@ text-decoration: none !important; } +.new-repo .required-plan { + margin: 10px; + margin-top: 20px; + margin-left: 50px; +} + +.new-repo .required-plan .alert { + color: #444 !important; +} + .new-repo .new-header { font-size: 22px; } diff --git a/static/js/app.js b/static/js/app.js index 69b3399aa..9d3e615be 100644 --- a/static/js/app.js +++ b/static/js/app.js @@ -54,7 +54,7 @@ quayApp = angular.module('quay', ['restangular', 'angularMoment', 'angulartics', return keyService; }]); - $provide.factory('PlanService', [function() { + $provide.factory('PlanService', ['Restangular', 'KeyService', function(Restangular, KeyService) { var plans = [ { title: 'Open Source', @@ -96,11 +96,54 @@ quayApp = angular.module('quay', ['restangular', 'angularMoment', 'angulartics', planService.planList = function() { return plans; - } + }; planService.getPlan = function(planId) { return planDict[planId]; - } + }; + + planService.getMinimumPlan = function(privateCount) { + for (var i = 0; i < plans.length; i++) { + var plan = plans[i]; + if (plan.privateRepos >= privateCount) { + return plan; + } + } + + return null; + }; + + planService.showSubscribeDialog = function($scope, planId, started, success, failed) { + var submitToken = function(token) { + $scope.$apply(function() { + started(); + }); + + mixpanel.track('plan_subscribe'); + + var subscriptionDetails = { + token: token.id, + plan: planId, + }; + + var createSubscriptionRequest = Restangular.one('user/plan'); + $scope.$apply(function() { + createSubscriptionRequest.customPUT(subscriptionDetails).then(success, failed); + }); + }; + + var planDetails = planService.getPlan(planId) + StripeCheckout.open({ + key: KeyService.stripePublishableKey, + address: false, // TODO change to true + amount: planDetails.price, + currency: 'usd', + name: 'Quay ' + planDetails.title + ' Subscription', + description: 'Up to ' + planDetails.privateRepos + ' private repositories', + panelLabel: 'Subscribe', + token: submitToken + }); + }; return planService; }]); diff --git a/static/js/controllers.js b/static/js/controllers.js index 334311918..df1ea6e22 100644 --- a/static/js/controllers.js +++ b/static/js/controllers.js @@ -701,6 +701,8 @@ function UserAdminCtrl($scope, $timeout, Restangular, PlanService, UserService, if (sub.usedPrivateRepos > $scope.subscribedPlan.privateRepos) { $scope.errorMessage = 'You are using more private repositories than your plan allows, please upgrate your subscription to avoid disruptions in your service.'; + } else { + $scope.errorMessage = null; } $scope.planLoading = false; @@ -709,7 +711,7 @@ function UserAdminCtrl($scope, $timeout, Restangular, PlanService, UserService, mixpanel.people.set({ 'plan': sub.plan }); - } + }; $scope.planLoading = true; var getSubscription = Restangular.one('user/plan'); @@ -720,36 +722,16 @@ function UserAdminCtrl($scope, $timeout, Restangular, PlanService, UserService, $scope.planChanging = false; $scope.subscribe = function(planId) { - var submitToken = function(token) { - $scope.$apply(function() { - mixpanel.track('plan_subscribe'); - - $scope.planChanging = true; - $scope.errorMessage = undefined; - - var subscriptionDetails = { - token: token.id, - plan: planId, - }; - - var createSubscriptionRequest = Restangular.one('user/plan'); - createSubscriptionRequest.customPUT(subscriptionDetails).then(subscribedToPlan, function() { - // Failure - $scope.errorMessage = 'Unable to subscribe.'; - }); - }); - }; - - var planDetails = PlanService.getPlan(planId) - StripeCheckout.open({ - key: KeyService.stripePublishableKey, - address: false, // TODO change to true - amount: planDetails.price, - currency: 'usd', - name: 'Quay ' + planDetails.title + ' Subscription', - description: 'Up to ' + planDetails.privateRepos + ' private repositories', - panelLabel: 'Subscribe', - token: submitToken + PlanService.showSubscribeDialog($scope, planId, function() { + // Subscribing. + $scope.planChanging = true; + }, function(plan) { + // Subscribed. + subscribedToPlan(plan); + }, function() { + // Failure. + $scope.errorMessage = 'Unable to subscribe.'; + $scope.planChanging = false; }); }; @@ -920,7 +902,7 @@ function V1Ctrl($scope, $location, UserService) { }; } -function NewRepoCtrl($scope, $location, $http, UserService, Restangular) { +function NewRepoCtrl($scope, $location, $http, UserService, Restangular, PlanService) { $scope.repo = { 'is_public': 1, 'description': '', @@ -999,6 +981,16 @@ function NewRepoCtrl($scope, $location, $http, UserService, Restangular) { }); }; + var subscribedToPlan = function(sub) { + $scope.planChanging = false; + $scope.subscription = sub; + $scope.subscribedPlan = PlanService.getPlan(sub.plan); + $scope.planRequired = null; + if ($scope.subscription.usedPrivateRepos >= $scope.subscribedPlan.privateRepos) { + $scope.planRequired = PlanService.getMinimumPlan($scope.subscription.usedPrivateRepos); + } + }; + $scope.$watch( function () { return UserService.currentUser(); }, function (currentUser) { $scope.user = currentUser; @@ -1061,4 +1053,28 @@ function NewRepoCtrl($scope, $location, $http, UserService, Restangular) { $scope.creating = false; }); }; + + $scope.upgradePlan = function() { + PlanService.showSubscribeDialog($scope, $scope.planRequired.stripeId, function() { + // Subscribing. + $scope.planChanging = true; + }, function(plan) { + // Subscribed. + subscribedToPlan(plan); + }, function() { + // Failure. + $('#couldnotsubscribeModal').modal(); + $scope.planChanging = false; + }); + }; + + $scope.plans = PlanService.planList(); + + // Load the user's subscription information in case they want to create a private + // repository. + var getSubscription = Restangular.one('user/plan'); + getSubscription.get().then(subscribedToPlan, function() { + // User has no subscription + $scope.planRequired = PlanService.getMinimumPlan(1); + }); } \ No newline at end of file diff --git a/static/partials/new-repo.html b/static/partials/new-repo.html index cde232546..dd02d2faa 100644 --- a/static/partials/new-repo.html +++ b/static/partials/new-repo.html @@ -62,6 +62,15 @@ You choose who can see, pull and push from/to this repository.
    + + +
    +
    + In order to make this repository private, you’ll need to upgrade your plan from {{ subscribedPlan.title }} to {{ planRequired.title }}. This will cost ${{ planRequired.price / 100 }}/month. +
    + Upgrade now + +
    @@ -91,7 +100,7 @@
    - +
    @@ -179,3 +188,22 @@
    + + + + From a313a77a6b7d1551f6ea8e8773960a6818c530f0 Mon Sep 17 00:00:00 2001 From: yackob03 Date: Tue, 29 Oct 2013 13:41:26 -0400 Subject: [PATCH 32/57] Switch some unnecessary button calls over to links. --- static/js/controllers.js | 12 ------------ static/partials/landing.html | 4 ++-- static/partials/v1-page.html | 2 +- static/partials/view-repo.html | 8 ++++---- 4 files changed, 7 insertions(+), 19 deletions(-) diff --git a/static/js/controllers.js b/static/js/controllers.js index df1ea6e22..b83be8ca8 100644 --- a/static/js/controllers.js +++ b/static/js/controllers.js @@ -229,14 +229,6 @@ function LandingCtrl($scope, $timeout, $location, Restangular, UserService, KeyS return getMarkedDown(getFirstTextLine(commentString)); }; - $scope.browseRepos = function() { - $location.path('/repository/'); - }; - - $scope.createRepo = function() { - $location.path('/new/'); - }; - $scope.register = function() { $('.form-signup').popover('hide'); $scope.registering = true; @@ -896,10 +888,6 @@ function V1Ctrl($scope, $location, UserService) { $scope.$watch( function () { return UserService.currentUser(); }, function (currentUser) { $scope.user = currentUser; }, true); - - $scope.browseRepos = function() { - $location.path('/repository/'); - }; } function NewRepoCtrl($scope, $location, $http, UserService, Restangular, PlanService) { diff --git a/static/partials/landing.html b/static/partials/landing.html index b53b4504f..4e3426972 100644 --- a/static/partials/landing.html +++ b/static/partials/landing.html @@ -61,8 +61,8 @@
    Welcome {{ user.username }}!
    - - + Browse all repositories + Create a new repository
    diff --git a/static/partials/v1-page.html b/static/partials/v1-page.html index 821654fdd..e81c3d598 100644 --- a/static/partials/v1-page.html +++ b/static/partials/v1-page.html @@ -3,7 +3,7 @@

    Welcome {{ user.username }}. Your account is fully activated!

    - + Browse all repositories
    diff --git a/static/partials/view-repo.html b/static/partials/view-repo.html index a10c609a3..b2293d2e4 100644 --- a/static/partials/view-repo.html +++ b/static/partials/view-repo.html @@ -75,7 +75,7 @@
    - +
    {{getTagCount(repo)}} {{currentTag.name}} - +
    Tags
    @@ -103,7 +103,7 @@
    - +
    {{imageHistory.length}} {{currentImage.id.substr(0, 12)}} - +
    Image
    From c91b40f3561648321ca94584247516d3707fd227 Mon Sep 17 00:00:00 2001 From: yackob03 Date: Tue, 29 Oct 2013 13:42:26 -0400 Subject: [PATCH 33/57] Switch to using environment variables to pass information to the build node, this closes down a security loophole with the admin endpoint of the server. --- buildserver/buildserver.py | 78 ++++++++++++++---------------- workers/dockerfilebuild.py | 97 ++++++++++++++++---------------------- 2 files changed, 76 insertions(+), 99 deletions(-) diff --git a/buildserver/buildserver.py b/buildserver/buildserver.py index 0c9a6f081..8ee7f4089 100644 --- a/buildserver/buildserver.py +++ b/buildserver/buildserver.py @@ -6,7 +6,7 @@ import re import requests import json -from flask import Flask, request, jsonify, url_for, abort, make_response +from flask import Flask, jsonify, url_for, abort, make_response from zipfile import ZipFile from tempfile import TemporaryFile, mkdtemp from uuid import uuid4 @@ -134,16 +134,19 @@ MIME_PROCESSORS = { } -builds = {} +build = { + 'total_commands': None, + 'total_images': None, + 'current_command': None, + 'current_image': None, + 'image_completion_percent': None, + 'status': 'waiting', + 'message': None, +} pool = ThreadPool(1) -@app.route('/build/', methods=['POST']) -def start_build(): - resource_url = request.values['resource_url'] - tag_name = request.values['tag'] - acccess_token = request.values['token'] - +def start_build(resource_url, tag_name, acccess_token): # Save the token host = re.match(r'([a-z0-9.:]+)/.+/.+$', tag_name) if host: @@ -160,8 +163,7 @@ def start_build(): dockercfg.write(json.dumps(payload)) else: - logger.warning('Invalid tag name: %s' % tag_name) - abort(400) + raise Exception('Invalid tag name: %s' % tag_name) docker_resource = requests.get(resource_url) c_type = docker_resource.headers['content-type'] @@ -170,8 +172,7 @@ def start_build(): (c_type, tag_name)) if c_type not in MIME_PROCESSORS: - logger.error('Invalid dockerfile content type: %s' % c_type) - abort(400) + raise Exception('Invalid dockerfile content type: %s' % c_type) build_dir = MIME_PROCESSORS[c_type](docker_resource) @@ -179,43 +180,34 @@ def start_build(): num_steps = count_steps(dockerfile_path) logger.debug('Dockerfile had %s steps' % num_steps) - job_id = str(uuid4()) - logger.info('Sending job to builder pool: %s' % job_id) + logger.info('Sending job to builder pool.') + build['total_commands'] = num_steps - result_object = { - 'id': job_id, - 'total_commands': num_steps, - 'total_images': None, - 'current_command': None, - 'current_image': None, - 'image_completion_percent': None, - 'status': 'waiting', - 'message': None, - } - builds[job_id] = result_object pool.apply_async(build_image, [build_dir, tag_name, num_steps, - result_object]) - - resp = make_response('Created', 201) - resp.headers['Location'] = url_for('get_status', job_id=job_id) - return resp - - -@app.route('/build/') -def get_status(job_id): - if job_id not in builds: - abort(400) - - return jsonify(builds[job_id]) + build]) @app.route('/build/', methods=['GET']) -def get_all_status(): - return jsonify({ - 'builds': builds, - }) +def get_status(): + if build: + return jsonify(build) + abort(404) + + +@app.route('/status/', methods=['GET']) +def health_check(): + return make_response('Running') if __name__ == '__main__': logging.basicConfig(level=logging.DEBUG, format=LOG_FORMAT) - app.run(host='0.0.0.0', port=5002, debug=True, threaded=True) + resource_url = os.environ['RESOURCE_URL'] + tag_name = os.environ['TAG'] + acccess_token = os.environ['TOKEN'] + + logger.debug('Starting job with resource url: %s tag: %s and token: %s' % + (resource_url, tag_name, acccess_token)) + + start_build(resource_url, tag_name, acccess_token) + + app.run(host='0.0.0.0', port=5002) diff --git a/workers/dockerfilebuild.py b/workers/dockerfilebuild.py index 1d4a7a828..a1bd7ede3 100644 --- a/workers/dockerfilebuild.py +++ b/workers/dockerfilebuild.py @@ -26,30 +26,19 @@ formatter = logging.Formatter(FORMAT) logger = logging.getLogger(__name__) +BUILD_SERVER_CMD = ('docker run -d -lxc-conf="lxc.aa_profile=unconfined" ' + + '-privileged -e \'RESOURCE_URL=%s\' -e \'TAG=%s\' ' + + '-e \'TOKEN=%s\' quay.io/quay/buildserver') -def try_connection(url, retries=5, period=5): + +def retry_command(to_call, args=[], kwargs={}, retries=5, period=5): try: - return requests.get(url) - except ConnectionError as ex: - if retries: - logger.debug('Retrying connection to url: %s after %ss' % (url, period)) - time.sleep(period) - return try_connection(url, retries-1, period) - raise ex - - -def try_connect_ssh(client, ip_addr, port, user, key_filename, retries=5, - period=5): - try: - client.connect(ip_addr, port, user, look_for_keys=False, - key_filename=key_filename) + return to_call(*args, **kwargs) except Exception as ex: if retries: - logger.debug('Retrying connection to ssh ip: %s:%s after %ss' % - (ip_addr, port, period)) + logger.debug('Retrying command after %ss' % period) time.sleep(period) - return try_connect_ssh(client, ip_addr, port, user, key_filename, - retries-1, period) + return retry_command(to_call, args, kwargs, retries-1, period) raise ex @@ -77,11 +66,12 @@ def babysit_builder(request): logger.debug('Cleaning up old DO node: %s' % old_id) old_droplet = digitalocean.Droplet(id=old_id, client_id=do_client_id, api_key=do_api_key) - old_droplet.destroy() + retry_command(old_droplet.destroy) # Pick the region for the new droplet allowed_regions = app.config['DO_ALLOWED_REGIONS'] - available_regions = {region.id for region in manager.get_all_regions()} + regions = retry_command(manager.get_all_regions) + available_regions = {region.id for region in regions} regions = available_regions.intersection(allowed_regions) if not regions: logger.error('No droplets in our allowed regtions, available: %s' % @@ -98,19 +88,21 @@ def babysit_builder(request): image_id=1004145, # Docker on 13.04 size_id=66, # 512MB, backup_active=False) - droplet.create(ssh_key_ids=[app.config['DO_SSH_KEY_ID']]) + retry_command(droplet.create, [], + {'ssh_key_ids': [app.config['DO_SSH_KEY_ID']]}) repository_build.build_node_id = droplet.id repository_build.phase = 'starting' repository_build.save() - startup = droplet.get_events()[0] - startup.load() + logger.debug('Waiting for DO node to be available.') + + startup = retry_command(droplet.get_events)[0] while not startup.percentage or int(startup.percentage) != 100: logger.debug('Droplet startup percentage: %s' % startup.percentage) time.sleep(5) - startup.load() + retry_command(startup.load) - droplet.load() + retry_command(droplet.load) logger.debug('Droplet started at ip address: %s' % droplet.ip_address) # connect to it with ssh @@ -119,8 +111,10 @@ def babysit_builder(request): ssh_client = paramiko.SSHClient() ssh_client.set_missing_host_key_policy(paramiko.AutoAddPolicy()) - try_connect_ssh(ssh_client, droplet.ip_address, 22, 'root', - key_filename=app.config['DO_SSH_PRIVATE_KEY_FILENAME']) + + retry_command(ssh_client.connect, [droplet.ip_address, 22, 'root'], + {'look_for_keys': False, + 'key_filename': app.config['DO_SSH_PRIVATE_KEY_FILENAME']}) # Load the node with the pull token token = app.config['BUILD_NODE_PULL_TOKEN'] @@ -142,6 +136,7 @@ def babysit_builder(request): if pull_status != 0: logger.error('Pull command failed for host: %s' % droplet.ip_address) + return False else: logger.debug('Pull status was: %s' % pull_status) @@ -149,56 +144,46 @@ def babysit_builder(request): remove_auth_cmd = 'rm .dockercfg' ssh_client.exec_command(remove_auth_cmd) - # Start the build server - start_cmd = 'docker run -d -privileged -lxc-conf="lxc.aa_profile=unconfined" quay.io/quay/buildserver' - ssh_client.exec_command(start_cmd) - - # wait for the server to be ready - logger.debug('Waiting for buildserver to be ready') - build_endpoint = 'http://%s:5002/build/' % droplet.ip_address - try: - try_connection(build_endpoint) - except ConnectionError: - #TODO cleanup - pass - - # send it the job - logger.debug('Sending build server request') - + # Prepare the signed resource url the build node can fetch the job from user_files = UserRequestFiles(app.config['AWS_ACCESS_KEY'], app.config['AWS_SECRET_KEY'], app.config['REGISTRY_S3_BUCKET']) + resource_url = user_files.get_file_url(repository_build.resource_key) - repo = repository_build.repository - payload = { - 'tag': repository_build.tag, - 'resource_url': user_files.get_file_url(repository_build.resource_key), - 'token': repository_build.access_token.code, - } - start_build = requests.post(build_endpoint, data=payload) + # Start the build server + start_cmd = BUILD_SERVER_CMD % (resource_url, repository_build.tag, + repository_build.access_token.code) + logger.debug('Sending build server request with command: %s' % start_cmd) + ssh_client.exec_command(start_cmd) + + status_endpoint = 'http://%s:5002/build/' % droplet.ip_address + # wait for the server to be ready + logger.debug('Waiting for buildserver to be ready') + retry_command(requests.get, [status_endpoint]) # wait for the job to be complete - status_url = start_build.headers['Location'] repository_build.phase = 'building' - repository_build.status_url = status_url + repository_build.status_url = status_endpoint repository_build.save() logger.debug('Waiting for job to be complete') - status = get_status(status_url) + status = get_status(status_endpoint) while status != 'error' and status != 'complete': logger.debug('Job status is: %s' % status) time.sleep(5) - status = get_status(status_url) + status = get_status(status_endpoint) logger.debug('Job complete with status: %s' % status) if status == 'error': + error_message = requests.get(status_endpoint).json()['message'] + logger.warning('Job error: %s' % error_message) repository_build.phase = 'error' else: repository_build.phase = 'complete' # clean up the DO node logger.debug('Cleaning up DO node.') - droplet.destroy() + # retry_command(droplet.destroy) repository_build.status_url = None repository_build.build_node_id = None; From 7f17c6da24fc84ee6df44d06e8611779deaaf91a Mon Sep 17 00:00:00 2001 From: yackob03 Date: Tue, 29 Oct 2013 14:42:16 -0400 Subject: [PATCH 34/57] Fix the forking problem in the buildserver. --- buildserver/buildserver.py | 19 +++++++++---------- 1 file changed, 9 insertions(+), 10 deletions(-) diff --git a/buildserver/buildserver.py b/buildserver/buildserver.py index 8ee7f4089..82cb24ac7 100644 --- a/buildserver/buildserver.py +++ b/buildserver/buildserver.py @@ -146,7 +146,15 @@ build = { pool = ThreadPool(1) -def start_build(resource_url, tag_name, acccess_token): +@app.before_first_request +def start_build(): + resource_url = os.environ['RESOURCE_URL'] + tag_name = os.environ['TAG'] + acccess_token = os.environ['TOKEN'] + + logger.debug('Starting job with resource url: %s tag: %s and token: %s' % + (resource_url, tag_name, acccess_token)) + # Save the token host = re.match(r'([a-z0-9.:]+)/.+/.+$', tag_name) if host: @@ -201,13 +209,4 @@ def health_check(): if __name__ == '__main__': logging.basicConfig(level=logging.DEBUG, format=LOG_FORMAT) - resource_url = os.environ['RESOURCE_URL'] - tag_name = os.environ['TAG'] - acccess_token = os.environ['TOKEN'] - - logger.debug('Starting job with resource url: %s tag: %s and token: %s' % - (resource_url, tag_name, acccess_token)) - - start_build(resource_url, tag_name, acccess_token) - app.run(host='0.0.0.0', port=5002) From 798cf781728a786f26440070bed30109ad93ec6d Mon Sep 17 00:00:00 2001 From: yackob03 Date: Tue, 29 Oct 2013 15:39:44 -0400 Subject: [PATCH 35/57] Re-add cleaning up the DO node. --- workers/dockerfilebuild.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/workers/dockerfilebuild.py b/workers/dockerfilebuild.py index a1bd7ede3..dc4209d35 100644 --- a/workers/dockerfilebuild.py +++ b/workers/dockerfilebuild.py @@ -183,7 +183,7 @@ def babysit_builder(request): # clean up the DO node logger.debug('Cleaning up DO node.') - # retry_command(droplet.destroy) + retry_command(droplet.destroy) repository_build.status_url = None repository_build.build_node_id = None; From 6f105326aa931471941f9e0ea33672d8e7d7fa5e Mon Sep 17 00:00:00 2001 From: yackob03 Date: Tue, 29 Oct 2013 15:42:19 -0400 Subject: [PATCH 36/57] Return unfinished items to the queue when they can be retried. --- data/queue.py | 4 ++++ workers/dockerfilebuild.py | 4 ++++ 2 files changed, 8 insertions(+) diff --git a/data/queue.py b/data/queue.py index 3038b8389..8e63c4d17 100644 --- a/data/queue.py +++ b/data/queue.py @@ -54,6 +54,10 @@ class WorkQueue(object): def complete(self, completed_item): completed_item.delete_instance() + def incomplete(self, incomplete_item): + incomplete_item.available = True + incomplete_item.save() + image_diff_queue = WorkQueue('imagediff') dockerfile_build_queue = WorkQueue('dockerfilebuild') diff --git a/workers/dockerfilebuild.py b/workers/dockerfilebuild.py index dc4209d35..2f7aea2da 100644 --- a/workers/dockerfilebuild.py +++ b/workers/dockerfilebuild.py @@ -212,6 +212,10 @@ def process_work_items(pool): def complete_callback(completed): if completed: dockerfile_build_queue.complete(local_item) + else: + # We have a retryable error, add the job back to the queue + dockerfile_build_queue.incomplete(local_item) + return complete_callback logger.debug('Sending work item to thread pool: %s' % pool) From 78d6c9d9a38d86bf1a04c170c059df195401b8f8 Mon Sep 17 00:00:00 2001 From: yackob03 Date: Tue, 29 Oct 2013 16:11:54 -0400 Subject: [PATCH 37/57] Just use the current server as the registry server, this will work for all externally available addresses. --- config.py | 3 --- endpoints/index.py | 5 ++++- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/config.py b/config.py index 3647c886e..de81bdfe6 100644 --- a/config.py +++ b/config.py @@ -103,7 +103,6 @@ class BuildNodeConfig(object): class DebugConfig(FlaskConfig, MailConfig, LocalStorage, SQLiteDB, StripeTestConfig, MixpanelTestConfig, GitHubTestConfig, DigitalOceanConfig, AWSCredentials, BuildNodeConfig): - REGISTRY_SERVER = 'localhost:5000' LOGGING_CONFIG = { 'level': logging.DEBUG, 'format': LOG_FORMAT @@ -117,7 +116,6 @@ class LocalHostedConfig(FlaskConfig, MailConfig, S3Storage, RDSMySQL, StripeLiveConfig, MixpanelTestConfig, GitHubProdConfig, DigitalOceanConfig, BuildNodeConfig): - REGISTRY_SERVER = 'localhost:5000' LOGGING_CONFIG = { 'level': logging.DEBUG, 'format': LOG_FORMAT @@ -128,7 +126,6 @@ class LocalHostedConfig(FlaskConfig, MailConfig, S3Storage, RDSMySQL, class ProductionConfig(FlaskConfig, MailConfig, S3Storage, RDSMySQL, StripeLiveConfig, MixpanelProdConfig, GitHubProdConfig, DigitalOceanConfig, BuildNodeConfig): - REGISTRY_SERVER = 'quay.io' LOGGING_CONFIG = { 'stream': sys.stderr, 'level': logging.DEBUG, diff --git a/endpoints/index.py b/endpoints/index.py index 375fa94b4..4920e2b15 100644 --- a/endpoints/index.py +++ b/endpoints/index.py @@ -2,6 +2,7 @@ import json import urllib import json import logging +import urlparse from flask import request, make_response, jsonify, abort from functools import wraps @@ -25,7 +26,9 @@ def generate_headers(role='read'): def wrapper(namespace, repository, *args, **kwargs): response = f(namespace, repository, *args, **kwargs) - response.headers['X-Docker-Endpoints'] = app.config['REGISTRY_SERVER'] + # We run our index and registry on the same hosts for now + registry_server = urlparse.urlparse(request.url).netloc + response.headers['X-Docker-Endpoints'] = registry_server has_token_request = request.headers.get('X-Docker-Token', '') From 6b6a238d3acc5fe83cdfd76f3cfec690f2b8fa8d Mon Sep 17 00:00:00 2001 From: yackob03 Date: Tue, 29 Oct 2013 18:13:27 -0400 Subject: [PATCH 38/57] Not all hosts may have apt-add-repository. --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index 2f608b81b..1e658d180 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,7 @@ to prepare a new host: ``` +sudo apt-get install software-properties-common sudo apt-add-repository -y ppa:nginx/stable sudo apt-get update sudo apt-get install -y git python-virtualenv python-dev phantomjs From ba5553e9a816224bec832434d10a5b8ce9eb54a5 Mon Sep 17 00:00:00 2001 From: yackob03 Date: Tue, 29 Oct 2013 18:14:00 -0400 Subject: [PATCH 39/57] Make the nginx config ALMOST work on digital ocean. --- nginx.conf | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/nginx.conf b/nginx.conf index 93e5a1ce1..68b3ed5c5 100644 --- a/nginx.conf +++ b/nginx.conf @@ -10,6 +10,7 @@ events { } http { + types_hash_max_size 2048; include /etc/nginx/mime.types; default_type application/octet-stream; @@ -36,8 +37,8 @@ http { keepalive_timeout 5; ssl on; - ssl_certificate /home/ubuntu/quay/certs/unified.cert; - ssl_certificate_key /home/ubuntu/quay/certs/quay.key; + ssl_certificate ./certs/unified.cert; + ssl_certificate_key ./certs/quay.key; ssl_session_timeout 5m; ssl_protocols SSLv3 TLSv1; ssl_ciphers ALL:!ADH:!EXPORT56:RC4+RSA:+HIGH:+MEDIUM:+LOW:+SSLv3:+EXP; From d3d593f7e134f9bce41e92cbdba8f024c0b55168 Mon Sep 17 00:00:00 2001 From: yackob03 Date: Tue, 29 Oct 2013 18:14:22 -0400 Subject: [PATCH 40/57] Add some missing dependencies to for the workers. --- requirements-nover.txt | 3 ++- requirements.txt | 4 ++++ 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/requirements-nover.txt b/requirements-nover.txt index a12714454..8d95e3e48 100644 --- a/requirements-nover.txt +++ b/requirements-nover.txt @@ -15,4 +15,5 @@ beautifulsoup4 marisa-trie apscheduler python-daemon -paramiko \ No newline at end of file +paramiko +python-digitalocean \ No newline at end of file diff --git a/requirements.txt b/requirements.txt index 171930241..494ed9198 100644 --- a/requirements.txt +++ b/requirements.txt @@ -12,6 +12,7 @@ beautifulsoup4==4.3.2 blinker==1.3 boto==2.15.0 distribute==0.6.34 +ecdsa==0.10 eventlet==0.14.0 greenlet==0.4.1 gunicorn==18.0 @@ -19,10 +20,13 @@ itsdangerous==0.23 lockfile==0.9.1 marisa-trie==0.5.1 mixpanel-py==3.0.0 +paramiko==1.12.0 peewee==2.1.4 py-bcrypt==0.4 +pycrypto==2.6.1 python-daemon==1.6 python-dateutil==2.1 +python-digitalocean==0.5 requests==2.0.0 six==1.4.1 stripe==1.9.8 From 5e81f999d1bab74d0fa24cd18ef4a8c4f6b22566 Mon Sep 17 00:00:00 2001 From: yackob03 Date: Tue, 29 Oct 2013 18:15:12 -0400 Subject: [PATCH 41/57] Make the build server and build worker slightly more robust to errors. --- buildserver/buildserver.py | 2 +- workers/dockerfilebuild.py | 6 ++++-- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/buildserver/buildserver.py b/buildserver/buildserver.py index 82cb24ac7..ebd039cdc 100644 --- a/buildserver/buildserver.py +++ b/buildserver/buildserver.py @@ -124,7 +124,7 @@ def build_image(build_dir, tag_name, num_steps, result_object): except Exception as e: logger.exception('Exception when processing request.') result_object['status'] = 'error' - result_object['message'] = e.message + result_object['message'] = str(e.message) MIME_PROCESSORS = { diff --git a/workers/dockerfilebuild.py b/workers/dockerfilebuild.py index 2f7aea2da..5112145b0 100644 --- a/workers/dockerfilebuild.py +++ b/workers/dockerfilebuild.py @@ -43,7 +43,7 @@ def retry_command(to_call, args=[], kwargs={}, retries=5, period=5): def get_status(url): - return requests.get(url).json()['status'] + return retry_command(requests.get, [url]).json()['status'] def babysit_builder(request): @@ -112,8 +112,10 @@ def babysit_builder(request): ssh_client = paramiko.SSHClient() ssh_client.set_missing_host_key_policy(paramiko.AutoAddPolicy()) + logger.debug('Connecting to droplet through ssh at ip: %s' % + droplet.ip_address) retry_command(ssh_client.connect, [droplet.ip_address, 22, 'root'], - {'look_for_keys': False, + {'look_for_keys': False, 'timeout': 10.0, 'key_filename': app.config['DO_SSH_PRIVATE_KEY_FILENAME']}) # Load the node with the pull token From da71aa7254ddea90a2448219c1c5582ec25495b6 Mon Sep 17 00:00:00 2001 From: Joseph Schorr Date: Tue, 29 Oct 2013 18:56:56 -0400 Subject: [PATCH 42/57] Clear the timer that watches build status when the page is changed --- static/js/controllers.js | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/static/js/controllers.js b/static/js/controllers.js index b83be8ca8..a74f606a7 100644 --- a/static/js/controllers.js +++ b/static/js/controllers.js @@ -336,13 +336,21 @@ function RepoCtrl($scope, Restangular, $routeParams, $rootScope, $location, $tim }); var startBuildInfoTimer = function(repo) { + if ($scope.interval) { return; } + getBuildInfo(repo); - setInterval(function() { + $scope.interval = setInterval(function() { var container = document.getElementById('build-status-container'); if (container != null && container.offsetWidth > 0) { $scope.$apply(function() { getBuildInfo(repo); }); } }, 2000); + + $scope.$on("$destroy", function() { + if ($scope.interval) { + clearInterval($scope.interval); + } + }); }; var getBuildInfo = function(repo) { From 7adf37e6b57c43e3ac5fd6d1b24c7a290df24cfb Mon Sep 17 00:00:00 2001 From: yackob03 Date: Tue, 29 Oct 2013 18:56:57 -0400 Subject: [PATCH 43/57] Move the parmiko import to work around a bug with python daemonization and paramiko. --- workers/dockerfilebuild.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/workers/dockerfilebuild.py b/workers/dockerfilebuild.py index 5112145b0..7a3daa12f 100644 --- a/workers/dockerfilebuild.py +++ b/workers/dockerfilebuild.py @@ -5,7 +5,6 @@ import time import argparse import digitalocean import requests -import paramiko from apscheduler.scheduler import Scheduler from multiprocessing.pool import ThreadPool @@ -109,6 +108,9 @@ def babysit_builder(request): repository_build.phase = 'initializing' repository_build.save() + # We wait until here to import paramiko because otherwise it doesn't work + # under the daemon context. + import paramiko ssh_client = paramiko.SSHClient() ssh_client.set_missing_host_key_policy(paramiko.AutoAddPolicy()) From 3a6f90dc9b3e924780ad6ea8778e3335708f04d2 Mon Sep 17 00:00:00 2001 From: Joseph Schorr Date: Tue, 29 Oct 2013 19:03:36 -0400 Subject: [PATCH 44/57] Move markdown dependencies to the base, since they are needed by search on all pages --- templates/base.html | 4 ++++ templates/index.html | 6 +----- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/templates/base.html b/templates/base.html index a5d9091e6..03563404e 100644 --- a/templates/base.html +++ b/templates/base.html @@ -51,6 +51,10 @@ + + + + {% block added_dependencies %} {% endblock %} diff --git a/templates/index.html b/templates/index.html index bd531dae9..22e993024 100644 --- a/templates/index.html +++ b/templates/index.html @@ -24,14 +24,10 @@ - - - - {% endblock %} {% block body_content %}
    -{% endblock %} \ No newline at end of file +{% endblock %} From 41fb93de5cb3ec793730b20dd9cf091fc2e03e68 Mon Sep 17 00:00:00 2001 From: Joseph Schorr Date: Tue, 29 Oct 2013 19:06:13 -0400 Subject: [PATCH 45/57] Checkin staging keys --- certs/quay-staging-enc.key | 30 ++++++++++++++++++++++++++++++ certs/quay-staging.cert | 36 ++++++++++++++++++++++++++++++++++++ certs/quay-staging.key | 27 +++++++++++++++++++++++++++ 3 files changed, 93 insertions(+) create mode 100644 certs/quay-staging-enc.key create mode 100644 certs/quay-staging.cert create mode 100644 certs/quay-staging.key diff --git a/certs/quay-staging-enc.key b/certs/quay-staging-enc.key new file mode 100644 index 000000000..2e4e0ec73 --- /dev/null +++ b/certs/quay-staging-enc.key @@ -0,0 +1,30 @@ +-----BEGIN RSA PRIVATE KEY----- +Proc-Type: 4,ENCRYPTED +DEK-Info: AES-256-CBC,2E92F159A4A60E842528201723AF241C + +fThdbyOk4whle4mTLn3fDt/eV/JqKXxs18yQl6OdS26sxB80MmwIAF6dyZI3AXHC +CBS6mmdZtFgqodXs125XXPijJ0m0V5s2yx7EHeqPMEYLnzu21QfsbtRGXtl5EX3m +il+F9Y/+Y3B0ZtvMcRsGPjnCkLeQZwMOVc8DsgC92bqgu3sCDuz70W6Na6wvIlWU +2/OViNDSI6QUo90bkdf1H/+pCr8Dge9MBBMnU2G7DqsaLR+ehUMj7sZs/MJjUOK5 +zERfQMEoywqBatQUlP203GWe3hsMAqAKnc5VUc52mpT7uC8afLuSqqu2SpP9W0f2 +Fu6jv0D9JqFI7tf11sGYyWdMoP7zSxcquRC+NMVfvEvlRfjourZ8LAkdkQioeGQO +uLUoXe55kDfcJMEahUNMbmrwA7pqfsfGkKCMnBTCATvjtghTTCL4xVrK7jwXk7Sv +q54TQe27sryYCGnCLErGxEwjs0lnIWsZ+ePa4qgF05JBty3psrFHZlubG+CT8sb7 +uzBwzjsIW4rgaBdxKzBgcYoUDhxGMqsg3DxcXh9EomsK/ka+SlVnSihYpxnuerxG +LYgSQhfSRZRTtHUzQcQuvT5sfa1UQIWWWxeUKxuTTcyE3g1DY3/osatchW9+CCW2 +z10daPvfdq70OiEnPLSDvD1Gs49QQlfsoaq4lhNU7VBqznr/5bdC4iiQssXbUOjM +odREtw0d8Ox72/V/z01/QjTiOIg5tk9tGORVFPmLHC/Db3OUkLHsp/ls8E2IkIMU +wrbmvA9ABLgfpLlgivTtYjkG+H9BtSbi4jGHjYGR40unHqzY+6EQfoL7hu6zzacB +4jOhKKtR1yskK42VI6vrihT3GaA3MslCBNmEUv/4FkbvRkVifDd7hMylHCkjlUSm +zpaydx2gVrRYlum/Ipd+yPb95iniVlaaNe9ZLqtpZt15mEgLwWfrTq470NGLKwbg +7IyIrN6TjHSy1BQMS7kQVzc8Mgafa5O3pXr3zJS2JCg13FC7DarRZfNlqQZ7ErRA +imF5jxMQC8agbairlrGQ2i+ckY2wm8OEKUu3f8+O1Rq6rHy7SLQq1PSoQQetstFz +jLDPRFvtymZb0e510nbsLOcQvaWjda2sU367ed99TkVm0J7PJG3J38B7Sz0Q82hh +jfoYsu+kyNCcR2eS7F3E/MAr85r5qW8/0oewV0ZNxZLyAqJLFHKaeEaGcW0q/wb3 +oDcli8kbX0YnMfm4p7YaR++VYtfiQrhzOkzYNmRV4xbVF3eE6f3lSjGXmeqWhM5c +o0+I75UvbjZzvfWjekkwKhzuIHRJV6g5ldxuaJ290xQrdbpzO0xmYKbOHkEtIjow +pBIaA6vgmOTnREdQQSjcNcD+eZDuzaKGwCa2IH5k5b025IOWS4XyZ2JNmRzgw2d8 +mF6fO8ZDpcQUMIv1Hbn4Yc7kE1vOPkUaB3pKumaQW16COoi+EtY+umWf2/SJb+Cd +PGfSzcagiR5Fs0wKgHCVTMuoKR/xhaSqLUkCgQoXKRNoT5cO/sTiYygDWyeyRnCE +CA3NKDsDnaoCAeDU5XS1tmNJSIxlQjlnFHP71otxMKtV/g42vwTkXmJ+6gH5g9TG +-----END RSA PRIVATE KEY----- \ No newline at end of file diff --git a/certs/quay-staging.cert b/certs/quay-staging.cert new file mode 100644 index 000000000..f02c8de26 --- /dev/null +++ b/certs/quay-staging.cert @@ -0,0 +1,36 @@ +-----BEGIN CERTIFICATE----- +MIIGTjCCBTagAwIBAgIDDK/RMA0GCSqGSIb3DQEBBQUAMIGMMQswCQYDVQQGEwJJ +TDEWMBQGA1UEChMNU3RhcnRDb20gTHRkLjErMCkGA1UECxMiU2VjdXJlIERpZ2l0 +YWwgQ2VydGlmaWNhdGUgU2lnbmluZzE4MDYGA1UEAxMvU3RhcnRDb20gQ2xhc3Mg +MSBQcmltYXJ5IEludGVybWVkaWF0ZSBTZXJ2ZXIgQ0EwHhcNMTMxMDI5MTIxNjM2 +WhcNMTQxMDMwMTUyMDI0WjBlMRkwFwYDVQQNExBXOHJYcjhsNVBEMkpMQ0VRMQsw +CQYDVQQGEwJVUzEYMBYGA1UEAxMPc3RhZ2luZy5xdWF5LmlvMSEwHwYJKoZIhvcN +AQkBFhJob3N0bWFzdGVyQHF1YXkuaW8wggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAw +ggEKAoIBAQDq45PoBSyAniiglXyt5yI3kbLcwLXRTrWNCv0rxi+57Elxs/ix7RII +Ig1iO1FARJP/ipRRHFB8GNuG+eIdJAEaeB39eyjvGvsOcE8hlK1Hu3Hd3PcKAwaV +JpVZyTblUYXy55kw9okwNZJpVJPOHxKaOjNYrJynw92VJ21WeeGk+kh0EZKQ4vtp +sMMYIJapQk1CYDdreZoA0TEGZixJG8laUfX+S+CJf9KY7qH8LefjK9fr6x7R+qd4 +Hvj6lwtGV5UEBkGtU2bzTAOSEMOJBcxOfrPovFFLVvtbYCRAIY2Y5PPvV1Wna9sB +h52hxRhoJpwU3/g+LXhJ6UEGFOMvfa+hAgMBAAGjggLdMIIC2TAJBgNVHRMEAjAA +MAsGA1UdDwQEAwIDqDATBgNVHSUEDDAKBggrBgEFBQcDATAdBgNVHQ4EFgQU6rqK +WYOkW9Ya0nerdfpJGA5GbxcwHwYDVR0jBBgwFoAU60I00Jiwq5/0G2sI98xkLu8O +LEUwIwYDVR0RBBwwGoIPc3RhZ2luZy5xdWF5LmlvggdxdWF5LmlvMIIBVgYDVR0g +BIIBTTCCAUkwCAYGZ4EMAQIBMIIBOwYLKwYBBAGBtTcBAgMwggEqMC4GCCsGAQUF +BwIBFiJodHRwOi8vd3d3LnN0YXJ0c3NsLmNvbS9wb2xpY3kucGRmMIH3BggrBgEF +BQcCAjCB6jAnFiBTdGFydENvbSBDZXJ0aWZpY2F0aW9uIEF1dGhvcml0eTADAgEB +GoG+VGhpcyBjZXJ0aWZpY2F0ZSB3YXMgaXNzdWVkIGFjY29yZGluZyB0byB0aGUg +Q2xhc3MgMSBWYWxpZGF0aW9uIHJlcXVpcmVtZW50cyBvZiB0aGUgU3RhcnRDb20g +Q0EgcG9saWN5LCByZWxpYW5jZSBvbmx5IGZvciB0aGUgaW50ZW5kZWQgcHVycG9z +ZSBpbiBjb21wbGlhbmNlIG9mIHRoZSByZWx5aW5nIHBhcnR5IG9ibGlnYXRpb25z +LjA1BgNVHR8ELjAsMCqgKKAmhiRodHRwOi8vY3JsLnN0YXJ0c3NsLmNvbS9jcnQx +LWNybC5jcmwwgY4GCCsGAQUFBwEBBIGBMH8wOQYIKwYBBQUHMAGGLWh0dHA6Ly9v +Y3NwLnN0YXJ0c3NsLmNvbS9zdWIvY2xhc3MxL3NlcnZlci9jYTBCBggrBgEFBQcw +AoY2aHR0cDovL2FpYS5zdGFydHNzbC5jb20vY2VydHMvc3ViLmNsYXNzMS5zZXJ2 +ZXIuY2EuY3J0MCMGA1UdEgQcMBqGGGh0dHA6Ly93d3cuc3RhcnRzc2wuY29tLzAN +BgkqhkiG9w0BAQUFAAOCAQEAfhP/++WewlphCojZwXijpFIy+XX1gR0p4kSfxVgA +Anl3khFL/xAvhk6pbWjGQM/9FWb/PFDRgj4fvMKGR8F9bMKNfBOrT+SyWDuI1Ax3 +y0unu0vZjEfUJmMktrr2aN3NI/bBmdVixNntsHRB0yrrl7Zk0TjQM3I1egfygoxa +tfARn5QOO/sReYJXlJdwmFMH0dpMT3++p5RhMZPDVeAdUUK/KzSdlPkVrLPJTKEY +d+IAIWjZq5CGOjM9052+CDhyAMvdywJQpxuhO/BzmPrt0ZQwuMdTUutPT2ijDGCB +J7nCUCVEtF25KJrJxeXY6oLxgXoaqqU1ZGivAS1oCtnocg== +-----END CERTIFICATE----- \ No newline at end of file diff --git a/certs/quay-staging.key b/certs/quay-staging.key new file mode 100644 index 000000000..2b41c1c6e --- /dev/null +++ b/certs/quay-staging.key @@ -0,0 +1,27 @@ +-----BEGIN RSA PRIVATE KEY----- +MIIEpAIBAAKCAQEA6uOT6AUsgJ4ooJV8reciN5Gy3MC10U61jQr9K8YvuexJcbP4 +se0SCCINYjtRQEST/4qUURxQfBjbhvniHSQBGngd/Xso7xr7DnBPIZStR7tx3dz3 +CgMGlSaVWck25VGF8ueZMPaJMDWSaVSTzh8SmjozWKycp8PdlSdtVnnhpPpIdBGS +kOL7abDDGCCWqUJNQmA3a3maANExBmYsSRvJWlH1/kvgiX/SmO6h/C3n4yvX6+se +0fqneB74+pcLRleVBAZBrVNm80wDkhDDiQXMTn6z6LxRS1b7W2AkQCGNmOTz71dV +p2vbAYedocUYaCacFN/4Pi14SelBBhTjL32voQIDAQABAoIBAGW2VIblLqcnVZps +AQhhDQ0ZF2XGQTU4qx8/QfAhqusMqaUF9Mw/R06kSD1gSEfXKms+vAj/hM6oCO/C +5yoNPDkVCI+KNGiNu2c+NNXqxrpILf+Pvp3kP4Z4pbWyjwXwLlvH9Csiprdsi1D3 +IeXgyLJmP3PHkzKGez4qS4tlzdMdBbJkdCQiE35yyF2os7F4HbehQ2Qyfw8PZk9o +T8uUEyh7SjYqmxJ2GfGXQd7+NXb3S1j7ehk/XTzlxgkhMW+eWk4hRAhd4j7FICAD +0UYx9K/j2TP8tNHgNd0k/BZkIbic6FD09YagPRu71Tc7MPvcSPm9SMDOj3WXJNC9 +/oDsOEECgYEA+w6mPpPxlspaEPsCYNo1/FdnmOYecLhruzqUU6lbSBhzW6p0R7H2 +GmfLeE4mGvkPJbx/zU13sRRhtzRB7QjuzZhKgrO8c/UoJeSaFimI3NYwjhtvszU6 +ActNQOpq5WvBXEOi+FegNW5+6vTnmxGG+gj8nSsu0JBPc2Db6jeFQbkCgYEA74Nw +X/iqRtuz+yabc4ALyowHJHdI5FUHv8uPv9Fk9KVRvxwq8Ak0ZcWnO0Sc+0eUC/v+ +VDVSvf1O+pMli+zIoAzGmLQFt/Is3E4frbBI7D3tWFjCzyduyVNbQBwhzfCDjk7z +Xr/vQ1tLlll5QhABtUdJlWIvZFRfm1Qi/un/8SkCgYAaui+Gn/drRzWZcy+IohJ3 +P9LemzkIZQnLD+x0j6YRIdE+JAJnE5IQs5Ycw60Y2AT9zniIocOpTXMtrtmJ45aQ +urLMAViBu8q/ZfvlehyA7iiTKGaW3IbFZCBgVdR1gig+q1CxQZrjtVS7rMDvaElH +WyeRj+RW/dYHgXtIDwsXuQKBgQDNYCOH5636vIGEJgK981opABFPz4kNYWwXpfFJ +RcAPl4KVIQ4gbYQkkGtpgtgpD6N+80GN63tbtk12x542cX9G3i2c2yDcLikRb1vy +j2q4SBGw48uH3gQ9VeC1BGpoMIheCozc/i+nzizuayJy507PpqUOFvcUTNT+WBL6 +CqSQoQKBgQCAtGhhTn35n/MlNKSrJ6mo8uKmkjX04trlusE+Ub9J2Bb+MT+Vdy/h +vsxAJWH9xcTPr6fTGmsiHBP2q8HzU36CFJPsk8mLnnT2Q5rd4nc3KMmXGu/NUwxA +AzNbRNJ45O+BZfm6YfZUXhHq4YmNauNwoWzAWmdP1701xI/1Jtmfeg== +-----END RSA PRIVATE KEY----- From 00bc4946cf513f0c878e15f068c9c9232f240c32 Mon Sep 17 00:00:00 2001 From: yackob03 Date: Tue, 29 Oct 2013 19:14:45 -0400 Subject: [PATCH 46/57] Create a unified cert from the quay-staging cert. --- certs/quay-staging-unified.cert | 113 ++++++++++++++++++++++ certs/{unified.cert => quay-unified.cert} | 0 nginx.conf | 2 +- 3 files changed, 114 insertions(+), 1 deletion(-) create mode 100644 certs/quay-staging-unified.cert rename certs/{unified.cert => quay-unified.cert} (100%) diff --git a/certs/quay-staging-unified.cert b/certs/quay-staging-unified.cert new file mode 100644 index 000000000..1dc06022c --- /dev/null +++ b/certs/quay-staging-unified.cert @@ -0,0 +1,113 @@ +-----BEGIN CERTIFICATE----- +MIIHRjCCBi6gAwIBAgIDDDb4MA0GCSqGSIb3DQEBBQUAMIGMMQswCQYDVQQGEwJJ +TDEWMBQGA1UEChMNU3RhcnRDb20gTHRkLjErMCkGA1UECxMiU2VjdXJlIERpZ2l0 +YWwgQ2VydGlmaWNhdGUgU2lnbmluZzE4MDYGA1UEAxMvU3RhcnRDb20gQ2xhc3Mg +MSBQcmltYXJ5IEludGVybWVkaWF0ZSBTZXJ2ZXIgQ0EwHhcNMTMwOTMwMTUxMTI3 +WhcNMTQxMDAxMDYyMDAxWjBhMRkwFwYDVQQNExBlNEZTNTBhYmNYcmQyZnlJMQsw +CQYDVQQGEwJVUzEUMBIGA1UEAxMLd3d3LnF1YXkuaW8xITAfBgkqhkiG9w0BCQEW +Emhvc3RtYXN0ZXJAcXVheS5pbzCCAiIwDQYJKoZIhvcNAQEBBQADggIPADCCAgoC +ggIBANGOItO9zOeJES+cQjB/8scbkLghi8wIvFnw/VJUUYsFrRYF2PJ96nrd0hcM +te/cvlU9phw6zhlay1zb8OuIAhtgIYFcKw/t41F7DRZGj+JaT620D5jFebWgLbLf +pxWnqGfGR4x5XgZOvzpWUgFBnX+KzvzwqfZndRLBBjpq2Rau30zggS6ff2iUNwPZ +8vPHUv/RQ6XVzq0WtbJQ1B3KVwSwcd9Eclg15LrWBd6RQxIl84CYDO6vhl00D6C8 +x8lvTjW+nB8mnnGS4F8pa3i5euwCMXWepO8EFGpeK4QikOFTevYAx1BUHeE/MGJX +FfPVIjhFVzWSrCnE2YjUcUAYoOnv0ZltpBFgsPUKyWZ4ZN3vbToorm4OYu9SJYtJ +FP51OsTizuyC85hm9zA03D3pf7zOIwIWwTG2ZdmKW4g3gNt8EJv25QC9vSiPmLa4 +wWzHgeRiMc7W9+lEive7HDafVBZQ3DX05qRbsYijhXTW6iojw0YntP5o3ndK/9Id +WfuP0cQxwxtAy7ykmnPUZ0ES58Hmf63QQ+unWhqO2nfbw/741/zC+ryyf0hcJmac +lS0Yjnisk4R62MOiRzyYxw0h8UBHBJvAzsNi+ouLtkEm8F8ne6wawGcXixwHPQnc +52XCcYZsguVwa5Pohh6/rcisTTJ3P9NSouFw4l2ghcrbwPALAgMBAAGjggLZMIIC +1TAJBgNVHRMEAjAAMAsGA1UdDwQEAwIDqDATBgNVHSUEDDAKBggrBgEFBQcDATAd +BgNVHQ4EFgQUkty8z9tltZ1SV8qVzUHTjRewBwswHwYDVR0jBBgwFoAU60I00Jiw +q5/0G2sI98xkLu8OLEUwHwYDVR0RBBgwFoILd3d3LnF1YXkuaW+CB3F1YXkuaW8w +ggFWBgNVHSAEggFNMIIBSTAIBgZngQwBAgEwggE7BgsrBgEEAYG1NwECAzCCASow +LgYIKwYBBQUHAgEWImh0dHA6Ly93d3cuc3RhcnRzc2wuY29tL3BvbGljeS5wZGYw +gfcGCCsGAQUFBwICMIHqMCcWIFN0YXJ0Q29tIENlcnRpZmljYXRpb24gQXV0aG9y +aXR5MAMCAQEagb5UaGlzIGNlcnRpZmljYXRlIHdhcyBpc3N1ZWQgYWNjb3JkaW5n +IHRvIHRoZSBDbGFzcyAxIFZhbGlkYXRpb24gcmVxdWlyZW1lbnRzIG9mIHRoZSBT +dGFydENvbSBDQSBwb2xpY3ksIHJlbGlhbmNlIG9ubHkgZm9yIHRoZSBpbnRlbmRl +ZCBwdXJwb3NlIGluIGNvbXBsaWFuY2Ugb2YgdGhlIHJlbHlpbmcgcGFydHkgb2Js +aWdhdGlvbnMuMDUGA1UdHwQuMCwwKqAooCaGJGh0dHA6Ly9jcmwuc3RhcnRzc2wu +Y29tL2NydDEtY3JsLmNybDCBjgYIKwYBBQUHAQEEgYEwfzA5BggrBgEFBQcwAYYt +aHR0cDovL29jc3Auc3RhcnRzc2wuY29tL3N1Yi9jbGFzczEvc2VydmVyL2NhMEIG +CCsGAQUFBzAChjZodHRwOi8vYWlhLnN0YXJ0c3NsLmNvbS9jZXJ0cy9zdWIuY2xh +c3MxLnNlcnZlci5jYS5jcnQwIwYDVR0SBBwwGoYYaHR0cDovL3d3dy5zdGFydHNz +bC5jb20vMA0GCSqGSIb3DQEBBQUAA4IBAQAFwzBHJ7d/Lutu/ub6gSDdMDvwAmWp +gsspoEqJJwFLPiVC3s7Ejj2qM20t2FZ4hEHPoQlfqQa/AmxQCLQnY5EnfJLZawwn +dYCFAbkGWvxa11ROHHv5gBovltBGef9lHuNzIc9hkkSpUBzH+nakGAdMs5MD+NH1 +f6GnsIOXLWlB14WHFSyN/v/bTZGkoq/9Tgkl4v4AkoZz6Fpd1XmZ4EXrZVBW/wWc +vnhKe3Khx2xU5xlU0GdtAd7WhasZhzyy7OxUvLqlnJygBYTg2R7RYF8jF3r85PU+ +NPo9t8Xl8I+y7JiD10WRJQdd34yK6DJlmr358J0nWWk7PQv6p5Cpx475 +-----END CERTIFICATE----- +-----BEGIN CERTIFICATE----- +MIIGNDCCBBygAwIBAgIBGDANBgkqhkiG9w0BAQUFADB9MQswCQYDVQQGEwJJTDEW +MBQGA1UEChMNU3RhcnRDb20gTHRkLjErMCkGA1UECxMiU2VjdXJlIERpZ2l0YWwg +Q2VydGlmaWNhdGUgU2lnbmluZzEpMCcGA1UEAxMgU3RhcnRDb20gQ2VydGlmaWNh +dGlvbiBBdXRob3JpdHkwHhcNMDcxMDI0MjA1NDE3WhcNMTcxMDI0MjA1NDE3WjCB +jDELMAkGA1UEBhMCSUwxFjAUBgNVBAoTDVN0YXJ0Q29tIEx0ZC4xKzApBgNVBAsT +IlNlY3VyZSBEaWdpdGFsIENlcnRpZmljYXRlIFNpZ25pbmcxODA2BgNVBAMTL1N0 +YXJ0Q29tIENsYXNzIDEgUHJpbWFyeSBJbnRlcm1lZGlhdGUgU2VydmVyIENBMIIB +IjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAtonGrO8JUngHrJJj0PREGBiE +gFYfka7hh/oyULTTRwbw5gdfcA4Q9x3AzhA2NIVaD5Ksg8asWFI/ujjo/OenJOJA +pgh2wJJuniptTT9uYSAK21ne0n1jsz5G/vohURjXzTCm7QduO3CHtPn66+6CPAVv +kvek3AowHpNz/gfK11+AnSJYUq4G2ouHI2mw5CrY6oPSvfNx23BaKA+vWjhwRRI/ +ME3NO68X5Q/LoKldSKqxYVDLNM08XMML6BDAjJvwAwNi/rJsPnIO7hxDKslIDlc5 +xDEhyBDBLIf+VJVSH1I8MRKbf+fAoKVZ1eKPPvDVqOHXcDGpxLPPr21TLwb0pwID +AQABo4IBrTCCAakwDwYDVR0TAQH/BAUwAwEB/zAOBgNVHQ8BAf8EBAMCAQYwHQYD +VR0OBBYEFOtCNNCYsKuf9BtrCPfMZC7vDixFMB8GA1UdIwQYMBaAFE4L7xqkQFul +F2mHMMo0aEPQQa7yMGYGCCsGAQUFBwEBBFowWDAnBggrBgEFBQcwAYYbaHR0cDov +L29jc3Auc3RhcnRzc2wuY29tL2NhMC0GCCsGAQUFBzAChiFodHRwOi8vd3d3LnN0 +YXJ0c3NsLmNvbS9zZnNjYS5jcnQwWwYDVR0fBFQwUjAnoCWgI4YhaHR0cDovL3d3 +dy5zdGFydHNzbC5jb20vc2ZzY2EuY3JsMCegJaAjhiFodHRwOi8vY3JsLnN0YXJ0 +c3NsLmNvbS9zZnNjYS5jcmwwgYAGA1UdIAR5MHcwdQYLKwYBBAGBtTcBAgEwZjAu +BggrBgEFBQcCARYiaHR0cDovL3d3dy5zdGFydHNzbC5jb20vcG9saWN5LnBkZjA0 +BggrBgEFBQcCARYoaHR0cDovL3d3dy5zdGFydHNzbC5jb20vaW50ZXJtZWRpYXRl +LnBkZjANBgkqhkiG9w0BAQUFAAOCAgEAIQlJPqWIbuALi0jaMU2P91ZXouHTYlfp +tVbzhUV1O+VQHwSL5qBaPucAroXQ+/8gA2TLrQLhxpFy+KNN1t7ozD+hiqLjfDen +xk+PNdb01m4Ge90h2c9W/8swIkn+iQTzheWq8ecf6HWQTd35RvdCNPdFWAwRDYSw +xtpdPvkBnufh2lWVvnQce/xNFE+sflVHfXv0pQ1JHpXo9xLBzP92piVH0PN1Nb6X +t1gW66pceG/sUzCv6gRNzKkC4/C2BBL2MLERPZBOVmTX3DxDX3M570uvh+v2/miI +RHLq0gfGabDBoYvvF0nXYbFFSF87ICHpW7LM9NfpMfULFWE7epTj69m8f5SuauNi +YpaoZHy4h/OZMn6SolK+u/hlz8nyMPyLwcKmltdfieFcNID1j0cHL7SRv7Gifl9L +WtBbnySGBVFaaQNlQ0lxxeBvlDRr9hvYqbBMflPrj0jfyjO1SPo2ShpTpjMM0InN +SRXNiTE8kMBy12VLUjWKRhFEuT2OKGWmPnmeXAhEKa2wNREuIU640ucQPl2Eg7PD +wuTSxv0JS3QJ3fGz0xk+gA2iCxnwOOfFwq/iI9th4p1cbiCJSS4jarJiwUW0n6+L +p/EiO/h94pDQehn7Skzj0n1fSoMD7SfWI55rjbRZotnvbIIp3XUZPD9MEI3vu3Un +0q6Dp6jOW6c= +-----END CERTIFICATE----- +-----BEGIN CERTIFICATE----- +MIIGTjCCBTagAwIBAgIDDK/RMA0GCSqGSIb3DQEBBQUAMIGMMQswCQYDVQQGEwJJ +TDEWMBQGA1UEChMNU3RhcnRDb20gTHRkLjErMCkGA1UECxMiU2VjdXJlIERpZ2l0 +YWwgQ2VydGlmaWNhdGUgU2lnbmluZzE4MDYGA1UEAxMvU3RhcnRDb20gQ2xhc3Mg +MSBQcmltYXJ5IEludGVybWVkaWF0ZSBTZXJ2ZXIgQ0EwHhcNMTMxMDI5MTIxNjM2 +WhcNMTQxMDMwMTUyMDI0WjBlMRkwFwYDVQQNExBXOHJYcjhsNVBEMkpMQ0VRMQsw +CQYDVQQGEwJVUzEYMBYGA1UEAxMPc3RhZ2luZy5xdWF5LmlvMSEwHwYJKoZIhvcN +AQkBFhJob3N0bWFzdGVyQHF1YXkuaW8wggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAw +ggEKAoIBAQDq45PoBSyAniiglXyt5yI3kbLcwLXRTrWNCv0rxi+57Elxs/ix7RII +Ig1iO1FARJP/ipRRHFB8GNuG+eIdJAEaeB39eyjvGvsOcE8hlK1Hu3Hd3PcKAwaV +JpVZyTblUYXy55kw9okwNZJpVJPOHxKaOjNYrJynw92VJ21WeeGk+kh0EZKQ4vtp +sMMYIJapQk1CYDdreZoA0TEGZixJG8laUfX+S+CJf9KY7qH8LefjK9fr6x7R+qd4 +Hvj6lwtGV5UEBkGtU2bzTAOSEMOJBcxOfrPovFFLVvtbYCRAIY2Y5PPvV1Wna9sB +h52hxRhoJpwU3/g+LXhJ6UEGFOMvfa+hAgMBAAGjggLdMIIC2TAJBgNVHRMEAjAA +MAsGA1UdDwQEAwIDqDATBgNVHSUEDDAKBggrBgEFBQcDATAdBgNVHQ4EFgQU6rqK +WYOkW9Ya0nerdfpJGA5GbxcwHwYDVR0jBBgwFoAU60I00Jiwq5/0G2sI98xkLu8O +LEUwIwYDVR0RBBwwGoIPc3RhZ2luZy5xdWF5LmlvggdxdWF5LmlvMIIBVgYDVR0g +BIIBTTCCAUkwCAYGZ4EMAQIBMIIBOwYLKwYBBAGBtTcBAgMwggEqMC4GCCsGAQUF +BwIBFiJodHRwOi8vd3d3LnN0YXJ0c3NsLmNvbS9wb2xpY3kucGRmMIH3BggrBgEF +BQcCAjCB6jAnFiBTdGFydENvbSBDZXJ0aWZpY2F0aW9uIEF1dGhvcml0eTADAgEB +GoG+VGhpcyBjZXJ0aWZpY2F0ZSB3YXMgaXNzdWVkIGFjY29yZGluZyB0byB0aGUg +Q2xhc3MgMSBWYWxpZGF0aW9uIHJlcXVpcmVtZW50cyBvZiB0aGUgU3RhcnRDb20g +Q0EgcG9saWN5LCByZWxpYW5jZSBvbmx5IGZvciB0aGUgaW50ZW5kZWQgcHVycG9z +ZSBpbiBjb21wbGlhbmNlIG9mIHRoZSByZWx5aW5nIHBhcnR5IG9ibGlnYXRpb25z +LjA1BgNVHR8ELjAsMCqgKKAmhiRodHRwOi8vY3JsLnN0YXJ0c3NsLmNvbS9jcnQx +LWNybC5jcmwwgY4GCCsGAQUFBwEBBIGBMH8wOQYIKwYBBQUHMAGGLWh0dHA6Ly9v +Y3NwLnN0YXJ0c3NsLmNvbS9zdWIvY2xhc3MxL3NlcnZlci9jYTBCBggrBgEFBQcw +AoY2aHR0cDovL2FpYS5zdGFydHNzbC5jb20vY2VydHMvc3ViLmNsYXNzMS5zZXJ2 +ZXIuY2EuY3J0MCMGA1UdEgQcMBqGGGh0dHA6Ly93d3cuc3RhcnRzc2wuY29tLzAN +BgkqhkiG9w0BAQUFAAOCAQEAfhP/++WewlphCojZwXijpFIy+XX1gR0p4kSfxVgA +Anl3khFL/xAvhk6pbWjGQM/9FWb/PFDRgj4fvMKGR8F9bMKNfBOrT+SyWDuI1Ax3 +y0unu0vZjEfUJmMktrr2aN3NI/bBmdVixNntsHRB0yrrl7Zk0TjQM3I1egfygoxa +tfARn5QOO/sReYJXlJdwmFMH0dpMT3++p5RhMZPDVeAdUUK/KzSdlPkVrLPJTKEY +d+IAIWjZq5CGOjM9052+CDhyAMvdywJQpxuhO/BzmPrt0ZQwuMdTUutPT2ijDGCB +J7nCUCVEtF25KJrJxeXY6oLxgXoaqqU1ZGivAS1oCtnocg== +-----END CERTIFICATE----- diff --git a/certs/unified.cert b/certs/quay-unified.cert similarity index 100% rename from certs/unified.cert rename to certs/quay-unified.cert diff --git a/nginx.conf b/nginx.conf index 68b3ed5c5..10d316c44 100644 --- a/nginx.conf +++ b/nginx.conf @@ -37,7 +37,7 @@ http { keepalive_timeout 5; ssl on; - ssl_certificate ./certs/unified.cert; + ssl_certificate ./certs/quay-unified.cert; ssl_certificate_key ./certs/quay.key; ssl_session_timeout 5m; ssl_protocols SSLv3 TLSv1; From 9436ae39414e2c068338d72175c79f60647311e4 Mon Sep 17 00:00:00 2001 From: yackob03 Date: Tue, 29 Oct 2013 19:32:15 -0400 Subject: [PATCH 47/57] Fix the stagig unified cert. --- certs/quay-staging-unified.cert | 145 ++++++++++++++++---------------- 1 file changed, 74 insertions(+), 71 deletions(-) diff --git a/certs/quay-staging-unified.cert b/certs/quay-staging-unified.cert index 1dc06022c..b2c0b5fce 100644 --- a/certs/quay-staging-unified.cert +++ b/certs/quay-staging-unified.cert @@ -1,43 +1,38 @@ -----BEGIN CERTIFICATE----- -MIIHRjCCBi6gAwIBAgIDDDb4MA0GCSqGSIb3DQEBBQUAMIGMMQswCQYDVQQGEwJJ +MIIGTjCCBTagAwIBAgIDDK/RMA0GCSqGSIb3DQEBBQUAMIGMMQswCQYDVQQGEwJJ TDEWMBQGA1UEChMNU3RhcnRDb20gTHRkLjErMCkGA1UECxMiU2VjdXJlIERpZ2l0 YWwgQ2VydGlmaWNhdGUgU2lnbmluZzE4MDYGA1UEAxMvU3RhcnRDb20gQ2xhc3Mg -MSBQcmltYXJ5IEludGVybWVkaWF0ZSBTZXJ2ZXIgQ0EwHhcNMTMwOTMwMTUxMTI3 -WhcNMTQxMDAxMDYyMDAxWjBhMRkwFwYDVQQNExBlNEZTNTBhYmNYcmQyZnlJMQsw -CQYDVQQGEwJVUzEUMBIGA1UEAxMLd3d3LnF1YXkuaW8xITAfBgkqhkiG9w0BCQEW -Emhvc3RtYXN0ZXJAcXVheS5pbzCCAiIwDQYJKoZIhvcNAQEBBQADggIPADCCAgoC -ggIBANGOItO9zOeJES+cQjB/8scbkLghi8wIvFnw/VJUUYsFrRYF2PJ96nrd0hcM -te/cvlU9phw6zhlay1zb8OuIAhtgIYFcKw/t41F7DRZGj+JaT620D5jFebWgLbLf -pxWnqGfGR4x5XgZOvzpWUgFBnX+KzvzwqfZndRLBBjpq2Rau30zggS6ff2iUNwPZ -8vPHUv/RQ6XVzq0WtbJQ1B3KVwSwcd9Eclg15LrWBd6RQxIl84CYDO6vhl00D6C8 -x8lvTjW+nB8mnnGS4F8pa3i5euwCMXWepO8EFGpeK4QikOFTevYAx1BUHeE/MGJX -FfPVIjhFVzWSrCnE2YjUcUAYoOnv0ZltpBFgsPUKyWZ4ZN3vbToorm4OYu9SJYtJ -FP51OsTizuyC85hm9zA03D3pf7zOIwIWwTG2ZdmKW4g3gNt8EJv25QC9vSiPmLa4 -wWzHgeRiMc7W9+lEive7HDafVBZQ3DX05qRbsYijhXTW6iojw0YntP5o3ndK/9Id -WfuP0cQxwxtAy7ykmnPUZ0ES58Hmf63QQ+unWhqO2nfbw/741/zC+ryyf0hcJmac -lS0Yjnisk4R62MOiRzyYxw0h8UBHBJvAzsNi+ouLtkEm8F8ne6wawGcXixwHPQnc -52XCcYZsguVwa5Pohh6/rcisTTJ3P9NSouFw4l2ghcrbwPALAgMBAAGjggLZMIIC -1TAJBgNVHRMEAjAAMAsGA1UdDwQEAwIDqDATBgNVHSUEDDAKBggrBgEFBQcDATAd -BgNVHQ4EFgQUkty8z9tltZ1SV8qVzUHTjRewBwswHwYDVR0jBBgwFoAU60I00Jiw -q5/0G2sI98xkLu8OLEUwHwYDVR0RBBgwFoILd3d3LnF1YXkuaW+CB3F1YXkuaW8w -ggFWBgNVHSAEggFNMIIBSTAIBgZngQwBAgEwggE7BgsrBgEEAYG1NwECAzCCASow -LgYIKwYBBQUHAgEWImh0dHA6Ly93d3cuc3RhcnRzc2wuY29tL3BvbGljeS5wZGYw -gfcGCCsGAQUFBwICMIHqMCcWIFN0YXJ0Q29tIENlcnRpZmljYXRpb24gQXV0aG9y -aXR5MAMCAQEagb5UaGlzIGNlcnRpZmljYXRlIHdhcyBpc3N1ZWQgYWNjb3JkaW5n -IHRvIHRoZSBDbGFzcyAxIFZhbGlkYXRpb24gcmVxdWlyZW1lbnRzIG9mIHRoZSBT -dGFydENvbSBDQSBwb2xpY3ksIHJlbGlhbmNlIG9ubHkgZm9yIHRoZSBpbnRlbmRl -ZCBwdXJwb3NlIGluIGNvbXBsaWFuY2Ugb2YgdGhlIHJlbHlpbmcgcGFydHkgb2Js -aWdhdGlvbnMuMDUGA1UdHwQuMCwwKqAooCaGJGh0dHA6Ly9jcmwuc3RhcnRzc2wu -Y29tL2NydDEtY3JsLmNybDCBjgYIKwYBBQUHAQEEgYEwfzA5BggrBgEFBQcwAYYt -aHR0cDovL29jc3Auc3RhcnRzc2wuY29tL3N1Yi9jbGFzczEvc2VydmVyL2NhMEIG -CCsGAQUFBzAChjZodHRwOi8vYWlhLnN0YXJ0c3NsLmNvbS9jZXJ0cy9zdWIuY2xh -c3MxLnNlcnZlci5jYS5jcnQwIwYDVR0SBBwwGoYYaHR0cDovL3d3dy5zdGFydHNz -bC5jb20vMA0GCSqGSIb3DQEBBQUAA4IBAQAFwzBHJ7d/Lutu/ub6gSDdMDvwAmWp -gsspoEqJJwFLPiVC3s7Ejj2qM20t2FZ4hEHPoQlfqQa/AmxQCLQnY5EnfJLZawwn -dYCFAbkGWvxa11ROHHv5gBovltBGef9lHuNzIc9hkkSpUBzH+nakGAdMs5MD+NH1 -f6GnsIOXLWlB14WHFSyN/v/bTZGkoq/9Tgkl4v4AkoZz6Fpd1XmZ4EXrZVBW/wWc -vnhKe3Khx2xU5xlU0GdtAd7WhasZhzyy7OxUvLqlnJygBYTg2R7RYF8jF3r85PU+ -NPo9t8Xl8I+y7JiD10WRJQdd34yK6DJlmr358J0nWWk7PQv6p5Cpx475 +MSBQcmltYXJ5IEludGVybWVkaWF0ZSBTZXJ2ZXIgQ0EwHhcNMTMxMDI5MTIxNjM2 +WhcNMTQxMDMwMTUyMDI0WjBlMRkwFwYDVQQNExBXOHJYcjhsNVBEMkpMQ0VRMQsw +CQYDVQQGEwJVUzEYMBYGA1UEAxMPc3RhZ2luZy5xdWF5LmlvMSEwHwYJKoZIhvcN +AQkBFhJob3N0bWFzdGVyQHF1YXkuaW8wggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAw +ggEKAoIBAQDq45PoBSyAniiglXyt5yI3kbLcwLXRTrWNCv0rxi+57Elxs/ix7RII +Ig1iO1FARJP/ipRRHFB8GNuG+eIdJAEaeB39eyjvGvsOcE8hlK1Hu3Hd3PcKAwaV +JpVZyTblUYXy55kw9okwNZJpVJPOHxKaOjNYrJynw92VJ21WeeGk+kh0EZKQ4vtp +sMMYIJapQk1CYDdreZoA0TEGZixJG8laUfX+S+CJf9KY7qH8LefjK9fr6x7R+qd4 +Hvj6lwtGV5UEBkGtU2bzTAOSEMOJBcxOfrPovFFLVvtbYCRAIY2Y5PPvV1Wna9sB +h52hxRhoJpwU3/g+LXhJ6UEGFOMvfa+hAgMBAAGjggLdMIIC2TAJBgNVHRMEAjAA +MAsGA1UdDwQEAwIDqDATBgNVHSUEDDAKBggrBgEFBQcDATAdBgNVHQ4EFgQU6rqK +WYOkW9Ya0nerdfpJGA5GbxcwHwYDVR0jBBgwFoAU60I00Jiwq5/0G2sI98xkLu8O +LEUwIwYDVR0RBBwwGoIPc3RhZ2luZy5xdWF5LmlvggdxdWF5LmlvMIIBVgYDVR0g +BIIBTTCCAUkwCAYGZ4EMAQIBMIIBOwYLKwYBBAGBtTcBAgMwggEqMC4GCCsGAQUF +BwIBFiJodHRwOi8vd3d3LnN0YXJ0c3NsLmNvbS9wb2xpY3kucGRmMIH3BggrBgEF +BQcCAjCB6jAnFiBTdGFydENvbSBDZXJ0aWZpY2F0aW9uIEF1dGhvcml0eTADAgEB +GoG+VGhpcyBjZXJ0aWZpY2F0ZSB3YXMgaXNzdWVkIGFjY29yZGluZyB0byB0aGUg +Q2xhc3MgMSBWYWxpZGF0aW9uIHJlcXVpcmVtZW50cyBvZiB0aGUgU3RhcnRDb20g +Q0EgcG9saWN5LCByZWxpYW5jZSBvbmx5IGZvciB0aGUgaW50ZW5kZWQgcHVycG9z +ZSBpbiBjb21wbGlhbmNlIG9mIHRoZSByZWx5aW5nIHBhcnR5IG9ibGlnYXRpb25z +LjA1BgNVHR8ELjAsMCqgKKAmhiRodHRwOi8vY3JsLnN0YXJ0c3NsLmNvbS9jcnQx +LWNybC5jcmwwgY4GCCsGAQUFBwEBBIGBMH8wOQYIKwYBBQUHMAGGLWh0dHA6Ly9v +Y3NwLnN0YXJ0c3NsLmNvbS9zdWIvY2xhc3MxL3NlcnZlci9jYTBCBggrBgEFBQcw +AoY2aHR0cDovL2FpYS5zdGFydHNzbC5jb20vY2VydHMvc3ViLmNsYXNzMS5zZXJ2 +ZXIuY2EuY3J0MCMGA1UdEgQcMBqGGGh0dHA6Ly93d3cuc3RhcnRzc2wuY29tLzAN +BgkqhkiG9w0BAQUFAAOCAQEAfhP/++WewlphCojZwXijpFIy+XX1gR0p4kSfxVgA +Anl3khFL/xAvhk6pbWjGQM/9FWb/PFDRgj4fvMKGR8F9bMKNfBOrT+SyWDuI1Ax3 +y0unu0vZjEfUJmMktrr2aN3NI/bBmdVixNntsHRB0yrrl7Zk0TjQM3I1egfygoxa +tfARn5QOO/sReYJXlJdwmFMH0dpMT3++p5RhMZPDVeAdUUK/KzSdlPkVrLPJTKEY +d+IAIWjZq5CGOjM9052+CDhyAMvdywJQpxuhO/BzmPrt0ZQwuMdTUutPT2ijDGCB +J7nCUCVEtF25KJrJxeXY6oLxgXoaqqU1ZGivAS1oCtnocg== -----END CERTIFICATE----- -----BEGIN CERTIFICATE----- MIIGNDCCBBygAwIBAgIBGDANBgkqhkiG9w0BAQUFADB9MQswCQYDVQQGEwJJTDEW @@ -76,38 +71,46 @@ p/EiO/h94pDQehn7Skzj0n1fSoMD7SfWI55rjbRZotnvbIIp3XUZPD9MEI3vu3Un 0q6Dp6jOW6c= -----END CERTIFICATE----- -----BEGIN CERTIFICATE----- -MIIGTjCCBTagAwIBAgIDDK/RMA0GCSqGSIb3DQEBBQUAMIGMMQswCQYDVQQGEwJJ -TDEWMBQGA1UEChMNU3RhcnRDb20gTHRkLjErMCkGA1UECxMiU2VjdXJlIERpZ2l0 -YWwgQ2VydGlmaWNhdGUgU2lnbmluZzE4MDYGA1UEAxMvU3RhcnRDb20gQ2xhc3Mg -MSBQcmltYXJ5IEludGVybWVkaWF0ZSBTZXJ2ZXIgQ0EwHhcNMTMxMDI5MTIxNjM2 -WhcNMTQxMDMwMTUyMDI0WjBlMRkwFwYDVQQNExBXOHJYcjhsNVBEMkpMQ0VRMQsw -CQYDVQQGEwJVUzEYMBYGA1UEAxMPc3RhZ2luZy5xdWF5LmlvMSEwHwYJKoZIhvcN -AQkBFhJob3N0bWFzdGVyQHF1YXkuaW8wggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAw -ggEKAoIBAQDq45PoBSyAniiglXyt5yI3kbLcwLXRTrWNCv0rxi+57Elxs/ix7RII -Ig1iO1FARJP/ipRRHFB8GNuG+eIdJAEaeB39eyjvGvsOcE8hlK1Hu3Hd3PcKAwaV -JpVZyTblUYXy55kw9okwNZJpVJPOHxKaOjNYrJynw92VJ21WeeGk+kh0EZKQ4vtp -sMMYIJapQk1CYDdreZoA0TEGZixJG8laUfX+S+CJf9KY7qH8LefjK9fr6x7R+qd4 -Hvj6lwtGV5UEBkGtU2bzTAOSEMOJBcxOfrPovFFLVvtbYCRAIY2Y5PPvV1Wna9sB -h52hxRhoJpwU3/g+LXhJ6UEGFOMvfa+hAgMBAAGjggLdMIIC2TAJBgNVHRMEAjAA -MAsGA1UdDwQEAwIDqDATBgNVHSUEDDAKBggrBgEFBQcDATAdBgNVHQ4EFgQU6rqK -WYOkW9Ya0nerdfpJGA5GbxcwHwYDVR0jBBgwFoAU60I00Jiwq5/0G2sI98xkLu8O -LEUwIwYDVR0RBBwwGoIPc3RhZ2luZy5xdWF5LmlvggdxdWF5LmlvMIIBVgYDVR0g -BIIBTTCCAUkwCAYGZ4EMAQIBMIIBOwYLKwYBBAGBtTcBAgMwggEqMC4GCCsGAQUF -BwIBFiJodHRwOi8vd3d3LnN0YXJ0c3NsLmNvbS9wb2xpY3kucGRmMIH3BggrBgEF -BQcCAjCB6jAnFiBTdGFydENvbSBDZXJ0aWZpY2F0aW9uIEF1dGhvcml0eTADAgEB -GoG+VGhpcyBjZXJ0aWZpY2F0ZSB3YXMgaXNzdWVkIGFjY29yZGluZyB0byB0aGUg -Q2xhc3MgMSBWYWxpZGF0aW9uIHJlcXVpcmVtZW50cyBvZiB0aGUgU3RhcnRDb20g -Q0EgcG9saWN5LCByZWxpYW5jZSBvbmx5IGZvciB0aGUgaW50ZW5kZWQgcHVycG9z -ZSBpbiBjb21wbGlhbmNlIG9mIHRoZSByZWx5aW5nIHBhcnR5IG9ibGlnYXRpb25z -LjA1BgNVHR8ELjAsMCqgKKAmhiRodHRwOi8vY3JsLnN0YXJ0c3NsLmNvbS9jcnQx -LWNybC5jcmwwgY4GCCsGAQUFBwEBBIGBMH8wOQYIKwYBBQUHMAGGLWh0dHA6Ly9v -Y3NwLnN0YXJ0c3NsLmNvbS9zdWIvY2xhc3MxL3NlcnZlci9jYTBCBggrBgEFBQcw -AoY2aHR0cDovL2FpYS5zdGFydHNzbC5jb20vY2VydHMvc3ViLmNsYXNzMS5zZXJ2 -ZXIuY2EuY3J0MCMGA1UdEgQcMBqGGGh0dHA6Ly93d3cuc3RhcnRzc2wuY29tLzAN -BgkqhkiG9w0BAQUFAAOCAQEAfhP/++WewlphCojZwXijpFIy+XX1gR0p4kSfxVgA -Anl3khFL/xAvhk6pbWjGQM/9FWb/PFDRgj4fvMKGR8F9bMKNfBOrT+SyWDuI1Ax3 -y0unu0vZjEfUJmMktrr2aN3NI/bBmdVixNntsHRB0yrrl7Zk0TjQM3I1egfygoxa -tfARn5QOO/sReYJXlJdwmFMH0dpMT3++p5RhMZPDVeAdUUK/KzSdlPkVrLPJTKEY -d+IAIWjZq5CGOjM9052+CDhyAMvdywJQpxuhO/BzmPrt0ZQwuMdTUutPT2ijDGCB -J7nCUCVEtF25KJrJxeXY6oLxgXoaqqU1ZGivAS1oCtnocg== +MIIHyTCCBbGgAwIBAgIBATANBgkqhkiG9w0BAQUFADB9MQswCQYDVQQGEwJJTDEW +MBQGA1UEChMNU3RhcnRDb20gTHRkLjErMCkGA1UECxMiU2VjdXJlIERpZ2l0YWwg +Q2VydGlmaWNhdGUgU2lnbmluZzEpMCcGA1UEAxMgU3RhcnRDb20gQ2VydGlmaWNh +dGlvbiBBdXRob3JpdHkwHhcNMDYwOTE3MTk0NjM2WhcNMzYwOTE3MTk0NjM2WjB9 +MQswCQYDVQQGEwJJTDEWMBQGA1UEChMNU3RhcnRDb20gTHRkLjErMCkGA1UECxMi +U2VjdXJlIERpZ2l0YWwgQ2VydGlmaWNhdGUgU2lnbmluZzEpMCcGA1UEAxMgU3Rh +cnRDb20gQ2VydGlmaWNhdGlvbiBBdXRob3JpdHkwggIiMA0GCSqGSIb3DQEBAQUA +A4ICDwAwggIKAoICAQDBiNsJvGxGfHiflXu1M5DycmLWwTYgIiRezul38kMKogZk +pMyONvg45iPwbm2xPN1yo4UcodM9tDMr0y+v/uqwQVlntsQGfQqedIXWeUyAN3rf +OQVSWff0G0ZDpNKFhdLDcfN1YjS6LIp/Ho/u7TTQEceWzVI9ujPW3U3eCztKS5/C +Ji/6tRYccjV3yjxd5srhJosaNnZcAdt0FCX+7bWgiA/deMotHweXMAEtcnn6RtYT +Kqi5pquDSR3l8u/d5AGOGAqPY1MWhWKpDhk6zLVmpsJrdAfkK+F2PrRt2PZE4XNi +HzvEvqBTViVsUQn3qqvKv3b9bZvzndu/PWa8DFaqr5hIlTpL36dYUNk4dalb6kMM +Av+Z6+hsTXBbKWWc3apdzK8BMewM69KN6Oqce+Zu9ydmDBpI125C4z/eIT574Q1w ++2OqqGwaVLRcJXrJosmLFqa7LH4XXgVNWG4SHQHuEhANxjJ/GP/89PrNbpHoNkm+ +Gkhpi8KWTRoSsmkXwQqQ1vp5Iki/untp+HDH+no32NgN0nZPV/+Qt+OR0t3vwmC3 +Zzrd/qqc8NSLf3Iizsafl7b4r4qgEKjZ+xjGtrVcUjyJthkqcwEKDwOzEmDyei+B +26Nu/yYwl/WL3YlXtq09s68rxbd2AvCl1iuahhQqcvbjM4xdCUsT37uMdBNSSwID +AQABo4ICUjCCAk4wDAYDVR0TBAUwAwEB/zALBgNVHQ8EBAMCAa4wHQYDVR0OBBYE +FE4L7xqkQFulF2mHMMo0aEPQQa7yMGQGA1UdHwRdMFswLKAqoCiGJmh0dHA6Ly9j +ZXJ0LnN0YXJ0Y29tLm9yZy9zZnNjYS1jcmwuY3JsMCugKaAnhiVodHRwOi8vY3Js +LnN0YXJ0Y29tLm9yZy9zZnNjYS1jcmwuY3JsMIIBXQYDVR0gBIIBVDCCAVAwggFM +BgsrBgEEAYG1NwEBATCCATswLwYIKwYBBQUHAgEWI2h0dHA6Ly9jZXJ0LnN0YXJ0 +Y29tLm9yZy9wb2xpY3kucGRmMDUGCCsGAQUFBwIBFilodHRwOi8vY2VydC5zdGFy +dGNvbS5vcmcvaW50ZXJtZWRpYXRlLnBkZjCB0AYIKwYBBQUHAgIwgcMwJxYgU3Rh +cnQgQ29tbWVyY2lhbCAoU3RhcnRDb20pIEx0ZC4wAwIBARqBl0xpbWl0ZWQgTGlh +YmlsaXR5LCByZWFkIHRoZSBzZWN0aW9uICpMZWdhbCBMaW1pdGF0aW9ucyogb2Yg +dGhlIFN0YXJ0Q29tIENlcnRpZmljYXRpb24gQXV0aG9yaXR5IFBvbGljeSBhdmFp +bGFibGUgYXQgaHR0cDovL2NlcnQuc3RhcnRjb20ub3JnL3BvbGljeS5wZGYwEQYJ +YIZIAYb4QgEBBAQDAgAHMDgGCWCGSAGG+EIBDQQrFilTdGFydENvbSBGcmVlIFNT +TCBDZXJ0aWZpY2F0aW9uIEF1dGhvcml0eTANBgkqhkiG9w0BAQUFAAOCAgEAFmyZ +9GYMNPXQhV59CuzaEE44HF7fpiUFS5Eyweg78T3dRAlbB0mKKctmArexmvclmAk8 +jhvh3TaHK0u7aNM5Zj2gJsfyOZEdUauCe37Vzlrk4gNXcGmXCPleWKYK34wGmkUW +FjgKXlf2Ysd6AgXmvB618p70qSmD+LIU424oh0TDkBreOKk8rENNZEXO3SipXPJz +ewT4F+irsfMuXGRuczE6Eri8sxHkfY+BUZo7jYn0TZNmezwD7dOaHZrzZVD1oNB1 +ny+v8OqCQ5j4aZyJecRDjkZy42Q2Eq/3JR44iZB3fsNrarnDy0RLrHiQi+fHLB5L +EUTINFInzQpdn4XBidUaePKVEFMy3YCEZnXZtWgo+2EuvoSoOMCZEoalHmdkrQYu +L6lwhceWD3yJZfWOQ1QOq92lgDmUYMA0yZZwLKMS9R9Ie70cfmu3nZD0Ijuu+Pwq +yvqCUqDvr0tVk+vBtfAii6w0TiYiBKGHLHVKt+V9E9e4DGTANtLJL4YSjCMJwRuC +O3NJo2pXh5Tl1njFmUNj403gdy3hZZlyaQQaRwnmDwFWJPsfvw55qVguucQJAX6V +um0ABj6y6koQOdjQK/W/7HW/lwLFCRsI3FU34oH7N4RDYiDK51ZLZer+bMEkkySh +NOsF/5oirpt9P/FlUQqmMGqz9IgcgA38corog14= -----END CERTIFICATE----- From 1bbcd91ae0fda0fb790c99320e9fb913e1c3f379 Mon Sep 17 00:00:00 2001 From: yackob03 Date: Tue, 29 Oct 2013 19:59:29 -0400 Subject: [PATCH 48/57] Make sure the daemon starts up in the right directory to access the certs file. --- buildserver/buildserver.py | 2 +- workers/dockerfilebuild.py | 4 +++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/buildserver/buildserver.py b/buildserver/buildserver.py index ebd039cdc..1b159173b 100644 --- a/buildserver/buildserver.py +++ b/buildserver/buildserver.py @@ -209,4 +209,4 @@ def health_check(): if __name__ == '__main__': logging.basicConfig(level=logging.DEBUG, format=LOG_FORMAT) - app.run(host='0.0.0.0', port=5002) + app.run(host='0.0.0.0', port=5002, threaded=True) diff --git a/workers/dockerfilebuild.py b/workers/dockerfilebuild.py index 7a3daa12f..e472e2116 100644 --- a/workers/dockerfilebuild.py +++ b/workers/dockerfilebuild.py @@ -5,6 +5,7 @@ import time import argparse import digitalocean import requests +import os from apscheduler.scheduler import Scheduler from multiprocessing.pool import ThreadPool @@ -257,7 +258,8 @@ if args.D: handler = logging.FileHandler(args.log) handler.setFormatter(formatter) root_logger.addHandler(handler) - with daemon.DaemonContext(files_preserve=[handler.stream]): + with daemon.DaemonContext(files_preserve=[handler.stream], + working_directory=os.getcwd()): start_worker() else: From dbbec5a853638a5405697271e5c5f180d9abeb46 Mon Sep 17 00:00:00 2001 From: Joseph Schorr Date: Tue, 29 Oct 2013 20:21:18 -0400 Subject: [PATCH 49/57] Fix description handling in new repo view --- endpoints/api.py | 2 ++ static/js/controllers.js | 3 ++- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/endpoints/api.py b/endpoints/api.py index c5592d8b8..853c7498f 100644 --- a/endpoints/api.py +++ b/endpoints/api.py @@ -191,6 +191,8 @@ def create_repo_api(): repo = model.create_repository(namespace_name, repository_name, owner, visibility) + repo.description = request.get_json()['description'] + repo.save() return jsonify({ 'namespace': namespace_name, diff --git a/static/js/controllers.js b/static/js/controllers.js index a74f606a7..0282c460b 100644 --- a/static/js/controllers.js +++ b/static/js/controllers.js @@ -1028,7 +1028,8 @@ function NewRepoCtrl($scope, $location, $http, UserService, Restangular, PlanSer var repo = $scope.repo; var data = { 'repository': repo.name, - 'visibility': repo.is_public == '1' ? 'public' : 'private' + 'visibility': repo.is_public == '1' ? 'public' : 'private', + 'description': repo.description }; var createPost = Restangular.one('repository'); From 081041b85fe4201d41bf183fbccf6a7f83276c00 Mon Sep 17 00:00:00 2001 From: Joseph Schorr Date: Tue, 29 Oct 2013 20:54:36 -0400 Subject: [PATCH 50/57] - Update the repo page when a build is complete - Change the polling to be 5s - Show a message on an empty repo when building --- static/js/controllers.js | 78 +++++++++++++++++--------- static/partials/build-status-item.html | 4 ++ static/partials/view-repo.html | 4 ++ 3 files changed, 59 insertions(+), 27 deletions(-) diff --git a/static/js/controllers.js b/static/js/controllers.js index 0282c460b..f3954c6a9 100644 --- a/static/js/controllers.js +++ b/static/js/controllers.js @@ -335,28 +335,69 @@ function RepoCtrl($scope, Restangular, $routeParams, $rootScope, $location, $tim } }); + var fetchRepository = function() { + var repositoryFetch = Restangular.one('repository/' + namespace + '/' + name); + repositoryFetch.get().then(function(repo) { + $rootScope.title = namespace + '/' + name; + $scope.repo = repo; + + $scope.setTag($routeParams.tag); + + $('#copyClipboard').clipboardCopy(); + $scope.loading = false; + + if (repo.is_building) { + startBuildInfoTimer(repo); + } + }, function() { + $scope.repo = null; + $scope.loading = false; + $rootScope.title = 'Unknown Repository'; + }); + }; + var startBuildInfoTimer = function(repo) { if ($scope.interval) { return; } getBuildInfo(repo); $scope.interval = setInterval(function() { - var container = document.getElementById('build-status-container'); - if (container != null && container.offsetWidth > 0) { - $scope.$apply(function() { getBuildInfo(repo); }); - } - }, 2000); + $scope.$apply(function() { getBuildInfo(repo); }); + }, 5000); $scope.$on("$destroy", function() { - if ($scope.interval) { - clearInterval($scope.interval); - } + cancelBuildInfoTimer(); }); }; + var cancelBuildInfoTimer = function() { + if ($scope.interval) { + clearInterval($scope.interval); + } + }; + var getBuildInfo = function(repo) { var buildInfo = Restangular.one('repository/' + repo.namespace + '/' + repo.name + '/build/'); buildInfo.get().then(function(resp) { - $scope.buildsInfo = resp.builds; + var runningBuilds = []; + for (var i = 0; i < resp.builds.length; ++i) { + var build = resp.builds[i]; + if (build.status != 'complete') { + runningBuilds.push(build); + } + } + + $scope.buildsInfo = runningBuilds; + if (!runningBuilds.length) { + // Cancel the build timer. + cancelBuildInfoTimer(); + + // Mark the repo as no longer building. + $scope.repo.is_building = false; + + // Reload the repo information. + fetchRepository(); + listImages(); + } }); }; @@ -455,24 +496,7 @@ function RepoCtrl($scope, Restangular, $routeParams, $rootScope, $location, $tim $scope.loading = true; // Fetch the repo. - var repositoryFetch = Restangular.one('repository/' + namespace + '/' + name); - repositoryFetch.get().then(function(repo) { - $rootScope.title = namespace + '/' + name; - $scope.repo = repo; - - $scope.setTag($routeParams.tag); - - $('#copyClipboard').clipboardCopy(); - $scope.loading = false; - - if (repo.is_building) { - startBuildInfoTimer(repo); - } - }, function() { - $scope.repo = null; - $scope.loading = false; - $rootScope.title = 'Unknown Repository'; - }); + fetchRepository(); // Fetch the image history. listImages(); diff --git a/static/partials/build-status-item.html b/static/partials/build-status-item.html index a85af075d..be1c06878 100644 --- a/static/partials/build-status-item.html +++ b/static/partials/build-status-item.html @@ -2,4 +2,8 @@
    + +
    + All Dockerfile builds complete +
    diff --git a/static/partials/view-repo.html b/static/partials/view-repo.html index b2293d2e4..77a574b5a 100644 --- a/static/partials/view-repo.html +++ b/static/partials/view-repo.html @@ -64,6 +64,10 @@
    (This repository is empty)
    +
    +
    Your build is currently processing, if this takes longer than an hour, please contact quay support
    +
    +
    From 8a0d6d05b4618992f738fab078496bb08d892f49 Mon Sep 17 00:00:00 2001 From: Joseph Schorr Date: Wed, 30 Oct 2013 13:03:44 -0400 Subject: [PATCH 51/57] Fix case where image changes load results in a 404 --- static/js/controllers.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/static/js/controllers.js b/static/js/controllers.js index f3954c6a9..1ea6fd055 100644 --- a/static/js/controllers.js +++ b/static/js/controllers.js @@ -433,7 +433,7 @@ function RepoCtrl($scope, Restangular, $routeParams, $rootScope, $location, $tim changesFetch.get().then(function(changeInfo) { $scope.currentImageChanges = changeInfo; }, function() { - $scope.currentImageChanges = {}; + $scope.currentImageChanges = {'added': [], 'removed': [], 'changed': []}; }); }; From f8f4c55f893db68f3b4100ce9f020b609a5c6c3e Mon Sep 17 00:00:00 2001 From: yackob03 Date: Wed, 30 Oct 2013 13:22:00 -0400 Subject: [PATCH 52/57] Close the db connections on the workers occasionally. --- workers/diffsworker.py | 5 +++++ workers/dockerfilebuild.py | 10 ++++++++++ 2 files changed, 15 insertions(+) diff --git a/workers/diffsworker.py b/workers/diffsworker.py index 62ea36d81..ce81b96f0 100644 --- a/workers/diffsworker.py +++ b/workers/diffsworker.py @@ -7,6 +7,7 @@ import argparse from apscheduler.scheduler import Scheduler from data.queue import image_diff_queue +from data.database import db as db_connection from endpoints.registry import process_image_changes @@ -37,6 +38,10 @@ def process_work_items(): logger.debug('No more work.') + if not db_connection.is_closed(): + logger.debug('Closing thread db connection.') + db_connection.close() + def start_worker(): logger.debug("Scheduling worker.") diff --git a/workers/dockerfilebuild.py b/workers/dockerfilebuild.py index e472e2116..4cdeb0e70 100644 --- a/workers/dockerfilebuild.py +++ b/workers/dockerfilebuild.py @@ -15,6 +15,7 @@ from requests.exceptions import ConnectionError from data.queue import dockerfile_build_queue from data.userfiles import UserRequestFiles from data import model +from data.database import db as db_connection from app import app @@ -201,6 +202,11 @@ def babysit_builder(request): logger.exception('Exception processing job: %s' % outer_ex.message) return False + finally: + if not db_connection.is_closed(): + logger.debug('Closing thread db connection.') + db_connection.close() + def process_work_items(pool): logger.debug('Getting work item from queue.') @@ -231,6 +237,10 @@ def process_work_items(pool): logger.debug('No more work.') + if not db_connection.is_closed(): + logger.debug('Closing thread db connection.') + db_connection.close() + def start_worker(): pool = ThreadPool(3) From b9cdf93758a5d24f0562c74b7154fcaa54b613d5 Mon Sep 17 00:00:00 2001 From: yackob03 Date: Wed, 30 Oct 2013 13:23:09 -0400 Subject: [PATCH 53/57] Make the quay support a mailto link. --- static/partials/view-repo.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/static/partials/view-repo.html b/static/partials/view-repo.html index 77a574b5a..2efe29be1 100644 --- a/static/partials/view-repo.html +++ b/static/partials/view-repo.html @@ -65,7 +65,7 @@
    -
    Your build is currently processing, if this takes longer than an hour, please contact quay support
    +
    Your build is currently processing, if this takes longer than an hour, please contact Quay.io support
    From 9dcf427f99c9f6f44b6df5d437e88b90521b0747 Mon Sep 17 00:00:00 2001 From: yackob03 Date: Wed, 30 Oct 2013 13:40:55 -0400 Subject: [PATCH 54/57] Remove an unnecessary check from the listImages function. --- static/js/controllers.js | 2 -- 1 file changed, 2 deletions(-) diff --git a/static/js/controllers.js b/static/js/controllers.js index 1ea6fd055..d7940ece6 100644 --- a/static/js/controllers.js +++ b/static/js/controllers.js @@ -402,8 +402,6 @@ function RepoCtrl($scope, Restangular, $routeParams, $rootScope, $location, $tim }; var listImages = function() { - if ($scope.imageHistory) { return; } - var imageFetch = Restangular.one('repository/' + namespace + '/' + name + '/image/'); imageFetch.get().then(function(resp) { $scope.imageHistory = resp.images; From e047b5e11e279f9a7e01df0da6934c4e3c1531dd Mon Sep 17 00:00:00 2001 From: Joseph Schorr Date: Wed, 30 Oct 2013 13:55:44 -0400 Subject: [PATCH 55/57] Dispose of the existing image tree so that a new one can be drawn in its place --- static/js/controllers.js | 11 ++++++++--- static/js/graphing.js | 10 ++++++++++ 2 files changed, 18 insertions(+), 3 deletions(-) diff --git a/static/js/controllers.js b/static/js/controllers.js index 1ea6fd055..25745ff0e 100644 --- a/static/js/controllers.js +++ b/static/js/controllers.js @@ -402,11 +402,16 @@ function RepoCtrl($scope, Restangular, $routeParams, $rootScope, $location, $tim }; var listImages = function() { - if ($scope.imageHistory) { return; } - var imageFetch = Restangular.one('repository/' + namespace + '/' + name + '/image/'); imageFetch.get().then(function(resp) { $scope.imageHistory = resp.images; + + // Dispose of any existing tree. + if ($scope.tree) { + $scope.tree.dispose(); + } + + // Create the new tree. $scope.tree = new ImageHistoryTree(namespace, name, resp.images, $scope.getCommentFirstLine, $scope.getTimeSince); @@ -418,7 +423,7 @@ function RepoCtrl($scope, Restangular, $routeParams, $rootScope, $location, $tim } $($scope.tree).bind('tagChanged', function(e) { - $scope.$apply(function() { $scope.setTag(e.tag, true); }); + $scope.$apply(function() { $scope.setTag(e.tag, true); }); }); $($scope.tree).bind('imageChanged', function(e) { $scope.$apply(function() { $scope.setImage(e.image); }); diff --git a/static/js/graphing.js b/static/js/graphing.js index c38ee273e..88aedc5da 100644 --- a/static/js/graphing.js +++ b/static/js/graphing.js @@ -809,6 +809,16 @@ ImageFileChangeTree.prototype.notifyResized = function() { }; +/** + * Disposes of the tree. + */ +ImageFileChangeTree.prototype.dispose = function() { + var container = this.container_ ; + $('#' + container).removeOverscroll(); + document.getElememtById(container).innerHTML = ''; +}; + + /** * Draws the tree. */ From 6a4341e2cfd0979f4bc0b34ce56b17553e0a4143 Mon Sep 17 00:00:00 2001 From: Joseph Schorr Date: Wed, 30 Oct 2013 15:44:01 -0400 Subject: [PATCH 56/57] Show a please sign in message instead of redirecting on the new repo page --- static/js/controllers.js | 12 ++++-------- static/partials/new-repo.html | 10 +++++++--- 2 files changed, 11 insertions(+), 11 deletions(-) diff --git a/static/js/controllers.js b/static/js/controllers.js index 25745ff0e..5abee4f49 100644 --- a/static/js/controllers.js +++ b/static/js/controllers.js @@ -928,6 +928,10 @@ function V1Ctrl($scope, $location, UserService) { } function NewRepoCtrl($scope, $location, $http, UserService, Restangular, PlanService) { + $scope.$watch( function () { return UserService.currentUser(); }, function (currentUser) { + $scope.user = currentUser; + }, true); + $scope.repo = { 'is_public': 1, 'description': '', @@ -1016,14 +1020,6 @@ function NewRepoCtrl($scope, $location, $http, UserService, Restangular, PlanSer } }; - $scope.$watch( function () { return UserService.currentUser(); }, function (currentUser) { - $scope.user = currentUser; - - if ($scope.user.anonymous) { - $location.path('/signin/'); - } - }, true); - $scope.editDescription = function() { if (!$scope.markdownDescriptionEditor) { var converter = Markdown.getSanitizingConverter(); diff --git a/static/partials/new-repo.html b/static/partials/new-repo.html index dd02d2faa..30cdd2316 100644 --- a/static/partials/new-repo.html +++ b/static/partials/new-repo.html @@ -1,12 +1,16 @@ -
    +
    +

    Please sign in

    +
    + +
    -
    +
    -
    +
    Uploading file {{ upload_file }}
    From ac88ca4ac23c7cc349071f79870ce05265c36f7f Mon Sep 17 00:00:00 2001 From: Joseph Schorr Date: Wed, 30 Oct 2013 16:13:34 -0400 Subject: [PATCH 57/57] Add dispose to the correct tree --- static/js/graphing.js | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/static/js/graphing.js b/static/js/graphing.js index 88aedc5da..5581dc3ce 100644 --- a/static/js/graphing.js +++ b/static/js/graphing.js @@ -715,7 +715,17 @@ ImageHistoryTree.prototype.toggle_ = function(d) { } }; -///////////////////////////////////////////////////////////////////////////////////////// + +/** + * Disposes of the tree. + */ +ImageHistoryTree.prototype.dispose = function() { + var container = this.container_ ; + $('#' + container).removeOverscroll(); + document.getElementById(container).innerHTML = ''; +}; + +//////////////////////////////////////////////////////////////////////////////// /** * Based off of http://bl.ocks.org/mbostock/1093025 by Mike Bostock (@mbostock) @@ -814,8 +824,7 @@ ImageFileChangeTree.prototype.notifyResized = function() { */ ImageFileChangeTree.prototype.dispose = function() { var container = this.container_ ; - $('#' + container).removeOverscroll(); - document.getElememtById(container).innerHTML = ''; + document.getElementById(container).innerHTML = ''; };