diff --git a/README.md b/README.md index 01ca6b799..1e658d180 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,7 @@ to prepare a new host: ``` +sudo apt-get install software-properties-common sudo apt-add-repository -y ppa:nginx/stable sudo apt-get update sudo apt-get install -y git python-virtualenv python-dev phantomjs @@ -30,6 +31,7 @@ start the workers: ``` STACK=prod python -m workers.diffsworker -D +STACK=prod python -m workers.dockerfilebuild -D ``` bouncing the servers: diff --git a/application.py b/application.py index d8d29a177..d3feefcb0 100644 --- a/application.py +++ b/application.py @@ -2,16 +2,25 @@ import logging from app import app as application + +logging.basicConfig(**application.config['LOGGING_CONFIG']) + + import endpoints.index import endpoints.api import endpoints.web import endpoints.tags import endpoints.registry + +logger = logging.getLogger(__name__) + +if application.config.get('INCLUDE_TEST_ENDPOINTS', False): + logger.debug('Loading test endpoints.') + import endpoints.test + # Remove this for prod config application.debug = True -logging.basicConfig(**application.config['LOGGING_CONFIG']) - if __name__ == '__main__': - application.run(port=5000, debug=True, host='0.0.0.0') + application.run(port=5000, debug=True, threaded=True, host='0.0.0.0') diff --git a/buildserver/Dockerfile b/buildserver/Dockerfile new file mode 100644 index 000000000..0a190a7cd --- /dev/null +++ b/buildserver/Dockerfile @@ -0,0 +1,26 @@ +FROM lopter/raring-base +MAINTAINER jake@devtable.com + +RUN echo deb http://archive.ubuntu.com/ubuntu precise universe > /etc/apt/sources.list.d/universe.list +RUN apt-get update -qq +RUN apt-get install -qqy iptables ca-certificates lxc python-virtualenv git python-dev xz-utils aufs-tools + +# This will use the latest public release. To use your own, comment it out... +ADD https://get.docker.io/builds/Linux/x86_64/docker-latest /usr/local/bin/docker +# ...then uncomment the following line, and copy your docker binary to current dir. +#ADD ./docker /usr/local/bin/docker + +# Install the files +ADD ./startserver /usr/local/bin/startserver +ADD ./buildserver.py ./buildserver.py +ADD ./requirements.txt ./requirements.txt + +RUN chmod +x /usr/local/bin/docker /usr/local/bin/startserver + +RUN virtualenv --distribute venv +RUN venv/bin/pip install -r requirements.txt + +VOLUME /var/lib/docker + +EXPOSE 5002:5002 +CMD startserver \ No newline at end of file diff --git a/buildserver/Readme.md b/buildserver/Readme.md new file mode 100644 index 000000000..8dc0dff56 --- /dev/null +++ b/buildserver/Readme.md @@ -0,0 +1,13 @@ +To build: + +``` +sudo docker build -t quay.io/quay/buildserver . +sudo docker push quay.io/quay/buildserver +``` + +To run: + +``` +sudo docker pull quay.io/quay/buildserver +sudo docker run -d -privileged -lxc-conf="lxc.aa_profile=unconfined" quay.io/quay/buildserver +``` \ No newline at end of file diff --git a/buildserver/buildserver.py b/buildserver/buildserver.py new file mode 100644 index 000000000..1b159173b --- /dev/null +++ b/buildserver/buildserver.py @@ -0,0 +1,212 @@ +import docker +import logging +import shutil +import os +import re +import requests +import json + +from flask import Flask, jsonify, url_for, abort, make_response +from zipfile import ZipFile +from tempfile import TemporaryFile, mkdtemp +from uuid import uuid4 +from multiprocessing.pool import ThreadPool +from base64 import b64encode + + +BUFFER_SIZE = 8 * 1024 +LOG_FORMAT = '%(asctime)-15s - %(levelname)s - %(pathname)s - ' + \ + '%(funcName)s - %(message)s' + +app = Flask(__name__) +logger = logging.getLogger(__name__) + + +def count_steps(dockerfile_path): + with open(dockerfile_path, 'r') as dockerfileobj: + steps = 0 + for line in dockerfileobj.readlines(): + stripped = line.strip() + if stripped and stripped[0] is not '#': + steps += 1 + return steps + + +def prepare_zip(request_file): + build_dir = mkdtemp(prefix='docker-build-') + + # Save the zip file to temp somewhere + with TemporaryFile() as zip_file: + zip_file.write(request_file.content) + to_extract = ZipFile(zip_file) + to_extract.extractall(build_dir) + + return build_dir + + +def prepare_dockerfile(request_file): + build_dir = mkdtemp(prefix='docker-build-') + dockerfile_path = os.path.join(build_dir, "Dockerfile") + with open(dockerfile_path, 'w') as dockerfile: + dockerfile.write(request_file.content) + + return build_dir + + +def build_image(build_dir, tag_name, num_steps, result_object): + try: + logger.debug('Starting build.') + docker_cl = docker.Client(version='1.5') + result_object['status'] = 'building' + build_status = docker_cl.build(path=build_dir, tag=tag_name) + + current_step = 0 + built_image = None + for status in build_status: + step_increment = re.search(r'Step ([0-9]+) :', status) + if step_increment: + current_step = int(step_increment.group(1)) + logger.debug('Step now: %s/%s' % (current_step, num_steps)) + result_object['current_command'] = current_step + continue + + complete = re.match(r'Successfully built ([a-z0-9]+)$', status) + if complete: + built_image = complete.group(1) + logger.debug('Final image ID is: %s' % built_image) + continue + + shutil.rmtree(build_dir) + + # Get the image count + if not built_image: + result_object['status'] = 'error' + result_object['message'] = 'Unable to build dockerfile.' + return + + history = docker_cl.history(built_image) + num_images = len(history) + result_object['total_images'] = num_images + + result_object['status'] = 'pushing' + logger.debug('Pushing to tag name: %s' % tag_name) + resp = docker_cl.push(tag_name) + + current_image = 0 + image_progress = 0 + for status in resp: + if u'status' in status: + status_msg = status[u'status'] + next_image = r'(Pushing|Image) [a-z0-9]+( already pushed, skipping)?$' + match = re.match(next_image, status_msg) + if match: + current_image += 1 + image_progress = 0 + logger.debug('Now pushing image %s/%s' % + (current_image, num_images)) + + elif status_msg == u'Pushing' and u'progress' in status: + percent = r'\(([0-9]+)%\)' + match = re.search(percent, status[u'progress']) + if match: + image_progress = int(match.group(1)) + + result_object['current_image'] = current_image + result_object['image_completion_percent'] = image_progress + + elif u'errorDetail' in status: + result_object['status'] = 'error' + if u'message' in status[u'errorDetail']: + result_object['message'] = status[u'errorDetail'][u'message'] + return + + result_object['status'] = 'complete' + except Exception as e: + logger.exception('Exception when processing request.') + result_object['status'] = 'error' + result_object['message'] = str(e.message) + + +MIME_PROCESSORS = { + 'application/zip': prepare_zip, + 'text/plain': prepare_dockerfile, + 'application/octet-stream': prepare_dockerfile, +} + + +build = { + 'total_commands': None, + 'total_images': None, + 'current_command': None, + 'current_image': None, + 'image_completion_percent': None, + 'status': 'waiting', + 'message': None, +} +pool = ThreadPool(1) + + +@app.before_first_request +def start_build(): + resource_url = os.environ['RESOURCE_URL'] + tag_name = os.environ['TAG'] + acccess_token = os.environ['TOKEN'] + + logger.debug('Starting job with resource url: %s tag: %s and token: %s' % + (resource_url, tag_name, acccess_token)) + + # Save the token + host = re.match(r'([a-z0-9.:]+)/.+/.+$', tag_name) + if host: + docker_endpoint = 'http://%s/v1/' % host.group(1) + dockercfg_path = os.path.join(os.environ.get('HOME', '.'), '.dockercfg') + token = b64encode('$token:%s' % acccess_token) + with open(dockercfg_path, 'w') as dockercfg: + payload = { + docker_endpoint: { + 'auth': token, + 'email': '', + } + } + dockercfg.write(json.dumps(payload)) + + else: + raise Exception('Invalid tag name: %s' % tag_name) + + docker_resource = requests.get(resource_url) + c_type = docker_resource.headers['content-type'] + + logger.info('Request to build file of type: %s with tag: %s' % + (c_type, tag_name)) + + if c_type not in MIME_PROCESSORS: + raise Exception('Invalid dockerfile content type: %s' % c_type) + + build_dir = MIME_PROCESSORS[c_type](docker_resource) + + dockerfile_path = os.path.join(build_dir, "Dockerfile") + num_steps = count_steps(dockerfile_path) + logger.debug('Dockerfile had %s steps' % num_steps) + + logger.info('Sending job to builder pool.') + build['total_commands'] = num_steps + + pool.apply_async(build_image, [build_dir, tag_name, num_steps, + build]) + + +@app.route('/build/', methods=['GET']) +def get_status(): + if build: + return jsonify(build) + abort(404) + + +@app.route('/status/', methods=['GET']) +def health_check(): + return make_response('Running') + + +if __name__ == '__main__': + logging.basicConfig(level=logging.DEBUG, format=LOG_FORMAT) + app.run(host='0.0.0.0', port=5002, threaded=True) diff --git a/buildserver/requirements.txt b/buildserver/requirements.txt new file mode 100644 index 000000000..3093eeb14 --- /dev/null +++ b/buildserver/requirements.txt @@ -0,0 +1,5 @@ +mock==1.0.1 +requests==1.2.3 +six==1.3.0 +flask==0.10.1 +-e git+git://github.com/DevTable/docker-py.git#egg=docker-py \ No newline at end of file diff --git a/buildserver/startserver b/buildserver/startserver new file mode 100644 index 000000000..9eb2e1114 --- /dev/null +++ b/buildserver/startserver @@ -0,0 +1,48 @@ +#!/bin/bash + +# First, make sure that cgroups are mounted correctly. +CGROUP=/sys/fs/cgroup + +[ -d $CGROUP ] || + mkdir $CGROUP + +mountpoint -q $CGROUP || + mount -n -t tmpfs -o uid=0,gid=0,mode=0755 cgroup $CGROUP || { + echo "Could not make a tmpfs mount. Did you use -privileged?" + exit 1 + } + +# Mount the cgroup hierarchies exactly as they are in the parent system. +for SUBSYS in $(cut -d: -f2 /proc/1/cgroup) +do + [ -d $CGROUP/$SUBSYS ] || mkdir $CGROUP/$SUBSYS + mountpoint -q $CGROUP/$SUBSYS || + mount -n -t cgroup -o $SUBSYS cgroup $CGROUP/$SUBSYS +done + +# Note: as I write those lines, the LXC userland tools cannot setup +# a "sub-container" properly if the "devices" cgroup is not in its +# own hierarchy. Let's detect this and issue a warning. +grep -q :devices: /proc/1/cgroup || + echo "WARNING: the 'devices' cgroup should be in its own hierarchy." +grep -qw devices /proc/1/cgroup || + echo "WARNING: it looks like the 'devices' cgroup is not mounted." + +# Now, close extraneous file descriptors. +pushd /proc/self/fd +for FD in * +do + case "$FD" in + # Keep stdin/stdout/stderr + [012]) + ;; + # Nuke everything else + *) + eval exec "$FD>&-" + ;; + esac +done +popd + +docker -d & +exec venv/bin/python buildserver.py \ No newline at end of file diff --git a/certs/digital_ocean b/certs/digital_ocean new file mode 100644 index 000000000..700f2b586 --- /dev/null +++ b/certs/digital_ocean @@ -0,0 +1,27 @@ +-----BEGIN RSA PRIVATE KEY----- +MIIEowIBAAKCAQEAwjlIK0HodmDNrZAmaALtr9RLriRSeeLh76gV8KHmjRweeT7v +dmhKeGP1nOAs17caZkcwsW0tiDbCeIv2MisV405sScjPOxFivWpY8tL72sgVuOAl +ReZauOGZ4M1ZcSa/YbT7tnFCIayYE9pde4ih5LmYZqKsBsaNq3ErcMnAzqG77D95 +8swuVwhz/INioBWwe4FjO76/0DqS357hT5yHDWthJD6UUH12VajPKBtXEvGNUtNL +vdq+drm9omt2y0seMn47fZXiNIulLv7ojsWKwtRMTsGcjnv6VMZAVAuX11u4cJd+ +oPTbDl0D+02B7XYcxABqdMZcOc1/7VTUlFFd4wIDAQABAoIBAAs4V+z3z8AW84rV +SwKzOJvxvbV/r6wO6VJ4+Vt/XtxEBZanhhnnCHZP//5iDPUhRMsnza5SSlEWKMHi +BAT97DPHcgYJLb+Rz4x1ulG80oPfDzIw8LZLCm6nycXs1v/sZx3z4J63iER9vgNX +mBLs371g42b6esmhasm+re3EGflV0LeY1IX0MY40pqGndmW8Fly1QH179TrMzVUJ +btu3i2JrwWmKk5zO5YGm0SYY5QQGCdjPj6SL+idDniAefEvbjJYz2qOaPOF3wj/7 +r8dAnmyaP10Q3JojT01Et5ltMfr0oF2/pic9tWYGrgn/aIuoXUXj0SF3Pfgrb/4L +Et1kzFECgYEA8Tb/9bYzQgtaQTQfzFU/KnsIKKnrxh73rZwnIxG59WvN0Ws41Byf +rv8fEbXWU8Yj0drxRSud9fADr99lZGWFxle8rSW5+qqoUxG8n/fkktzHxyPE/9Mh +pZW7un7a5/glKgUpHpjaOCZj9rhdF1AwdUXLSo1sFc7VBsKvKiKJAT0CgYEAziDt +A9h5lOgiLGf1xdBq3qmLIlARz7fivAcZ5acSGN5k6MFFxjHNqhcXRusqs7g+hvCN +eRupdwfgSdLwrTfvxuY4pCcddfYIZO3uUZYs/glvYRtIxaP2kMBkZTs9KzI02Bjv +zT3NPReR/46SqW0zvYTlRFSY7VZ0eRED/5xnjZ8CgYAZdlrSjyceA6DFXUE2CpGe +ZFpaIIW45i/y7ZbcBtUAaR7SymS3T0Yz7M5UykMTmMjTMC9jw9Tqzyk0eXp0fJsA +cuaByIe3RCh8jFTC9iH0tsWH6eizsI/OsN2eNCHbdsBFjUHn7u6qGrNWqeN5wIc8 ++d8ZwY/1RV4LVqWy5u5baQKBgHLFvJMWluQFuPl2zU9etBLU3ma1pKU/I11EqvPH +afk044UCEKLBml1pzAkt6jH1lcM2798OOvbPCOCyNlaMvdLG36TvLqU+3/+qx7bf +4p90i3LLaWK64BBLP9tp9640n13vzJ5AGiY5GI7uSNVTu6p789hvLlOAfwvmII7T +/IjLAoGBAO6iU8i6pAOaKa7+/uExXx6xwk3vqQtovxByo1/m7NpyUtT+ElDSq+t9 +7f+3TzzPB6ggdMl8d+PSyHR3o7KjVPgOSe7zld7eePhUrLjwZ4lh5ohcvhvYfaRL +0EgRTaTb+zLtCAvJS/ilNnJoIcxUmD8u5uSXpY7vAleSOiQTJRTh +-----END RSA PRIVATE KEY----- diff --git a/certs/digital_ocean.pub b/certs/digital_ocean.pub new file mode 100644 index 000000000..95db83601 --- /dev/null +++ b/certs/digital_ocean.pub @@ -0,0 +1 @@ +ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDCOUgrQeh2YM2tkCZoAu2v1EuuJFJ54uHvqBXwoeaNHB55Pu92aEp4Y/Wc4CzXtxpmRzCxbS2INsJ4i/YyKxXjTmxJyM87EWK9aljy0vvayBW44CVF5lq44ZngzVlxJr9htPu2cUIhrJgT2l17iKHkuZhmoqwGxo2rcStwycDOobvsP3nyzC5XCHP8g2KgFbB7gWM7vr/QOpLfnuFPnIcNa2EkPpRQfXZVqM8oG1cS8Y1S00u92r52ub2ia3bLSx4yfjt9leI0i6Uu/uiOxYrC1ExOwZyOe/pUxkBUC5fXW7hwl36g9NsOXQP7TYHtdhzEAGp0xlw5zX/tVNSUUV3j jake@coreserver diff --git a/certs/quay-staging-enc.key b/certs/quay-staging-enc.key new file mode 100644 index 000000000..2e4e0ec73 --- /dev/null +++ b/certs/quay-staging-enc.key @@ -0,0 +1,30 @@ +-----BEGIN RSA PRIVATE KEY----- +Proc-Type: 4,ENCRYPTED +DEK-Info: AES-256-CBC,2E92F159A4A60E842528201723AF241C + +fThdbyOk4whle4mTLn3fDt/eV/JqKXxs18yQl6OdS26sxB80MmwIAF6dyZI3AXHC +CBS6mmdZtFgqodXs125XXPijJ0m0V5s2yx7EHeqPMEYLnzu21QfsbtRGXtl5EX3m +il+F9Y/+Y3B0ZtvMcRsGPjnCkLeQZwMOVc8DsgC92bqgu3sCDuz70W6Na6wvIlWU +2/OViNDSI6QUo90bkdf1H/+pCr8Dge9MBBMnU2G7DqsaLR+ehUMj7sZs/MJjUOK5 +zERfQMEoywqBatQUlP203GWe3hsMAqAKnc5VUc52mpT7uC8afLuSqqu2SpP9W0f2 +Fu6jv0D9JqFI7tf11sGYyWdMoP7zSxcquRC+NMVfvEvlRfjourZ8LAkdkQioeGQO +uLUoXe55kDfcJMEahUNMbmrwA7pqfsfGkKCMnBTCATvjtghTTCL4xVrK7jwXk7Sv +q54TQe27sryYCGnCLErGxEwjs0lnIWsZ+ePa4qgF05JBty3psrFHZlubG+CT8sb7 +uzBwzjsIW4rgaBdxKzBgcYoUDhxGMqsg3DxcXh9EomsK/ka+SlVnSihYpxnuerxG +LYgSQhfSRZRTtHUzQcQuvT5sfa1UQIWWWxeUKxuTTcyE3g1DY3/osatchW9+CCW2 +z10daPvfdq70OiEnPLSDvD1Gs49QQlfsoaq4lhNU7VBqznr/5bdC4iiQssXbUOjM +odREtw0d8Ox72/V/z01/QjTiOIg5tk9tGORVFPmLHC/Db3OUkLHsp/ls8E2IkIMU +wrbmvA9ABLgfpLlgivTtYjkG+H9BtSbi4jGHjYGR40unHqzY+6EQfoL7hu6zzacB +4jOhKKtR1yskK42VI6vrihT3GaA3MslCBNmEUv/4FkbvRkVifDd7hMylHCkjlUSm +zpaydx2gVrRYlum/Ipd+yPb95iniVlaaNe9ZLqtpZt15mEgLwWfrTq470NGLKwbg +7IyIrN6TjHSy1BQMS7kQVzc8Mgafa5O3pXr3zJS2JCg13FC7DarRZfNlqQZ7ErRA +imF5jxMQC8agbairlrGQ2i+ckY2wm8OEKUu3f8+O1Rq6rHy7SLQq1PSoQQetstFz +jLDPRFvtymZb0e510nbsLOcQvaWjda2sU367ed99TkVm0J7PJG3J38B7Sz0Q82hh +jfoYsu+kyNCcR2eS7F3E/MAr85r5qW8/0oewV0ZNxZLyAqJLFHKaeEaGcW0q/wb3 +oDcli8kbX0YnMfm4p7YaR++VYtfiQrhzOkzYNmRV4xbVF3eE6f3lSjGXmeqWhM5c +o0+I75UvbjZzvfWjekkwKhzuIHRJV6g5ldxuaJ290xQrdbpzO0xmYKbOHkEtIjow +pBIaA6vgmOTnREdQQSjcNcD+eZDuzaKGwCa2IH5k5b025IOWS4XyZ2JNmRzgw2d8 +mF6fO8ZDpcQUMIv1Hbn4Yc7kE1vOPkUaB3pKumaQW16COoi+EtY+umWf2/SJb+Cd +PGfSzcagiR5Fs0wKgHCVTMuoKR/xhaSqLUkCgQoXKRNoT5cO/sTiYygDWyeyRnCE +CA3NKDsDnaoCAeDU5XS1tmNJSIxlQjlnFHP71otxMKtV/g42vwTkXmJ+6gH5g9TG +-----END RSA PRIVATE KEY----- \ No newline at end of file diff --git a/certs/quay-staging-unified.cert b/certs/quay-staging-unified.cert new file mode 100644 index 000000000..b2c0b5fce --- /dev/null +++ b/certs/quay-staging-unified.cert @@ -0,0 +1,116 @@ +-----BEGIN CERTIFICATE----- +MIIGTjCCBTagAwIBAgIDDK/RMA0GCSqGSIb3DQEBBQUAMIGMMQswCQYDVQQGEwJJ +TDEWMBQGA1UEChMNU3RhcnRDb20gTHRkLjErMCkGA1UECxMiU2VjdXJlIERpZ2l0 +YWwgQ2VydGlmaWNhdGUgU2lnbmluZzE4MDYGA1UEAxMvU3RhcnRDb20gQ2xhc3Mg +MSBQcmltYXJ5IEludGVybWVkaWF0ZSBTZXJ2ZXIgQ0EwHhcNMTMxMDI5MTIxNjM2 +WhcNMTQxMDMwMTUyMDI0WjBlMRkwFwYDVQQNExBXOHJYcjhsNVBEMkpMQ0VRMQsw +CQYDVQQGEwJVUzEYMBYGA1UEAxMPc3RhZ2luZy5xdWF5LmlvMSEwHwYJKoZIhvcN +AQkBFhJob3N0bWFzdGVyQHF1YXkuaW8wggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAw +ggEKAoIBAQDq45PoBSyAniiglXyt5yI3kbLcwLXRTrWNCv0rxi+57Elxs/ix7RII +Ig1iO1FARJP/ipRRHFB8GNuG+eIdJAEaeB39eyjvGvsOcE8hlK1Hu3Hd3PcKAwaV +JpVZyTblUYXy55kw9okwNZJpVJPOHxKaOjNYrJynw92VJ21WeeGk+kh0EZKQ4vtp +sMMYIJapQk1CYDdreZoA0TEGZixJG8laUfX+S+CJf9KY7qH8LefjK9fr6x7R+qd4 +Hvj6lwtGV5UEBkGtU2bzTAOSEMOJBcxOfrPovFFLVvtbYCRAIY2Y5PPvV1Wna9sB +h52hxRhoJpwU3/g+LXhJ6UEGFOMvfa+hAgMBAAGjggLdMIIC2TAJBgNVHRMEAjAA +MAsGA1UdDwQEAwIDqDATBgNVHSUEDDAKBggrBgEFBQcDATAdBgNVHQ4EFgQU6rqK +WYOkW9Ya0nerdfpJGA5GbxcwHwYDVR0jBBgwFoAU60I00Jiwq5/0G2sI98xkLu8O +LEUwIwYDVR0RBBwwGoIPc3RhZ2luZy5xdWF5LmlvggdxdWF5LmlvMIIBVgYDVR0g +BIIBTTCCAUkwCAYGZ4EMAQIBMIIBOwYLKwYBBAGBtTcBAgMwggEqMC4GCCsGAQUF +BwIBFiJodHRwOi8vd3d3LnN0YXJ0c3NsLmNvbS9wb2xpY3kucGRmMIH3BggrBgEF +BQcCAjCB6jAnFiBTdGFydENvbSBDZXJ0aWZpY2F0aW9uIEF1dGhvcml0eTADAgEB +GoG+VGhpcyBjZXJ0aWZpY2F0ZSB3YXMgaXNzdWVkIGFjY29yZGluZyB0byB0aGUg +Q2xhc3MgMSBWYWxpZGF0aW9uIHJlcXVpcmVtZW50cyBvZiB0aGUgU3RhcnRDb20g +Q0EgcG9saWN5LCByZWxpYW5jZSBvbmx5IGZvciB0aGUgaW50ZW5kZWQgcHVycG9z +ZSBpbiBjb21wbGlhbmNlIG9mIHRoZSByZWx5aW5nIHBhcnR5IG9ibGlnYXRpb25z +LjA1BgNVHR8ELjAsMCqgKKAmhiRodHRwOi8vY3JsLnN0YXJ0c3NsLmNvbS9jcnQx +LWNybC5jcmwwgY4GCCsGAQUFBwEBBIGBMH8wOQYIKwYBBQUHMAGGLWh0dHA6Ly9v +Y3NwLnN0YXJ0c3NsLmNvbS9zdWIvY2xhc3MxL3NlcnZlci9jYTBCBggrBgEFBQcw +AoY2aHR0cDovL2FpYS5zdGFydHNzbC5jb20vY2VydHMvc3ViLmNsYXNzMS5zZXJ2 +ZXIuY2EuY3J0MCMGA1UdEgQcMBqGGGh0dHA6Ly93d3cuc3RhcnRzc2wuY29tLzAN +BgkqhkiG9w0BAQUFAAOCAQEAfhP/++WewlphCojZwXijpFIy+XX1gR0p4kSfxVgA +Anl3khFL/xAvhk6pbWjGQM/9FWb/PFDRgj4fvMKGR8F9bMKNfBOrT+SyWDuI1Ax3 +y0unu0vZjEfUJmMktrr2aN3NI/bBmdVixNntsHRB0yrrl7Zk0TjQM3I1egfygoxa +tfARn5QOO/sReYJXlJdwmFMH0dpMT3++p5RhMZPDVeAdUUK/KzSdlPkVrLPJTKEY +d+IAIWjZq5CGOjM9052+CDhyAMvdywJQpxuhO/BzmPrt0ZQwuMdTUutPT2ijDGCB +J7nCUCVEtF25KJrJxeXY6oLxgXoaqqU1ZGivAS1oCtnocg== +-----END CERTIFICATE----- +-----BEGIN CERTIFICATE----- +MIIGNDCCBBygAwIBAgIBGDANBgkqhkiG9w0BAQUFADB9MQswCQYDVQQGEwJJTDEW +MBQGA1UEChMNU3RhcnRDb20gTHRkLjErMCkGA1UECxMiU2VjdXJlIERpZ2l0YWwg +Q2VydGlmaWNhdGUgU2lnbmluZzEpMCcGA1UEAxMgU3RhcnRDb20gQ2VydGlmaWNh +dGlvbiBBdXRob3JpdHkwHhcNMDcxMDI0MjA1NDE3WhcNMTcxMDI0MjA1NDE3WjCB +jDELMAkGA1UEBhMCSUwxFjAUBgNVBAoTDVN0YXJ0Q29tIEx0ZC4xKzApBgNVBAsT +IlNlY3VyZSBEaWdpdGFsIENlcnRpZmljYXRlIFNpZ25pbmcxODA2BgNVBAMTL1N0 +YXJ0Q29tIENsYXNzIDEgUHJpbWFyeSBJbnRlcm1lZGlhdGUgU2VydmVyIENBMIIB +IjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAtonGrO8JUngHrJJj0PREGBiE +gFYfka7hh/oyULTTRwbw5gdfcA4Q9x3AzhA2NIVaD5Ksg8asWFI/ujjo/OenJOJA +pgh2wJJuniptTT9uYSAK21ne0n1jsz5G/vohURjXzTCm7QduO3CHtPn66+6CPAVv +kvek3AowHpNz/gfK11+AnSJYUq4G2ouHI2mw5CrY6oPSvfNx23BaKA+vWjhwRRI/ +ME3NO68X5Q/LoKldSKqxYVDLNM08XMML6BDAjJvwAwNi/rJsPnIO7hxDKslIDlc5 +xDEhyBDBLIf+VJVSH1I8MRKbf+fAoKVZ1eKPPvDVqOHXcDGpxLPPr21TLwb0pwID +AQABo4IBrTCCAakwDwYDVR0TAQH/BAUwAwEB/zAOBgNVHQ8BAf8EBAMCAQYwHQYD +VR0OBBYEFOtCNNCYsKuf9BtrCPfMZC7vDixFMB8GA1UdIwQYMBaAFE4L7xqkQFul +F2mHMMo0aEPQQa7yMGYGCCsGAQUFBwEBBFowWDAnBggrBgEFBQcwAYYbaHR0cDov +L29jc3Auc3RhcnRzc2wuY29tL2NhMC0GCCsGAQUFBzAChiFodHRwOi8vd3d3LnN0 +YXJ0c3NsLmNvbS9zZnNjYS5jcnQwWwYDVR0fBFQwUjAnoCWgI4YhaHR0cDovL3d3 +dy5zdGFydHNzbC5jb20vc2ZzY2EuY3JsMCegJaAjhiFodHRwOi8vY3JsLnN0YXJ0 +c3NsLmNvbS9zZnNjYS5jcmwwgYAGA1UdIAR5MHcwdQYLKwYBBAGBtTcBAgEwZjAu +BggrBgEFBQcCARYiaHR0cDovL3d3dy5zdGFydHNzbC5jb20vcG9saWN5LnBkZjA0 +BggrBgEFBQcCARYoaHR0cDovL3d3dy5zdGFydHNzbC5jb20vaW50ZXJtZWRpYXRl +LnBkZjANBgkqhkiG9w0BAQUFAAOCAgEAIQlJPqWIbuALi0jaMU2P91ZXouHTYlfp +tVbzhUV1O+VQHwSL5qBaPucAroXQ+/8gA2TLrQLhxpFy+KNN1t7ozD+hiqLjfDen +xk+PNdb01m4Ge90h2c9W/8swIkn+iQTzheWq8ecf6HWQTd35RvdCNPdFWAwRDYSw +xtpdPvkBnufh2lWVvnQce/xNFE+sflVHfXv0pQ1JHpXo9xLBzP92piVH0PN1Nb6X +t1gW66pceG/sUzCv6gRNzKkC4/C2BBL2MLERPZBOVmTX3DxDX3M570uvh+v2/miI +RHLq0gfGabDBoYvvF0nXYbFFSF87ICHpW7LM9NfpMfULFWE7epTj69m8f5SuauNi +YpaoZHy4h/OZMn6SolK+u/hlz8nyMPyLwcKmltdfieFcNID1j0cHL7SRv7Gifl9L +WtBbnySGBVFaaQNlQ0lxxeBvlDRr9hvYqbBMflPrj0jfyjO1SPo2ShpTpjMM0InN +SRXNiTE8kMBy12VLUjWKRhFEuT2OKGWmPnmeXAhEKa2wNREuIU640ucQPl2Eg7PD +wuTSxv0JS3QJ3fGz0xk+gA2iCxnwOOfFwq/iI9th4p1cbiCJSS4jarJiwUW0n6+L +p/EiO/h94pDQehn7Skzj0n1fSoMD7SfWI55rjbRZotnvbIIp3XUZPD9MEI3vu3Un +0q6Dp6jOW6c= +-----END CERTIFICATE----- +-----BEGIN CERTIFICATE----- +MIIHyTCCBbGgAwIBAgIBATANBgkqhkiG9w0BAQUFADB9MQswCQYDVQQGEwJJTDEW +MBQGA1UEChMNU3RhcnRDb20gTHRkLjErMCkGA1UECxMiU2VjdXJlIERpZ2l0YWwg +Q2VydGlmaWNhdGUgU2lnbmluZzEpMCcGA1UEAxMgU3RhcnRDb20gQ2VydGlmaWNh +dGlvbiBBdXRob3JpdHkwHhcNMDYwOTE3MTk0NjM2WhcNMzYwOTE3MTk0NjM2WjB9 +MQswCQYDVQQGEwJJTDEWMBQGA1UEChMNU3RhcnRDb20gTHRkLjErMCkGA1UECxMi +U2VjdXJlIERpZ2l0YWwgQ2VydGlmaWNhdGUgU2lnbmluZzEpMCcGA1UEAxMgU3Rh +cnRDb20gQ2VydGlmaWNhdGlvbiBBdXRob3JpdHkwggIiMA0GCSqGSIb3DQEBAQUA +A4ICDwAwggIKAoICAQDBiNsJvGxGfHiflXu1M5DycmLWwTYgIiRezul38kMKogZk +pMyONvg45iPwbm2xPN1yo4UcodM9tDMr0y+v/uqwQVlntsQGfQqedIXWeUyAN3rf +OQVSWff0G0ZDpNKFhdLDcfN1YjS6LIp/Ho/u7TTQEceWzVI9ujPW3U3eCztKS5/C +Ji/6tRYccjV3yjxd5srhJosaNnZcAdt0FCX+7bWgiA/deMotHweXMAEtcnn6RtYT +Kqi5pquDSR3l8u/d5AGOGAqPY1MWhWKpDhk6zLVmpsJrdAfkK+F2PrRt2PZE4XNi +HzvEvqBTViVsUQn3qqvKv3b9bZvzndu/PWa8DFaqr5hIlTpL36dYUNk4dalb6kMM +Av+Z6+hsTXBbKWWc3apdzK8BMewM69KN6Oqce+Zu9ydmDBpI125C4z/eIT574Q1w ++2OqqGwaVLRcJXrJosmLFqa7LH4XXgVNWG4SHQHuEhANxjJ/GP/89PrNbpHoNkm+ +Gkhpi8KWTRoSsmkXwQqQ1vp5Iki/untp+HDH+no32NgN0nZPV/+Qt+OR0t3vwmC3 +Zzrd/qqc8NSLf3Iizsafl7b4r4qgEKjZ+xjGtrVcUjyJthkqcwEKDwOzEmDyei+B +26Nu/yYwl/WL3YlXtq09s68rxbd2AvCl1iuahhQqcvbjM4xdCUsT37uMdBNSSwID +AQABo4ICUjCCAk4wDAYDVR0TBAUwAwEB/zALBgNVHQ8EBAMCAa4wHQYDVR0OBBYE +FE4L7xqkQFulF2mHMMo0aEPQQa7yMGQGA1UdHwRdMFswLKAqoCiGJmh0dHA6Ly9j +ZXJ0LnN0YXJ0Y29tLm9yZy9zZnNjYS1jcmwuY3JsMCugKaAnhiVodHRwOi8vY3Js +LnN0YXJ0Y29tLm9yZy9zZnNjYS1jcmwuY3JsMIIBXQYDVR0gBIIBVDCCAVAwggFM +BgsrBgEEAYG1NwEBATCCATswLwYIKwYBBQUHAgEWI2h0dHA6Ly9jZXJ0LnN0YXJ0 +Y29tLm9yZy9wb2xpY3kucGRmMDUGCCsGAQUFBwIBFilodHRwOi8vY2VydC5zdGFy +dGNvbS5vcmcvaW50ZXJtZWRpYXRlLnBkZjCB0AYIKwYBBQUHAgIwgcMwJxYgU3Rh +cnQgQ29tbWVyY2lhbCAoU3RhcnRDb20pIEx0ZC4wAwIBARqBl0xpbWl0ZWQgTGlh +YmlsaXR5LCByZWFkIHRoZSBzZWN0aW9uICpMZWdhbCBMaW1pdGF0aW9ucyogb2Yg +dGhlIFN0YXJ0Q29tIENlcnRpZmljYXRpb24gQXV0aG9yaXR5IFBvbGljeSBhdmFp +bGFibGUgYXQgaHR0cDovL2NlcnQuc3RhcnRjb20ub3JnL3BvbGljeS5wZGYwEQYJ +YIZIAYb4QgEBBAQDAgAHMDgGCWCGSAGG+EIBDQQrFilTdGFydENvbSBGcmVlIFNT +TCBDZXJ0aWZpY2F0aW9uIEF1dGhvcml0eTANBgkqhkiG9w0BAQUFAAOCAgEAFmyZ +9GYMNPXQhV59CuzaEE44HF7fpiUFS5Eyweg78T3dRAlbB0mKKctmArexmvclmAk8 +jhvh3TaHK0u7aNM5Zj2gJsfyOZEdUauCe37Vzlrk4gNXcGmXCPleWKYK34wGmkUW +FjgKXlf2Ysd6AgXmvB618p70qSmD+LIU424oh0TDkBreOKk8rENNZEXO3SipXPJz +ewT4F+irsfMuXGRuczE6Eri8sxHkfY+BUZo7jYn0TZNmezwD7dOaHZrzZVD1oNB1 +ny+v8OqCQ5j4aZyJecRDjkZy42Q2Eq/3JR44iZB3fsNrarnDy0RLrHiQi+fHLB5L +EUTINFInzQpdn4XBidUaePKVEFMy3YCEZnXZtWgo+2EuvoSoOMCZEoalHmdkrQYu +L6lwhceWD3yJZfWOQ1QOq92lgDmUYMA0yZZwLKMS9R9Ie70cfmu3nZD0Ijuu+Pwq +yvqCUqDvr0tVk+vBtfAii6w0TiYiBKGHLHVKt+V9E9e4DGTANtLJL4YSjCMJwRuC +O3NJo2pXh5Tl1njFmUNj403gdy3hZZlyaQQaRwnmDwFWJPsfvw55qVguucQJAX6V +um0ABj6y6koQOdjQK/W/7HW/lwLFCRsI3FU34oH7N4RDYiDK51ZLZer+bMEkkySh +NOsF/5oirpt9P/FlUQqmMGqz9IgcgA38corog14= +-----END CERTIFICATE----- diff --git a/certs/quay-staging.cert b/certs/quay-staging.cert new file mode 100644 index 000000000..f02c8de26 --- /dev/null +++ b/certs/quay-staging.cert @@ -0,0 +1,36 @@ +-----BEGIN CERTIFICATE----- +MIIGTjCCBTagAwIBAgIDDK/RMA0GCSqGSIb3DQEBBQUAMIGMMQswCQYDVQQGEwJJ +TDEWMBQGA1UEChMNU3RhcnRDb20gTHRkLjErMCkGA1UECxMiU2VjdXJlIERpZ2l0 +YWwgQ2VydGlmaWNhdGUgU2lnbmluZzE4MDYGA1UEAxMvU3RhcnRDb20gQ2xhc3Mg +MSBQcmltYXJ5IEludGVybWVkaWF0ZSBTZXJ2ZXIgQ0EwHhcNMTMxMDI5MTIxNjM2 +WhcNMTQxMDMwMTUyMDI0WjBlMRkwFwYDVQQNExBXOHJYcjhsNVBEMkpMQ0VRMQsw +CQYDVQQGEwJVUzEYMBYGA1UEAxMPc3RhZ2luZy5xdWF5LmlvMSEwHwYJKoZIhvcN +AQkBFhJob3N0bWFzdGVyQHF1YXkuaW8wggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAw +ggEKAoIBAQDq45PoBSyAniiglXyt5yI3kbLcwLXRTrWNCv0rxi+57Elxs/ix7RII +Ig1iO1FARJP/ipRRHFB8GNuG+eIdJAEaeB39eyjvGvsOcE8hlK1Hu3Hd3PcKAwaV +JpVZyTblUYXy55kw9okwNZJpVJPOHxKaOjNYrJynw92VJ21WeeGk+kh0EZKQ4vtp +sMMYIJapQk1CYDdreZoA0TEGZixJG8laUfX+S+CJf9KY7qH8LefjK9fr6x7R+qd4 +Hvj6lwtGV5UEBkGtU2bzTAOSEMOJBcxOfrPovFFLVvtbYCRAIY2Y5PPvV1Wna9sB +h52hxRhoJpwU3/g+LXhJ6UEGFOMvfa+hAgMBAAGjggLdMIIC2TAJBgNVHRMEAjAA +MAsGA1UdDwQEAwIDqDATBgNVHSUEDDAKBggrBgEFBQcDATAdBgNVHQ4EFgQU6rqK +WYOkW9Ya0nerdfpJGA5GbxcwHwYDVR0jBBgwFoAU60I00Jiwq5/0G2sI98xkLu8O +LEUwIwYDVR0RBBwwGoIPc3RhZ2luZy5xdWF5LmlvggdxdWF5LmlvMIIBVgYDVR0g +BIIBTTCCAUkwCAYGZ4EMAQIBMIIBOwYLKwYBBAGBtTcBAgMwggEqMC4GCCsGAQUF +BwIBFiJodHRwOi8vd3d3LnN0YXJ0c3NsLmNvbS9wb2xpY3kucGRmMIH3BggrBgEF +BQcCAjCB6jAnFiBTdGFydENvbSBDZXJ0aWZpY2F0aW9uIEF1dGhvcml0eTADAgEB +GoG+VGhpcyBjZXJ0aWZpY2F0ZSB3YXMgaXNzdWVkIGFjY29yZGluZyB0byB0aGUg +Q2xhc3MgMSBWYWxpZGF0aW9uIHJlcXVpcmVtZW50cyBvZiB0aGUgU3RhcnRDb20g +Q0EgcG9saWN5LCByZWxpYW5jZSBvbmx5IGZvciB0aGUgaW50ZW5kZWQgcHVycG9z +ZSBpbiBjb21wbGlhbmNlIG9mIHRoZSByZWx5aW5nIHBhcnR5IG9ibGlnYXRpb25z +LjA1BgNVHR8ELjAsMCqgKKAmhiRodHRwOi8vY3JsLnN0YXJ0c3NsLmNvbS9jcnQx +LWNybC5jcmwwgY4GCCsGAQUFBwEBBIGBMH8wOQYIKwYBBQUHMAGGLWh0dHA6Ly9v +Y3NwLnN0YXJ0c3NsLmNvbS9zdWIvY2xhc3MxL3NlcnZlci9jYTBCBggrBgEFBQcw +AoY2aHR0cDovL2FpYS5zdGFydHNzbC5jb20vY2VydHMvc3ViLmNsYXNzMS5zZXJ2 +ZXIuY2EuY3J0MCMGA1UdEgQcMBqGGGh0dHA6Ly93d3cuc3RhcnRzc2wuY29tLzAN +BgkqhkiG9w0BAQUFAAOCAQEAfhP/++WewlphCojZwXijpFIy+XX1gR0p4kSfxVgA +Anl3khFL/xAvhk6pbWjGQM/9FWb/PFDRgj4fvMKGR8F9bMKNfBOrT+SyWDuI1Ax3 +y0unu0vZjEfUJmMktrr2aN3NI/bBmdVixNntsHRB0yrrl7Zk0TjQM3I1egfygoxa +tfARn5QOO/sReYJXlJdwmFMH0dpMT3++p5RhMZPDVeAdUUK/KzSdlPkVrLPJTKEY +d+IAIWjZq5CGOjM9052+CDhyAMvdywJQpxuhO/BzmPrt0ZQwuMdTUutPT2ijDGCB +J7nCUCVEtF25KJrJxeXY6oLxgXoaqqU1ZGivAS1oCtnocg== +-----END CERTIFICATE----- \ No newline at end of file diff --git a/certs/quay-staging.key b/certs/quay-staging.key new file mode 100644 index 000000000..2b41c1c6e --- /dev/null +++ b/certs/quay-staging.key @@ -0,0 +1,27 @@ +-----BEGIN RSA PRIVATE KEY----- +MIIEpAIBAAKCAQEA6uOT6AUsgJ4ooJV8reciN5Gy3MC10U61jQr9K8YvuexJcbP4 +se0SCCINYjtRQEST/4qUURxQfBjbhvniHSQBGngd/Xso7xr7DnBPIZStR7tx3dz3 +CgMGlSaVWck25VGF8ueZMPaJMDWSaVSTzh8SmjozWKycp8PdlSdtVnnhpPpIdBGS +kOL7abDDGCCWqUJNQmA3a3maANExBmYsSRvJWlH1/kvgiX/SmO6h/C3n4yvX6+se +0fqneB74+pcLRleVBAZBrVNm80wDkhDDiQXMTn6z6LxRS1b7W2AkQCGNmOTz71dV +p2vbAYedocUYaCacFN/4Pi14SelBBhTjL32voQIDAQABAoIBAGW2VIblLqcnVZps +AQhhDQ0ZF2XGQTU4qx8/QfAhqusMqaUF9Mw/R06kSD1gSEfXKms+vAj/hM6oCO/C +5yoNPDkVCI+KNGiNu2c+NNXqxrpILf+Pvp3kP4Z4pbWyjwXwLlvH9Csiprdsi1D3 +IeXgyLJmP3PHkzKGez4qS4tlzdMdBbJkdCQiE35yyF2os7F4HbehQ2Qyfw8PZk9o +T8uUEyh7SjYqmxJ2GfGXQd7+NXb3S1j7ehk/XTzlxgkhMW+eWk4hRAhd4j7FICAD +0UYx9K/j2TP8tNHgNd0k/BZkIbic6FD09YagPRu71Tc7MPvcSPm9SMDOj3WXJNC9 +/oDsOEECgYEA+w6mPpPxlspaEPsCYNo1/FdnmOYecLhruzqUU6lbSBhzW6p0R7H2 +GmfLeE4mGvkPJbx/zU13sRRhtzRB7QjuzZhKgrO8c/UoJeSaFimI3NYwjhtvszU6 +ActNQOpq5WvBXEOi+FegNW5+6vTnmxGG+gj8nSsu0JBPc2Db6jeFQbkCgYEA74Nw +X/iqRtuz+yabc4ALyowHJHdI5FUHv8uPv9Fk9KVRvxwq8Ak0ZcWnO0Sc+0eUC/v+ +VDVSvf1O+pMli+zIoAzGmLQFt/Is3E4frbBI7D3tWFjCzyduyVNbQBwhzfCDjk7z +Xr/vQ1tLlll5QhABtUdJlWIvZFRfm1Qi/un/8SkCgYAaui+Gn/drRzWZcy+IohJ3 +P9LemzkIZQnLD+x0j6YRIdE+JAJnE5IQs5Ycw60Y2AT9zniIocOpTXMtrtmJ45aQ +urLMAViBu8q/ZfvlehyA7iiTKGaW3IbFZCBgVdR1gig+q1CxQZrjtVS7rMDvaElH +WyeRj+RW/dYHgXtIDwsXuQKBgQDNYCOH5636vIGEJgK981opABFPz4kNYWwXpfFJ +RcAPl4KVIQ4gbYQkkGtpgtgpD6N+80GN63tbtk12x542cX9G3i2c2yDcLikRb1vy +j2q4SBGw48uH3gQ9VeC1BGpoMIheCozc/i+nzizuayJy507PpqUOFvcUTNT+WBL6 +CqSQoQKBgQCAtGhhTn35n/MlNKSrJ6mo8uKmkjX04trlusE+Ub9J2Bb+MT+Vdy/h +vsxAJWH9xcTPr6fTGmsiHBP2q8HzU36CFJPsk8mLnnT2Q5rd4nc3KMmXGu/NUwxA +AzNbRNJ45O+BZfm6YfZUXhHq4YmNauNwoWzAWmdP1701xI/1Jtmfeg== +-----END RSA PRIVATE KEY----- diff --git a/certs/unified.cert b/certs/quay-unified.cert similarity index 100% rename from certs/unified.cert rename to certs/quay-unified.cert diff --git a/config.py b/config.py index 3ae57769a..de81bdfe6 100644 --- a/config.py +++ b/config.py @@ -42,10 +42,13 @@ class RDSMySQL(object): DB_DRIVER = MySQLDatabase -class S3Storage(object): +class AWSCredentials(object): AWS_ACCESS_KEY = 'AKIAJWZWUIS24TWSMWRA' AWS_SECRET_KEY = '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' REGISTRY_S3_BUCKET = 'quay-registry' + + +class S3Storage(AWSCredentials): STORAGE_KIND = 's3' @@ -85,21 +88,34 @@ class GitHubProdConfig(GitHubTestConfig): GITHUB_CLIENT_SECRET = 'f89d8bb28ea3bd4e1c68808500d185a816be53b1' +class DigitalOceanConfig(object): + DO_CLIENT_ID = 'LJ44y2wwYj1MD0BRxS6qHA' + DO_CLIENT_SECRET = 'b9357a6f6ff45a33bb03f6dbbad135f9' + DO_SSH_KEY_ID = '46986' + DO_SSH_PRIVATE_KEY_FILENAME = 'certs/digital_ocean' + DO_ALLOWED_REGIONS = {1, 4} + + +class BuildNodeConfig(object): + BUILD_NODE_PULL_TOKEN = 'F02O2E86CQLKZUQ0O81J8XDHQ6F0N1V36L9JTOEEK6GKKMT1GI8PTJQT4OU88Y6G' + + class DebugConfig(FlaskConfig, MailConfig, LocalStorage, SQLiteDB, - StripeTestConfig, MixpanelTestConfig, GitHubTestConfig): - REGISTRY_SERVER = 'localhost:5000' + StripeTestConfig, MixpanelTestConfig, GitHubTestConfig, + DigitalOceanConfig, AWSCredentials, BuildNodeConfig): LOGGING_CONFIG = { 'level': logging.DEBUG, 'format': LOG_FORMAT } SEND_FILE_MAX_AGE_DEFAULT = 0 POPULATE_DB_TEST_DATA = True + INCLUDE_TEST_ENDPOINTS = True class LocalHostedConfig(FlaskConfig, MailConfig, S3Storage, RDSMySQL, StripeLiveConfig, MixpanelTestConfig, - GitHubProdConfig): - REGISTRY_SERVER = 'localhost:5000' + GitHubProdConfig, DigitalOceanConfig, + BuildNodeConfig): LOGGING_CONFIG = { 'level': logging.DEBUG, 'format': LOG_FORMAT @@ -109,8 +125,7 @@ class LocalHostedConfig(FlaskConfig, MailConfig, S3Storage, RDSMySQL, class ProductionConfig(FlaskConfig, MailConfig, S3Storage, RDSMySQL, StripeLiveConfig, MixpanelProdConfig, - GitHubProdConfig): - REGISTRY_SERVER = 'quay.io' + GitHubProdConfig, DigitalOceanConfig, BuildNodeConfig): LOGGING_CONFIG = { 'stream': sys.stderr, 'level': logging.DEBUG, diff --git a/data/database.py b/data/database.py index 0f3aef888..31581d443 100644 --- a/data/database.py +++ b/data/database.py @@ -150,6 +150,16 @@ class RepositoryTag(BaseModel): ) +class RepositoryBuild(BaseModel): + repository = ForeignKeyField(Repository) + access_token = ForeignKeyField(AccessToken) + resource_key = CharField() + tag = CharField() + build_node_id = IntegerField(null=True) + phase = CharField(default='waiting') + status_url = CharField(null=True) + + class QueueItem(BaseModel): queue_name = CharField(index=True) body = TextField() @@ -162,7 +172,7 @@ def initialize_db(): create_model_tables([User, Repository, Image, AccessToken, Role, RepositoryPermission, Visibility, RepositoryTag, EmailConfirmation, FederatedLogin, LoginService, - QueueItem]) + QueueItem, RepositoryBuild]) Role.create(name='admin') Role.create(name='write') Role.create(name='read') diff --git a/data/model.py b/data/model.py index 15827cca4..a46ca1912 100644 --- a/data/model.py +++ b/data/model.py @@ -30,6 +30,10 @@ class InvalidTokenException(DataModelException): pass +class InvalidRepositoryBuildException(DataModelException): + pass + + def create_user(username, password, email): if not validate_email(email): raise InvalidEmailAddressException('Invalid email address: %s' % email) @@ -283,8 +287,8 @@ def set_repository_visibility(repo, visibility): repo.save() -def create_repository(namespace, name, owner): - private = Visibility.get(name='private') +def create_repository(namespace, name, owner, visibility='private'): + private = Visibility.get(name=visibility) repo = Repository.create(namespace=namespace, name=name, visibility=private) admin = Role.get(name='admin') @@ -548,3 +552,28 @@ def load_token_data(code): return fetched[0] else: raise InvalidTokenException('Invalid delegate token code: %s' % code) + + +def get_repository_build(request_dbid): + try: + return RepositoryBuild.get(RepositoryBuild.id == request_dbid) + except RepositoryBuild.DoesNotExist: + msg = 'Unable to locate a build by id: %s' % request_dbid + raise InvalidRepositoryBuildException(msg) + + +def list_repository_builds(namespace_name, repository_name, + include_inactive=True): + joined = RepositoryBuild.select().join(Repository) + filtered = joined + if not include_inactive: + filtered = filtered.where(RepositoryBuild.phase != 'error', + RepositoryBuild.phase != 'complete') + fetched = list(filtered.where(Repository.name == repository_name, + Repository.namespace == namespace_name)) + return fetched + + +def create_repository_build(repo, access_token, resource_key, tag): + return RepositoryBuild.create(repository=repo, access_token=access_token, + resource_key=resource_key, tag=tag) diff --git a/data/queue.py b/data/queue.py index 0e1383aa2..8e63c4d17 100644 --- a/data/queue.py +++ b/data/queue.py @@ -54,5 +54,10 @@ class WorkQueue(object): def complete(self, completed_item): completed_item.delete_instance() + def incomplete(self, incomplete_item): + incomplete_item.available = True + incomplete_item.save() + image_diff_queue = WorkQueue('imagediff') +dockerfile_build_queue = WorkQueue('dockerfilebuild') diff --git a/data/userfiles.py b/data/userfiles.py new file mode 100644 index 000000000..b8ddd0d90 --- /dev/null +++ b/data/userfiles.py @@ -0,0 +1,56 @@ +import boto +import os +import logging + +from boto.s3.key import Key +from uuid import uuid4 +import hmac +import time +import urllib +import base64 +import sha + + +logger = logging.getLogger(__name__) + + +class S3FileWriteException(Exception): + pass + + +class UserRequestFiles(object): + def __init__(self, s3_access_key, s3_secret_key, bucket_name): + self._s3_conn = boto.connect_s3(s3_access_key, s3_secret_key, + is_secure=False) + self._bucket_name = bucket_name + self._bucket = self._s3_conn.get_bucket(bucket_name) + self._access_key = s3_access_key + self._secret_key = s3_secret_key + self._prefix = 'userfiles' + + def prepare_for_drop(self, mime_type): + """ Returns a signed URL to upload a file to our bucket. """ + logger.debug('Requested upload url with content type: %s' % mime_type) + file_id = str(uuid4()) + full_key = os.path.join(self._prefix, file_id) + k = Key(self._bucket, full_key) + url = k.generate_url(300, 'PUT', headers={'Content-Type': mime_type}) + return (url, file_id) + + def store_file(self, flask_file): + file_id = str(uuid4()) + full_key = os.path.join(self._prefix, file_id) + k = Key(self._bucket, full_key) + logger.debug('Setting s3 content type to: %s' % flask_file.content_type) + k.set_metadata('Content-Type', flask_file.content_type) + bytes_written = k.set_contents_from_file(flask_file) + + if bytes_written == 0: + raise S3FileWriteException('Unable to write file to S3') + + return file_id + + def get_file_url(self, file_id, expires_in=300): + full_key = os.path.join(self._prefix, file_id) + k = Key(self._bucket, full_key) + return k.generate_url(expires_in) diff --git a/endpoints/api.py b/endpoints/api.py index ab69b71b9..853c7498f 100644 --- a/endpoints/api.py +++ b/endpoints/api.py @@ -1,8 +1,11 @@ import logging import stripe import re +import requests +import urlparse +import json -from flask import request, make_response, jsonify, abort +from flask import request, make_response, jsonify, abort, url_for from flask.ext.login import login_required, current_user, logout_user from flask.ext.principal import identity_changed, AnonymousIdentity from functools import wraps @@ -11,6 +14,8 @@ from collections import defaultdict import storage from data import model +from data.userfiles import UserRequestFiles +from data.queue import dockerfile_build_queue from app import app from util.email import send_confirmation_email, send_recovery_email from util.names import parse_repository_name @@ -170,10 +175,29 @@ def get_matching_users(prefix): }) -@app.route('/api/repository/', methods=['POST']) +user_files = UserRequestFiles(app.config['AWS_ACCESS_KEY'], + app.config['AWS_SECRET_KEY'], + app.config['REGISTRY_S3_BUCKET']) + + +@app.route('/api/repository', methods=['POST']) @api_login_required def create_repo_api(): - pass + owner = current_user.db_user() + + namespace_name = owner.username + repository_name = request.get_json()['repository'] + visibility = request.get_json()['visibility'] + + repo = model.create_repository(namespace_name, repository_name, owner, + visibility) + repo.description = request.get_json()['description'] + repo.save() + + return jsonify({ + 'namespace': namespace_name, + 'name': repository_name + }) @app.route('/api/find/repository', methods=['GET']) @@ -323,6 +347,9 @@ def get_repo_api(namespace, repository): tag_dict = {tag.name: tag_view(tag) for tag in tags} can_write = ModifyRepositoryPermission(namespace, repository).can() can_admin = AdministerRepositoryPermission(namespace, repository).can() + active_builds = model.list_repository_builds(namespace, repository, + include_inactive=False) + return jsonify({ 'namespace': namespace, 'name': repository, @@ -330,13 +357,83 @@ def get_repo_api(namespace, repository): 'tags': tag_dict, 'can_write': can_write, 'can_admin': can_admin, - 'is_public': is_public + 'is_public': is_public, + 'is_building': len(active_builds) > 0, }) abort(404) # Not fount abort(403) # Permission denied +@app.route('/api/repository//build/', methods=['GET']) +@api_login_required +@parse_repository_name +def get_repo_builds(namespace, repository): + permission = ModifyRepositoryPermission(namespace, repository) + if permission.can(): + def build_view(build_obj): + if build_obj.status_url: + # Delegate the status to the build node + node_status = requests.get(build_obj.status_url).json() + node_status['id'] = build_obj.id + return node_status + + # If there was no status url, do the best we can + return { + 'id': build_obj.id, + 'total_commands': None, + 'total_images': None, + 'current_command': None, + 'current_image': None, + 'image_completion_percent': None, + 'status': build_obj.phase, + 'message': None, + } + + builds = model.list_repository_builds(namespace, repository) + return jsonify({ + 'builds': [build_view(build) for build in builds] + }) + + abort(403) # Permissions denied + + + +@app.route('/api/filedrop/', methods=['POST']) +def get_filedrop_url(): + mime_type = request.get_json()['mimeType'] + (url, file_id) = user_files.prepare_for_drop(mime_type) + return jsonify({ + 'url': url, + 'file_id': file_id + }) + + +@app.route('/api/repository//build/', methods=['POST']) +@api_login_required +@parse_repository_name +def request_repo_build(namespace, repository): + permission = ModifyRepositoryPermission(namespace, repository) + if permission.can(): + logger.debug('User requested repository initialization.') + dockerfile_id = request.get_json()['file_id'] + + repo = model.get_repository(namespace, repository) + token = model.create_access_token(repo, 'write') + + host = urlparse.urlparse(request.url).netloc + tag = '%s/%s/%s' % (host, repo.namespace, repo.name) + build_request = model.create_repository_build(repo, token, dockerfile_id, + tag) + dockerfile_build_queue.put(json.dumps({'build_id': build_request.id})) + + return jsonify({ + 'started': True + }) + + abort(403) # Permissions denied + + def role_view(repo_perm_obj): return { 'role': repo_perm_obj.role.name diff --git a/endpoints/index.py b/endpoints/index.py index 375fa94b4..4920e2b15 100644 --- a/endpoints/index.py +++ b/endpoints/index.py @@ -2,6 +2,7 @@ import json import urllib import json import logging +import urlparse from flask import request, make_response, jsonify, abort from functools import wraps @@ -25,7 +26,9 @@ def generate_headers(role='read'): def wrapper(namespace, repository, *args, **kwargs): response = f(namespace, repository, *args, **kwargs) - response.headers['X-Docker-Endpoints'] = app.config['REGISTRY_SERVER'] + # We run our index and registry on the same hosts for now + registry_server = urlparse.urlparse(request.url).netloc + response.headers['X-Docker-Endpoints'] = registry_server has_token_request = request.headers.get('X-Docker-Token', '') diff --git a/endpoints/test.py b/endpoints/test.py new file mode 100644 index 000000000..ff5e42c37 --- /dev/null +++ b/endpoints/test.py @@ -0,0 +1,48 @@ +from random import SystemRandom +from flask import jsonify, send_file +from app import app + + +@app.route('/test/build/status', methods=['GET']) +def generate_random_build_status(): + response = { + 'id': 1, + 'total_commands': None, + 'total_images': None, + 'current_command': None, + 'current_image': None, + 'image_completion_percent': None, + 'status': None, + 'message': None, + } + + random = SystemRandom() + phases = { + 'waiting': {}, + 'starting': { + 'total_commands': 7, + 'current_command': 0, + }, + 'initializing': {}, + 'error': { + 'message': 'Oops!' + }, + 'complete': {}, + 'building': { + 'total_commands': 7, + 'current_command': random.randint(1, 7), + }, + 'pushing': { + 'total_commands': 7, + 'current_command': 7, + 'total_images': 11, + 'current_image': random.randint(1, 11), + 'image_completion_percent': random.randint(0, 100), + }, + } + + phase = random.choice(phases.keys()) + response['status'] = phase + response.update(phases[phase]) + + return jsonify(response) diff --git a/endpoints/web.py b/endpoints/web.py index 3e6b2fc4d..ddfbca3d6 100644 --- a/endpoints/web.py +++ b/endpoints/web.py @@ -67,6 +67,11 @@ def signin(): return index('') +@app.route('/new/') +def new(): + return index('') + + @app.route('/repository/') def repository(): return index('') diff --git a/initdb.py b/initdb.py index 562a29e16..c5afbdc93 100644 --- a/initdb.py +++ b/initdb.py @@ -5,6 +5,7 @@ import os import hashlib from datetime import datetime, timedelta +from flask import url_for import storage @@ -95,6 +96,8 @@ def __generate_repository(user, name, description, is_public, permissions, create_subtree(repo, structure, None) + return repo + if __name__ == '__main__': initialize_db() @@ -141,6 +144,15 @@ if __name__ == '__main__': 'Shared repository, another user can write.', False, [(new_user_2, 'write')], (5, [], 'latest')) - __generate_repository(new_user_1, 'empty', - 'Empty repository with no images or tags.', False, - [], (0, [], None)) \ No newline at end of file + building = __generate_repository(new_user_1, 'building', + 'Empty repository which is building.', + False, [], (0, [], None)) + + token = model.create_access_token(building, 'write') + tag = 'ci.devtable.com:5000/%s/%s' % (building.namespace, building.name) + build = model.create_repository_build(building, token, '123-45-6789', tag) + + build.build_node_id = 1 + build.phase = 'building' + build.status_url = 'http://localhost:5000/test/build/status' + build.save() diff --git a/nginx.conf b/nginx.conf index 93e5a1ce1..10d316c44 100644 --- a/nginx.conf +++ b/nginx.conf @@ -10,6 +10,7 @@ events { } http { + types_hash_max_size 2048; include /etc/nginx/mime.types; default_type application/octet-stream; @@ -36,8 +37,8 @@ http { keepalive_timeout 5; ssl on; - ssl_certificate /home/ubuntu/quay/certs/unified.cert; - ssl_certificate_key /home/ubuntu/quay/certs/quay.key; + ssl_certificate ./certs/quay-unified.cert; + ssl_certificate_key ./certs/quay.key; ssl_session_timeout 5m; ssl_protocols SSLv3 TLSv1; ssl_ciphers ALL:!ADH:!EXPORT56:RC4+RSA:+HIGH:+MEDIUM:+LOW:+SSLv3:+EXP; diff --git a/requirements-nover.txt b/requirements-nover.txt index fe43012cd..8d95e3e48 100644 --- a/requirements-nover.txt +++ b/requirements-nover.txt @@ -14,4 +14,6 @@ mixpanel-py beautifulsoup4 marisa-trie apscheduler -python-daemon \ No newline at end of file +python-daemon +paramiko +python-digitalocean \ No newline at end of file diff --git a/requirements.txt b/requirements.txt index 171930241..494ed9198 100644 --- a/requirements.txt +++ b/requirements.txt @@ -12,6 +12,7 @@ beautifulsoup4==4.3.2 blinker==1.3 boto==2.15.0 distribute==0.6.34 +ecdsa==0.10 eventlet==0.14.0 greenlet==0.4.1 gunicorn==18.0 @@ -19,10 +20,13 @@ itsdangerous==0.23 lockfile==0.9.1 marisa-trie==0.5.1 mixpanel-py==3.0.0 +paramiko==1.12.0 peewee==2.1.4 py-bcrypt==0.4 +pycrypto==2.6.1 python-daemon==1.6 python-dateutil==2.1 +python-digitalocean==0.5 requests==2.0.0 six==1.4.1 stripe==1.9.8 diff --git a/static/css/quay.css b/static/css/quay.css index f6de60079..9f5001fc4 100644 --- a/static/css/quay.css +++ b/static/css/quay.css @@ -36,13 +36,57 @@ 100% { -webkit-transform: scale(1); } } - @keyframes scaleup { 0% { transform: scale(0); } 100% { transform: scale(1); } } +.user-tools .user-tool { + font-size: 24px; + margin-top: 14px; + color: #aaa; +} +.user-tools i.user-tool:hover { + cursor: pointer; + color: #428bca; +} + +.status-boxes .popover { + margin-right: 20px; +} + +.status-boxes .popover-content { + width: 260px; +} + +.build-statuses { +} + +.build-status-container { + padding: 4px; + margin-bottom: 10px; + border-bottom: 1px solid #eee; + width: 230px; +} + +.build-status-container .build-message { + display: block; + white-space: nowrap; + font-size: 12px; +} + +.build-status-container .progress { + height: 12px; + margin: 0px; + margin-top: 10px; + width: 230px; +} + +.build-status-container:last-child { + margin-bottom: 0px; + border-bottom: 0px solid white; +} .repo-circle { position: relative; @@ -57,15 +101,23 @@ .repo-circle.no-background { background: transparent; + position: relative; height: 40px; width: 40px; } -.repo-circle .icon-lock { - font-size: 50%; +.repo-circle .fa-hdd { + font-size: 36px; +} + +.repo-circle.no-background .fa-hdd { + font-size: 30px; +} + +.repo-circle .fa-lock { position: absolute; - bottom: -6px; - right: 0px; + bottom: -2px; + right: -4px; background: rgb(253, 191, 191); width: 20px; display: inline-block; @@ -73,11 +125,11 @@ text-align: center; height: 20px; line-height: 21px; - font-size: 12px; + font-size: 16px !important; } -.repo-circle.no-background .icon-lock { - bottom: -4px; +.repo-circle.no-background .fa-lock { + bottom: -2px; right: -6px; color: #444; } @@ -117,6 +169,85 @@ text-decoration: none !important; } +.new-repo .required-plan { + margin: 10px; + margin-top: 20px; + margin-left: 50px; +} + +.new-repo .required-plan .alert { + color: #444 !important; +} + +.new-repo .new-header { + font-size: 22px; +} + +.new-repo .new-header .repo-circle { + margin-right: 14px; +} + +.new-repo .new-header .name-container { + display: inline-block; + width: 300px; +} + +.new-repo .description { + margin-left: 10px; + margin-top: 10px; +} + +.new-repo .section { + padding-bottom: 20px; + border-bottom: 1px solid #eee; + margin-bottom: 16px; +} + +.new-repo .repo-option { + margin: 6px; + margin-top: 16px; +} + +.new-repo .repo-option i { + font-size: 18px; + padding-left: 10px; + padding-right: 10px; + width: 42px; + display: inline-block; + text-align: center; +} + +.new-repo .option-description { + display: inline-block; + vertical-align: top; +} + +.new-repo .option-description label { + display: block; +} + +.new-repo .cbox { + margin: 10px; +} + +.new-repo .initialize-repo { + margin: 10px; + margin-top: 16px; + margin-left: 20px; + padding: 10px; + border: 1px dashed #ccc; +} + +.new-repo .initialize-repo .init-description { + color: #444; + font-size: 12px; + text-align: center; +} + +.new-repo .initialize-repo .file-drop { + margin: 10px; +} + .user-guide h3 { margin-bottom: 20px; } @@ -532,6 +663,7 @@ p.editable:hover i { } .repo .description { + margin-top: 10px; margin-bottom: 40px; } @@ -561,22 +693,70 @@ p.editable:hover i { display: inline-block; } +.repo .status-boxes { + float: right; + margin-bottom: 20px; +} + +.repo .status-boxes .status-box { + cursor: pointer; + display: inline-block; + border: 1px solid #eee; + border-radius: 4px; +} + +.repo .status-boxes .status-box .title { + padding: 4px; + display: inline-block; + padding-left: 10px; + padding-right: 10px; +} + +.repo .status-boxes .status-box .title i { + margin-right: 6px; +} + +.repo .status-boxes .status-box .count { + display: inline-block; + background-image: linear-gradient(to bottom,#e8e8e8 0,#f5f5f5 100%); + padding: 4px; + padding-left: 10px; + padding-right: 10px; + font-weight: bold; + + transform: scaleX(0); + -webkit-transform: scaleX(0); + -moz-transform: scaleX(0); + + transition: transform 500ms ease-in-out; + -webkit-transition: -webkit-transform 500ms ease-in-out; + -moz-transition: -moz-transform 500ms ease-in-out; +} + +.repo .status-boxes .status-box .count.visible { + transform: scaleX(1); + -webkit-transform: scaleX(1); + -moz-transform: scaleX(1); +} + .repo .pull-command { float: right; display: inline-block; - font-size: 1.2em; + font-size: 0.8em; position: relative; - margin-right: 10px; + margin-top: 30px; + margin-right: 26px; } -.repo .pull-command .pull-container { +.repo .pull-container { display: inline-block; width: 300px; + margin-left: 10px; margin-right: 10px; vertical-align: middle; } -.repo .pull-command input { +.repo .pull-container input { cursor: default; background: white; color: #666; @@ -702,7 +882,6 @@ p.editable:hover i { } .repo-listing i { - font-size: 1.5em; color: #999; display: inline-block; margin-right: 6px; @@ -742,8 +921,22 @@ p.editable:hover i { cursor: pointer; } -.repo .description p { - margin-bottom: 6px; + +.repo .build-info { + padding: 10px; + margin: 0px; +} + +.repo .build-info .progress { + margin: 0px; + margin-top: 10px; +} + +.repo .section { + display: block; + margin-bottom: 20px; + padding-bottom: 20px; + border-bottom: 1px solid #eee; } .repo .description p:last-child { @@ -793,15 +986,15 @@ p.editable:hover i { margin-right: 10px; } -.repo .changes-container i.icon-plus-sign-alt { +.repo .changes-container i.fa-plus-square { color: rgb(73, 209, 73); } -.repo .changes-container i.icon-minus-sign-alt { +.repo .changes-container i.fa-minus-square { color: rgb(209, 73, 73); } -.repo .changes-container i.icon-edit-sign { +.repo .changes-container i.fa-pencil-square { color: rgb(73, 100, 209); } @@ -929,11 +1122,11 @@ p.editable:hover i { width: 580px; } -.repo-admin .repo-access-state .state-icon i.icon-lock { +.repo-admin .repo-access-state .state-icon i.fa-lock { background: rgb(253, 191, 191); } -.repo-admin .repo-access-state .state-icon i.icon-unlock-alt { +.repo-admin .repo-access-state .state-icon i.fa-unlock { background: rgb(170, 236, 170); } @@ -1105,6 +1298,7 @@ p.editable:hover i { border: 1px solid #eee; margin-top: 10px; padding: 10px; + min-height: 50px; } /* Overrides for typeahead to work with bootstrap 3. */ diff --git a/static/directives/build-status.html b/static/directives/build-status.html new file mode 100644 index 000000000..8c27dba53 --- /dev/null +++ b/static/directives/build-status.html @@ -0,0 +1,8 @@ +
+ {{ getBuildMessage(build) }} +
+
+
+
+ +
diff --git a/static/directives/repo-circle.html b/static/directives/repo-circle.html index 62986d4bf..49d60ce56 100644 --- a/static/directives/repo-circle.html +++ b/static/directives/repo-circle.html @@ -1,2 +1,2 @@ - - + + diff --git a/static/js/app.js b/static/js/app.js index 73161f17e..9d3e615be 100644 --- a/static/js/app.js +++ b/static/js/app.js @@ -1,5 +1,5 @@ // Start the application code itself. -quayApp = angular.module('quay', ['restangular', 'angularMoment', 'angulartics', 'angulartics.mixpanel'], function($provide) { +quayApp = angular.module('quay', ['restangular', 'angularMoment', 'angulartics', 'angulartics.mixpanel', '$strap.directives'], function($provide) { $provide.factory('UserService', ['Restangular', function(Restangular) { var userResponse = { verified: false, @@ -54,7 +54,7 @@ quayApp = angular.module('quay', ['restangular', 'angularMoment', 'angulartics', return keyService; }]); - $provide.factory('PlanService', [function() { + $provide.factory('PlanService', ['Restangular', 'KeyService', function(Restangular, KeyService) { var plans = [ { title: 'Open Source', @@ -96,11 +96,54 @@ quayApp = angular.module('quay', ['restangular', 'angularMoment', 'angulartics', planService.planList = function() { return plans; - } + }; planService.getPlan = function(planId) { return planDict[planId]; - } + }; + + planService.getMinimumPlan = function(privateCount) { + for (var i = 0; i < plans.length; i++) { + var plan = plans[i]; + if (plan.privateRepos >= privateCount) { + return plan; + } + } + + return null; + }; + + planService.showSubscribeDialog = function($scope, planId, started, success, failed) { + var submitToken = function(token) { + $scope.$apply(function() { + started(); + }); + + mixpanel.track('plan_subscribe'); + + var subscriptionDetails = { + token: token.id, + plan: planId, + }; + + var createSubscriptionRequest = Restangular.one('user/plan'); + $scope.$apply(function() { + createSubscriptionRequest.customPUT(subscriptionDetails).then(success, failed); + }); + }; + + var planDetails = planService.getPlan(planId) + StripeCheckout.open({ + key: KeyService.stripePublishableKey, + address: false, // TODO change to true + amount: planDetails.price, + currency: 'usd', + name: 'Quay ' + planDetails.title + ' Subscription', + description: 'Up to ' + planDetails.privateRepos + ' private repositories', + panelLabel: 'Subscribe', + token: submitToken + }); + }; return planService; }]); @@ -155,6 +198,7 @@ quayApp = angular.module('quay', ['restangular', 'angularMoment', 'angulartics', when('/guide/', {title: 'User Guide', templateUrl: '/static/partials/guide.html', controller: GuideCtrl}). when('/plans/', {title: 'Plans and Pricing', templateUrl: '/static/partials/plans.html', controller: PlansCtrl}). when('/signin/', {title: 'Signin', templateUrl: '/static/partials/signin.html', controller: SigninCtrl}). + when('/new/', {title: 'Create new repository', templateUrl: '/static/partials/new-repo.html', controller: NewRepoCtrl}). when('/v1/', {title: 'Activation information', templateUrl: '/static/partials/v1-page.html', controller: V1Ctrl}). @@ -176,7 +220,73 @@ quayApp.directive('repoCircle', function () { 'repo': '=repo' }, controller: function($scope, $element) { - window.console.log($scope); + } + }; + return directiveDefinitionObject; +}); + + +quayApp.directive('buildStatus', function () { + var directiveDefinitionObject = { + priority: 0, + templateUrl: '/static/directives/build-status.html', + replace: false, + transclude: false, + restrict: 'C', + scope: { + 'build': '=build' + }, + controller: function($scope, $element) { + $scope.getBuildProgress = function(buildInfo) { + switch (buildInfo.status) { + case 'building': + return (buildInfo.current_command / buildInfo.total_commands) * 100; + break; + + case 'pushing': + var imagePercentDecimal = (buildInfo.image_completion_percent / 100); + return ((buildInfo.current_image + imagePercentDecimal) / buildInfo.total_images) * 100; + break; + + case 'complete': + return 100; + break; + + case 'initializing': + case 'starting': + case 'waiting': + return 0; + break; + } + + return -1; + }; + + $scope.getBuildMessage = function(buildInfo) { + switch (buildInfo.status) { + case 'initializing': + return 'Starting Dockerfile build'; + break; + + case 'starting': + case 'waiting': + case 'building': + return 'Building image from Dockerfile'; + break; + + case 'pushing': + return 'Pushing image built from Dockerfile'; + break; + + case 'complete': + return 'Dockerfile build completed and pushed'; + break; + + case 'error': + return 'Dockerfile build failed: ' + buildInfo.message; + break; + } + }; } }; return directiveDefinitionObject; diff --git a/static/js/controllers.js b/static/js/controllers.js index b0e1f315e..0b7a8c3f4 100644 --- a/static/js/controllers.js +++ b/static/js/controllers.js @@ -92,7 +92,7 @@ function HeaderCtrl($scope, $location, UserService, Restangular) { }, template: function (datum) { template = '
'; - template += '' + template += '' template += '' + datum.repo.namespace +'/' + datum.repo.name + '' if (datum.repo.description) { template += '' + getFirstTextLine(datum.repo.description) + '' @@ -211,7 +211,7 @@ function RepoListCtrl($scope, Restangular, UserService) { }); } -function LandingCtrl($scope, $timeout, Restangular, UserService, KeyService) { +function LandingCtrl($scope, $timeout, $location, Restangular, UserService, KeyService) { $('.form-signup').popover(); $scope.$watch( function () { return UserService.currentUser(); }, function (currentUser) { @@ -236,10 +236,6 @@ function LandingCtrl($scope, $timeout, Restangular, UserService, KeyService) { return getMarkedDown(getFirstTextLine(commentString)); }; - $scope.browseRepos = function() { - document.location = '/repository/'; - }; - $scope.register = function() { $('.form-signup').popover('hide'); $scope.registering = true; @@ -346,12 +342,83 @@ function RepoCtrl($scope, Restangular, $routeParams, $rootScope, $location, $tim } }); - var listImages = function() { - if ($scope.imageHistory) { return; } + var fetchRepository = function() { + var repositoryFetch = Restangular.one('repository/' + namespace + '/' + name); + repositoryFetch.get().then(function(repo) { + $rootScope.title = namespace + '/' + name; + $scope.repo = repo; + $scope.setTag($routeParams.tag); + + $('#copyClipboard').clipboardCopy(); + $scope.loading = false; + + if (repo.is_building) { + startBuildInfoTimer(repo); + } + }, function() { + $scope.repo = null; + $scope.loading = false; + $rootScope.title = 'Unknown Repository'; + }); + }; + + var startBuildInfoTimer = function(repo) { + if ($scope.interval) { return; } + + getBuildInfo(repo); + $scope.interval = setInterval(function() { + $scope.$apply(function() { getBuildInfo(repo); }); + }, 5000); + + $scope.$on("$destroy", function() { + cancelBuildInfoTimer(); + }); + }; + + var cancelBuildInfoTimer = function() { + if ($scope.interval) { + clearInterval($scope.interval); + } + }; + + var getBuildInfo = function(repo) { + var buildInfo = Restangular.one('repository/' + repo.namespace + '/' + repo.name + '/build/'); + buildInfo.get().then(function(resp) { + var runningBuilds = []; + for (var i = 0; i < resp.builds.length; ++i) { + var build = resp.builds[i]; + if (build.status != 'complete') { + runningBuilds.push(build); + } + } + + $scope.buildsInfo = runningBuilds; + if (!runningBuilds.length) { + // Cancel the build timer. + cancelBuildInfoTimer(); + + // Mark the repo as no longer building. + $scope.repo.is_building = false; + + // Reload the repo information. + fetchRepository(); + listImages(); + } + }); + }; + + var listImages = function() { var imageFetch = Restangular.one('repository/' + namespace + '/' + name + '/image/'); imageFetch.get().then(function(resp) { $scope.imageHistory = resp.images; + + // Dispose of any existing tree. + if ($scope.tree) { + $scope.tree.dispose(); + } + + // Create the new tree. $scope.tree = new ImageHistoryTree(namespace, name, resp.images, $scope.getCommentFirstLine, $scope.getTimeSince); @@ -363,7 +430,7 @@ function RepoCtrl($scope, Restangular, $routeParams, $rootScope, $location, $tim } $($scope.tree).bind('tagChanged', function(e) { - $scope.$apply(function() { $scope.setTag(e.tag, true); }); + $scope.$apply(function() { $scope.setTag(e.tag, true); }); }); $($scope.tree).bind('imageChanged', function(e) { $scope.$apply(function() { $scope.setImage(e.image); }); @@ -378,7 +445,7 @@ function RepoCtrl($scope, Restangular, $routeParams, $rootScope, $location, $tim changesFetch.get().then(function(changeInfo) { $scope.currentImageChanges = changeInfo; }, function() { - $scope.currentImageChanges = {}; + $scope.currentImageChanges = {'added': [], 'removed': [], 'changed': []}; }); }; @@ -441,20 +508,7 @@ function RepoCtrl($scope, Restangular, $routeParams, $rootScope, $location, $tim $scope.loading = true; // Fetch the repo. - var repositoryFetch = Restangular.one('repository/' + namespace + '/' + name); - repositoryFetch.get().then(function(repo) { - $rootScope.title = namespace + '/' + name; - $scope.repo = repo; - - $scope.setTag($routeParams.tag); - - $('#copyClipboard').clipboardCopy(); - $scope.loading = false; - }, function() { - $scope.repo = null; - $scope.loading = false; - $rootScope.title = 'Unknown Repository'; - }); + fetchRepository(); // Fetch the image history. listImages(); @@ -492,7 +546,7 @@ function RepoAdminCtrl($scope, Restangular, $routeParams, $rootScope) { }, template: function (datum) { template = '
'; - template += '' + template += '' template += '' + datum.username + '' template += '
' return template; @@ -702,6 +756,8 @@ function UserAdminCtrl($scope, $timeout, Restangular, PlanService, UserService, if (sub.usedPrivateRepos > $scope.subscribedPlan.privateRepos) { $scope.errorMessage = 'You are using more private repositories than your plan allows, please upgrate your subscription to avoid disruptions in your service.'; + } else { + $scope.errorMessage = null; } $scope.planLoading = false; @@ -710,7 +766,7 @@ function UserAdminCtrl($scope, $timeout, Restangular, PlanService, UserService, mixpanel.people.set({ 'plan': sub.plan }); - } + }; $scope.planLoading = true; var getSubscription = Restangular.one('user/plan'); @@ -721,36 +777,16 @@ function UserAdminCtrl($scope, $timeout, Restangular, PlanService, UserService, $scope.planChanging = false; $scope.subscribe = function(planId) { - var submitToken = function(token) { - $scope.$apply(function() { - mixpanel.track('plan_subscribe'); - - $scope.planChanging = true; - $scope.errorMessage = undefined; - - var subscriptionDetails = { - token: token.id, - plan: planId, - }; - - var createSubscriptionRequest = Restangular.one('user/plan'); - createSubscriptionRequest.customPUT(subscriptionDetails).then(subscribedToPlan, function() { - // Failure - $scope.errorMessage = 'Unable to subscribe.'; - }); - }); - }; - - var planDetails = PlanService.getPlan(planId) - StripeCheckout.open({ - key: KeyService.stripePublishableKey, - address: false, // TODO change to true - amount: planDetails.price, - currency: 'usd', - name: 'Quay ' + planDetails.title + ' Subscription', - description: 'Up to ' + planDetails.privateRepos + ' private repositories', - panelLabel: 'Subscribe', - token: submitToken + PlanService.showSubscribeDialog($scope, planId, function() { + // Subscribing. + $scope.planChanging = true; + }, function(plan) { + // Subscribed. + subscribedToPlan(plan); + }, function() { + // Failure. + $scope.errorMessage = 'Unable to subscribe.'; + $scope.planChanging = false; }); }; @@ -911,12 +947,182 @@ function ImageViewCtrl($scope, $routeParams, $rootScope, Restangular) { }); } -function V1Ctrl($scope, UserService) { +function V1Ctrl($scope, $location, UserService) { + $scope.$watch( function () { return UserService.currentUser(); }, function (currentUser) { + $scope.user = currentUser; + }, true); +} + +function NewRepoCtrl($scope, $location, $http, UserService, Restangular, PlanService) { $scope.$watch( function () { return UserService.currentUser(); }, function (currentUser) { $scope.user = currentUser; }, true); - $scope.browseRepos = function() { - document.location = '/repository/'; + $scope.repo = { + 'is_public': 1, + 'description': '', + 'initialize': false }; + + $('#couldnotbuildModal').on('hidden.bs.modal', function() { + $scope.$apply(function() { + $location.path('/repository/' + $scope.created.namespace + '/' + $scope.created.name); + }); + }); + + var startBuild = function(repo, fileId) { + $scope.building = true; + + var data = { + 'file_id': fileId + }; + + var startBuildCall = Restangular.one('repository/' + repo.namespace + '/' + repo.name + '/build/'); + startBuildCall.customPOST(data).then(function(resp) { + $location.path('/repository/' + repo.namespace + '/' + repo.name); + }, function() { + $('#couldnotbuildModal').modal(); + }); + }; + + var conductUpload = function(repo, file, url, fileId, mimeType) { + var request = new XMLHttpRequest(); + request.open('PUT', url, true); + request.setRequestHeader('Content-Type', mimeType); + request.onprogress = function(e) { + $scope.$apply(function() { + var percentLoaded; + if (e.lengthComputable) { + $scope.upload_progress = (e.loaded / e.total) * 100; + } + }); + }; + request.onerror = function() { + $scope.$apply(function() { + $('#couldnotbuildModal').modal(); + }); + }; + request.onreadystatechange = function() { + var state = request.readyState; + if (state == 4) { + $scope.$apply(function() { + $scope.uploading = false; + startBuild(repo, fileId); + }); + return; + } + }; + request.send(file); + }; + + var startFileUpload = function(repo) { + $scope.uploading = true; + $scope.uploading_progress = 0; + + var uploader = $('#file-drop')[0]; + var file = uploader.files[0]; + $scope.upload_file = file.name; + + var mimeType = file.type || 'application/octet-stream'; + var data = { + 'mimeType': mimeType + }; + + var getUploadUrl = Restangular.one('filedrop/'); + getUploadUrl.customPOST(data).then(function(resp) { + conductUpload(repo, file, resp.url, resp.file_id, mimeType); + }, function() { + $('#couldnotbuildModal').modal(); + }); + }; + + var subscribedToPlan = function(sub) { + $scope.planChanging = false; + $scope.subscription = sub; + $scope.subscribedPlan = PlanService.getPlan(sub.plan); + $scope.planRequired = null; + if ($scope.subscription.usedPrivateRepos >= $scope.subscribedPlan.privateRepos) { + $scope.planRequired = PlanService.getMinimumPlan($scope.subscription.usedPrivateRepos); + } + }; + + $scope.editDescription = function() { + if (!$scope.markdownDescriptionEditor) { + var converter = Markdown.getSanitizingConverter(); + var editor = new Markdown.Editor(converter, '-description'); + editor.run(); + $scope.markdownDescriptionEditor = editor; + } + + $('#wmd-input-description')[0].value = $scope.repo.description; + $('#editModal').modal({}); + }; + + $scope.getMarkedDown = function(string) { + if (!string) { return ''; } + return getMarkedDown(string); + }; + + $scope.saveDescription = function() { + $('#editModal').modal('hide'); + $scope.repo.description = $('#wmd-input-description')[0].value; + }; + + $scope.createNewRepo = function() { + var uploader = $('#file-drop')[0]; + if ($scope.repo.initialize && uploader.files.length < 1) { + $('#missingfileModal').modal(); + return; + } + + $scope.creating = true; + var repo = $scope.repo; + var data = { + 'repository': repo.name, + 'visibility': repo.is_public == '1' ? 'public' : 'private', + 'description': repo.description + }; + + var createPost = Restangular.one('repository'); + createPost.customPOST(data).then(function(created) { + $scope.creating = false; + $scope.created = created; + + // Repository created. Start the upload process if applicable. + if ($scope.repo.initialize) { + startFileUpload(created); + return; + } + + // Otherwise, redirect to the repo page. + $location.path('/repository/' + created.namespace + '/' + created.name); + }, function() { + $('#cannotcreateModal').modal(); + $scope.creating = false; + }); + }; + + $scope.upgradePlan = function() { + PlanService.showSubscribeDialog($scope, $scope.planRequired.stripeId, function() { + // Subscribing. + $scope.planChanging = true; + }, function(plan) { + // Subscribed. + subscribedToPlan(plan); + }, function() { + // Failure. + $('#couldnotsubscribeModal').modal(); + $scope.planChanging = false; + }); + }; + + $scope.plans = PlanService.planList(); + + // Load the user's subscription information in case they want to create a private + // repository. + var getSubscription = Restangular.one('user/plan'); + getSubscription.get().then(subscribedToPlan, function() { + // User has no subscription + $scope.planRequired = PlanService.getMinimumPlan(1); + }); } \ No newline at end of file diff --git a/static/js/graphing.js b/static/js/graphing.js index 1066c0594..5581dc3ce 100644 --- a/static/js/graphing.js +++ b/static/js/graphing.js @@ -715,7 +715,17 @@ ImageHistoryTree.prototype.toggle_ = function(d) { } }; -///////////////////////////////////////////////////////////////////////////////////////// + +/** + * Disposes of the tree. + */ +ImageHistoryTree.prototype.dispose = function() { + var container = this.container_ ; + $('#' + container).removeOverscroll(); + document.getElementById(container).innerHTML = ''; +}; + +//////////////////////////////////////////////////////////////////////////////// /** * Based off of http://bl.ocks.org/mbostock/1093025 by Mike Bostock (@mbostock) @@ -809,6 +819,15 @@ ImageFileChangeTree.prototype.notifyResized = function() { }; +/** + * Disposes of the tree. + */ +ImageFileChangeTree.prototype.dispose = function() { + var container = this.container_ ; + document.getElementById(container).innerHTML = ''; +}; + + /** * Draws the tree. */ @@ -1039,17 +1058,17 @@ ImageFileChangeTree.prototype.update_ = function(source) { node.select('.node-icon') .html(function(d) { if (!d.kind) { - var folder = d._children ? 'icon-folder-close' : 'icon-folder-open'; + var folder = d._children ? 'fa fa-folder' : 'fa fa-folder-open'; return ''; } var icon = { - 'added': 'plus-sign-alt', - 'removed': 'minus-sign-alt', - 'changed': 'edit-sign' + 'added': 'plus-square', + 'removed': 'minus-square', + 'changed': 'pencil-square' }; - return ''; + return ''; }); // Transition exiting nodes to the parent's new position. diff --git a/static/lib/angular-strap.min.js b/static/lib/angular-strap.min.js new file mode 100644 index 000000000..7bf8788fb --- /dev/null +++ b/static/lib/angular-strap.min.js @@ -0,0 +1,8 @@ +/** + * AngularStrap - Twitter Bootstrap directives for AngularJS + * @version v0.7.5 - 2013-07-21 + * @link http://mgcrea.github.com/angular-strap + * @author Olivier Louvignes + * @license MIT License, http://www.opensource.org/licenses/MIT + */ +angular.module("$strap.config",[]).value("$strapConfig",{}),angular.module("$strap.filters",["$strap.config"]),angular.module("$strap.directives",["$strap.config"]),angular.module("$strap",["$strap.filters","$strap.directives","$strap.config"]),angular.module("$strap.directives").directive("bsAlert",["$parse","$timeout","$compile",function(t,e,n){return{restrict:"A",link:function(a,i,o){var r=t(o.bsAlert),s=(r.assign,r(a)),l=function(t){e(function(){i.alert("close")},1*t)};o.bsAlert?a.$watch(o.bsAlert,function(t,e){s=t,i.html((t.title?""+t.title+" ":"")+t.content||""),t.closed&&i.hide(),n(i.contents())(a),(t.type||e.type)&&(e.type&&i.removeClass("alert-"+e.type),t.type&&i.addClass("alert-"+t.type)),angular.isDefined(t.closeAfter)?l(t.closeAfter):o.closeAfter&&l(o.closeAfter),(angular.isUndefined(o.closeButton)||"0"!==o.closeButton&&"false"!==o.closeButton)&&i.prepend('')},!0):((angular.isUndefined(o.closeButton)||"0"!==o.closeButton&&"false"!==o.closeButton)&&i.prepend(''),o.closeAfter&&l(o.closeAfter)),i.addClass("alert").alert(),i.hasClass("fade")&&(i.removeClass("in"),setTimeout(function(){i.addClass("in")}));var u=o.ngRepeat&&o.ngRepeat.split(" in ").pop();i.on("close",function(t){var e;u?(t.preventDefault(),i.removeClass("in"),e=function(){i.trigger("closed"),a.$parent&&a.$parent.$apply(function(){for(var t=u.split("."),e=a.$parent,n=0;t.length>n;++n)e&&(e=e[t[n]]);e&&e.splice(a.$index,1)})},$.support.transition&&i.hasClass("fade")?i.on($.support.transition.end,e):e()):s&&(t.preventDefault(),i.removeClass("in"),e=function(){i.trigger("closed"),a.$apply(function(){s.closed=!0})},$.support.transition&&i.hasClass("fade")?i.on($.support.transition.end,e):e())})}}}]),angular.module("$strap.directives").directive("bsButton",["$parse","$timeout",function(t){return{restrict:"A",require:"?ngModel",link:function(e,n,a,i){if(i){n.parent('[data-toggle="buttons-checkbox"], [data-toggle="buttons-radio"]').length||n.attr("data-toggle","button");var o=!!e.$eval(a.ngModel);o&&n.addClass("active"),e.$watch(a.ngModel,function(t,e){var a=!!t,i=!!e;a!==i?$.fn.button.Constructor.prototype.toggle.call(r):a&&!o&&n.addClass("active")})}n.hasClass("btn")||n.on("click.button.data-api",function(){n.button("toggle")}),n.button();var r=n.data("button");r.toggle=function(){if(!i)return $.fn.button.Constructor.prototype.toggle.call(this);var a=n.parent('[data-toggle="buttons-radio"]');a.length?(n.siblings("[ng-model]").each(function(n,a){t($(a).attr("ng-model")).assign(e,!1)}),e.$digest(),i.$modelValue||(i.$setViewValue(!i.$modelValue),e.$digest())):e.$apply(function(){i.$setViewValue(!i.$modelValue)})}}}}]).directive("bsButtonsCheckbox",["$parse",function(){return{restrict:"A",require:"?ngModel",compile:function(t){t.attr("data-toggle","buttons-checkbox").find("a, button").each(function(t,e){$(e).attr("bs-button","")})}}}]).directive("bsButtonsRadio",["$timeout",function(t){return{restrict:"A",require:"?ngModel",compile:function(e,n){return e.attr("data-toggle","buttons-radio"),n.ngModel||e.find("a, button").each(function(t,e){$(e).attr("bs-button","")}),function(e,n,a,i){i&&(t(function(){n.find("[value]").button().filter('[value="'+i.$viewValue+'"]').addClass("active")}),n.on("click.button.data-api",function(t){e.$apply(function(){i.$setViewValue($(t.target).closest("button").attr("value"))})}),e.$watch(a.ngModel,function(t,i){if(t!==i){var o=n.find('[value="'+e.$eval(a.ngModel)+'"]');o.length&&o.button("toggle")}}))}}}}]),angular.module("$strap.directives").directive("bsButtonSelect",["$parse","$timeout",function(t){return{restrict:"A",require:"?ngModel",link:function(e,n,a,i){var o=t(a.bsButtonSelect);o.assign,i&&(n.text(e.$eval(a.ngModel)),e.$watch(a.ngModel,function(t){n.text(t)}));var r,s,l,u;n.bind("click",function(){r=o(e),s=i?e.$eval(a.ngModel):n.text(),l=r.indexOf(s),u=l>r.length-2?r[0]:r[l+1],e.$apply(function(){n.text(u),i&&i.$setViewValue(u)})})}}}]),angular.module("$strap.directives").directive("bsDatepicker",["$timeout","$strapConfig",function(t,e){var n=/(iP(a|o)d|iPhone)/g.test(navigator.userAgent),a=function a(t){return t=t||"en",{"/":"[\\/]","-":"[-]",".":"[.]"," ":"[\\s]",dd:"(?:(?:[0-2]?[0-9]{1})|(?:[3][01]{1}))",d:"(?:(?:[0-2]?[0-9]{1})|(?:[3][01]{1}))",mm:"(?:[0]?[1-9]|[1][012])",m:"(?:[0]?[1-9]|[1][012])",DD:"(?:"+$.fn.datepicker.dates[t].days.join("|")+")",D:"(?:"+$.fn.datepicker.dates[t].daysShort.join("|")+")",MM:"(?:"+$.fn.datepicker.dates[t].months.join("|")+")",M:"(?:"+$.fn.datepicker.dates[t].monthsShort.join("|")+")",yyyy:"(?:(?:[1]{1}[0-9]{1}[0-9]{1}[0-9]{1})|(?:[2]{1}[0-9]{3}))(?![[0-9]])",yy:"(?:(?:[0-9]{1}[0-9]{1}))(?![[0-9]])"}},i=function i(t,e){var n,i=t,o=a(e);return n=0,angular.forEach(o,function(t,e){i=i.split(e).join("${"+n+"}"),n++}),n=0,angular.forEach(o,function(t){i=i.split("${"+n+"}").join(t),n++}),RegExp("^"+i+"$",["i"])};return{restrict:"A",require:"?ngModel",link:function(t,a,o,r){var s=angular.extend({autoclose:!0},e.datepicker||{}),l=o.dateType||s.type||"date";angular.forEach(["format","weekStart","calendarWeeks","startDate","endDate","daysOfWeekDisabled","autoclose","startView","minViewMode","todayBtn","todayHighlight","keyboardNavigation","language","forceParse"],function(t){angular.isDefined(o[t])&&(s[t]=o[t])});var u=s.language||"en",c=o.dateFormat||s.format||$.fn.datepicker.dates[u]&&$.fn.datepicker.dates[u].format||"mm/dd/yyyy",d=n?"yyyy-mm-dd":c,p=i(d,u);r&&(r.$formatters.unshift(function(t){return"date"===l&&angular.isString(t)&&t?$.fn.datepicker.DPGlobal.parseDate(t,$.fn.datepicker.DPGlobal.parseFormat(c),u):t}),r.$parsers.unshift(function(t){return t?"date"===l&&angular.isDate(t)?(r.$setValidity("date",!0),t):angular.isString(t)&&p.test(t)?(r.$setValidity("date",!0),n?new Date(t):"string"===l?t:$.fn.datepicker.DPGlobal.parseDate(t,$.fn.datepicker.DPGlobal.parseFormat(d),u)):(r.$setValidity("date",!1),void 0):(r.$setValidity("date",!0),null)}),r.$render=function(){if(n){var t=r.$viewValue?$.fn.datepicker.DPGlobal.formatDate(r.$viewValue,$.fn.datepicker.DPGlobal.parseFormat(d),u):"";return a.val(t),t}return r.$viewValue||a.val(""),a.datepicker("update",r.$viewValue)}),n?a.prop("type","date").css("-webkit-appearance","textfield"):(r&&a.on("changeDate",function(e){t.$apply(function(){r.$setViewValue("string"===l?a.val():e.date)})}),a.datepicker(angular.extend(s,{format:d,language:u})),t.$on("$destroy",function(){var t=a.data("datepicker");t&&(t.picker.remove(),a.data("datepicker",null))}),o.$observe("startDate",function(t){a.datepicker("setStartDate",t)}),o.$observe("endDate",function(t){a.datepicker("setEndDate",t)}));var f=a.siblings('[data-toggle="datepicker"]');f.length&&f.on("click",function(){a.prop("disabled")||a.trigger("focus")})}}}]),angular.module("$strap.directives").directive("bsDropdown",["$parse","$compile","$timeout",function(t,e,n){var a=function(t,e){return e||(e=['"]),angular.forEach(t,function(t,n){if(t.divider)return e.splice(n+1,0,'
  • ');var i=""+'"+(t.text||"")+"";t.submenu&&t.submenu.length&&(i+=a(t.submenu).join("\n")),i+="",e.splice(n+1,0,i)}),e};return{restrict:"EA",scope:!0,link:function(i,o,r){var s=t(r.bsDropdown),l=s(i);n(function(){!angular.isArray(l);var t=angular.element(a(l).join(""));t.insertAfter(o),e(o.next("ul.dropdown-menu"))(i)}),o.addClass("dropdown-toggle").attr("data-toggle","dropdown")}}}]),angular.module("$strap.directives").factory("$modal",["$rootScope","$compile","$http","$timeout","$q","$templateCache","$strapConfig",function(t,e,n,a,i,o,r){var s=function s(s){function l(s){var l=angular.extend({show:!0},r.modal,s),u=l.scope?l.scope:t.$new(),c=l.template;return i.when(o.get(c)||n.get(c,{cache:!0}).then(function(t){return t.data})).then(function(t){var n=c.replace(".html","").replace(/[\/|\.|:]/g,"-")+"-"+u.$id,i=$('
    ').attr("id",n).addClass("fade").html(t);return l.modalClass&&i.addClass(l.modalClass),$("body").append(i),a(function(){e(i)(u)}),u.$modal=function(t){i.modal(t)},angular.forEach(["show","hide"],function(t){u[t]=function(){i.modal(t)}}),u.dismiss=u.hide,angular.forEach(["show","shown","hide","hidden"],function(t){i.on(t,function(e){u.$emit("modal-"+t,e)})}),i.on("shown",function(){$("input[autofocus], textarea[autofocus]",i).first().trigger("focus")}),i.on("hidden",function(){l.persist||u.$destroy()}),u.$on("$destroy",function(){i.remove()}),i.modal(l),i})}return new l(s)};return s}]).directive("bsModal",["$q","$modal",function(t,e){return{restrict:"A",scope:!0,link:function(n,a,i){var o={template:n.$eval(i.bsModal),persist:!0,show:!1,scope:n};angular.forEach(["modalClass","backdrop","keyboard"],function(t){angular.isDefined(i[t])&&(o[t]=i[t])}),t.when(e(o)).then(function(t){a.attr("data-target","#"+t.attr("id")).attr("data-toggle","modal")})}}}]),angular.module("$strap.directives").directive("bsNavbar",["$location",function(t){return{restrict:"A",link:function(e,n){e.$watch(function(){return t.path()},function(t){$("li[data-match-route]",n).each(function(e,n){var a=angular.element(n),i=a.attr("data-match-route"),o=RegExp("^"+i+"$",["i"]);o.test(t)?a.addClass("active").find(".collapse.in").collapse("hide"):a.removeClass("active")})})}}}]),angular.module("$strap.directives").directive("bsPopover",["$parse","$compile","$http","$timeout","$q","$templateCache",function(t,e,n,a,i,o){return $("body").on("keyup",function(t){27===t.keyCode&&$(".popover.in").each(function(){$(this).popover("hide")})}),{restrict:"A",scope:!0,link:function(r,s,l){var u=t(l.bsPopover),c=(u.assign,u(r)),d={};angular.isObject(c)&&(d=c),i.when(d.content||o.get(c)||n.get(c,{cache:!0})).then(function(t){angular.isObject(t)&&(t=t.data),l.unique&&s.on("show",function(){$(".popover.in").each(function(){var t=$(this),e=t.data("bs.popover");e&&!e.$element.is(s)&&t.popover("hide")})}),l.hide&&r.$watch(l.hide,function(t,e){t?n.hide():t!==e&&n.show()}),l.show&&r.$watch(l.show,function(t,e){t?a(function(){n.show()}):t!==e&&n.hide()}),s.popover(angular.extend({},d,{content:t,html:!0}));var n=s.data("bs.popover");n.hasContent=function(){return this.getTitle()||t},n.getPosition=function(){var t=$.fn.popover.Constructor.prototype.getPosition.apply(this,arguments);return e(this.$tip)(r),r.$digest(),this.$tip.data("bs.popover",this),t},r.$popover=function(t){n(t)},angular.forEach(["show","hide"],function(t){r[t]=function(){n[t]()}}),r.dismiss=r.hide,angular.forEach(["show","shown","hide","hidden"],function(t){s.on(t,function(e){r.$emit("popover-"+t,e)})})})}}}]),angular.module("$strap.directives").directive("bsSelect",["$timeout",function(t){return{restrict:"A",require:"?ngModel",link:function(e,n,a,i){var o=e.$eval(a.bsSelect)||{};t(function(){n.selectpicker(o),n.next().removeClass("ng-scope")}),i&&e.$watch(a.ngModel,function(t,e){angular.equals(t,e)||n.selectpicker("refresh")})}}}]),angular.module("$strap.directives").directive("bsTabs",["$parse","$compile","$timeout",function(t,e,n){var a='
    ';return{restrict:"A",require:"?ngModel",priority:0,scope:!0,template:a,replace:!0,transclude:!0,compile:function(){return function(e,a,i,o){var r=t(i.bsTabs);r.assign,r(e),e.panes=[];var s,l,u,c=a.find("ul.nav-tabs"),d=a.find("div.tab-content"),p=0;n(function(){d.find("[data-title], [data-tab]").each(function(t){var n=angular.element(this);s="tab-"+e.$id+"-"+t,l=n.data("title")||n.data("tab"),u=!u&&n.hasClass("active"),n.attr("id",s).addClass("tab-pane"),i.fade&&n.addClass("fade"),e.panes.push({id:s,title:l,content:this.innerHTML,active:u})}),e.panes.length&&!u&&(d.find(".tab-pane:first-child").addClass("active"+(i.fade?" in":"")),e.panes[0].active=!0)}),o&&(a.on("show",function(t){var n=$(t.target);e.$apply(function(){o.$setViewValue(n.data("index"))})}),e.$watch(i.ngModel,function(t){angular.isUndefined(t)||(p=t,setTimeout(function(){var e=$(c[0].querySelectorAll("li")[1*t]);e.hasClass("active")||e.children("a").tab("show")}))}))}}}}]),angular.module("$strap.directives").directive("bsTimepicker",["$timeout","$strapConfig",function(t,e){var n="((?:(?:[0-1][0-9])|(?:[2][0-3])|(?:[0-9])):(?:[0-5][0-9])(?::[0-5][0-9])?(?:\\s?(?:am|AM|pm|PM))?)";return{restrict:"A",require:"?ngModel",link:function(a,i,o,r){if(r){i.on("changeTime.timepicker",function(){t(function(){r.$setViewValue(i.val())})});var s=RegExp("^"+n+"$",["i"]);r.$parsers.unshift(function(t){return!t||s.test(t)?(r.$setValidity("time",!0),t):(r.$setValidity("time",!1),void 0)})}i.attr("data-toggle","timepicker"),i.parent().addClass("bootstrap-timepicker"),i.timepicker(e.timepicker||{});var l=i.data("timepicker"),u=i.siblings('[data-toggle="timepicker"]');u.length&&u.on("click",$.proxy(l.showWidget,l))}}}]),angular.module("$strap.directives").directive("bsTooltip",["$parse","$compile",function(t){return{restrict:"A",scope:!0,link:function(e,n,a){var i=t(a.bsTooltip),o=(i.assign,i(e));e.$watch(a.bsTooltip,function(t,e){t!==e&&(o=t)}),a.unique&&n.on("show",function(){$(".tooltip.in").each(function(){var t=$(this),e=t.data("tooltip");e&&!e.$element.is(n)&&t.tooltip("hide")})}),n.tooltip({title:function(){return angular.isFunction(o)?o.apply(null,arguments):o},html:!0});var r=n.data("tooltip");r.show=function(){var t=$.fn.tooltip.Constructor.prototype.show.apply(this,arguments);return this.tip().data("tooltip",this),t},e._tooltip=function(t){n.tooltip(t)},e.hide=function(){n.tooltip("hide")},e.show=function(){n.tooltip("show")},e.dismiss=e.hide}}}]),angular.module("$strap.directives").directive("bsTypeahead",["$parse",function(t){return{restrict:"A",require:"?ngModel",link:function(e,n,a,i){var o=t(a.bsTypeahead),r=(o.assign,o(e));e.$watch(a.bsTypeahead,function(t,e){t!==e&&(r=t)}),n.attr("data-provide","typeahead"),n.typeahead({source:function(){return angular.isFunction(r)?r.apply(null,arguments):r},minLength:a.minLength||1,items:a.items,updater:function(t){return i&&e.$apply(function(){i.$setViewValue(t)}),e.$emit("typeahead-updated",t),t}});var s=n.data("typeahead");s.lookup=function(){var t;return this.query=this.$element.val()||"",this.query.length +
    +
    +
    + +
    + All Dockerfile builds complete +
    +
    diff --git a/static/partials/header.html b/static/partials/header.html index 07f142c98..2fa772a4f 100644 --- a/static/partials/header.html +++ b/static/partials/header.html @@ -2,9 +2,9 @@
    @@ -27,9 +27,11 @@
    -
  • - + + + +
  • {{ user.username }} diff --git a/static/partials/image-view.html b/static/partials/image-view.html index 99523f61e..5e4477116 100644 --- a/static/partials/image-view.html +++ b/static/partials/image-view.html @@ -3,14 +3,14 @@
    • - +
    - +

    - + {{repo.namespace}} / {{repo.name}} @@ -31,7 +31,7 @@
    - +

    @@ -75,7 +75,7 @@ No matching changes
    - + {{folder}}/{{getFilename(change.file)}} diff --git a/static/partials/landing.html b/static/partials/landing.html index 890566750..4e3426972 100644 --- a/static/partials/landing.html +++ b/static/partials/landing.html @@ -10,7 +10,7 @@
    - +

    Your Top Repositories

    @@ -44,10 +44,10 @@ @@ -61,14 +61,15 @@
    Welcome {{ user.username }}!
    - + Browse all repositories + Create a new repository
    - + Secure Store your private docker containers where only you and your team @@ -77,7 +78,7 @@
    - + Shareable Have to share a repository? No problem! Share with anyone you choose @@ -85,7 +86,7 @@
    - + Cloud Hosted Accessible from anywhere, anytime @@ -97,7 +98,7 @@
    -
    +
    Take a tour of Quay
    diff --git a/static/partials/new-repo.html b/static/partials/new-repo.html new file mode 100644 index 000000000..30cdd2316 --- /dev/null +++ b/static/partials/new-repo.html @@ -0,0 +1,213 @@ +
    +

    Please sign in

    +
    + +
    + +
    + +
    + +
    + +
    + Uploading file {{ upload_file }} +
    +
    +
    +
    + +
    + +
    +
    + + +
    +
    +
    +
    +
    + + {{user.username}} / +
    +
    + +
    + Description:
    +

    + + +

    +
    +
    +
    + + +
    +
    +
    +
    +
    + + + +
    + + Anyone can see and pull from this repository. You choose who can push. +
    +
    +
    + + + +
    + + You choose who can see, pull and push from/to this repository. +
    +
    + + +
    +
    + In order to make this repository private, you’ll need to upgrade your plan from {{ subscribedPlan.title }} to {{ planRequired.title }}. This will cost ${{ planRequired.price / 100 }}/month. +
    + Upgrade now + +
    +
    +
    +
    + + +
    +
    +
    +
    + +
    + + Automatically populate your repository with a new image constructed from a Dockerfile +
    + +
    +
    + Upload a Dockerfile or a zip file containing a Dockerfile in the root directory +
    + + +
    +
    +
    +
    + +
    +
    +
    + +
    +
    + +
    +
    + + + + + + + + + + + + + + + + + + + + diff --git a/static/partials/repo-admin.html b/static/partials/repo-admin.html index a800cf08c..e3d341cd3 100644 --- a/static/partials/repo-admin.html +++ b/static/partials/repo-admin.html @@ -3,7 +3,7 @@
    - +
    @@ -12,7 +12,7 @@
    - +

    {{repo.namespace}} / {{repo.name}}

    @@ -22,7 +22,7 @@
    User Access Permissions - +
    @@ -37,7 +37,7 @@ - + {{username}} @@ -50,7 +50,7 @@ - + @@ -68,7 +68,7 @@
    Access Token Permissions - +
    @@ -83,7 +83,7 @@ - + {{ token.friendlyName }} @@ -95,7 +95,7 @@ - + @@ -118,7 +118,7 @@
    Repository Settings
    -
    +
    This repository is currently private. Only users on the above access list may view and interact with it. @@ -128,7 +128,7 @@
    -
    +
    This repository is currently public and is visible to all users, and may be pulled by all users. @@ -175,7 +175,7 @@
    -

    {{ shownToken.friendlyName }}

    +

    {{ shownToken.friendlyName }}

    The docker username is $token and the password is the token. You may use any value for email.
    diff --git a/static/partials/repo-list.html b/static/partials/repo-list.html index a1292e6f2..5edd390d0 100644 --- a/static/partials/repo-list.html +++ b/static/partials/repo-list.html @@ -1,9 +1,16 @@
    - +
    + + + +

    Your Repositories

    diff --git a/static/partials/signin.html b/static/partials/signin.html index da203d1c8..71821ec0a 100644 --- a/static/partials/signin.html +++ b/static/partials/signin.html @@ -18,11 +18,11 @@ - Sign In with GitHub + Sign In with GitHub
    Invalid username or password.
    diff --git a/static/partials/user-admin.html b/static/partials/user-admin.html index 16df0fe00..8408863fb 100644 --- a/static/partials/user-admin.html +++ b/static/partials/user-admin.html @@ -1,6 +1,6 @@
    - +
    @@ -18,7 +18,7 @@
    {{ plan.title }} - + Subscribed
    @@ -59,7 +59,7 @@
    - +
    diff --git a/static/partials/v1-page.html b/static/partials/v1-page.html index 821654fdd..e81c3d598 100644 --- a/static/partials/v1-page.html +++ b/static/partials/v1-page.html @@ -3,7 +3,7 @@

    Welcome {{ user.username }}. Your account is fully activated!

    - + Browse all repositories
    diff --git a/static/partials/view-repo.html b/static/partials/view-repo.html index fc0587ad4..2efe29be1 100644 --- a/static/partials/view-repo.html +++ b/static/partials/view-repo.html @@ -3,7 +3,7 @@
    - +
    @@ -16,20 +16,19 @@ - + - + -
    - Get this repository: - +
    + +
    +
    + + + Building Images + + {{ buildsInfo ? buildsInfo.length : '-' }} +
    +
    -

    - - -

    +
    +

    + + +

    +
    -
    + +
    (This repository is empty)
    +
    +
    Your build is currently processing, if this takes longer than an hour, please contact Quay.io support
    +
    + +
    @@ -61,22 +79,22 @@
    - - {{getTagCount(repo)}} +
    + {{getTagCount(repo)}} {{currentTag.name}} - +
    Tags
    - +
    @@ -89,15 +107,15 @@
    - - {{imageHistory.length}} +
    + {{imageHistory.length}} {{currentImage.id.substr(0, 12)}} - +
    Image
    @@ -117,22 +135,22 @@
    - +
    - + {{currentImageChanges.added.length}} - + {{currentImageChanges.removed.length}} - + {{currentImageChanges.changed.length}}
    @@ -140,15 +158,15 @@
    - + {{file}}
    - + {{file}}
    - + {{file}}
    diff --git a/templates/base.html b/templates/base.html index 50cb9b482..03563404e 100644 --- a/templates/base.html +++ b/templates/base.html @@ -12,7 +12,7 @@ - + @@ -43,6 +43,7 @@ + @@ -50,6 +51,10 @@ + + + + {% block added_dependencies %} {% endblock %} diff --git a/templates/index.html b/templates/index.html index bd531dae9..22e993024 100644 --- a/templates/index.html +++ b/templates/index.html @@ -24,14 +24,10 @@ - - - - {% endblock %} {% block body_content %}
    -{% endblock %} \ No newline at end of file +{% endblock %} diff --git a/test/data/test.db b/test/data/test.db index 96e878b95..759203fe9 100644 Binary files a/test/data/test.db and b/test/data/test.db differ diff --git a/workers/diffsworker.py b/workers/diffsworker.py index bdd9181a7..ce81b96f0 100644 --- a/workers/diffsworker.py +++ b/workers/diffsworker.py @@ -7,10 +7,10 @@ import argparse from apscheduler.scheduler import Scheduler from data.queue import image_diff_queue +from data.database import db as db_connection from endpoints.registry import process_image_changes - root_logger = logging.getLogger('') root_logger.setLevel(logging.DEBUG) @@ -38,6 +38,10 @@ def process_work_items(): logger.debug('No more work.') + if not db_connection.is_closed(): + logger.debug('Closing thread db connection.') + db_connection.close() + def start_worker(): logger.debug("Scheduling worker.") @@ -59,11 +63,6 @@ parser.add_argument('--log', default='diffsworker.log', args = parser.parse_args() -# if not args.D: -# else: -# logging.basicConfig(format=FORMAT, level=logging.DEBUG) -# start_worker(args) - if args.D: handler = logging.FileHandler(args.log) handler.setFormatter(formatter) diff --git a/workers/dockerfilebuild.py b/workers/dockerfilebuild.py new file mode 100644 index 000000000..4cdeb0e70 --- /dev/null +++ b/workers/dockerfilebuild.py @@ -0,0 +1,279 @@ +import logging +import json +import daemon +import time +import argparse +import digitalocean +import requests +import os + +from apscheduler.scheduler import Scheduler +from multiprocessing.pool import ThreadPool +from base64 import b64encode +from requests.exceptions import ConnectionError + +from data.queue import dockerfile_build_queue +from data.userfiles import UserRequestFiles +from data import model +from data.database import db as db_connection +from app import app + + +root_logger = logging.getLogger('') +root_logger.setLevel(logging.DEBUG) + +FORMAT = '%(asctime)-15s - %(levelname)s - %(pathname)s - %(funcName)s - %(message)s' +formatter = logging.Formatter(FORMAT) + +logger = logging.getLogger(__name__) + +BUILD_SERVER_CMD = ('docker run -d -lxc-conf="lxc.aa_profile=unconfined" ' + + '-privileged -e \'RESOURCE_URL=%s\' -e \'TAG=%s\' ' + + '-e \'TOKEN=%s\' quay.io/quay/buildserver') + + +def retry_command(to_call, args=[], kwargs={}, retries=5, period=5): + try: + return to_call(*args, **kwargs) + except Exception as ex: + if retries: + logger.debug('Retrying command after %ss' % period) + time.sleep(period) + return retry_command(to_call, args, kwargs, retries-1, period) + raise ex + + +def get_status(url): + return retry_command(requests.get, [url]).json()['status'] + + +def babysit_builder(request): + """ Spin up a build node and ask it to build our job. Retryable errors + should return False, while fatal errors should return True. + """ + try: + logger.debug('Starting work item: %s' % request) + repository_build = model.get_repository_build(request['build_id']) + logger.debug('Request details: %s' % repository_build) + + # Initialize digital ocean API + do_client_id = app.config['DO_CLIENT_ID'] + do_api_key = app.config['DO_CLIENT_SECRET'] + manager = digitalocean.Manager(client_id=do_client_id, api_key=do_api_key) + + # check if there is already a DO node for this build, if so clean it up + old_id = repository_build.build_node_id + if old_id: + logger.debug('Cleaning up old DO node: %s' % old_id) + old_droplet = digitalocean.Droplet(id=old_id, client_id=do_client_id, + api_key=do_api_key) + retry_command(old_droplet.destroy) + + # Pick the region for the new droplet + allowed_regions = app.config['DO_ALLOWED_REGIONS'] + regions = retry_command(manager.get_all_regions) + available_regions = {region.id for region in regions} + regions = available_regions.intersection(allowed_regions) + if not regions: + logger.error('No droplets in our allowed regtions, available: %s' % + available_regions) + return False + + # start the DO node + name = 'dockerfile-build-%s' % repository_build.id + logger.debug('Starting DO node: %s' % name) + droplet = digitalocean.Droplet(client_id=do_client_id, + api_key=do_api_key, + name=name, + region_id=regions.pop(), + image_id=1004145, # Docker on 13.04 + size_id=66, # 512MB, + backup_active=False) + retry_command(droplet.create, [], + {'ssh_key_ids': [app.config['DO_SSH_KEY_ID']]}) + repository_build.build_node_id = droplet.id + repository_build.phase = 'starting' + repository_build.save() + + logger.debug('Waiting for DO node to be available.') + + startup = retry_command(droplet.get_events)[0] + while not startup.percentage or int(startup.percentage) != 100: + logger.debug('Droplet startup percentage: %s' % startup.percentage) + time.sleep(5) + retry_command(startup.load) + + retry_command(droplet.load) + logger.debug('Droplet started at ip address: %s' % droplet.ip_address) + + # connect to it with ssh + repository_build.phase = 'initializing' + repository_build.save() + + # We wait until here to import paramiko because otherwise it doesn't work + # under the daemon context. + import paramiko + ssh_client = paramiko.SSHClient() + ssh_client.set_missing_host_key_policy(paramiko.AutoAddPolicy()) + + logger.debug('Connecting to droplet through ssh at ip: %s' % + droplet.ip_address) + retry_command(ssh_client.connect, [droplet.ip_address, 22, 'root'], + {'look_for_keys': False, 'timeout': 10.0, + 'key_filename': app.config['DO_SSH_PRIVATE_KEY_FILENAME']}) + + # Load the node with the pull token + token = app.config['BUILD_NODE_PULL_TOKEN'] + basicauth = b64encode('%s:%s' % ('$token', token)) + auth_object = { + 'https://quay.io/v1/': { + 'auth': basicauth, + 'email': '', + }, + } + + create_auth_cmd = 'echo \'%s\' > .dockercfg' % json.dumps(auth_object) + ssh_client.exec_command(create_auth_cmd) + + # Pull and run the buildserver + pull_cmd = 'docker pull quay.io/quay/buildserver' + _, stdout, _ = ssh_client.exec_command(pull_cmd) + pull_status = stdout.channel.recv_exit_status() + + if pull_status != 0: + logger.error('Pull command failed for host: %s' % droplet.ip_address) + return False + else: + logger.debug('Pull status was: %s' % pull_status) + + # Remove the credentials we used to pull so crafty users cant steal them + remove_auth_cmd = 'rm .dockercfg' + ssh_client.exec_command(remove_auth_cmd) + + # Prepare the signed resource url the build node can fetch the job from + user_files = UserRequestFiles(app.config['AWS_ACCESS_KEY'], + app.config['AWS_SECRET_KEY'], + app.config['REGISTRY_S3_BUCKET']) + resource_url = user_files.get_file_url(repository_build.resource_key) + + # Start the build server + start_cmd = BUILD_SERVER_CMD % (resource_url, repository_build.tag, + repository_build.access_token.code) + logger.debug('Sending build server request with command: %s' % start_cmd) + ssh_client.exec_command(start_cmd) + + status_endpoint = 'http://%s:5002/build/' % droplet.ip_address + # wait for the server to be ready + logger.debug('Waiting for buildserver to be ready') + retry_command(requests.get, [status_endpoint]) + + # wait for the job to be complete + repository_build.phase = 'building' + repository_build.status_url = status_endpoint + repository_build.save() + + logger.debug('Waiting for job to be complete') + status = get_status(status_endpoint) + while status != 'error' and status != 'complete': + logger.debug('Job status is: %s' % status) + time.sleep(5) + status = get_status(status_endpoint) + + logger.debug('Job complete with status: %s' % status) + if status == 'error': + error_message = requests.get(status_endpoint).json()['message'] + logger.warning('Job error: %s' % error_message) + repository_build.phase = 'error' + else: + repository_build.phase = 'complete' + + # clean up the DO node + logger.debug('Cleaning up DO node.') + retry_command(droplet.destroy) + + repository_build.status_url = None + repository_build.build_node_id = None; + repository_build.save() + + return True + + except Exception as outer_ex: + # We don't really know what these are, but they are probably retryable + logger.exception('Exception processing job: %s' % outer_ex.message) + return False + + finally: + if not db_connection.is_closed(): + logger.debug('Closing thread db connection.') + db_connection.close() + + +def process_work_items(pool): + logger.debug('Getting work item from queue.') + + item = dockerfile_build_queue.get(processing_time=60*60) # allow 1 hr + + while item: + logger.debug('Queue gave us some work: %s' % item.body) + + request = json.loads(item.body) + + def build_callback(item): + local_item = item + def complete_callback(completed): + if completed: + dockerfile_build_queue.complete(local_item) + else: + # We have a retryable error, add the job back to the queue + dockerfile_build_queue.incomplete(local_item) + + return complete_callback + + logger.debug('Sending work item to thread pool: %s' % pool) + pool.apply_async(babysit_builder, [request], + callback=build_callback(item)) + + item = dockerfile_build_queue.get() + + logger.debug('No more work.') + + if not db_connection.is_closed(): + logger.debug('Closing thread db connection.') + db_connection.close() + + +def start_worker(): + pool = ThreadPool(3) + logger.debug('Scheduling worker.') + + sched = Scheduler() + sched.start() + + sched.add_interval_job(process_work_items, args=[pool], seconds=30) + + while True: + time.sleep(60 * 60 * 24) # sleep one day, basically forever + + +desc = 'Worker daemon to monitor dockerfile build' +parser = argparse.ArgumentParser(description=desc) +parser.add_argument('-D', action='store_true', default=False, + help='Run the worker in daemon mode.') +parser.add_argument('--log', default='dockerfilebuild.log', + help='Specify the log file for the worker as a daemon.') +args = parser.parse_args() + + +if args.D: + handler = logging.FileHandler(args.log) + handler.setFormatter(formatter) + root_logger.addHandler(handler) + with daemon.DaemonContext(files_preserve=[handler.stream], + working_directory=os.getcwd()): + start_worker() + +else: + handler = logging.StreamHandler() + handler.setFormatter(formatter) + root_logger.addHandler(handler) + start_worker() \ No newline at end of file