Switch to using environment variables to pass information to the build node, this closes down a security loophole with the admin endpoint of the server.
This commit is contained in:
yackob03
parent
a313a77a6b
commit
c91b40f356
2 changed files with 76 additions and 99 deletions
|
|
@ -26,30 +26,19 @@ formatter = logging.Formatter(FORMAT)
|
|||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
BUILD_SERVER_CMD = ('docker run -d -lxc-conf="lxc.aa_profile=unconfined" ' +
|
||||
'-privileged -e \'RESOURCE_URL=%s\' -e \'TAG=%s\' ' +
|
||||
'-e \'TOKEN=%s\' quay.io/quay/buildserver')
|
||||
|
||||
def try_connection(url, retries=5, period=5):
|
||||
|
||||
def retry_command(to_call, args=[], kwargs={}, retries=5, period=5):
|
||||
try:
|
||||
return requests.get(url)
|
||||
except ConnectionError as ex:
|
||||
if retries:
|
||||
logger.debug('Retrying connection to url: %s after %ss' % (url, period))
|
||||
time.sleep(period)
|
||||
return try_connection(url, retries-1, period)
|
||||
raise ex
|
||||
|
||||
|
||||
def try_connect_ssh(client, ip_addr, port, user, key_filename, retries=5,
|
||||
period=5):
|
||||
try:
|
||||
client.connect(ip_addr, port, user, look_for_keys=False,
|
||||
key_filename=key_filename)
|
||||
return to_call(*args, **kwargs)
|
||||
except Exception as ex:
|
||||
if retries:
|
||||
logger.debug('Retrying connection to ssh ip: %s:%s after %ss' %
|
||||
(ip_addr, port, period))
|
||||
logger.debug('Retrying command after %ss' % period)
|
||||
time.sleep(period)
|
||||
return try_connect_ssh(client, ip_addr, port, user, key_filename,
|
||||
retries-1, period)
|
||||
return retry_command(to_call, args, kwargs, retries-1, period)
|
||||
raise ex
|
||||
|
||||
|
||||
|
|
@ -77,11 +66,12 @@ def babysit_builder(request):
|
|||
logger.debug('Cleaning up old DO node: %s' % old_id)
|
||||
old_droplet = digitalocean.Droplet(id=old_id, client_id=do_client_id,
|
||||
api_key=do_api_key)
|
||||
old_droplet.destroy()
|
||||
retry_command(old_droplet.destroy)
|
||||
|
||||
# Pick the region for the new droplet
|
||||
allowed_regions = app.config['DO_ALLOWED_REGIONS']
|
||||
available_regions = {region.id for region in manager.get_all_regions()}
|
||||
regions = retry_command(manager.get_all_regions)
|
||||
available_regions = {region.id for region in regions}
|
||||
regions = available_regions.intersection(allowed_regions)
|
||||
if not regions:
|
||||
logger.error('No droplets in our allowed regtions, available: %s' %
|
||||
|
|
@ -98,19 +88,21 @@ def babysit_builder(request):
|
|||
image_id=1004145, # Docker on 13.04
|
||||
size_id=66, # 512MB,
|
||||
backup_active=False)
|
||||
droplet.create(ssh_key_ids=[app.config['DO_SSH_KEY_ID']])
|
||||
retry_command(droplet.create, [],
|
||||
{'ssh_key_ids': [app.config['DO_SSH_KEY_ID']]})
|
||||
repository_build.build_node_id = droplet.id
|
||||
repository_build.phase = 'starting'
|
||||
repository_build.save()
|
||||
|
||||
startup = droplet.get_events()[0]
|
||||
startup.load()
|
||||
logger.debug('Waiting for DO node to be available.')
|
||||
|
||||
startup = retry_command(droplet.get_events)[0]
|
||||
while not startup.percentage or int(startup.percentage) != 100:
|
||||
logger.debug('Droplet startup percentage: %s' % startup.percentage)
|
||||
time.sleep(5)
|
||||
startup.load()
|
||||
retry_command(startup.load)
|
||||
|
||||
droplet.load()
|
||||
retry_command(droplet.load)
|
||||
logger.debug('Droplet started at ip address: %s' % droplet.ip_address)
|
||||
|
||||
# connect to it with ssh
|
||||
|
|
@ -119,8 +111,10 @@ def babysit_builder(request):
|
|||
|
||||
ssh_client = paramiko.SSHClient()
|
||||
ssh_client.set_missing_host_key_policy(paramiko.AutoAddPolicy())
|
||||
try_connect_ssh(ssh_client, droplet.ip_address, 22, 'root',
|
||||
key_filename=app.config['DO_SSH_PRIVATE_KEY_FILENAME'])
|
||||
|
||||
retry_command(ssh_client.connect, [droplet.ip_address, 22, 'root'],
|
||||
{'look_for_keys': False,
|
||||
'key_filename': app.config['DO_SSH_PRIVATE_KEY_FILENAME']})
|
||||
|
||||
# Load the node with the pull token
|
||||
token = app.config['BUILD_NODE_PULL_TOKEN']
|
||||
|
|
@ -142,6 +136,7 @@ def babysit_builder(request):
|
|||
|
||||
if pull_status != 0:
|
||||
logger.error('Pull command failed for host: %s' % droplet.ip_address)
|
||||
return False
|
||||
else:
|
||||
logger.debug('Pull status was: %s' % pull_status)
|
||||
|
||||
|
|
@ -149,56 +144,46 @@ def babysit_builder(request):
|
|||
remove_auth_cmd = 'rm .dockercfg'
|
||||
ssh_client.exec_command(remove_auth_cmd)
|
||||
|
||||
# Start the build server
|
||||
start_cmd = 'docker run -d -privileged -lxc-conf="lxc.aa_profile=unconfined" quay.io/quay/buildserver'
|
||||
ssh_client.exec_command(start_cmd)
|
||||
|
||||
# wait for the server to be ready
|
||||
logger.debug('Waiting for buildserver to be ready')
|
||||
build_endpoint = 'http://%s:5002/build/' % droplet.ip_address
|
||||
try:
|
||||
try_connection(build_endpoint)
|
||||
except ConnectionError:
|
||||
#TODO cleanup
|
||||
pass
|
||||
|
||||
# send it the job
|
||||
logger.debug('Sending build server request')
|
||||
|
||||
# Prepare the signed resource url the build node can fetch the job from
|
||||
user_files = UserRequestFiles(app.config['AWS_ACCESS_KEY'],
|
||||
app.config['AWS_SECRET_KEY'],
|
||||
app.config['REGISTRY_S3_BUCKET'])
|
||||
resource_url = user_files.get_file_url(repository_build.resource_key)
|
||||
|
||||
repo = repository_build.repository
|
||||
payload = {
|
||||
'tag': repository_build.tag,
|
||||
'resource_url': user_files.get_file_url(repository_build.resource_key),
|
||||
'token': repository_build.access_token.code,
|
||||
}
|
||||
start_build = requests.post(build_endpoint, data=payload)
|
||||
# Start the build server
|
||||
start_cmd = BUILD_SERVER_CMD % (resource_url, repository_build.tag,
|
||||
repository_build.access_token.code)
|
||||
logger.debug('Sending build server request with command: %s' % start_cmd)
|
||||
ssh_client.exec_command(start_cmd)
|
||||
|
||||
status_endpoint = 'http://%s:5002/build/' % droplet.ip_address
|
||||
# wait for the server to be ready
|
||||
logger.debug('Waiting for buildserver to be ready')
|
||||
retry_command(requests.get, [status_endpoint])
|
||||
|
||||
# wait for the job to be complete
|
||||
status_url = start_build.headers['Location']
|
||||
repository_build.phase = 'building'
|
||||
repository_build.status_url = status_url
|
||||
repository_build.status_url = status_endpoint
|
||||
repository_build.save()
|
||||
|
||||
logger.debug('Waiting for job to be complete')
|
||||
status = get_status(status_url)
|
||||
status = get_status(status_endpoint)
|
||||
while status != 'error' and status != 'complete':
|
||||
logger.debug('Job status is: %s' % status)
|
||||
time.sleep(5)
|
||||
status = get_status(status_url)
|
||||
status = get_status(status_endpoint)
|
||||
|
||||
logger.debug('Job complete with status: %s' % status)
|
||||
if status == 'error':
|
||||
error_message = requests.get(status_endpoint).json()['message']
|
||||
logger.warning('Job error: %s' % error_message)
|
||||
repository_build.phase = 'error'
|
||||
else:
|
||||
repository_build.phase = 'complete'
|
||||
|
||||
# clean up the DO node
|
||||
logger.debug('Cleaning up DO node.')
|
||||
droplet.destroy()
|
||||
# retry_command(droplet.destroy)
|
||||
|
||||
repository_build.status_url = None
|
||||
repository_build.build_node_id = None;
|
||||
|
|
|
|||
Reference in a new issue