vbatts/quay
Archived
1
0
Fork 0
Code Issues Pull requests Releases Wiki Activity

Merge branch 'bobthe' of https://bitbucket.org/yackob03/quay into bobthe

Browse source
This commit is contained in:
Joseph Schorr 2014-02-10 19:13:40 -05:00
commit 7c081f029c
7 changed files with 213 additions and 140 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

4
application.py
View file

@ -21,10 +21,6 @@ from endpoints.webhooks import webhooks
logger = logging.getLogger(__name__)
if application.config.get('INCLUDE_TEST_ENDPOINTS', False):
logger.debug('Loading test endpoints.')
import endpoints.test
application.register_blueprint(web)
application.register_blueprint(index, url_prefix='/v1')
application.register_blueprint(tags, url_prefix='/v1')

15
config.py
View file

@ -11,6 +11,7 @@ from util import analytics
from test.teststorage import FakeStorage, FakeUserfiles
from test import analytics as fake_analytics
from test.testlogs import TestBuildLogs
class FlaskConfig(object):
@ -91,6 +92,10 @@ class RedisBuildLogs(object):
BUILDLOGS = BuildLogs('logs.quay.io')
class TestBuildLogs(object):
BUILDLOGS = TestBuildLogs('logs.quay.io')
class StripeTestConfig(object):
STRIPE_SECRET_KEY = 'sk_test_PEbmJCYrLXPW0VRLSnWUiZ7Y'
STRIPE_PUBLISHABLE_KEY = 'pk_test_uEDHANKm9CHCvVa2DLcipGRh'
@ -140,13 +145,13 @@ class BuildNodeConfig(object):
BUILD_NODE_PULL_TOKEN = 'F02O2E86CQLKZUQ0O81J8XDHQ6F0N1V36L9JTOEEK6GKKMT1GI8PTJQT4OU88Y6G'
def logs_init_builder(level=logging.DEBUG):
def logs_init_builder(level=logging.DEBUG,
formatter=logstash_formatter.LogstashFormatter()):
@staticmethod
def init_logs():
handler = logging.StreamHandler()
root_logger = logging.getLogger('')
root_logger.setLevel(level)
formatter = logstash_formatter.LogstashFormatter()
handler.setFormatter(formatter)
root_logger.addHandler(handler)
@ -158,17 +163,15 @@ class TestConfig(FlaskConfig, FakeStorage, EphemeralDB, FakeUserfiles,
LOGGING_CONFIG = logs_init_builder(logging.WARN)
POPULATE_DB_TEST_DATA = True
TESTING = True
INCLUDE_TEST_ENDPOINTS = True
class DebugConfig(FlaskConfig, MailConfig, LocalStorage, SQLiteDB,
StripeTestConfig, MixpanelTestConfig, GitHubTestConfig,
DigitalOceanConfig, BuildNodeConfig, S3Userfiles,
RedisBuildLogs):
LOGGING_CONFIG = logs_init_builder()
TestBuildLogs):
LOGGING_CONFIG = logs_init_builder(formatter=logging.Formatter())
SEND_FILE_MAX_AGE_DEFAULT = 0
POPULATE_DB_TEST_DATA = True
INCLUDE_TEST_ENDPOINTS = True
class LocalHostedConfig(FlaskConfig, MailConfig, S3Storage, RDSMySQL,

44
data/buildlogs.py
View file

@ -10,6 +10,10 @@ class BuildLogs(object):
def _logs_key(build_id):
return 'builds/%s/logs' % build_id
@staticmethod
def _commands_key(build_id):
return 'builds/%s/commands' % build_id
def append_log_entry(self, build_id, log_obj):
"""
Appends the serialized form of log_obj to the end of the log entry list
@ -20,12 +24,30 @@ class BuildLogs(object):
def append_log_message(self, build_id, log_message):
"""
Wraps the message in an envelope and push it to the end of the log entry
list and returns the new length of the list.
list and returns the index at which it was inserted.
"""
log_obj = {
'message': log_message
}
return self._redis.rpush(self._logs_key(build_id), json.dumps(log_obj))
return self._redis.rpush(self._logs_key(build_id), json.dumps(log_obj)) - 1
def append_command_message(self, build_id, command_message):
"""
Wraps the message in an envelope and push it to the end of the log entry
list, to the commands list, and returns the new length of the list.
"""
log_obj = {
'message': command_message,
'is_command': True,
}
idx = self._redis.rpush(self._logs_key(build_id), json.dumps(log_obj)) - 1
cmd_obj = {
'message': command_message,
'index': idx,
}
self._redis.rpush(self._commands_key(build_id), json.dumps(cmd_obj))
return idx
def get_log_entries(self, build_id, start_index, end_index):
"""
@ -37,6 +59,24 @@ class BuildLogs(object):
end_index)
return (llen, (json.loads(entry) for entry in log_entries))
def get_commands(self, build_id):
"""
Returns a list of all Dockerfile commands that have passed through the
specified build thus far.
"""
commands = self._redis.lrange(self._commands_key(build_id), 0, -1)
return (json.loads(cmd) for cmd in commands)
def get_last_command(self, build_id):
"""
Returns only the last command from the list of commands.
"""
commands = self._redis.lrange(self._commands_key(build_id), -1, -1)
if commands:
return json.loads(commands[-1])
else:
return None
@staticmethod
def _status_key(build_id):
return 'builds/%s/status' % build_id

27
endpoints/api.py
View file

@ -1184,10 +1184,31 @@ def get_repo_build_status(namespace, repository, build_uuid):
def get_repo_build_logs(namespace, repository, build_uuid):
permission = ModifyRepositoryPermission(namespace, repository)
if permission.can():
response_obj = {}
build = model.get_repository_build(namespace, repository, build_uuid)
start = int(request.args.get('start', -1000))
start_param = request.args.get('start', None)
end = int(request.args.get('end', -1))
last_command = None
include_commands = request.args.get('commands', 'false')
if include_commands.lower() not in {'0', 'false'}:
commands = [cmd for cmd in build_logs.get_commands(build.uuid)]
response_obj['commands'] = commands
if commands:
last_command = commands[-1]
elif start_param is None:
last_command = build_logs.get_last_command(build.uuid)
if start_param is None:
if last_command:
start = last_command['index']
else:
start = 0
else:
start = int(start_param)
count, logs = build_logs.get_log_entries(build.uuid, start, end)
if start < 0:
@ -1196,13 +1217,15 @@ def get_repo_build_logs(namespace, repository, build_uuid):
if end < 0:
end = count + end
return jsonify({
response_obj.update({
'start': start,
'end': end,
'total': count,
'logs': [log for log in logs],
})
return jsonify(response_obj)
abort(403) # Permission denied

124
endpoints/test.py
View file

@ -1,124 +0,0 @@
import math
from random import SystemRandom
from flask import jsonify, request
from loremipsum import get_sentences
from endpoints.api import api
def generate_image_completion(rand_func):
images = {}
for image_id in range(rand_func.randint(1, 11)):
total = int(math.pow(abs(rand_func.gauss(0, 1000)), 2))
current = rand_func.randint(0, total)
image_id = 'image_id_%s' % image_id
images[image_id] = {
'total': total,
'current': current,
}
return images
def generate_fake_status():
response = {
'id': 'deadbeef-dead-beef-dead-beefdeadbeef',
'status': None,
}
random = SystemRandom()
phases = {
'waiting': {},
'starting': {
'total_commands': 7,
'current_command': 0,
},
'initializing': {},
'error': {},
'complete': {},
'building': {
'total_commands': 7,
'current_command': random.randint(1, 7),
},
'pushing': {
'total_commands': 7,
'current_command': 7,
'push_completion': random.random(),
'image_completion': generate_image_completion(random),
},
}
phase = random.choice(phases.keys())
response['phase'] = phase
response['status'] = (phases[phase])
return response
BASE_BUILDING_URL = '/repository/devtable/building/build/'
@api.route(BASE_BUILDING_URL, methods=['GET'])
def get_fake_repo_build_status_list():
return jsonify({'builds': [generate_fake_status()]})
@api.route(BASE_BUILDING_URL + 'deadbeef-dead-beef-dead-beefdeadbeef/logs',
methods=['GET'])
def get_fake_repo_build_logs():
start = int(request.args.get('start', 0))
end = int(request.args.get('end', 0))
had_start = 'start' in request.args
had_end = 'end' in request.args
adv_start = 0
adv_end = 0
adv_total = 0
lorem_logs = []
if had_start and had_end:
numlogs = end - start + 1
adv_start = start
adv_end = end
adv_total = end + 1
lorem_logs = get_sentences(numlogs)
elif had_start:
adv_start = start
adv_end = start + 9
lorem_logs = get_sentences(10)
adv_total = adv_end + 1
elif had_end:
adv_start = max(0, (end - 9))
adv_end = end
adv_total = end + 1
lorem_logs = get_sentences(adv_end - adv_start + 1)
else:
adv_start = 100
adv_end = 109
adv_total = 110
lorem_logs = get_sentences(10)
def wrap_log_message(rand, msg):
if rand.randint(1, 10) == 1:
block = {
'is_command': True,
'message': 'Step %s : %s' % (rand.randint(1, 10), msg)
}
else:
block = {
'message': msg,
}
return block
rnd = SystemRandom()
return jsonify({
'start': adv_start,
'end': adv_end,
'total': adv_total,
'logs': [wrap_log_message(rnd, sentence) for sentence in lorem_logs]
})
@api.route(BASE_BUILDING_URL + '/deadbeef-dead-beef-dead-beefdeadbeef/status',
methods=['GET'])
def get_fake_repo_build_status():
return jsonify(generate_fake_status())

136
test/testlogs.py Normal file
View file

@ -0,0 +1,136 @@
import math
import logging
from random import SystemRandom
from loremipsum import get_sentence
from data.buildlogs import BuildLogs
logger = logging.getLogger(__name__)
class TestBuildLogs(BuildLogs):
TEST_BUILD_ID = 'deadbeef-dead-beef-dead-beefdeadbeef'
def __init__(self, redis_host):
super(TestBuildLogs, self).__init__(redis_host)
self.last_command = 0
self.logs = [self._generate_command()]
self.commands = [{
'index': 0,
'message': self.logs[0]['message'],
}]
self.request_counter = 0
self._generate_logs()
def _generate_command(self):
self.last_command += 1
return {
'message': 'Step %s : %s' % (self.last_command, get_sentence()),
'is_command': True,
}
def _generate_logs(self):
rand = SystemRandom()
num_logs = rand.randint(1, 500)
for _ in range(num_logs):
if rand.randint(1, 50) == 1:
cmd = self._generate_command()
self.commands.append({
'message': cmd['message'],
'index': len(self.logs),
})
self.logs.append(cmd)
else:
self.logs.append({
'message': get_sentence(),
})
@staticmethod
def generate_image_completion(rand_func):
images = {}
for image_id in range(rand_func.randint(1, 11)):
total = int(math.pow(abs(rand_func.gauss(0, 1000)), 2))
current = rand_func.randint(0, total)
image_id = 'image_id_%s' % image_id
images[image_id] = {
'total': total,
'current': current,
}
return images
@staticmethod
def generate_fake_status():
response = {
'id': 'deadbeef-dead-beef-dead-beefdeadbeef',
'status': None,
}
random = SystemRandom()
phases = {
'waiting': {},
'starting': {
'total_commands': 7,
'current_command': 0,
},
'initializing': {},
'error': {},
'complete': {},
'building': {
'total_commands': 7,
'current_command': random.randint(1, 7),
},
'pushing': {
'total_commands': 7,
'current_command': 7,
'push_completion': random.random(),
'image_completion': TestBuildLogs.generate_image_completion(random),
},
}
phase = random.choice(phases.keys())
response['phase'] = phase
response['status'] = (phases[phase])
return response
def get_log_entries(self, build_id, start_index, end_index):
if build_id == self.TEST_BUILD_ID:
self.request_counter += 1
if self.request_counter % 10 == 0:
self._generate_logs()
logger.debug('Returning logs %s:%s', start_index, end_index)
if end_index >= 0:
end_index += 1
return (len(self.logs), self.logs[start_index:end_index])
else:
return super(TestBuildLogs, self).get_log_entries(build_id, start_index,
end_index)
def get_commands(self, build_id):
if build_id == self.TEST_BUILD_ID:
self.request_counter += 1
if self.request_counter % 10 == 0:
self._generate_logs()
return self.commands
else:
return super(TestBuildLogs, self).get_commands(build_id)
def get_last_command(self, build_id):
if build_id == self.TEST_BUILD_ID:
self.request_counter += 1
if self.request_counter % 10 == 0:
self._generate_logs()
return self.commands[-1]
else:
return super(TestBuildLogs, self).get_last_command(build_id)
def get_status(self, build_id):
if build_id == self.TEST_BUILD_ID:
self.request_counter += 1
if self.request_counter % 10 == 0:
self._generate_logs()
return self.generate_fake_status()
else:
return super(TestBuildLogs, self).get_status(build_id)

3
workers/dockerfilebuild.py
View file

@ -103,8 +103,7 @@ class DockerfileBuildContext(object):
logger.debug('Status: %s', str(status.encode('utf-8')))
step_increment = re.search(r'Step ([0-9]+) :', status)
if step_increment:
build_logs.append_log_entry({'message': str(status),
'is_command': True})
build_logs.append_command_message(str(status))
current_step = int(step_increment.group(1))
logger.debug('Step now: %s/%s' % (current_step, self._num_steps))
with self._status as status: