From 6fd343741b64f234cfdd3a525965942960f5b43d Mon Sep 17 00:00:00 2001 From: yackob03 Date: Mon, 10 Feb 2014 19:12:43 -0500 Subject: [PATCH] Change to the new paging format with the commands available at the top. --- application.py | 4 -- config.py | 15 ++-- data/buildlogs.py | 44 +++++++++++- endpoints/api.py | 27 +++++++- endpoints/test.py | 124 --------------------------------- test/testlogs.py | 136 +++++++++++++++++++++++++++++++++++++ workers/dockerfilebuild.py | 3 +- 7 files changed, 213 insertions(+), 140 deletions(-) delete mode 100644 endpoints/test.py create mode 100644 test/testlogs.py diff --git a/application.py b/application.py index 80411d50f..d3d95e846 100644 --- a/application.py +++ b/application.py @@ -21,10 +21,6 @@ from endpoints.webhooks import webhooks logger = logging.getLogger(__name__) -if application.config.get('INCLUDE_TEST_ENDPOINTS', False): - logger.debug('Loading test endpoints.') - import endpoints.test - application.register_blueprint(web) application.register_blueprint(index, url_prefix='/v1') application.register_blueprint(tags, url_prefix='/v1') diff --git a/config.py b/config.py index 9be2c86a6..6f013d4ae 100644 --- a/config.py +++ b/config.py @@ -11,6 +11,7 @@ from util import analytics from test.teststorage import FakeStorage, FakeUserfiles from test import analytics as fake_analytics +from test.testlogs import TestBuildLogs class FlaskConfig(object): @@ -91,6 +92,10 @@ class RedisBuildLogs(object): BUILDLOGS = BuildLogs('logs.quay.io') +class TestBuildLogs(object): + BUILDLOGS = TestBuildLogs('logs.quay.io') + + class StripeTestConfig(object): STRIPE_SECRET_KEY = 'sk_test_PEbmJCYrLXPW0VRLSnWUiZ7Y' STRIPE_PUBLISHABLE_KEY = 'pk_test_uEDHANKm9CHCvVa2DLcipGRh' @@ -140,13 +145,13 @@ class BuildNodeConfig(object): BUILD_NODE_PULL_TOKEN = 'F02O2E86CQLKZUQ0O81J8XDHQ6F0N1V36L9JTOEEK6GKKMT1GI8PTJQT4OU88Y6G' -def logs_init_builder(level=logging.DEBUG): +def logs_init_builder(level=logging.DEBUG, + formatter=logstash_formatter.LogstashFormatter()): @staticmethod def init_logs(): handler = logging.StreamHandler() root_logger = logging.getLogger('') root_logger.setLevel(level) - formatter = logstash_formatter.LogstashFormatter() handler.setFormatter(formatter) root_logger.addHandler(handler) @@ -158,17 +163,15 @@ class TestConfig(FlaskConfig, FakeStorage, EphemeralDB, FakeUserfiles, LOGGING_CONFIG = logs_init_builder(logging.WARN) POPULATE_DB_TEST_DATA = True TESTING = True - INCLUDE_TEST_ENDPOINTS = True class DebugConfig(FlaskConfig, MailConfig, LocalStorage, SQLiteDB, StripeTestConfig, MixpanelTestConfig, GitHubTestConfig, DigitalOceanConfig, BuildNodeConfig, S3Userfiles, - RedisBuildLogs): - LOGGING_CONFIG = logs_init_builder() + TestBuildLogs): + LOGGING_CONFIG = logs_init_builder(formatter=logging.Formatter()) SEND_FILE_MAX_AGE_DEFAULT = 0 POPULATE_DB_TEST_DATA = True - INCLUDE_TEST_ENDPOINTS = True class LocalHostedConfig(FlaskConfig, MailConfig, S3Storage, RDSMySQL, diff --git a/data/buildlogs.py b/data/buildlogs.py index ff09934f7..d0317c121 100644 --- a/data/buildlogs.py +++ b/data/buildlogs.py @@ -10,6 +10,10 @@ class BuildLogs(object): def _logs_key(build_id): return 'builds/%s/logs' % build_id + @staticmethod + def _commands_key(build_id): + return 'builds/%s/commands' % build_id + def append_log_entry(self, build_id, log_obj): """ Appends the serialized form of log_obj to the end of the log entry list @@ -20,12 +24,30 @@ class BuildLogs(object): def append_log_message(self, build_id, log_message): """ Wraps the message in an envelope and push it to the end of the log entry - list and returns the new length of the list. + list and returns the index at which it was inserted. """ log_obj = { 'message': log_message } - return self._redis.rpush(self._logs_key(build_id), json.dumps(log_obj)) + return self._redis.rpush(self._logs_key(build_id), json.dumps(log_obj)) - 1 + + def append_command_message(self, build_id, command_message): + """ + Wraps the message in an envelope and push it to the end of the log entry + list, to the commands list, and returns the new length of the list. + """ + log_obj = { + 'message': command_message, + 'is_command': True, + } + idx = self._redis.rpush(self._logs_key(build_id), json.dumps(log_obj)) - 1 + + cmd_obj = { + 'message': command_message, + 'index': idx, + } + self._redis.rpush(self._commands_key(build_id), json.dumps(cmd_obj)) + return idx def get_log_entries(self, build_id, start_index, end_index): """ @@ -37,6 +59,24 @@ class BuildLogs(object): end_index) return (llen, (json.loads(entry) for entry in log_entries)) + def get_commands(self, build_id): + """ + Returns a list of all Dockerfile commands that have passed through the + specified build thus far. + """ + commands = self._redis.lrange(self._commands_key(build_id), 0, -1) + return (json.loads(cmd) for cmd in commands) + + def get_last_command(self, build_id): + """ + Returns only the last command from the list of commands. + """ + commands = self._redis.lrange(self._commands_key(build_id), -1, -1) + if commands: + return json.loads(commands[-1]) + else: + return None + @staticmethod def _status_key(build_id): return 'builds/%s/status' % build_id diff --git a/endpoints/api.py b/endpoints/api.py index a6a69f433..49bdc3060 100644 --- a/endpoints/api.py +++ b/endpoints/api.py @@ -1184,10 +1184,31 @@ def get_repo_build_status(namespace, repository, build_uuid): def get_repo_build_logs(namespace, repository, build_uuid): permission = ModifyRepositoryPermission(namespace, repository) if permission.can(): + response_obj = {} + build = model.get_repository_build(namespace, repository, build_uuid) - start = int(request.args.get('start', -1000)) + start_param = request.args.get('start', None) end = int(request.args.get('end', -1)) + + last_command = None + include_commands = request.args.get('commands', 'false') + if include_commands.lower() not in {'0', 'false'}: + commands = [cmd for cmd in build_logs.get_commands(build.uuid)] + response_obj['commands'] = commands + if commands: + last_command = commands[-1] + elif start_param is None: + last_command = build_logs.get_last_command(build.uuid) + + if start_param is None: + if last_command: + start = last_command['index'] + else: + start = 0 + else: + start = int(start_param) + count, logs = build_logs.get_log_entries(build.uuid, start, end) if start < 0: @@ -1196,13 +1217,15 @@ def get_repo_build_logs(namespace, repository, build_uuid): if end < 0: end = count + end - return jsonify({ + response_obj.update({ 'start': start, 'end': end, 'total': count, 'logs': [log for log in logs], }) + return jsonify(response_obj) + abort(403) # Permission denied diff --git a/endpoints/test.py b/endpoints/test.py deleted file mode 100644 index 0ebd953ed..000000000 --- a/endpoints/test.py +++ /dev/null @@ -1,124 +0,0 @@ -import math - -from random import SystemRandom -from flask import jsonify, request -from loremipsum import get_sentences - -from endpoints.api import api - -def generate_image_completion(rand_func): - images = {} - for image_id in range(rand_func.randint(1, 11)): - total = int(math.pow(abs(rand_func.gauss(0, 1000)), 2)) - current = rand_func.randint(0, total) - image_id = 'image_id_%s' % image_id - images[image_id] = { - 'total': total, - 'current': current, - } - return images - - -def generate_fake_status(): - response = { - 'id': 'deadbeef-dead-beef-dead-beefdeadbeef', - 'status': None, - } - - random = SystemRandom() - phases = { - 'waiting': {}, - 'starting': { - 'total_commands': 7, - 'current_command': 0, - }, - 'initializing': {}, - 'error': {}, - 'complete': {}, - 'building': { - 'total_commands': 7, - 'current_command': random.randint(1, 7), - }, - 'pushing': { - 'total_commands': 7, - 'current_command': 7, - 'push_completion': random.random(), - 'image_completion': generate_image_completion(random), - }, - } - - phase = random.choice(phases.keys()) - response['phase'] = phase - response['status'] = (phases[phase]) - - return response - - -BASE_BUILDING_URL = '/repository/devtable/building/build/' - - -@api.route(BASE_BUILDING_URL, methods=['GET']) -def get_fake_repo_build_status_list(): - return jsonify({'builds': [generate_fake_status()]}) - - -@api.route(BASE_BUILDING_URL + 'deadbeef-dead-beef-dead-beefdeadbeef/logs', - methods=['GET']) -def get_fake_repo_build_logs(): - start = int(request.args.get('start', 0)) - end = int(request.args.get('end', 0)) - had_start = 'start' in request.args - had_end = 'end' in request.args - - adv_start = 0 - adv_end = 0 - adv_total = 0 - lorem_logs = [] - - if had_start and had_end: - numlogs = end - start + 1 - adv_start = start - adv_end = end - adv_total = end + 1 - lorem_logs = get_sentences(numlogs) - elif had_start: - adv_start = start - adv_end = start + 9 - lorem_logs = get_sentences(10) - adv_total = adv_end + 1 - elif had_end: - adv_start = max(0, (end - 9)) - adv_end = end - adv_total = end + 1 - lorem_logs = get_sentences(adv_end - adv_start + 1) - else: - adv_start = 100 - adv_end = 109 - adv_total = 110 - lorem_logs = get_sentences(10) - - def wrap_log_message(rand, msg): - if rand.randint(1, 10) == 1: - block = { - 'is_command': True, - 'message': 'Step %s : %s' % (rand.randint(1, 10), msg) - } - else: - block = { - 'message': msg, - } - return block - - rnd = SystemRandom() - return jsonify({ - 'start': adv_start, - 'end': adv_end, - 'total': adv_total, - 'logs': [wrap_log_message(rnd, sentence) for sentence in lorem_logs] - }) - - -@api.route(BASE_BUILDING_URL + '/deadbeef-dead-beef-dead-beefdeadbeef/status', - methods=['GET']) -def get_fake_repo_build_status(): - return jsonify(generate_fake_status()) diff --git a/test/testlogs.py b/test/testlogs.py new file mode 100644 index 000000000..1df202e68 --- /dev/null +++ b/test/testlogs.py @@ -0,0 +1,136 @@ +import math +import logging + +from random import SystemRandom +from loremipsum import get_sentence + +from data.buildlogs import BuildLogs + + +logger = logging.getLogger(__name__) + + +class TestBuildLogs(BuildLogs): + TEST_BUILD_ID = 'deadbeef-dead-beef-dead-beefdeadbeef' + + def __init__(self, redis_host): + super(TestBuildLogs, self).__init__(redis_host) + self.last_command = 0 + self.logs = [self._generate_command()] + self.commands = [{ + 'index': 0, + 'message': self.logs[0]['message'], + }] + self.request_counter = 0 + self._generate_logs() + + def _generate_command(self): + self.last_command += 1 + return { + 'message': 'Step %s : %s' % (self.last_command, get_sentence()), + 'is_command': True, + } + + def _generate_logs(self): + rand = SystemRandom() + num_logs = rand.randint(1, 500) + for _ in range(num_logs): + if rand.randint(1, 50) == 1: + cmd = self._generate_command() + self.commands.append({ + 'message': cmd['message'], + 'index': len(self.logs), + }) + self.logs.append(cmd) + else: + self.logs.append({ + 'message': get_sentence(), + }) + + @staticmethod + def generate_image_completion(rand_func): + images = {} + for image_id in range(rand_func.randint(1, 11)): + total = int(math.pow(abs(rand_func.gauss(0, 1000)), 2)) + current = rand_func.randint(0, total) + image_id = 'image_id_%s' % image_id + images[image_id] = { + 'total': total, + 'current': current, + } + return images + + @staticmethod + def generate_fake_status(): + response = { + 'id': 'deadbeef-dead-beef-dead-beefdeadbeef', + 'status': None, + } + + random = SystemRandom() + phases = { + 'waiting': {}, + 'starting': { + 'total_commands': 7, + 'current_command': 0, + }, + 'initializing': {}, + 'error': {}, + 'complete': {}, + 'building': { + 'total_commands': 7, + 'current_command': random.randint(1, 7), + }, + 'pushing': { + 'total_commands': 7, + 'current_command': 7, + 'push_completion': random.random(), + 'image_completion': TestBuildLogs.generate_image_completion(random), + }, + } + + phase = random.choice(phases.keys()) + response['phase'] = phase + response['status'] = (phases[phase]) + + return response + + def get_log_entries(self, build_id, start_index, end_index): + if build_id == self.TEST_BUILD_ID: + self.request_counter += 1 + if self.request_counter % 10 == 0: + self._generate_logs() + logger.debug('Returning logs %s:%s', start_index, end_index) + if end_index >= 0: + end_index += 1 + return (len(self.logs), self.logs[start_index:end_index]) + else: + return super(TestBuildLogs, self).get_log_entries(build_id, start_index, + end_index) + + def get_commands(self, build_id): + if build_id == self.TEST_BUILD_ID: + self.request_counter += 1 + if self.request_counter % 10 == 0: + self._generate_logs() + return self.commands + else: + return super(TestBuildLogs, self).get_commands(build_id) + + def get_last_command(self, build_id): + if build_id == self.TEST_BUILD_ID: + self.request_counter += 1 + if self.request_counter % 10 == 0: + self._generate_logs() + return self.commands[-1] + else: + return super(TestBuildLogs, self).get_last_command(build_id) + + def get_status(self, build_id): + if build_id == self.TEST_BUILD_ID: + self.request_counter += 1 + if self.request_counter % 10 == 0: + self._generate_logs() + return self.generate_fake_status() + else: + return super(TestBuildLogs, self).get_status(build_id) diff --git a/workers/dockerfilebuild.py b/workers/dockerfilebuild.py index 69cc89e64..ba4de4093 100644 --- a/workers/dockerfilebuild.py +++ b/workers/dockerfilebuild.py @@ -103,8 +103,7 @@ class DockerfileBuildContext(object): logger.debug('Status: %s', str(status.encode('utf-8'))) step_increment = re.search(r'Step ([0-9]+) :', status) if step_increment: - build_logs.append_log_entry({'message': str(status), - 'is_command': True}) + build_logs.append_command_message(str(status)) current_step = int(step_increment.group(1)) logger.debug('Step now: %s/%s' % (current_step, self._num_steps)) with self._status as status: