Change to the new paging format with the commands available at the top.

application.py

@@ -21,10 +21,6 @@ from endpoints.webhooks import webhooks
 
 logger = logging.getLogger(__name__)
 
-if application.config.get('INCLUDE_TEST_ENDPOINTS', False):
-  logger.debug('Loading test endpoints.')
-  import endpoints.test
-
 application.register_blueprint(web)
 application.register_blueprint(index, url_prefix='/v1')
 application.register_blueprint(tags, url_prefix='/v1')

config.py

@@ -11,6 +11,7 @@ from util import analytics
 
 from test.teststorage import FakeStorage, FakeUserfiles
 from test import analytics as fake_analytics
+from test.testlogs import TestBuildLogs
 
 
 class FlaskConfig(object):
@@ -91,6 +92,10 @@ class RedisBuildLogs(object):
   BUILDLOGS = BuildLogs('logs.quay.io')
 
 
+class TestBuildLogs(object):
+  BUILDLOGS = TestBuildLogs('logs.quay.io')
+
+
 class StripeTestConfig(object):
   STRIPE_SECRET_KEY = 'sk_test_PEbmJCYrLXPW0VRLSnWUiZ7Y'
   STRIPE_PUBLISHABLE_KEY = 'pk_test_uEDHANKm9CHCvVa2DLcipGRh'
@@ -140,13 +145,13 @@ class BuildNodeConfig(object):
   BUILD_NODE_PULL_TOKEN = 'F02O2E86CQLKZUQ0O81J8XDHQ6F0N1V36L9JTOEEK6GKKMT1GI8PTJQT4OU88Y6G'
 
 
-def logs_init_builder(level=logging.DEBUG):
+def logs_init_builder(level=logging.DEBUG,
+                      formatter=logstash_formatter.LogstashFormatter()):
   @staticmethod
   def init_logs():
     handler = logging.StreamHandler()
     root_logger = logging.getLogger('')
     root_logger.setLevel(level)
-    formatter = logstash_formatter.LogstashFormatter()
     handler.setFormatter(formatter)
     root_logger.addHandler(handler)
 
@@ -158,17 +163,15 @@ class TestConfig(FlaskConfig, FakeStorage, EphemeralDB, FakeUserfiles,
   LOGGING_CONFIG = logs_init_builder(logging.WARN)
   POPULATE_DB_TEST_DATA = True
   TESTING = True
-  INCLUDE_TEST_ENDPOINTS = True
 
 
 class DebugConfig(FlaskConfig, MailConfig, LocalStorage, SQLiteDB,
                   StripeTestConfig, MixpanelTestConfig, GitHubTestConfig,
                   DigitalOceanConfig, BuildNodeConfig, S3Userfiles,
-                  RedisBuildLogs):
+                  TestBuildLogs):
-  LOGGING_CONFIG = logs_init_builder()
+  LOGGING_CONFIG = logs_init_builder(formatter=logging.Formatter())
   SEND_FILE_MAX_AGE_DEFAULT = 0
   POPULATE_DB_TEST_DATA = True
-  INCLUDE_TEST_ENDPOINTS = True
 
 
 class LocalHostedConfig(FlaskConfig, MailConfig, S3Storage, RDSMySQL,

data/buildlogs.py

@@ -10,6 +10,10 @@ class BuildLogs(object):
   def _logs_key(build_id):
     return 'builds/%s/logs' % build_id
 
+  @staticmethod
+  def _commands_key(build_id):
+    return 'builds/%s/commands' % build_id
+
   def append_log_entry(self, build_id, log_obj):
     """
     Appends the serialized form of log_obj to the end of the log entry list
@@ -20,12 +24,30 @@ class BuildLogs(object):
   def append_log_message(self, build_id, log_message):
     """
     Wraps the message in an envelope and push it to the end of the log entry
-    list and returns the new length of the list.
+    list and returns the index at which it was inserted.
     """
     log_obj = {
       'message': log_message
     }
-    return self._redis.rpush(self._logs_key(build_id), json.dumps(log_obj))
+    return self._redis.rpush(self._logs_key(build_id), json.dumps(log_obj)) - 1
+
+  def append_command_message(self, build_id, command_message):
+    """
+    Wraps the message in an envelope and push it to the end of the log entry
+    list, to the commands list, and returns the new length of the list.
+    """
+    log_obj = {
+      'message': command_message,
+      'is_command': True,
+    }
+    idx = self._redis.rpush(self._logs_key(build_id), json.dumps(log_obj)) - 1
+
+    cmd_obj = {
+      'message': command_message,
+      'index': idx,
+    }
+    self._redis.rpush(self._commands_key(build_id), json.dumps(cmd_obj))
+    return idx
 
   def get_log_entries(self, build_id, start_index, end_index):
     """ 
@@ -37,6 +59,24 @@ class BuildLogs(object):
                                      end_index)
     return (llen, (json.loads(entry) for entry in log_entries))
 
+  def get_commands(self, build_id):
+    """
+    Returns a list of all Dockerfile commands that have passed through the
+    specified build thus far.
+    """
+    commands = self._redis.lrange(self._commands_key(build_id), 0, -1)
+    return (json.loads(cmd) for cmd in commands)
+
+  def get_last_command(self, build_id):
+    """
+    Returns only the last command from the list of commands.
+    """
+    commands = self._redis.lrange(self._commands_key(build_id), -1, -1)
+    if commands:
+      return json.loads(commands[-1])
+    else:
+      return None
+
   @staticmethod
   def _status_key(build_id):
     return 'builds/%s/status' % build_id

endpoints/api.py

@@ -1184,10 +1184,31 @@ def get_repo_build_status(namespace, repository, build_uuid):
 def get_repo_build_logs(namespace, repository, build_uuid):
   permission = ModifyRepositoryPermission(namespace, repository)
   if permission.can():
+    response_obj = {}
+
     build = model.get_repository_build(namespace, repository, build_uuid)
 
-    start = int(request.args.get('start', -1000))
+    start_param = request.args.get('start', None)
     end = int(request.args.get('end', -1))
+
+    last_command = None
+    include_commands = request.args.get('commands', 'false')
+    if include_commands.lower() not in {'0', 'false'}:
+      commands = [cmd for cmd in build_logs.get_commands(build.uuid)]
+      response_obj['commands'] = commands
+      if commands:
+        last_command = commands[-1]
+    elif start_param is None:
+      last_command = build_logs.get_last_command(build.uuid)
+
+    if start_param is None:
+      if last_command:
+        start = last_command['index']
+      else:
+        start = 0
+    else:
+      start = int(start_param)
+
     count, logs = build_logs.get_log_entries(build.uuid, start, end)
 
     if start < 0:
@@ -1196,13 +1217,15 @@ def get_repo_build_logs(namespace, repository, build_uuid):
     if end < 0:
       end = count + end
 
-    return jsonify({
+    response_obj.update({
       'start': start,
       'end': end,
       'total': count,
       'logs': [log for log in logs],
     })
 
+    return jsonify(response_obj)
+
   abort(403)  # Permission denied
 
 

endpoints/test.py

@@ -1,124 +0,0 @@
-import math
-
-from random import SystemRandom
-from flask import jsonify, request
-from loremipsum import get_sentences
-
-from endpoints.api import api
-
-def generate_image_completion(rand_func):
-  images = {}
-  for image_id in range(rand_func.randint(1, 11)):
-    total = int(math.pow(abs(rand_func.gauss(0, 1000)), 2))
-    current = rand_func.randint(0, total)
-    image_id = 'image_id_%s' % image_id
-    images[image_id] = {
-      'total': total,
-      'current': current,
-    }
-  return images
-
-
-def generate_fake_status():
-  response = {
-    'id': 'deadbeef-dead-beef-dead-beefdeadbeef',
-    'status': None,
-  }
-
-  random = SystemRandom()
-  phases = {
-    'waiting': {},
-    'starting': {
-      'total_commands': 7,
-      'current_command': 0,
-    },
-    'initializing': {},
-    'error': {},
-    'complete': {},
-    'building': {
-      'total_commands': 7,
-      'current_command': random.randint(1, 7),
-    },
-    'pushing': {
-      'total_commands': 7,
-      'current_command': 7,
-      'push_completion': random.random(),
-      'image_completion': generate_image_completion(random),
-    },
-  }
-
-  phase = random.choice(phases.keys())
-  response['phase'] = phase
-  response['status'] = (phases[phase])
-
-  return response
-
-
-BASE_BUILDING_URL = '/repository/devtable/building/build/'
-
-
-@api.route(BASE_BUILDING_URL, methods=['GET'])
-def get_fake_repo_build_status_list():
-  return jsonify({'builds': [generate_fake_status()]})
-
-
-@api.route(BASE_BUILDING_URL + 'deadbeef-dead-beef-dead-beefdeadbeef/logs',
-           methods=['GET'])
-def get_fake_repo_build_logs():
-  start = int(request.args.get('start', 0))
-  end = int(request.args.get('end', 0))
-  had_start = 'start' in request.args
-  had_end = 'end' in request.args
-
-  adv_start = 0
-  adv_end = 0
-  adv_total = 0
-  lorem_logs = []
-
-  if had_start and had_end:
-    numlogs = end - start + 1
-    adv_start = start
-    adv_end = end
-    adv_total = end + 1
-    lorem_logs = get_sentences(numlogs)
-  elif had_start:
-    adv_start = start
-    adv_end = start + 9
-    lorem_logs = get_sentences(10)
-    adv_total = adv_end + 1
-  elif had_end:
-    adv_start = max(0, (end - 9))
-    adv_end = end
-    adv_total = end + 1
-    lorem_logs = get_sentences(adv_end - adv_start + 1)
-  else:
-    adv_start = 100
-    adv_end = 109
-    adv_total = 110
-    lorem_logs = get_sentences(10)
-
-  def wrap_log_message(rand, msg):
-    if rand.randint(1, 10) == 1:
-      block = {
-        'is_command': True,
-        'message': 'Step %s : %s' % (rand.randint(1, 10), msg)
-      }
-    else:
-      block = {
-        'message': msg,
-      }
-    return block
-
-  rnd = SystemRandom()
-  return jsonify({
-    'start': adv_start,
-    'end': adv_end,
-    'total': adv_total,
-    'logs': [wrap_log_message(rnd, sentence) for sentence in lorem_logs]
-  })
-
-
-@api.route(BASE_BUILDING_URL + '/deadbeef-dead-beef-dead-beefdeadbeef/status',
-           methods=['GET'])
-def get_fake_repo_build_status():
-  return jsonify(generate_fake_status())

test/testlogs.py

@@ -0,0 +1,136 @@
+import math
+import logging
+
+from random import SystemRandom
+from loremipsum import get_sentence
+
+from data.buildlogs import BuildLogs
+
+
+logger = logging.getLogger(__name__)
+
+
+class TestBuildLogs(BuildLogs):
+  TEST_BUILD_ID = 'deadbeef-dead-beef-dead-beefdeadbeef'
+
+  def __init__(self, redis_host):
+    super(TestBuildLogs, self).__init__(redis_host)
+    self.last_command = 0
+    self.logs = [self._generate_command()]
+    self.commands = [{
+      'index': 0,
+      'message': self.logs[0]['message'],
+    }]
+    self.request_counter = 0
+    self._generate_logs()
+
+  def _generate_command(self):
+    self.last_command += 1
+    return {
+      'message': 'Step %s : %s' % (self.last_command, get_sentence()),
+      'is_command': True,
+    }
+
+  def _generate_logs(self):
+    rand = SystemRandom()
+    num_logs = rand.randint(1, 500)
+    for _ in range(num_logs):
+      if rand.randint(1, 50) == 1:
+        cmd = self._generate_command()
+        self.commands.append({
+          'message': cmd['message'],
+          'index': len(self.logs),
+        })
+        self.logs.append(cmd)
+      else:
+        self.logs.append({
+          'message': get_sentence(),
+        })
+
+  @staticmethod
+  def generate_image_completion(rand_func):
+    images = {}
+    for image_id in range(rand_func.randint(1, 11)):
+      total = int(math.pow(abs(rand_func.gauss(0, 1000)), 2))
+      current = rand_func.randint(0, total)
+      image_id = 'image_id_%s' % image_id
+      images[image_id] = {
+        'total': total,
+        'current': current,
+      }
+    return images
+
+  @staticmethod
+  def generate_fake_status():
+    response = {
+      'id': 'deadbeef-dead-beef-dead-beefdeadbeef',
+      'status': None,
+    }
+
+    random = SystemRandom()
+    phases = {
+      'waiting': {},
+      'starting': {
+        'total_commands': 7,
+        'current_command': 0,
+      },
+      'initializing': {},
+      'error': {},
+      'complete': {},
+      'building': {
+        'total_commands': 7,
+        'current_command': random.randint(1, 7),
+      },
+      'pushing': {
+        'total_commands': 7,
+        'current_command': 7,
+        'push_completion': random.random(),
+        'image_completion': TestBuildLogs.generate_image_completion(random),
+      },
+    }
+
+    phase = random.choice(phases.keys())
+    response['phase'] = phase
+    response['status'] = (phases[phase])
+
+    return response
+
+  def get_log_entries(self, build_id, start_index, end_index):
+    if build_id == self.TEST_BUILD_ID:
+      self.request_counter += 1
+      if self.request_counter % 10 == 0:
+        self._generate_logs()
+      logger.debug('Returning logs %s:%s', start_index, end_index)
+      if end_index >= 0:
+        end_index += 1
+      return (len(self.logs), self.logs[start_index:end_index])
+    else:
+      return super(TestBuildLogs, self).get_log_entries(build_id, start_index,
+                                                        end_index)
+
+  def get_commands(self, build_id):
+    if build_id == self.TEST_BUILD_ID:
+      self.request_counter += 1
+      if self.request_counter % 10 == 0:
+        self._generate_logs()
+      return self.commands
+    else:
+      return super(TestBuildLogs, self).get_commands(build_id)
+
+  def get_last_command(self, build_id):
+    if build_id == self.TEST_BUILD_ID:
+      self.request_counter += 1
+      if self.request_counter % 10 == 0:
+        self._generate_logs()
+      return self.commands[-1]
+    else:
+      return super(TestBuildLogs, self).get_last_command(build_id)
+
+  def get_status(self, build_id):
+    if build_id == self.TEST_BUILD_ID:
+      self.request_counter += 1
+      if self.request_counter % 10 == 0:
+        self._generate_logs()
+      return self.generate_fake_status()
+    else:
+      return super(TestBuildLogs, self).get_status(build_id)

workers/dockerfilebuild.py

@@ -103,8 +103,7 @@ class DockerfileBuildContext(object):
       logger.debug('Status: %s', str(status.encode('utf-8')))
       step_increment = re.search(r'Step ([0-9]+) :', status)
       if step_increment:
-        build_logs.append_log_entry({'message': str(status),
+        build_logs.append_command_message(str(status))
-                                     'is_command': True})
         current_step = int(step_increment.group(1))
         logger.debug('Step now: %s/%s' % (current_step, self._num_steps))
         with self._status as status: