Really try to emulate the logs format with the test logs. Switch to a simplified flat logs format.
This commit is contained in:
yackob03
parent
85694dd110
commit
b920a0cb1f
4 changed files with 187 additions and 196 deletions
262
test/testlogs.py
262
test/testlogs.py
|
|
@ -1,151 +1,191 @@
|
|||
import math
|
||||
import logging
|
||||
|
||||
from random import SystemRandom
|
||||
from loremipsum import get_sentence
|
||||
from functools import wraps
|
||||
from copy import deepcopy
|
||||
|
||||
from data.buildlogs import BuildLogs
|
||||
from random import choice
|
||||
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
random = SystemRandom()
|
||||
|
||||
|
||||
def maybe_advance_script(is_get_status=False):
|
||||
def inner_advance(func):
|
||||
@wraps(func)
|
||||
def wrapper(self, *args, **kwargs):
|
||||
advance_units = random.randint(1, 500)
|
||||
logger.debug('Advancing script %s units', advance_units)
|
||||
while advance_units > 0 and self.remaining_script:
|
||||
units = self.remaining_script[0][0]
|
||||
|
||||
if advance_units > units:
|
||||
advance_units -= units
|
||||
self.advance_script(is_get_status)
|
||||
else:
|
||||
break
|
||||
|
||||
return func(self, *args, **kwargs)
|
||||
return wrapper
|
||||
return inner_advance
|
||||
|
||||
|
||||
class TestBuildLogs(BuildLogs):
|
||||
COMMAND_TYPES = ['FROM', 'MAINTAINER', 'RUN', 'CMD', 'EXPOSE', 'ENV', 'ADD',
|
||||
'ENTRYPOINT', 'VOLUME', 'USER', 'WORKDIR']
|
||||
STATUS_TEMPLATE = {
|
||||
'total_commands': None,
|
||||
'current_command': None,
|
||||
'push_completion': 0.0,
|
||||
'image_completion': {},
|
||||
}
|
||||
|
||||
def __init__(self, redis_host, namespace, repository, test_build_id):
|
||||
super(TestBuildLogs, self).__init__(redis_host)
|
||||
self.namespace = namespace
|
||||
self.repository = repository
|
||||
self.test_build_id = test_build_id
|
||||
self.last_command = 0
|
||||
self.logs = [self._generate_command()]
|
||||
self.commands = [{
|
||||
'index': 0,
|
||||
'message': self.logs[0]['message'],
|
||||
}]
|
||||
self.request_counter = 0
|
||||
self._generate_logs()
|
||||
self.remaining_script = self._generate_script()
|
||||
logger.debug('Total script size: %s', len(self.remaining_script))
|
||||
print('Total script size: %s' % len(self.remaining_script))
|
||||
self._logs = []
|
||||
|
||||
self._status = {}
|
||||
self._last_status = {}
|
||||
|
||||
def _get_random_command(self):
|
||||
COMMANDS = ['FROM', 'MAINTAINER', 'RUN', 'CMD', 'EXPOSE', 'ENV', 'ADD',
|
||||
'ENTRYPOINT', 'VOLUME', 'USER', 'WORKDIR']
|
||||
def advance_script(self, is_get_status):
|
||||
(_, log, status_wrapper) = self.remaining_script.pop(0)
|
||||
if log is not None:
|
||||
logger.debug('Writing log line: %s', str(log))
|
||||
self._logs.append(log)
|
||||
|
||||
return choice(COMMANDS)
|
||||
if status_wrapper is not None:
|
||||
(phase, status) = status_wrapper
|
||||
logger.debug('New status: %s', str(status))
|
||||
|
||||
def _generate_command(self):
|
||||
self.last_command += 1
|
||||
from data import model
|
||||
build_obj = model.get_repository_build(self.namespace, self.repository,
|
||||
self.test_build_id)
|
||||
build_obj.phase = phase
|
||||
build_obj.save()
|
||||
|
||||
self._status = status
|
||||
if not is_get_status:
|
||||
self._last_status = status
|
||||
|
||||
def _generate_script(self):
|
||||
script = []
|
||||
|
||||
# generate the init phase
|
||||
script.append(self._generate_phase(400, 'initializing'))
|
||||
script.extend(self._generate_logs(random.randint(1, 3)))
|
||||
|
||||
# move to the building phase
|
||||
script.append(self._generate_phase(400, 'building'))
|
||||
total_commands = random.randint(5, 20)
|
||||
for command_num in range(1, total_commands + 1):
|
||||
command_weight = random.randint(50, 100)
|
||||
script.append(self._generate_command(command_num, total_commands,
|
||||
command_weight))
|
||||
|
||||
# we want 0 logs some percent of the time
|
||||
num_logs = max(0, random.randint(-50, 400))
|
||||
script.extend(self._generate_logs(num_logs))
|
||||
|
||||
# move to the pushing phase
|
||||
script.append(self._generate_phase(400, 'pushing'))
|
||||
script.extend(self._generate_push_statuses(total_commands))
|
||||
|
||||
# move to the error or complete phase
|
||||
if random.randint(0, 1) == 0:
|
||||
script.append(self._generate_phase(400, 'complete'))
|
||||
else:
|
||||
script.append(self._generate_phase(400, 'error'))
|
||||
script.append((1, {'message': 'Something bad happened! Oh noes!',
|
||||
'type': self.ERROR}, None))
|
||||
|
||||
return script
|
||||
|
||||
def _generate_phase(self, start_weight, phase_name):
|
||||
return (start_weight, {'message': phase_name, 'type': self.PHASE},
|
||||
(phase_name, deepcopy(self.STATUS_TEMPLATE)))
|
||||
|
||||
def _generate_command(self, command_num, total_commands, command_weight):
|
||||
sentence = get_sentence()
|
||||
command = self._get_random_command()
|
||||
command = random.choice(self.COMMAND_TYPES)
|
||||
if command == 'FROM':
|
||||
sentence = choice(['ubuntu', 'quay.io/devtable/simple', 'quay.io/buynlarge/orgrepo', 'stackbrew/ubuntu:precise'])
|
||||
sentence = random.choice(['ubuntu', 'quay.io/devtable/simple',
|
||||
'quay.io/buynlarge/orgrepo',
|
||||
'stackbrew/ubuntu:precise'])
|
||||
|
||||
return {
|
||||
'message': 'Step %s: %s %s' % (self.last_command, command, sentence),
|
||||
'is_command': True,
|
||||
msg = {
|
||||
'message': 'Step %s: %s %s' % (command_num, command, sentence),
|
||||
'type': self.COMMAND,
|
||||
}
|
||||
|
||||
def _generate_logs(self):
|
||||
rand = SystemRandom()
|
||||
num_logs = rand.randint(1, 500)
|
||||
for _ in range(num_logs):
|
||||
if rand.randint(1, 50) == 1:
|
||||
cmd = self._generate_command()
|
||||
self.commands.append({
|
||||
'message': cmd['message'],
|
||||
'index': len(self.logs),
|
||||
})
|
||||
self.logs.append(cmd)
|
||||
else:
|
||||
self.logs.append({
|
||||
'message': get_sentence(),
|
||||
})
|
||||
status = deepcopy(self.STATUS_TEMPLATE)
|
||||
status['total_commands'] = total_commands
|
||||
status['current_command'] = command_num
|
||||
return (command_weight, msg, ('building', status))
|
||||
|
||||
@staticmethod
|
||||
def _generate_image_completion(rand_func):
|
||||
images = {}
|
||||
for image_id in range(rand_func.randint(1, 11)):
|
||||
total = int(math.pow(abs(rand_func.gauss(0, 1000)), 2))
|
||||
current = rand_func.randint(0, total)
|
||||
image_id = 'image_id_%s' % image_id
|
||||
images[image_id] = {
|
||||
'total': total,
|
||||
'current': current,
|
||||
def _generate_logs(count):
|
||||
return [(1, {'message': get_sentence()}, None) for _ in range(count)]
|
||||
|
||||
@staticmethod
|
||||
def _compute_total_completion(statuses, total_images):
|
||||
percentage_with_sizes = float(len(statuses.values()))/total_images
|
||||
sent_bytes = sum([status[u'current'] for status in statuses.values()])
|
||||
total_bytes = sum([status[u'total'] for status in statuses.values()])
|
||||
return float(sent_bytes)/total_bytes*percentage_with_sizes
|
||||
|
||||
@staticmethod
|
||||
def _generate_push_statuses(total_commands):
|
||||
push_status_template = deepcopy(TestBuildLogs.STATUS_TEMPLATE)
|
||||
push_status_template['current_command'] = total_commands
|
||||
push_status_template['total_commands'] = total_commands
|
||||
|
||||
push_statuses = []
|
||||
|
||||
one_mb = 1 * 1024 * 1024
|
||||
|
||||
num_images = random.randint(2, 7)
|
||||
sizes = [random.randint(one_mb, one_mb * 5) for _ in range(num_images)]
|
||||
|
||||
image_completion = {}
|
||||
for image_num, image_size in enumerate(sizes):
|
||||
image_id = 'image_id_%s' % image_num
|
||||
|
||||
image_completion[image_id] = {
|
||||
'current': 0,
|
||||
'total': image_size,
|
||||
}
|
||||
return images
|
||||
|
||||
def _generate_fake_status(self):
|
||||
random = SystemRandom()
|
||||
phases = {
|
||||
'waiting': {},
|
||||
'starting': {
|
||||
'total_commands': 7,
|
||||
'current_command': 0,
|
||||
},
|
||||
'initializing': {},
|
||||
'error': {},
|
||||
'complete': {},
|
||||
'building': {
|
||||
'total_commands': 7,
|
||||
'current_command': random.randint(1, 7),
|
||||
},
|
||||
'pushing': {
|
||||
'total_commands': 7,
|
||||
'current_command': 7,
|
||||
'push_completion': random.random(),
|
||||
'image_completion': self._generate_image_completion(random),
|
||||
},
|
||||
}
|
||||
for i in range(one_mb, image_size, one_mb):
|
||||
image_completion[image_id]['current'] = i
|
||||
new_status = deepcopy(push_status_template)
|
||||
new_status['image_completion'] = deepcopy(image_completion)
|
||||
|
||||
phase = random.choice(phases.keys())
|
||||
completion = TestBuildLogs._compute_total_completion(image_completion,
|
||||
num_images)
|
||||
new_status['push_completion'] = completion
|
||||
push_statuses.append((250, None, ('pushing', new_status)))
|
||||
|
||||
from data import model
|
||||
build_obj = model.get_repository_build(self.namespace, self.repository,
|
||||
self.test_build_id)
|
||||
build_obj.phase = phase
|
||||
build_obj.save()
|
||||
return push_statuses
|
||||
|
||||
return phases[phase]
|
||||
|
||||
def get_log_entries(self, build_id, start_index, end_index):
|
||||
@maybe_advance_script()
|
||||
def get_log_entries(self, build_id, start_index):
|
||||
if build_id == self.test_build_id:
|
||||
self.request_counter += 1
|
||||
if self.request_counter % 10 == 0:
|
||||
self._generate_logs()
|
||||
logger.debug('Returning logs %s:%s', start_index, end_index)
|
||||
if end_index >= 0:
|
||||
end_index += 1
|
||||
return (len(self.logs), self.logs[start_index:end_index])
|
||||
return (len(self._logs), self._logs[start_index:])
|
||||
else:
|
||||
return super(TestBuildLogs, self).get_log_entries(build_id, start_index,
|
||||
end_index)
|
||||
|
||||
def get_commands(self, build_id):
|
||||
if build_id == self.test_build_id:
|
||||
self.request_counter += 1
|
||||
if self.request_counter % 10 == 0:
|
||||
self._generate_logs()
|
||||
return self.commands
|
||||
else:
|
||||
return super(TestBuildLogs, self).get_commands(build_id)
|
||||
|
||||
def get_last_command(self, build_id):
|
||||
if build_id == self.test_build_id:
|
||||
self.request_counter += 1
|
||||
if self.request_counter % 10 == 0:
|
||||
self._generate_logs()
|
||||
return self.commands[-1]
|
||||
else:
|
||||
return super(TestBuildLogs, self).get_last_command(build_id)
|
||||
return super(TestBuildLogs, self).get_log_entries(build_id, start_index)
|
||||
|
||||
@maybe_advance_script(True)
|
||||
def get_status(self, build_id):
|
||||
if build_id == self.test_build_id:
|
||||
self.request_counter += 1
|
||||
if self.request_counter % 10 == 0:
|
||||
self._generate_logs()
|
||||
last_status = self._status
|
||||
self._status = self._generate_fake_status()
|
||||
return last_status
|
||||
returnable_status = self._last_status
|
||||
self._last_status = self._status
|
||||
return returnable_status
|
||||
else:
|
||||
return super(TestBuildLogs, self).get_status(build_id)
|
||||
|
|
|
|||
Reference in a new issue