From 0e55fb2ea35b9b978cdf166ccf0aaac7233d921a Mon Sep 17 00:00:00 2001 From: Chris Evich Date: Wed, 20 Sep 2017 17:52:27 -0400 Subject: [PATCH] integration: Add color + readable logging When run by hand, it's much easier to spot things going wrong when they're colored in red. Add an ansible.cfg to make that happen. This also sets a default output log file (``$ARTIFACTS/main.log``) - that doesn't contain color-codes. When executing against multple hosts, the output can sometimes become difficult to read, esp. with lots of async. tasks. The callback_plugin script reorganizes how the console and log is organized, making it clearer which host did what and when. Signed-off-by: Chris Evich --- contrib/test/integration/ansible.cfg | 359 ++++++++++++++++++ .../integration/callback_plugins/default.py | 156 ++++++++ contrib/test/venv-ansible-playbook.sh | 4 +- 3 files changed, 518 insertions(+), 1 deletion(-) create mode 100644 contrib/test/integration/ansible.cfg create mode 100644 contrib/test/integration/callback_plugins/default.py diff --git a/contrib/test/integration/ansible.cfg b/contrib/test/integration/ansible.cfg new file mode 100644 index 00000000..92a13a5f --- /dev/null +++ b/contrib/test/integration/ansible.cfg @@ -0,0 +1,359 @@ +# config file for ansible -- http://ansible.com/ +# ============================================== + +# nearly all parameters can be overridden in ansible-playbook +# or with command line flags. ansible will read ANSIBLE_CONFIG, +# ansible.cfg in the current working directory, .ansible.cfg in +# the home directory or /etc/ansible/ansible.cfg, whichever it +# finds first + +[defaults] + +# some basic default values... + +#inventory = inventory +#library = /usr/share/my_modules/ +#remote_tmp = $HOME/.ansible/tmp +#local_tmp = .ansible/tmp +#forks = 5 +forks = 10 +#poll_interval = 15 +#sudo_user = root +#ask_sudo_pass = True +ask_sudo_pass = False +#ask_pass = True +ask_pass = False +#transport = smart +#remote_port = 22 +#module_lang = C +#module_set_locale = True + +# plays will gather facts by default, which contain information about +# the remote system. +# +# smart - gather by default, but don't regather if already gathered +# implicit - gather by default, turn off with gather_facts: False +# explicit - do not gather by default, must say gather_facts: True +#gathering = implicit +gathering = smart + +# by default retrieve all facts subsets +# all - gather all subsets +# network - gather min and network facts +# hardware - gather hardware facts (longest facts to retrieve) +# virtual - gather min and virtual facts +# facter - import facts from facter +# ohai - import facts from ohai +# You can combine them using comma (ex: network,virtual) +# You can negate them using ! (ex: !hardware,!facter,!ohai) +# A minimal set of facts is always gathered. +gather_subset = network + +# additional paths to search for roles in, colon separated +# N/B: This depends on how ansible is called +#roles_path = $WORKSPACE/kommandir_workspace/roles + +# uncomment this to disable SSH key host checking +#host_key_checking = False +host_key_checking = False + +# change the default callback +#stdout_callback = skippy +# enable additional callbacks +#callback_whitelist = timer, mail + +# Determine whether includes in tasks and handlers are "static" by +# default. As of 2.0, includes are dynamic by default. Setting these +# values to True will make includes behave more like they did in the +# 1.x versions. +task_includes_static = True +handler_includes_static = True + +# change this for alternative sudo implementations +#sudo_exe = sudo + +# What flags to pass to sudo +# WARNING: leaving out the defaults might create unexpected behaviours +#sudo_flags = -H -S -n + +# SSH timeout +#timeout = 10 + +# default user to use for playbooks if user is not specified +# (/usr/bin/ansible will use current user as default) +#remote_user = root +remote_user = root + +# logging is off by default unless this path is defined +# if so defined, consider logrotate +log_path = $ARTIFACTS/main.log + +# default module name for /usr/bin/ansible +#module_name = command + +# use this shell for commands executed under sudo +# you may need to change this to bin/bash in rare instances +# if sudo is constrained +# executable = /bin/sh + +# if inventory variables overlap, does the higher precedence one win +# or are hash values merged together? The default is 'replace' but +# this can also be set to 'merge'. +hash_behaviour = replace + +# by default, variables from roles will be visible in the global variable +# scope. To prevent this, the following option can be enabled, and only +# tasks and handlers within the role will see the variables there +private_role_vars = False + +# list any Jinja2 extensions to enable here: +#jinja2_extensions = jinja2.ext.do,jinja2.ext.i18n + +# if set, always use this private key file for authentication, same as +# if passing --private-key to ansible or ansible-playbook +#private_key_file = /path/to/file + +# If set, configures the path to the Vault password file as an alternative to +# specifying --vault-password-file on the command line. +#vault_password_file = /path/to/vault_password_file + +# format of string {{ ansible_managed }} available within Jinja2 +# templates indicates to users editing templates files will be replaced. +# replacing {file}, {host} and {uid} and strftime codes with proper values. +#ansible_managed = Ansible managed: {file} modified on %Y-%m-%d %H:%M:%S by {uid} on {host} +# This short version is better used in templates as it won't flag the file as changed every run. +#ansible_managed = Ansible managed: {file} on {host} + +# by default, ansible-playbook will display "Skipping [host]" if it determines a task +# should not be run on a host. Set this to "False" if you don't want to see these "Skipping" +# messages. NOTE: the task header will still be shown regardless of whether or not the +# task is skipped. +#display_skipped_hosts = True +display_skipped_hosts = False + +# by default, if a task in a playbook does not include a name: field then +# ansible-playbook will construct a header that includes the task's action but +# not the task's args. This is a security feature because ansible cannot know +# if the *module* considers an argument to be no_log at the time that the +# header is printed. If your environment doesn't have a problem securing +# stdout from ansible-playbook (or you have manually specified no_log in your +# playbook on all of the tasks where you have secret information) then you can +# safely set this to True to get more informative messages. +display_args_to_stdout = False + +# by default (as of 1.3), Ansible will raise errors when attempting to dereference +# Jinja2 variables that are not set in templates or action lines. Uncomment this line +# to revert the behavior to pre-1.3. +#error_on_undefined_vars = False + +# by default (as of 1.6), Ansible may display warnings based on the configuration of the +# system running ansible itself. This may include warnings about 3rd party packages or +# other conditions that should be resolved if possible. +# to disable these warnings, set the following value to False: +system_warnings = False + +# by default (as of 1.4), Ansible may display deprecation warnings for language +# features that should no longer be used and will be removed in future versions. +# to disable these warnings, set the following value to False: +deprecation_warnings = False + +# (as of 1.8), Ansible can optionally warn when usage of the shell and +# command module appear to be simplified by using a default Ansible module +# instead. These warnings can be silenced by adjusting the following +# setting or adding warn=yes or warn=no to the end of the command line +# parameter string. This will for example suggest using the git module +# instead of shelling out to the git command. +command_warnings = False + + +# set plugin path directories here, separate with colons +#action_plugins = /usr/share/ansible/plugins/action +#callback_plugins = /usr/share/ansible/plugins/callback +#connection_plugins = /usr/share/ansible/plugins/connection +#lookup_plugins = /usr/share/ansible/plugins/lookup +#vars_plugins = /usr/share/ansible/plugins/vars +#filter_plugins = /usr/share/ansible/plugins/filter +#test_plugins = /usr/share/ansible/plugins/test +#strategy_plugins = /usr/share/ansible/plugins/strategy + +# Most callbacks shipped with Ansible are disabled by default +# and need to be whitelisted in your ansible.cfg file in order to function. +callback_whitelist = default + +# by default callbacks are not loaded for /bin/ansible, enable this if you +# want, for example, a notification or logging callback to also apply to +# /bin/ansible runs +#bin_ansible_callbacks = False + + +# don't like cows? that's unfortunate. +# set to 1 if you don't want cowsay support or export ANSIBLE_NOCOWS=1 +#nocows = 1 + +# set which cowsay stencil you'd like to use by default. When set to 'random', +# a random stencil will be selected for each task. The selection will be filtered +# against the `cow_whitelist` option below. +#cow_selection = default +#cow_selection = random + +# when using the 'random' option for cowsay, stencils will be restricted to this list. +# it should be formatted as a comma-separated list with no spaces between names. +# NOTE: line continuations here are for formatting purposes only, as the INI parser +# in python does not support them. +#cow_whitelist=bud-frogs,bunny,cheese,daemon,default,dragon,elephant-in-snake,elephant,eyes,\ +# hellokitty,kitty,luke-koala,meow,milk,moofasa,moose,ren,sheep,small,stegosaurus,\ +# stimpy,supermilker,three-eyes,turkey,turtle,tux,udder,vader-koala,vader,www + +# don't like colors either? +# set to 1 if you don't want colors, or export ANSIBLE_NOCOLOR=1 +nocolor = 0 + +# if set to a persistent type (not 'memory', for example 'redis') fact values +# from previous runs in Ansible will be stored. This may be useful when +# wanting to use, for example, IP information from one group of servers +# without having to talk to them in the same playbook run to get their +# current IP information. +#fact_caching = memory + +# retry files +# When a playbook fails by default a .retry file will be created in ~/ +# You can disable this feature by setting retry_files_enabled to False +# and you can change the location of the files by setting retry_files_save_path + +#retry_files_enabled = False +retry_files_enabled = False + +# squash actions +# Ansible can optimise actions that call modules with list parameters +# when looping. Instead of calling the module once per with_ item, the +# module is called once with all items at once. Currently this only works +# under limited circumstances, and only with parameters named 'name'. +squash_actions = apk,apt,dnf,package,pacman,pkgng,yum,zypper + +# prevents logging of task data, off by default +#no_log = False + +# prevents logging of tasks, but only on the targets, data is still logged on the master/controller +no_target_syslog = True + +# controls whether Ansible will raise an error or warning if a task has no +# choice but to create world readable temporary files to execute a module on +# the remote machine. This option is False by default for security. Users may +# turn this on to have behaviour more like Ansible prior to 2.1.x. See +# https://docs.ansible.com/ansible/become.html#becoming-an-unprivileged-user +# for more secure ways to fix this than enabling this option. +#allow_world_readable_tmpfiles = False + +# controls the compression level of variables sent to +# worker processes. At the default of 0, no compression +# is used. This value must be an integer from 0 to 9. +#var_compression_level = 9 + +# controls what compression method is used for new-style ansible modules when +# they are sent to the remote system. The compression types depend on having +# support compiled into both the controller's python and the client's python. +# The names should match with the python Zipfile compression types: +# * ZIP_STORED (no compression. available everywhere) +# * ZIP_DEFLATED (uses zlib, the default) +# These values may be set per host via the ansible_module_compression inventory +# variable +#module_compression = 'ZIP_DEFLATED' + +# This controls the cutoff point (in bytes) on --diff for files +# set to 0 for unlimited (RAM may suffer!). +#max_diff_size = 1048576 + +[privilege_escalation] +#become=True +#become_method=sudo +#become_user=root +become_user=root +#become_ask_pass=False + +[paramiko_connection] + +# uncomment this line to cause the paramiko connection plugin to not record new host +# keys encountered. Increases performance on new host additions. Setting works independently of the +# host key checking setting above. +#record_host_keys=False + +# by default, Ansible requests a pseudo-terminal for commands executed under sudo. Uncomment this +# line to disable this behaviour. +#pty=False + +[ssh_connection] + +# ssh arguments to use +# Leaving off ControlPersist will result in poor performance, so use +# paramiko on older platforms rather than removing it +ssh_args = -o ControlMaster=auto -o ControlPersist=60s -o UserKnownHostsFile=/dev/null -o PreferredAuthentications=publickey -o ConnectTimeout=13 + +# The path to use for the ControlPath sockets. This defaults to +# "%(directory)s/ansible-ssh-%%h-%%p-%%r", however on some systems with +# very long hostnames or very long path names (caused by long user names or +# deeply nested home directories) this can exceed the character limit on +# file socket names (108 characters for most platforms). In that case, you +# may wish to shorten the string below. +# +# Example: +# control_path = %(directory)s/%%h-%%r +#control_path = %(directory)s/ansible-ssh-%%h-%%p-%%r + +# Enabling pipelining reduces the number of SSH operations required to +# execute a module on the remote server. This can result in a significant +# performance improvement when enabled, however when using "sudo:" you must +# first disable 'requiretty' in /etc/sudoers +# +# By default, this option is disabled to preserve compatibility with +# sudoers configurations that have requiretty (the default on many distros). +# +#pipelining = False +pipelining=True + +# if True, make ansible use scp if the connection type is ssh +# (default is sftp) +#scp_if_ssh = True + +# if False, sftp will not use batch mode to transfer files. This may cause some +# types of file transfer failures impossible to catch however, and should +# only be disabled if your sftp version has problems with batch mode +#sftp_batch_mode = False + +[accelerate] +#accelerate_port = 5099 +#accelerate_timeout = 30 +#accelerate_connect_timeout = 5.0 + +# The daemon timeout is measured in minutes. This time is measured +# from the last activity to the accelerate daemon. +#accelerate_daemon_timeout = 30 + +# If set to yes, accelerate_multi_key will allow multiple +# private keys to be uploaded to it, though each user must +# have access to the system via SSH to add a new key. The default +# is "no". +#accelerate_multi_key = yes + +[selinux] +# file systems that require special treatment when dealing with security context +# the default behaviour that copies the existing context or uses the user default +# needs to be changed to use the file system dependent context. +#special_context_filesystems=nfs,vboxsf,fuse,ramfs + +# Set this to yes to allow libvirt_lxc connections to work without SELinux. +#libvirt_lxc_noseclabel = yes + +[colors] +#highlight = white +#verbose = blue +#warn = bright purple +#error = red +#debug = dark gray +#deprecate = purple +#skip = cyan +#unreachable = red +#ok = green +#changed = yellow +#diff_add = green +#diff_remove = red +#diff_lines = cyan diff --git a/contrib/test/integration/callback_plugins/default.py b/contrib/test/integration/callback_plugins/default.py new file mode 100644 index 00000000..99821f66 --- /dev/null +++ b/contrib/test/integration/callback_plugins/default.py @@ -0,0 +1,156 @@ +'''Plugin to override the default output logic.''' + +# upstream: https://gist.github.com/cliffano/9868180 + +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + + +# For some reason this has to be done +import imp +import os + +ANSIBLE_PATH = imp.find_module('ansible')[1] +DEFAULT_PATH = os.path.join(ANSIBLE_PATH, 'plugins/callback/default.py') +DEFAULT_MODULE = imp.load_source( + 'ansible.plugins.callback.default', + DEFAULT_PATH +) + +try: + from ansible.plugins.callback import CallbackBase + BASECLASS = CallbackBase +except ImportError: # < ansible 2.1 + BASECLASS = DEFAULT_MODULE.CallbackModule + + +class CallbackModule(DEFAULT_MODULE.CallbackModule): # pylint: disable=too-few-public-methods,no-init + ''' + Override for the default callback module. + + Render std err/out outside of the rest of the result which it prints with + indentation. + ''' + CALLBACK_VERSION = 2.0 + CALLBACK_TYPE = 'stdout' + CALLBACK_NAME = 'default' + + def __init__(self, *args, **kwargs): + # pylint: disable=non-parent-init-called + BASECLASS.__init__(self, *args, **kwargs) + self.failed_task = [] + self.result_file = os.environ.get('AHT_RESULT_FILE') + + def _dump_results(self, result): + '''Return the text to output for a result.''' + result['_ansible_verbose_always'] = True + + save = {} + for key in ['stdout', 'stdout_lines', 'stderr', 'stderr_lines', 'msg']: + if key in result: + save[key] = result.pop(key) + + output = BASECLASS._dump_results(self, result) # pylint: disable=protected-access + + for key in ['stdout', 'stderr', 'msg']: + if key in save and save[key]: + output += '\n\n%s:\n---\n%s\n---' % (key.upper(), save[key]) + + for key, value in save.items(): + result[key] = value + + return output + + def v2_runner_on_unreachable(self, result): + self.failed_task = result + + if self._play.strategy == 'free' and self._last_task_banner != result._task._uuid: + self._print_task_banner(result._task) + + delegated_vars = result._result.get('_ansible_delegated_vars', None) + if delegated_vars: + self._display.display("fatal: [%s -> %s]: UNREACHABLE! => %s" % (result._host.get_name(), delegated_vars['ansible_host'], self._dump_results(result._result)), color=C.COLOR_UNREACHABLE) + else: + self._display.display("fatal: [%s]: UNREACHABLE! => %s" % (result._host.get_name(), self._dump_results(result._result)), color=C.COLOR_UNREACHABLE) + + def v2_runner_on_failed(self,result, ignore_errors=False): + if ignore_errors is not True: + # Sets environment variable for test failures for use in playboks. + # Handlers tasks can conditionalize themselves using this variable + # to run only on failure. + os.environ["AHT_FAILURE"] = "1" + + # Save last failure + self.failed_task = result + + if self._play.strategy == 'free' and self._last_task_banner != result._task._uuid: + self._print_task_banner(result._task) + + delegated_vars = result._result.get('_ansible_delegated_vars', None) + if 'exception' in result._result: + if self._display.verbosity < 3: + # extract just the actual error message from the exception text + error = result._result['exception'].strip().split('\n')[-1] + msg = "An exception occurred during task execution. To see the full traceback, use -vvv. The error was: %s" % error + else: + msg = "An exception occurred during task execution. The full traceback is:\n" + result._result['exception'] + + self._display.display(msg, color=C.COLOR_ERROR) + + if result._task.loop and 'results' in result._result: + self._process_items(result) + + else: + if delegated_vars: + self._display.display("fatal: [%s -> %s]: FAILED! => %s" % (result._host.get_name(), delegated_vars['ansible_host'], self._dump_results(result._result)), color=C.COLOR_ERROR) + else: + self._display.display("fatal: [%s]: FAILED! => %s" % (result._host.get_name(), self._dump_results(result._result)), color=C.COLOR_ERROR) + + if ignore_errors: + self._display.display("...ignoring", color=C.COLOR_SKIP) + + def v2_playbook_on_stats(self, stats): + self._display.banner("PLAY RECAP") + + hosts = sorted(stats.processed.keys()) + for h in hosts: + t = stats.summarize(h) + + self._display.display(u"%s : %s %s %s %s" % ( + hostcolor(h, t), + colorize(u'ok', t['ok'], C.COLOR_OK), + colorize(u'changed', t['changed'], C.COLOR_CHANGED), + colorize(u'unreachable', t['unreachable'], C.COLOR_UNREACHABLE), + colorize(u'failed', t['failures'], C.COLOR_ERROR)), + screen_only=True + ) + + self._display.display(u"%s : %s %s %s %s" % ( + hostcolor(h, t, False), + colorize(u'ok', t['ok'], None), + colorize(u'changed', t['changed'], None), + colorize(u'unreachable', t['unreachable'], None), + colorize(u'failed', t['failures'], None)), + log_only=True + ) + + self._display.display("", screen_only=True) + # Save result to file if environment variable exists + if self.result_file is not None: + if self.failed_task: + with open(self.result_file, 'w') as f: + f.write("PLAY: %s\n%s\n%s" % (self._play, \ + self.failed_task._task, \ + self._dump_results(self.failed_task._result))) + else: + open(self.result_file, 'w').close() diff --git a/contrib/test/venv-ansible-playbook.sh b/contrib/test/venv-ansible-playbook.sh index a60357b2..1a336f3f 100755 --- a/contrib/test/venv-ansible-playbook.sh +++ b/contrib/test/venv-ansible-playbook.sh @@ -53,6 +53,8 @@ echo "Bootstrapping trusted virtual environment, this may take a few minutes, de echo "(logs: \"$WORKSPACE/crio_venv_setup_log.txt\")" echo +mkdir -p "$WORKSPACE/artifacts" + ( set -x cd "$WORKSPACE" @@ -89,7 +91,7 @@ echo rm -rf ./.venvbootstrap # Exit trusted virtualenv -) &> $WORKSPACE/crio_venv_setup_log.txt; +) &> $WORKSPACE/artifacts/crio_venv_setup_log.txt; echo echo "Executing \"$WORKSPACE/.cri-o_venv/bin/ansible-playbook $@\""