Introduce role to clone from git repos in parallel

For a large number of large repositories, running this in parallel is
***MUCH*** faster.  The role simply takes a list of Ansible ``git``
module arguments, builds a list of async. tasks, then ensures they
all complete before a 10-minute (default) timeout.

Signed-off-by: Chris Evich <cevich@redhat.com>
This commit is contained in:
Chris Evich 2017-11-06 15:29:43 -05:00
parent 6b6d634cfc
commit 717f01ff6d
5 changed files with 81 additions and 0 deletions

View file

@ -0,0 +1,18 @@
---
# Maximum time to wait (in seconds) for each git operation to complete
git_op_timeout: '{{ 60 * 30 }}'
# Interval time (in seconds) to wait between each git status check
git_op_status_delay: '10'
# List of dictionaries, with options to the git ansible module.
git_ops:
# Default depth to use when no depth option specified (above).
# 1: shallow clone only reference HEAD
# None: clone all commits
git_def_depth: 1
# When true, always force-clone, clobbering any posible local changes.
always_force: False

View file

@ -0,0 +1,25 @@
---
- name: The destination directory always exists
file:
path: "{{ git_op.dest }}"
state: directory
- name: git_op's options are fed to git module
git:
dest: "{{ git_op.dest }}"
repo: "{{ git_op.repo }}"
depth: "{{ git_op.depth | default(git_def_depth | int) }}"
recursive: "{{ git_op['recursive'] | default(omit) }}"
reference: "{{ git_op.reference | default(omit) }}"
refspec: "{{ git_op.refspec | default(omit) }}"
remote: "{{ git_op.remote | default(omit) }}"
version: "{{ git_op.version | default(omit) }}"
force: "{{ git_op.force | default(omit) if not always_force else True }}"
register: result
async: "{{ git_op_timeout }}"
poll: 0
- name: git_op async-state is included in async_results list
set_fact:
async_results: "{{ async_results | union([result]) }}"

View file

@ -0,0 +1,11 @@
---
- name: All async states in async_result completed or timed out
async_status:
jid: "{{ async_result.ansible_job_id }}"
failed_when: result | failed
register: result
until: result.finished | bool
# Guarantee at least one retry
retries: "{{ (git_op_timeout|int / git_op_status_delay|int) | round(method='ceil')|int }}"
delay: "{{ git_op_status_delay|int }}"

View file

@ -0,0 +1,23 @@
---
- assert:
that:
# Verify conformance with defaults, expected types, and values
- "git_op_timeout | default(0) >= 1"
- "git_op_status_delay | default(0) >= 1"
- "git_ops is defined"
# Verify vars were not overridden by command-line, include, or role override
- "async_results == []"
- "result == None"
- name: All git operations are run in parallel
include: "{{ role_path }}/tasks/async_git.yml"
with_items: "{{ git_ops }}"
loop_control:
loop_var: "git_op"
- name: All parallel git operations are completed and are successful
include: "{{ role_path }}/tasks/async_status.yml"
with_items: "{{ async_results }}"
loop_control:
loop_var: "async_result"

View file

@ -0,0 +1,4 @@
---
async_results: []
result: