Add an end-to-end test suite for different versions of Docker

This script, when run, will boot up a Container Linux VM (via vagrant) with a specific version of Docker installed, and then run a login, push and pull test against the host machine's Quay instance.

.dockerignore

@@ -31,3 +31,4 @@ static/build/**
 .gitlab-ci/*
 .gitlab-ci.*
 docker-compose.yaml
+test/dockerclients/**

.gitignore

@@ -27,3 +27,5 @@ build/
 *.iml
 .DS_Store
 .pytest_cache/*
+test/dockerclients/Vagrantfile
+test/dockerclients/.*

test/dockerclients/50-insecure-registry.conf

@@ -0,0 +1,2 @@
+[Service]
+Environment=DOCKER_OPTS='--insecure-registry="0.0.0.0/0"'

test/dockerclients/Dockerfile.test

@@ -0,0 +1,4 @@
+FROM quay.io/quay/busybox
+RUN date > somefile
+RUN date +%s%N > anotherfile
+RUN date +"%T.%N" > thirdfile

test/dockerclients/clients_test.py

@@ -0,0 +1,152 @@
+import subprocess
+import sys
+import time
+
+import unicodedata
+
+from threading import Thread
+
+from termcolor import colored
+
+def remove_control_characters(s):
+  return "".join(ch for ch in unicode(s) if unicodedata.category(ch)[0]!="C")
+
+BOXES = [
+  ("AntonioMeireles/coreos-stable --box-version=835.8.0", 'v1.8.3'),
+  ("AntonioMeireles/coreos-stable --box-version=766.3.0", 'v1.7.1'),
+  ("AntonioMeireles/coreos-stable --box-version=681.0.0", 'v1.6.2'),
+  ("AntonioMeireles/coreos-stable --box-version=607.0.0", 'v1.5.0'),
+  ("AntonioMeireles/coreos-stable --box-version=557.2.0", 'v1.4.1'),
+
+  ("yungsang/coreos --box-version=1.3.8", 'v1.3.3'),
+  ("yungsang/coreos --box-version=1.3.7", 'v1.3.2'),
+  ("yungsang/coreos --box-version=1.2.9", 'v1.2.0'),
+  ("yungsang/coreos --box-version=1.1.5", 'v1.1.2'),
+  ("yungsang/coreos --box-version=1.0.0", 'v1.0.1'),
+]
+
+class CommandFailedException(Exception):
+  pass
+
+
+class SpinOutputter(Thread):
+  def __init__(self, initial_message):
+    super(SpinOutputter, self).__init__()
+    self.previous_line = ''
+    self.next_line = initial_message
+    self.running = True
+    self.daemon = True
+
+  @staticmethod
+  def spinning_cursor():
+    while 1:
+      for cursor in '|/-\\':
+        yield cursor
+
+  def set_next(self, text):
+    first_line = text.split('\n')[0].strip()
+    first_line = remove_control_characters(first_line)
+    self.next_line = first_line[:80]
+
+  def _clear_line(self):
+    sys.stdout.write('\r')
+    sys.stdout.write(' ' * (len(self.previous_line) + 2))
+    sys.stdout.flush()
+
+    sys.stdout.write('\r')
+    sys.stdout.flush()
+    self.previous_line = ''
+
+  def stop(self):
+    self._clear_line()
+    self.running = False
+
+  def run(self):
+    spinner = SpinOutputter.spinning_cursor()
+    while self.running:
+      self._clear_line()
+      sys.stdout.write('\r')
+      sys.stdout.flush()
+
+      sys.stdout.write(next(spinner))
+      sys.stdout.write(" ")
+      sys.stdout.write(colored(self.next_line, attrs=['dark']))
+      sys.stdout.flush()
+
+      self.previous_line = self.next_line
+      time.sleep(0.25)
+
+
+def _run_and_wait(command, error_allowed=False):
+  # Run the command itself.
+  outputter = SpinOutputter('Running command %s' % command)
+  outputter.start()
+
+  output = ''
+  process = subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
+  for line in iter(process.stdout.readline, ''):
+    output += line
+    outputter.set_next(line)
+
+  result = process.wait()
+  outputter.stop()
+
+  if result != 0 and not error_allowed:
+    print colored('>>> Command `%s` Failed:' % command, 'red')
+    print output
+    raise CommandFailedException()
+
+
+def _run_box(box, docker_version, registry):
+  print (colored('>>> Box: %s' % box, attrs=['bold']) + ' | ' +
+         colored('Docker version: %s' % docker_version, 'cyan', attrs=['bold']))
+  print colored('>>> Starting box', 'yellow')
+  _run_and_wait(['vagrant', 'destroy', '-f'], error_allowed=True)
+  _run_and_wait(['rm', 'Vagrantfile'], error_allowed=True)
+  _run_and_wait(['vagrant', 'init'] + box.split(' '))
+  _run_and_wait(['vagrant', 'up', '--provider', 'virtualbox'])
+
+  print colored('>>> Setting up Docker', 'yellow')
+  _run_and_wait(['vagrant', 'ssh', '-c', 'sudo mkdir -p /etc/systemd/system/docker.service.d/'])
+  _run_and_wait(['vagrant', 'scp', '50-insecure-registry.conf', '/home/core'])
+  _run_and_wait(['vagrant', 'scp', 'Dockerfile.test', '/home/core/Dockerfile'])
+
+  cp_command = ('sudo cp /home/core/50-insecure-registry.conf ' + 
+                '/etc/systemd/system/docker.service.d/50-insecure-registry.conf')
+  _run_and_wait(['vagrant', 'ssh', '-c', cp_command])
+  _run_and_wait(['vagrant', 'ssh', '-c', 'sudo systemctl daemon-reload'])
+
+  print colored('>>> Building test image', 'yellow')
+  _run_and_wait(['vagrant', 'ssh', '-c', 'docker build -t %s/devtable/testrepo .' % registry])
+
+  print colored('>>> Testing login', 'cyan')
+  _run_and_wait(['vagrant', 'ssh', '-c',
+                 'docker login --username=devtable --password=password --email=foo %s' % registry])
+
+  print colored('>>> Testing push', 'cyan')
+  _run_and_wait(['vagrant', 'ssh', '-c', 'docker push %s/devtable/testrepo' % registry])
+
+  print colored('>>> Removing all images', 'yellow')
+  _run_and_wait(['vagrant', 'ssh', '-c', 'docker rmi -f %s/devtable/testrepo' % registry])
+  _run_and_wait(['vagrant', 'ssh', '-c', 'docker rmi -f quay.io/quay/busybox'])
+
+  print colored('>>> Testing pull', 'cyan')
+  _run_and_wait(['vagrant', 'ssh', '-c', 'docker pull %s/devtable/testrepo' % registry])
+
+  print colored('>>> Tearing down box', 'magenta')
+  _run_and_wait(['vagrant', 'destroy', '-f'], error_allowed=True)
+
+  print colored('>>> Successfully tested box %s' % box, 'green')
+  print ""
+
+def test_clients(registry='10.0.2.2:5000'):
+  print colored('>>> Running against registry ', attrs=['bold']) + colored(registry, 'cyan')
+  for box, docker_version in BOXES:
+    try:
+      _run_box(box, docker_version, registry)
+    except CommandFailedException:
+      sys.exit(-1)
+
+
+if __name__ == "__main__":
+  test_clients(sys.argv[1] if len(sys.argv) > 1 else '10.0.2.2:5000')