vbatts/quay
Archived
1
0
Fork 0
Code Issues Pull requests Releases Wiki Activity

Merge pull request #3346 from quay/client-test-improvements

Browse source 
Client test improvements
This commit is contained in:
Joseph Schorr 2019-02-20 16:25:13 -05:00 committed by GitHub
commit a8d64f153e
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
5 changed files with 186 additions and 59 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

0
test/dockerclients/50-insecure-registry.conf → test/clients/50-insecure-registry.conf
View file

1
test/dockerclients/Dockerfile.test → test/clients/Dockerfile.test
View file

@ -2,3 +2,4 @@ FROM quay.io/quay/busybox
RUN date > somefile
RUN date +%s%N > anotherfile
RUN date +"%T.%N" > thirdfile
RUN echo "testing 123" > testfile

0
test/clients/__init__.py Normal file
View file

119
test/clients/client.py Normal file
View file

@ -0,0 +1,119 @@
from abc import ABCMeta, abstractmethod
from collections import namedtuple
from six import add_metaclass
Command = namedtuple('Command', ['command'])
# NOTE: FileCopy is done via `scp`, instead of `ssh` which is how Command is run.
FileCopy = namedtuple('FileCopy', ['source', 'destination'])
@add_metaclass(ABCMeta)
class Client(object):
""" Client defines the interface for all clients being tested. """
@abstractmethod
def setup_client(self, registry_host):
""" Returns the commands necessary to setup the client inside the VM.
"""
@abstractmethod
def populate_test_image(self, registry_host, namespace, name):
""" Returns the commands necessary to populate the test image. """
@abstractmethod
def print_version(self):
""" Returns the commands necessary to print the version of the client. """
@abstractmethod
def login(self, registry_host, username, password):
""" Returns the commands necessary to login. """
@abstractmethod
def push(self, registry_host, namespace, name):
""" Returns the commands necessary to test pushing. """
@abstractmethod
def pre_pull_cleanup(self, registry_host, namespace, name):
""" Returns the commands necessary to cleanup before pulling. """
@abstractmethod
def pull(self, registry_host, namespace, name):
""" Returns the commands necessary to test pulling. """
@abstractmethod
def verify(self, registry_host, namespace, name):
""" Returns the commands necessary to verify the pulled image. """
class DockerClient(Client):
def __init__(self, requires_v1=False):
self.requires_v1 = requires_v1
def setup_client(self, registry_host):
cp_command = ('sudo cp /home/core/50-insecure-registry.conf ' +
'/etc/systemd/system/docker.service.d/50-insecure-registry.conf')
yield Command('sudo mkdir -p /etc/systemd/system/docker.service.d/')
yield FileCopy('50-insecure-registry.conf', '/home/core')
yield FileCopy('Dockerfile.test', '/home/core/Dockerfile')
yield Command(cp_command)
yield Command('sudo systemctl daemon-reload')
def populate_test_image(self, registry_host, namespace, name):
if self.requires_v1:
# These versions of Docker don't support the new TLS cert on quay.io, so we need to pull
# from v1.quay.io and then retag so the build works.
yield Command('docker pull v1.quay.io/quay/busybox')
yield Command('docker tag v1.quay.io/quay/busybox quay.io/quay/busybox')
yield Command('docker build -t %s/%s/%s .' % (registry_host, namespace, name))
def print_version(self):
yield Command('docker version')
def login(self, registry_host, username, password):
yield Command('docker login --username=%s --password=%s %s' %
(username, password, registry_host))
def push(self, registry_host, namespace, name):
yield Command('docker push %s/%s/%s' % (registry_host, namespace, name))
def pre_pull_cleanup(self, registry_host, namespace, name):
prefix = 'v1.' if self.requires_v1 else ''
yield Command('docker rmi -f %s/%s/%s' % (registry_host, namespace, name))
yield Command('docker rmi -f %squay.io/quay/busybox' % prefix)
def pull(self, registry_host, namespace, name):
yield Command('docker pull %s/%s/%s' % (registry_host, namespace, name))
def verify(self, registry_host, namespace, name):
yield Command('docker run %s/%s/%s echo testfile' % (registry_host, namespace, name))
class PodmanClient(Client):
def setup_client(self, registry_host):
yield FileCopy('Dockerfile.test', '/home/vagrant/Dockerfile')
def populate_test_image(self, registry_host, namespace, name):
yield Command('sudo podman build -t %s/%s/%s /home/vagrant/' % (registry_host, namespace, name))
def print_version(self):
yield Command('sudo podman version')
def login(self, registry_host, username, password):
yield Command('sudo podman login --tls-verify=false --username=%s --password=%s %s' %
(username, password, registry_host))
def push(self, registry_host, namespace, name):
yield Command('sudo podman push --tls-verify=false %s/%s/%s' % (registry_host, namespace, name))
def pre_pull_cleanup(self, registry_host, namespace, name):
yield Command('sudo podman rmi -f %s/%s/%s' % (registry_host, namespace, name))
yield Command('sudo podman rmi -f quay.io/quay/busybox')
def pull(self, registry_host, namespace, name):
yield Command('sudo podman pull --tls-verify=false %s/%s/%s' % (registry_host, namespace, name))
def verify(self, registry_host, namespace, name):
yield Command('sudo podman run %s/%s/%s echo testfile' % (registry_host, namespace, name))

123
test/dockerclients/clients_test.py → test/clients/clients_test.py
View file

@ -9,45 +9,50 @@ from threading import Thread
from termcolor import colored
from test.clients.client import DockerClient, PodmanClient, Command, FileCopy
def remove_control_characters(s):
return "".join(ch for ch in unicode(s) if unicodedata.category(ch)[0]!="C")
# These tuples are the box&version and whether it requires V1 registry protocol
# These tuples are the box&version and the client to use.
BOXES = [
("kleesc/coreos --box-version=1911.4.0", False), # docker 18.06.1
("kleesc/coreos --box-version=1800.7.0", False), # docker 18.03.1
("kleesc/coreos --box-version=1688.5.3", False), # docker 17.12.1
("kleesc/coreos --box-version=1632.3.0", False), # docker 17.09.1
("kleesc/coreos --box-version=1576.5.0", False), # docker 17.09.0
("kleesc/coreos --box-version=1520.9.0", False), # docker 1.12.6
("kleesc/coreos --box-version=1235.6.0", False), # docker 1.12.3
("kleesc/coreos --box-version=1185.5.0", False), # docker 1.11.2
("kleesc/centos7-podman --box-version=0.11.1.1", PodmanClient()), # podman 0.11.1.1
("kleesc/coreos --box-version=1122.3.0", False), # docker 1.10.3
("kleesc/coreos --box-version=899.17.0", False), # docker 1.9.1
("kleesc/coreos --box-version=835.13.0", False), # docker 1.8.3
("kleesc/coreos --box-version=766.5.0", False), # docker 1.7.1
("kleesc/coreos --box-version=717.3.0", False), # docker 1.6.2
("kleesc/coreos --box-version=647.2.0", False), # docker 1.5.0
("kleesc/coreos --box-version=557.2.0", False), # docker 1.4.1
("kleesc/coreos --box-version=522.6.0", False), # docker 1.3.3
("kleesc/coreos --box-version=1911.4.0", DockerClient()), # docker 18.06.1
("kleesc/coreos --box-version=1800.7.0", DockerClient()), # docker 18.03.1
("kleesc/coreos --box-version=1688.5.3", DockerClient()), # docker 17.12.1
("kleesc/coreos --box-version=1632.3.0", DockerClient()), # docker 17.09.1
("kleesc/coreos --box-version=1576.5.0", DockerClient()), # docker 17.09.0
("kleesc/coreos --box-version=1520.9.0", DockerClient()), # docker 1.12.6
("kleesc/coreos --box-version=1235.6.0", DockerClient()), # docker 1.12.3
("kleesc/coreos --box-version=1185.5.0", DockerClient()), # docker 1.11.2
("yungsang/coreos --box-version=1.3.7", False), # docker 1.3.2
("yungsang/coreos --box-version=1.2.9", False), # docker 1.2.0
("yungsang/coreos --box-version=1.1.5", False), # docker 1.1.2
("yungsang/coreos --box-version=1.0.0", False), # docker 1.0.1
("yungsang/coreos --box-version=0.9.10", False), # docker 1.0.0
("yungsang/coreos --box-version=0.9.6", False), # docker 0.11.1
("kleesc/coreos --box-version=1122.3.0", DockerClient()), # docker 1.10.3
("kleesc/coreos --box-version=899.17.0", DockerClient()), # docker 1.9.1
("kleesc/coreos --box-version=835.13.0", DockerClient()), # docker 1.8.3
("kleesc/coreos --box-version=766.5.0", DockerClient()), # docker 1.7.1
("kleesc/coreos --box-version=717.3.0", DockerClient()), # docker 1.6.2
("kleesc/coreos --box-version=647.2.0", DockerClient()), # docker 1.5.0
("kleesc/coreos --box-version=557.2.0", DockerClient()), # docker 1.4.1
("kleesc/coreos --box-version=522.6.0", DockerClient()), # docker 1.3.3
("yungsang/coreos --box-version=0.9.1", True), # docker 0.10.0
("yungsang/coreos --box-version=0.3.1", True),
("yungsang/coreos --box-version=1.3.7", DockerClient()), # docker 1.3.2
("yungsang/coreos --box-version=1.2.9", DockerClient()), # docker 1.2.0
("yungsang/coreos --box-version=1.1.5", DockerClient()), # docker 1.1.2
("yungsang/coreos --box-version=1.0.0", DockerClient()), # docker 1.0.1
("yungsang/coreos --box-version=0.9.10", DockerClient()), # docker 1.0.0
("yungsang/coreos --box-version=0.9.6", DockerClient()), # docker 0.11.1
("yungsang/coreos --box-version=0.9.1", DockerClient(True)), # docker 0.10.0
("yungsang/coreos --box-version=0.3.1", DockerClient(True)),
]
def _check_vagrant():
vagrant_command = 'vagrant'
vagrant = any(os.access(os.path.join(path, vagrant_command), os.X_OK) for path in os.environ.get('PATH').split(':'))
vagrant = any(os.access(os.path.join(path, vagrant_command), os.X_OK)
for path in os.environ.get('PATH').split(':'))
vagrant_plugins = subprocess.check_output([vagrant_command, 'plugin', 'list'])
return (vagrant, 'vagrant-scp' in vagrant_plugins)
@ -128,7 +133,18 @@ def _run_and_wait(command, error_allowed=False):
def _indent(text, amount):
return ''.join((' ' * amount) + line for line in text.splitlines(True))
def _run_box(box, requires_v1, registry):
def _run_commands(commands):
last_result = None
for command in commands:
if isinstance(command, Command):
last_result = _run_and_wait(['vagrant', 'ssh', '-c', command.command])
else:
last_result = _run_and_wait(['vagrant', 'scp', command.source, command.destination])
return last_result
def _run_box(box, client, registry):
vagrant, vagrant_scp = _check_vagrant()
if not vagrant:
print("vagrant command not found")
@ -138,6 +154,11 @@ def _run_box(box, requires_v1, registry):
print("vagrant-scp plugin not installed")
return
namespace = 'devtable'
repo_name = 'testrepo%s' % int(time.time())
username = 'devtable'
password = 'password'
print colored('>>> Box: %s' % box, attrs=['bold'])
print colored('>>> Starting box', 'yellow')
_run_and_wait(['vagrant', 'destroy', '-f'], error_allowed=True)
@ -145,44 +166,30 @@ def _run_box(box, requires_v1, registry):
_run_and_wait(['vagrant', 'init'] + box.split(' '))
_run_and_wait(['vagrant', 'up', '--provider', 'virtualbox'])
print colored('>>> Setting up Docker', 'yellow')
_run_and_wait(['vagrant', 'ssh', '-c', 'sudo mkdir -p /etc/systemd/system/docker.service.d/'])
_run_and_wait(['vagrant', 'scp', '50-insecure-registry.conf', '/home/core'])
_run_and_wait(['vagrant', 'scp', 'Dockerfile.test', '/home/core/Dockerfile'])
print colored('>>> Setting up runtime', 'yellow')
_run_commands(client.setup_client(registry))
cp_command = ('sudo cp /home/core/50-insecure-registry.conf ' +
'/etc/systemd/system/docker.service.d/50-insecure-registry.conf')
_run_and_wait(['vagrant', 'ssh', '-c', cp_command])
_run_and_wait(['vagrant', 'ssh', '-c', 'sudo systemctl daemon-reload'])
print colored('>>> Client version', 'cyan')
runtime_version = _run_commands(client.print_version())
print _indent(runtime_version, 4)
print colored('>>> Docker version', 'cyan')
docker_version = _run_and_wait(['vagrant', 'ssh', '-c', 'docker version'])
print _indent(docker_version, 4)
print colored('>>> Building test image', 'yellow')
if requires_v1:
# These versions of Docker don't support the new TLS cert on quay.io, so we need to pull
# from v1.quay.io and then retag so the build works.
_run_and_wait(['vagrant', 'ssh', '-c', 'docker pull v1.quay.io/quay/busybox'])
_run_and_wait(['vagrant', 'ssh', '-c',
'docker tag v1.quay.io/quay/busybox quay.io/quay/busybox'])
_run_and_wait(['vagrant', 'ssh', '-c', 'docker build -t %s/devtable/testrepo .' % registry])
print colored('>>> Populating test image', 'yellow')
_run_commands(client.populate_test_image(registry, namespace, repo_name))
print colored('>>> Testing login', 'cyan')
_run_and_wait(['vagrant', 'ssh', '-c',
'docker login --username=devtable --password=password %s' % registry])
_run_commands(client.login(registry, username, password))
print colored('>>> Testing push', 'cyan')
_run_and_wait(['vagrant', 'ssh', '-c', 'docker push %s/devtable/testrepo' % registry])
_run_commands(client.push(registry, namespace, repo_name))
print colored('>>> Removing all images', 'yellow')
prefix = 'v1.' if requires_v1 else ''
_run_and_wait(['vagrant', 'ssh', '-c', 'docker rmi -f %s/devtable/testrepo' % registry])
_run_and_wait(['vagrant', 'ssh', '-c', 'docker rmi -f %squay.io/quay/busybox' % prefix])
_run_commands(client.pre_pull_cleanup(registry, namespace, repo_name))
print colored('>>> Testing pull', 'cyan')
_run_and_wait(['vagrant', 'ssh', '-c', 'docker pull %s/devtable/testrepo' % registry])
_run_commands(client.pull(registry, namespace, repo_name))
print colored('>>> Verifying', 'cyan')
_run_commands(client.verify(registry, namespace, repo_name))
print colored('>>> Tearing down box', 'magenta')
_run_and_wait(['vagrant', 'destroy', '-f'], error_allowed=True)
@ -192,9 +199,9 @@ def _run_box(box, requires_v1, registry):
def test_clients(registry='10.0.2.2:5000'):
print colored('>>> Running against registry ', attrs=['bold']) + colored(registry, 'cyan')
for box, requires_v1 in BOXES:
for box, client in BOXES:
try:
_run_box(box, requires_v1, registry)
_run_box(box, client, registry)
except CommandFailedException:
sys.exit(-1)