From 1e15b6e001ae5294895e233ad94289a21916ddcc Mon Sep 17 00:00:00 2001 From: Aaron Lehmann Date: Wed, 5 Aug 2015 16:26:34 -0700 Subject: [PATCH] Redesign integration testing to copy images to nested containers Previously, the strategy for avoiding lots of rebuilding and repulling for each Docker version being tested was to use a mountpoint to persist /var/lib/docker. This was pretty broken, and may not be a reliable strategy. This commit changes the scripts to instead build/pull images outside the innermost container, and copy them to the final test environment with docker save/docker load. This requires a fair amount of changes, since run.sh must now communicate with the Docker engine that was formerly started by test_runner.sh. The code that starts this engine has been broken out to run_engine.sh so that starting the engine and running the tests under it can be done separately (with the images loaded in between these steps). Signed-off-by: Aaron Lehmann --- contrib/docker-integration/helpers.bash | 21 +++++++ contrib/docker-integration/install_certs.sh | 4 ++ contrib/docker-integration/run.sh | 57 +++++++++++++++++-- contrib/docker-integration/run_engine.sh | 12 ++++ .../docker-integration/run_multiversion.sh | 48 ++++------------ contrib/docker-integration/test_runner.sh | 32 ----------- 6 files changed, 102 insertions(+), 72 deletions(-) create mode 100644 contrib/docker-integration/helpers.bash create mode 100755 contrib/docker-integration/run_engine.sh diff --git a/contrib/docker-integration/helpers.bash b/contrib/docker-integration/helpers.bash new file mode 100644 index 00000000..60d96ae0 --- /dev/null +++ b/contrib/docker-integration/helpers.bash @@ -0,0 +1,21 @@ +# Start docker daemon +function start_daemon() { + # Drivers to use for Docker engines the tests are going to create. + STORAGE_DRIVER=${STORAGE_DRIVER:-overlay} + EXEC_DRIVER=${EXEC_DRIVER:-native} + + docker --daemon --log-level=panic \ + --storage-driver="$STORAGE_DRIVER" --exec-driver="$EXEC_DRIVER" & + DOCKER_PID=$! + + # Wait for it to become reachable. + tries=10 + until docker version &> /dev/null; do + (( tries-- )) + if [ $tries -le 0 ]; then + echo >&2 "error: daemon failed to start" + exit 1 + fi + sleep 1 + done +} diff --git a/contrib/docker-integration/install_certs.sh b/contrib/docker-integration/install_certs.sh index c1fa2b20..a23d8bee 100644 --- a/contrib/docker-integration/install_certs.sh +++ b/contrib/docker-integration/install_certs.sh @@ -1,6 +1,10 @@ #!/bin/sh set -e +# Set IP address in /etc/hosts for localregistry +IP=$(ifconfig eth0|grep "inet addr:"| cut -d: -f2 | awk '{ print $1}') +echo "$IP localregistry" >> /etc/hosts + hostname=$1 if [ "$hostname" = "" ]; then hostname="localhost" diff --git a/contrib/docker-integration/run.sh b/contrib/docker-integration/run.sh index 81ca2ad9..707dfa18 100755 --- a/contrib/docker-integration/run.sh +++ b/contrib/docker-integration/run.sh @@ -1,11 +1,20 @@ #!/usr/bin/env bash set -e +set -x + +source helpers.bash cd "$(dirname "$(readlink -f "$BASH_SOURCE")")" +# Port used by engine under test +ENGINE_PORT=5216 + # Root directory of Distribution DISTRIBUTION_ROOT=$(cd ../..; pwd -P) +DOCKER_GRAPHDRIVER=${DOCKER_GRAPHDRIVER:-overlay} +EXEC_DRIVER=${EXEC_DRIVER:-native} + volumeMount="" if [ "$DOCKER_VOLUME" != "" ]; then volumeMount="-v ${DOCKER_VOLUME}:/var/lib/docker" @@ -14,18 +23,58 @@ fi dockerMount="" if [ "$DOCKER_BINARY" != "" ]; then dockerMount="-v ${DOCKER_BINARY}:/usr/local/bin/docker" +else + DOCKER_BINARY=docker fi # Image containing the integration tests environment. INTEGRATION_IMAGE=${INTEGRATION_IMAGE:-distribution/docker-integration} +if [ "$1" == "-d" ]; then + start_daemon + shift +fi + +TESTS=${@:-.} + # Make sure we upgrade the integration environment. docker pull $INTEGRATION_IMAGE -# Start the integration tests in a Docker container. -docker run --rm -t --privileged $volumeMount $dockerMount \ +# Start a Docker engine inside a docker container +ID=$(docker run -d -it -p $ENGINE_PORT:$ENGINE_PORT --privileged $volumeMount $dockerMount \ -v ${DISTRIBUTION_ROOT}:/go/src/github.com/docker/distribution \ - -e "STORAGE_DRIVER=$DOCKER_GRAPHDRIVER" \ + -e "ENGINE_PORT=$ENGINE_PORT" \ + -e "DOCKER_GRAPHDRIVER=$DOCKER_GRAPHDRIVER" \ -e "EXEC_DRIVER=$EXEC_DRIVER" \ ${INTEGRATION_IMAGE} \ - ./test_runner.sh "$@" + ./run_engine.sh) + +# Wait for it to become reachable. +tries=10 +until "$DOCKER_BINARY" -H "127.0.0.1:$ENGINE_PORT" version &> /dev/null; do + (( tries-- )) + if [ $tries -le 0 ]; then + echo >&2 "error: daemon failed to start" + exit 1 + fi + sleep 1 +done + +# Make sure we have images outside the container, to transfer to the container. +# Not much will happen here if the images are already present. +docker-compose pull +docker-compose build + +# Transfer images to the inner container. +for image in "$INTEGRATION_IMAGE" registry:0.9.1 dockerintegration_nginx dockerintegration_registryv2; do + docker save "$image" | "$DOCKER_BINARY" -H "127.0.0.1:$ENGINE_PORT" load +done + +#DOCKER_HOST="tcp://127.0.0.1:$ENGINE_PORT" docker-compose pull +#DOCKER_HOST="tcp://127.0.0.1:$ENGINE_PORT" docker-compose build + +# Run the tests. +docker exec -it "$ID" sh -c "DOCKER_HOST=tcp://127.0.0.1:$ENGINE_PORT ./test_runner.sh $TESTS" + +# Stop container +docker rm -f -v "$ID" diff --git a/contrib/docker-integration/run_engine.sh b/contrib/docker-integration/run_engine.sh new file mode 100755 index 00000000..9c59aa26 --- /dev/null +++ b/contrib/docker-integration/run_engine.sh @@ -0,0 +1,12 @@ +#!/bin/sh +set -e +set -x + +# Set IP address in /etc/hosts for localregistry +IP=$(ifconfig eth0|grep "inet addr:"| cut -d: -f2 | awk '{ print $1}') +echo "$IP localregistry" >> /etc/hosts + +sh install_certs.sh localregistry + +docker --daemon -H "0.0.0.0:$ENGINE_PORT" --log-level=panic \ + --storage-driver="$DOCKER_GRAPHDRIVER" --exec-driver="$EXEC_DRIVER" diff --git a/contrib/docker-integration/run_multiversion.sh b/contrib/docker-integration/run_multiversion.sh index 1917b688..c6da0314 100755 --- a/contrib/docker-integration/run_multiversion.sh +++ b/contrib/docker-integration/run_multiversion.sh @@ -5,44 +5,20 @@ set -e set -x -# Don't use /tmp because this isn't available in boot2docker -tmpdir_template="`pwd`/docker-versions.XXXXX" +source helpers.bash + +if [ `uname` = "Linux" ]; then + tmpdir_template="$TMPDIR/docker-versions.XXXXX" +else + # /tmp isn't available for mounting in boot2docker + tmpdir_template="`pwd`/../../../docker-versions.XXXXX" +fi + tmpdir=`mktemp -d "$tmpdir_template"` trap "rm -rf $tmpdir" EXIT if [ "$1" == "-d" ]; then - # Start docker daemon - - # Drivers to use for Docker engines the tests are going to create. - STORAGE_DRIVER=${STORAGE_DRIVER:-overlay} - EXEC_DRIVER=${EXEC_DRIVER:-native} - - docker --daemon --log-level=panic \ - --storage-driver="$STORAGE_DRIVER" --exec-driver="$EXEC_DRIVER" & - DOCKER_PID=$! - - # Wait for it to become reachable. - tries=10 - until docker version &> /dev/null; do - (( tries-- )) - if [ $tries -le 0 ]; then - echo >&2 "error: daemon failed to start" - exit 1 - fi - sleep 1 - done -fi - -# If DOCKER_VOLUME is unset, create a temporary directory to cache containers -# between runs -# Only do this on Linux, because using /var/lib/docker from a host volume seems -# problematic with boot2docker. -if [ "$DOCKER_VOLUME" = "" -a `uname` = "Linux" ]; then - volumes_template="`pwd`/docker-versions.XXXXX" - volume=`mktemp -d "$volumes_template"` - trap "rm -rf $tmpdir $volume" EXIT -else - volume="$DOCKER_VOLUME" + start_daemon fi # Released versions @@ -56,7 +32,7 @@ for v in $versions; do docker cp "$ID:/usr/local/bin/docker" "$tmpdir/docker-$v" echo "Running tests with Docker $v" - DOCKER_BINARY="$binpath" DOCKER_VOLUME="$volume" ./run.sh + DOCKER_BINARY="$binpath" DOCKER_VOLUME="$DOCKER_VOLUME" DOCKER_GRAPHDRIVER="$DOCKER_GRAPHDRIVER" ./run.sh # Cleanup. docker rm -f "$ID" @@ -71,7 +47,7 @@ ID=$(docker create dockerswarm/dind-master) docker cp "$ID:/usr/local/bin/docker" "$tmpdir/docker-master" echo "Running tests with Docker master" -DOCKER_BINARY="$binpath" DOCKER_VOLUME="$volume" ./run.sh +DOCKER_BINARY="$binpath" DOCKER_VOLUME="$DOCKER_VOLUME" ./run.sh # Cleanup. docker rm -f "$ID" diff --git a/contrib/docker-integration/test_runner.sh b/contrib/docker-integration/test_runner.sh index 2c958c5e..0a628238 100755 --- a/contrib/docker-integration/test_runner.sh +++ b/contrib/docker-integration/test_runner.sh @@ -3,48 +3,16 @@ set -e cd "$(dirname "$(readlink -f "$BASH_SOURCE")")" -# Load the helpers. -#. helpers.bash - TESTS=${@:-.} -# Drivers to use for Docker engines the tests are going to create. -STORAGE_DRIVER=${STORAGE_DRIVER:-overlay} -EXEC_DRIVER=${EXEC_DRIVER:-native} - - function execute() { >&2 echo "++ $@" eval "$@" } -# Set IP address in /etc/hosts for localregistry -IP=$(ifconfig eth0|grep "inet addr:"| cut -d: -f2 | awk '{ print $1}') -execute echo "$IP localregistry" >> /etc/hosts - -# Setup certificates -execute sh install_certs.sh localregistry - -# Start the docker engine. -execute docker --daemon --log-level=panic \ - --storage-driver="$STORAGE_DRIVER" --exec-driver="$EXEC_DRIVER" & -DOCKER_PID=$! - -# Wait for it to become reachable. -tries=10 -until docker version &> /dev/null; do - (( tries-- )) - if [ $tries -le 0 ]; then - echo >&2 "error: daemon failed to start" - exit 1 - fi - sleep 1 -done - execute time docker-compose build execute docker-compose up -d # Run the tests. execute time bats -p $TESTS -