Revert "update to use containerd seccomp package"

This reverts commit 4f8e065faf055d3f0463a92622297ca3afac07f4.
This commit is contained in:
Jess Frazelle 2018-03-22 09:15:36 -04:00
parent 09243b740c
commit 60f032f6f5
8199 changed files with 1598219 additions and 30742 deletions

View file

@ -0,0 +1,60 @@
## About
This directory contains a collection of scripts used to build and manage this
repository. If there are any issues regarding the intention of a particular
script (or even part of a certain script), please reach out to us.
It may help us either refine our current scripts, or add on new ones
that are appropriate for a given use case.
## DinD (dind.sh)
DinD is a wrapper script which allows Docker to be run inside a Docker
container. DinD requires the container to
be run with privileged mode enabled.
## Generate Authors (generate-authors.sh)
Generates AUTHORS; a file with all the names and corresponding emails of
individual contributors. AUTHORS can be found in the home directory of
this repository.
## Make
There are two make files, each with different extensions. Neither are supposed
to be called directly; only invoke `make`. Both scripts run inside a Docker
container.
### make.ps1
- The Windows native build script that uses PowerShell semantics; it is limited
unlike `hack\make.sh` since it does not provide support for the full set of
operations provided by the Linux counterpart, `make.sh`. However, `make.ps1`
does provide support for local Windows development and Windows to Windows CI.
More information is found within `make.ps1` by the author, @jhowardmsft
### make.sh
- Referenced via `make test` when running tests on a local machine,
or directly referenced when running tests inside a Docker development container.
- When running on a local machine, `make test` to run all tests found in
`test`, `test-unit`, `test-integration`, and `test-docker-py` on
your local machine. The default timeout is set in `make.sh` to 60 minutes
(`${TIMEOUT:=60m}`), since it currently takes up to an hour to run
all of the tests.
- When running inside a Docker development container, `hack/make.sh` does
not have a single target that runs all the tests. You need to provide a
single command line with multiple targets that performs the same thing.
An example referenced from [Run targets inside a development container](https://docs.docker.com/opensource/project/test-and-docs/#run-targets-inside-a-development-container): `root@5f8630b873fe:/go/src/github.com/moby/moby# hack/make.sh dynbinary binary cross test-unit test-integration test-docker-py`
- For more information related to testing outside the scope of this README,
refer to
[Run tests and test documentation](https://docs.docker.com/opensource/project/test-and-docs/)
## Release (release.sh)
Releases any bundles built by `make` on a public AWS S3 bucket.
For information regarding configuration, please view `release.sh`.
## Vendor (vendor.sh)
A shell script that is a wrapper around Vndr. For information on how to use
this, please refer to [vndr's README](https://github.com/LK4D4/vndr/blob/master/README.md)

View file

@ -0,0 +1,10 @@
#!/usr/bin/env bash
# Entrypoint for jenkins arm CI build
set -eu -o pipefail
hack/test/unit
hack/make.sh \
binary-daemon \
dynbinary \
test-integration

View file

@ -0,0 +1,9 @@
#!/usr/bin/env bash
# Entrypoint for jenkins experimental CI
set -eu -o pipefail
export DOCKER_EXPERIMENTAL=y
hack/make.sh \
binary-daemon \
test-integration

View file

@ -0,0 +1,17 @@
#!/usr/bin/env bash
# Entrypoint for jenkins janky CI build
set -eu -o pipefail
hack/validate/default
hack/test/unit
bash <(curl -s https://codecov.io/bash) \
-f coverage.txt \
-C $GIT_SHA1 || \
echo 'Codecov failed to upload'
hack/make.sh \
binary-daemon \
dynbinary \
test-docker-py \
test-integration \
cross

View file

@ -0,0 +1,6 @@
#!/usr/bin/env bash
# Entrypoint for jenkins powerpc CI build
set -eu -o pipefail
hack/test/unit
hack/make.sh dynbinary test-integration

View file

@ -0,0 +1,6 @@
#!/usr/bin/env bash
# Entrypoint for jenkins s390x (z) CI build
set -eu -o pipefail
hack/test/unit
hack/make.sh dynbinary test-integration

View file

@ -0,0 +1,33 @@
#!/usr/bin/env bash
set -e
# DinD: a wrapper script which allows docker to be run inside a docker container.
# Original version by Jerome Petazzoni <jerome@docker.com>
# See the blog post: https://blog.docker.com/2013/09/docker-can-now-run-within-docker/
#
# This script should be executed inside a docker container in privileged mode
# ('docker run --privileged', introduced in docker 0.6).
# Usage: dind CMD [ARG...]
# apparmor sucks and Docker needs to know that it's in a container (c) @tianon
export container=docker
if [ -d /sys/kernel/security ] && ! mountpoint -q /sys/kernel/security; then
mount -t securityfs none /sys/kernel/security || {
echo >&2 'Could not mount /sys/kernel/security.'
echo >&2 'AppArmor detection and --privileged mode might break.'
}
fi
# Mount /tmp (conditionally)
if ! mountpoint -q /tmp; then
mount -t tmpfs none /tmp
fi
if [ $# -gt 0 ]; then
exec "$@"
fi
echo >&2 'ERROR: No command specified.'
echo >&2 'You probably want to run hack/make.sh, or maybe a shell?'

View file

@ -0,0 +1,36 @@
#!/bin/sh
# containerd is also pinned in vendor.conf. When updating the binary
# version you may also need to update the vendor version to pick up bug
# fixes or new APIs.
CONTAINERD_COMMIT=cfd04396dc68220d1cecbe686a6cc3aa5ce3667c # v1.0.2
install_containerd() {
echo "Install containerd version $CONTAINERD_COMMIT"
git clone https://github.com/containerd/containerd.git "$GOPATH/src/github.com/containerd/containerd"
cd "$GOPATH/src/github.com/containerd/containerd"
git checkout -q "$CONTAINERD_COMMIT"
(
export BUILDTAGS='static_build netgo'
export EXTRA_FLAGS='-buildmode=pie'
export EXTRA_LDFLAGS='-extldflags "-fno-PIC -static"'
# Reset build flags to nothing if we want a dynbinary
if [ "$1" == "dynamic" ]; then
export BUILDTAGS=''
export EXTRA_FLAGS=''
export EXTRA_LDFLAGS=''
fi
make
)
mkdir -p ${PREFIX}
cp bin/containerd ${PREFIX}/docker-containerd
cp bin/containerd-shim ${PREFIX}/docker-containerd-shim
cp bin/ctr ${PREFIX}/docker-containerd-ctr
}

View file

@ -0,0 +1,31 @@
#!/bin/sh
DOCKERCLI_CHANNEL=${DOCKERCLI_CHANNEL:-edge}
DOCKERCLI_VERSION=${DOCKERCLI_VERSION:-17.06.0-ce}
install_dockercli() {
echo "Install docker/cli version $DOCKERCLI_VERSION from $DOCKERCLI_CHANNEL"
arch=$(uname -m)
# No official release of these platforms
if [[ "$arch" != "x86_64" ]] && [[ "$arch" != "s390x" ]]; then
build_dockercli
return
fi
url=https://download.docker.com/linux/static
curl -Ls $url/$DOCKERCLI_CHANNEL/$arch/docker-$DOCKERCLI_VERSION.tgz | \
tar -xz docker/docker
mkdir -p ${PREFIX}
mv docker/docker ${PREFIX}/
rmdir docker
}
build_dockercli() {
git clone https://github.com/docker/docker-ce "$GOPATH/tmp/docker-ce"
cd "$GOPATH/tmp/docker-ce"
git checkout -q "v$DOCKERCLI_VERSION"
mkdir -p "$GOPATH/src/github.com/docker"
mv components/cli "$GOPATH/src/github.com/docker/cli"
go build -buildmode=pie -o ${PREFIX}/docker github.com/docker/cli/cmd/docker
}

View file

@ -0,0 +1,12 @@
#!/bin/sh
GOMETALINTER_COMMIT=bfcc1d6942136fd86eb6f1a6fb328de8398fbd80
install_gometalinter() {
echo "Installing gometalinter version $GOMETALINTER_COMMIT"
go get -d github.com/alecthomas/gometalinter
cd "$GOPATH/src/github.com/alecthomas/gometalinter"
git checkout -q "$GOMETALINTER_COMMIT"
go build -buildmode=pie -o ${PREFIX}/gometalinter github.com/alecthomas/gometalinter
GOBIN=${PREFIX} ${PREFIX}/gometalinter --install
}

View file

@ -0,0 +1,30 @@
#!/bin/bash
set -e
set -x
RM_GOPATH=0
TMP_GOPATH=${TMP_GOPATH:-""}
: ${PREFIX:="/usr/local/bin"}
if [ -z "$TMP_GOPATH" ]; then
export GOPATH="$(mktemp -d)"
RM_GOPATH=1
else
export GOPATH="$TMP_GOPATH"
fi
dir="$(dirname $0)"
bin=$1
shift
if [ ! -f "${dir}/${bin}.installer" ]; then
echo "Could not find installer for \"$bin\""
exit 1
fi
. $dir/$bin.installer
install_$bin "$@"

View file

@ -0,0 +1,38 @@
#!/bin/sh
# LIBNETWORK_COMMIT is used to build the docker-userland-proxy binary. When
# updating the binary version, consider updating github.com/docker/libnetwork
# in vendor.conf accordingly
LIBNETWORK_COMMIT=1b91bc94094ecfdae41daa465cc0c8df37dfb3dd
install_proxy() {
case "$1" in
"dynamic")
install_proxy_dynamic
return
;;
"")
export CGO_ENABLED=0
_install_proxy
;;
*)
echo 'Usage: $0 [dynamic]'
;;
esac
}
install_proxy_dynamic() {
export PROXY_LDFLAGS="-linkmode=external" install_proxy
export BUILD_MODE="-buildmode=pie"
_install_proxy
}
_install_proxy() {
echo "Install docker-proxy version $LIBNETWORK_COMMIT"
git clone https://github.com/docker/libnetwork.git "$GOPATH/src/github.com/docker/libnetwork"
cd "$GOPATH/src/github.com/docker/libnetwork"
git checkout -q "$LIBNETWORK_COMMIT"
go build $BUILD_MODE -ldflags="$PROXY_LDFLAGS" -o ${PREFIX}/docker-proxy github.com/docker/libnetwork/cmd/proxy
}

View file

@ -0,0 +1,22 @@
#!/bin/sh
# When updating RUNC_COMMIT, also update runc in vendor.conf accordingly
RUNC_COMMIT=4fc53a81fb7c994640722ac585fa9ca548971871
install_runc() {
# Do not build with ambient capabilities support
RUNC_BUILDTAGS="${RUNC_BUILDTAGS:-"seccomp apparmor selinux"}"
echo "Install runc version $RUNC_COMMIT"
git clone https://github.com/opencontainers/runc.git "$GOPATH/src/github.com/opencontainers/runc"
cd "$GOPATH/src/github.com/opencontainers/runc"
git checkout -q "$RUNC_COMMIT"
if [ -z "$1" ]; then
target=static
else
target="$1"
fi
make BUILDTAGS="$RUNC_BUILDTAGS" "$target"
mkdir -p ${PREFIX}
cp runc ${PREFIX}/docker-runc
}

View file

@ -0,0 +1,14 @@
#!/bin/sh
TINI_COMMIT=949e6facb77383876aeff8a6944dde66b3089574
install_tini() {
echo "Install tini version $TINI_COMMIT"
git clone https://github.com/krallin/tini.git "$GOPATH/tini"
cd "$GOPATH/tini"
git checkout -q "$TINI_COMMIT"
cmake .
make tini-static
mkdir -p ${PREFIX}
cp tini-static ${PREFIX}/docker-init
}

View file

@ -0,0 +1,12 @@
#!/bin/sh
# When updating TOMLV_COMMIT, consider updating github.com/BurntSushi/toml
# in vendor.conf accordingly
TOMLV_COMMIT=a368813c5e648fee92e5f6c30e3944ff9d5e8895
install_tomlv() {
echo "Install tomlv version $TOMLV_COMMIT"
git clone https://github.com/BurntSushi/toml.git "$GOPATH/src/github.com/BurntSushi/toml"
cd "$GOPATH/src/github.com/BurntSushi/toml" && git checkout -q "$TOMLV_COMMIT"
go build -v -buildmode=pie -o ${PREFIX}/tomlv github.com/BurntSushi/toml/cmd/tomlv
}

View file

@ -0,0 +1,11 @@
#!/bin/sh
VNDR_COMMIT=a6e196d8b4b0cbbdc29aebdb20c59ac6926bb384
install_vndr() {
echo "Install vndr version $VNDR_COMMIT"
git clone https://github.com/LK4D4/vndr.git "$GOPATH/src/github.com/LK4D4/vndr"
cd "$GOPATH/src/github.com/LK4D4/vndr"
git checkout -q "$VNDR_COMMIT"
go build -buildmode=pie -v -o ${PREFIX}/vndr .
}

View file

@ -0,0 +1,15 @@
#!/usr/bin/env bash
set -e
cd "$(dirname "$(readlink -f "$BASH_SOURCE")")/.."
# see also ".mailmap" for how email addresses and names are deduplicated
{
cat <<-'EOH'
# This file lists all individuals having contributed content to the repository.
# For how it is generated, see `hack/generate-authors.sh`.
EOH
echo
git log --format='%aN <%aE>' | LC_ALL=C.UTF-8 sort -uf
} > AUTHORS

View file

@ -0,0 +1,27 @@
#!/bin/sh
set -eu
swagger generate model -f api/swagger.yaml \
-t api -m types --skip-validator -C api/swagger-gen.yaml \
-n ErrorResponse \
-n GraphDriverData \
-n IdResponse \
-n ImageDeleteResponseItem \
-n ImageSummary \
-n Plugin -n PluginDevice -n PluginMount -n PluginEnv -n PluginInterfaceType \
-n Port \
-n ServiceUpdateResponse \
-n Volume
swagger generate operation -f api/swagger.yaml \
-t api -a types -m types -C api/swagger-gen.yaml \
-T api/templates --skip-responses --skip-parameters --skip-validator \
-n Authenticate \
-n ContainerChanges \
-n ContainerCreate \
-n ContainerTop \
-n ContainerUpdate \
-n ContainerWait \
-n ImageHistory \
-n VolumesCreate \
-n VolumesList

View file

@ -0,0 +1,69 @@
# Integration Testing on Swarm
IT on Swarm allows you to execute integration test in parallel across a Docker Swarm cluster
## Architecture
### Master service
- Works as a funker caller
- Calls a worker funker (`-worker-service`) with a chunk of `-check.f` filter strings (passed as a file via `-input` flag, typically `/mnt/input`)
### Worker service
- Works as a funker callee
- Executes an equivalent of `TESTFLAGS=-check.f TestFoo|TestBar|TestBaz ... make test-integration` using the bind-mounted API socket (`docker.sock`)
### Client
- Controls master and workers via `docker stack`
- No need to have a local daemon
Typically, the master and workers are supposed to be running on a cloud environment,
while the client is supposed to be running on a laptop, e.g. Docker for Mac/Windows.
## Requirement
- Docker daemon 1.13 or later
- Private registry for distributed execution with multiple nodes
## Usage
### Step 1: Prepare images
$ make build-integration-cli-on-swarm
Following environment variables are known to work in this step:
- `BUILDFLAGS`
- `DOCKER_INCREMENTAL_BINARY`
Note: during the transition into Moby Project, you might need to create a symbolic link `$GOPATH/src/github.com/docker/docker` to `$GOPATH/src/github.com/moby/moby`.
### Step 2: Execute tests
$ ./hack/integration-cli-on-swarm/integration-cli-on-swarm -replicas 40 -push-worker-image YOUR_REGISTRY.EXAMPLE.COM/integration-cli-worker:latest
Following environment variables are known to work in this step:
- `DOCKER_GRAPHDRIVER`
- `DOCKER_EXPERIMENTAL`
#### Flags
Basic flags:
- `-replicas N`: the number of worker service replicas. i.e. degree of parallelism.
- `-chunks N`: the number of chunks. By default, `chunks` == `replicas`.
- `-push-worker-image REGISTRY/IMAGE:TAG`: push the worker image to the registry. Note that if you have only single node and hence you do not need a private registry, you do not need to specify `-push-worker-image`.
Experimental flags for mitigating makespan nonuniformity:
- `-shuffle`: Shuffle the test filter strings
Flags for debugging IT on Swarm itself:
- `-rand-seed N`: the random seed. This flag is useful for deterministic replaying. By default(0), the timestamp is used.
- `-filters-file FILE`: the file contains `-check.f` strings. By default, the file is automatically generated.
- `-dry-run`: skip the actual workload
- `keep-executor`: do not auto-remove executor containers, which is used for running privileged programs on Swarm

View file

@ -0,0 +1,6 @@
# this Dockerfile is solely used for the master image.
# Please refer to the top-level Makefile for the worker image.
FROM golang:1.7
ADD . /go/src/github.com/docker/docker/hack/integration-cli-on-swarm/agent
RUN go build -buildmode=pie -o /master github.com/docker/docker/hack/integration-cli-on-swarm/agent/master
ENTRYPOINT ["/master"]

View file

@ -0,0 +1,132 @@
package main
import (
"encoding/json"
"fmt"
"log"
"strings"
"sync"
"sync/atomic"
"time"
"github.com/bfirsh/funker-go"
"github.com/docker/docker/hack/integration-cli-on-swarm/agent/types"
)
const (
// funkerRetryTimeout is for the issue https://github.com/bfirsh/funker/issues/3
// When all the funker replicas are busy in their own job, we cannot connect to funker.
funkerRetryTimeout = 1 * time.Hour
funkerRetryDuration = 1 * time.Second
)
// ticker is needed for some CI (e.g., on Travis, job is aborted when no output emitted for 10 minutes)
func ticker(d time.Duration) chan struct{} {
t := time.NewTicker(d)
stop := make(chan struct{})
go func() {
for {
select {
case <-t.C:
log.Printf("tick (just for keeping CI job active) per %s", d.String())
case <-stop:
t.Stop()
}
}
}()
return stop
}
func executeTests(funkerName string, testChunks [][]string) error {
tickerStopper := ticker(9*time.Minute + 55*time.Second)
defer func() {
close(tickerStopper)
}()
begin := time.Now()
log.Printf("Executing %d chunks in parallel, against %q", len(testChunks), funkerName)
var wg sync.WaitGroup
var passed, failed uint32
for chunkID, tests := range testChunks {
log.Printf("Executing chunk %d (contains %d test filters)", chunkID, len(tests))
wg.Add(1)
go func(chunkID int, tests []string) {
defer wg.Done()
chunkBegin := time.Now()
result, err := executeTestChunkWithRetry(funkerName, types.Args{
ChunkID: chunkID,
Tests: tests,
})
if result.RawLog != "" {
for _, s := range strings.Split(result.RawLog, "\n") {
log.Printf("Log (chunk %d): %s", chunkID, s)
}
}
if err != nil {
log.Printf("Error while executing chunk %d: %v",
chunkID, err)
atomic.AddUint32(&failed, 1)
} else {
if result.Code == 0 {
atomic.AddUint32(&passed, 1)
} else {
atomic.AddUint32(&failed, 1)
}
log.Printf("Finished chunk %d [%d/%d] with %d test filters in %s, code=%d.",
chunkID, passed+failed, len(testChunks), len(tests),
time.Since(chunkBegin), result.Code)
}
}(chunkID, tests)
}
wg.Wait()
// TODO: print actual tests rather than chunks
log.Printf("Executed %d chunks in %s. PASS: %d, FAIL: %d.",
len(testChunks), time.Since(begin), passed, failed)
if failed > 0 {
return fmt.Errorf("%d chunks failed", failed)
}
return nil
}
func executeTestChunk(funkerName string, args types.Args) (types.Result, error) {
ret, err := funker.Call(funkerName, args)
if err != nil {
return types.Result{}, err
}
tmp, err := json.Marshal(ret)
if err != nil {
return types.Result{}, err
}
var result types.Result
err = json.Unmarshal(tmp, &result)
return result, err
}
func executeTestChunkWithRetry(funkerName string, args types.Args) (types.Result, error) {
begin := time.Now()
for i := 0; time.Since(begin) < funkerRetryTimeout; i++ {
result, err := executeTestChunk(funkerName, args)
if err == nil {
log.Printf("executeTestChunk(%q, %d) returned code %d in trial %d", funkerName, args.ChunkID, result.Code, i)
return result, nil
}
if errorSeemsInteresting(err) {
log.Printf("Error while calling executeTestChunk(%q, %d), will retry (trial %d): %v",
funkerName, args.ChunkID, i, err)
}
// TODO: non-constant sleep
time.Sleep(funkerRetryDuration)
}
return types.Result{}, fmt.Errorf("could not call executeTestChunk(%q, %d) in %v", funkerName, args.ChunkID, funkerRetryTimeout)
}
// errorSeemsInteresting returns true if err does not seem about https://github.com/bfirsh/funker/issues/3
func errorSeemsInteresting(err error) bool {
boringSubstrs := []string{"connection refused", "connection reset by peer", "no such host", "transport endpoint is not connected", "no route to host"}
errS := err.Error()
for _, boringS := range boringSubstrs {
if strings.Contains(errS, boringS) {
return false
}
}
return true
}

View file

@ -0,0 +1,65 @@
package main
import (
"errors"
"flag"
"io/ioutil"
"log"
"strings"
)
func main() {
if err := xmain(); err != nil {
log.Fatalf("fatal error: %v", err)
}
}
func xmain() error {
workerService := flag.String("worker-service", "", "Name of worker service")
chunks := flag.Int("chunks", 0, "Number of chunks")
input := flag.String("input", "", "Path to input file")
randSeed := flag.Int64("rand-seed", int64(0), "Random seed")
shuffle := flag.Bool("shuffle", false, "Shuffle the input so as to mitigate makespan nonuniformity")
flag.Parse()
if *workerService == "" {
return errors.New("worker-service unset")
}
if *chunks == 0 {
return errors.New("chunks unset")
}
if *input == "" {
return errors.New("input unset")
}
tests, err := loadTests(*input)
if err != nil {
return err
}
testChunks := chunkTests(tests, *chunks, *shuffle, *randSeed)
log.Printf("Loaded %d tests (%d chunks)", len(tests), len(testChunks))
return executeTests(*workerService, testChunks)
}
func chunkTests(tests []string, numChunks int, shuffle bool, randSeed int64) [][]string {
// shuffling (experimental) mitigates makespan nonuniformity
// Not sure this can cause some locality problem..
if shuffle {
shuffleStrings(tests, randSeed)
}
return chunkStrings(tests, numChunks)
}
func loadTests(filename string) ([]string, error) {
b, err := ioutil.ReadFile(filename)
if err != nil {
return nil, err
}
var tests []string
for _, line := range strings.Split(string(b), "\n") {
s := strings.TrimSpace(line)
if s != "" {
tests = append(tests, s)
}
}
return tests, nil
}

View file

@ -0,0 +1,28 @@
package main
import (
"math/rand"
)
// chunkStrings chunks the string slice
func chunkStrings(x []string, numChunks int) [][]string {
var result [][]string
chunkSize := (len(x) + numChunks - 1) / numChunks
for i := 0; i < len(x); i += chunkSize {
ub := i + chunkSize
if ub > len(x) {
ub = len(x)
}
result = append(result, x[i:ub])
}
return result
}
// shuffleStrings shuffles strings
func shuffleStrings(x []string, seed int64) {
r := rand.New(rand.NewSource(seed))
for i := range x {
j := r.Intn(i + 1)
x[i], x[j] = x[j], x[i]
}
}

View file

@ -0,0 +1,63 @@
package main
import (
"fmt"
"reflect"
"testing"
"time"
)
func generateInput(inputLen int) []string {
input := []string{}
for i := 0; i < inputLen; i++ {
input = append(input, fmt.Sprintf("s%d", i))
}
return input
}
func testChunkStrings(t *testing.T, inputLen, numChunks int) {
t.Logf("inputLen=%d, numChunks=%d", inputLen, numChunks)
input := generateInput(inputLen)
result := chunkStrings(input, numChunks)
t.Logf("result has %d chunks", len(result))
inputReconstructedFromResult := []string{}
for i, chunk := range result {
t.Logf("chunk %d has %d elements", i, len(chunk))
inputReconstructedFromResult = append(inputReconstructedFromResult, chunk...)
}
if !reflect.DeepEqual(input, inputReconstructedFromResult) {
t.Fatal("input != inputReconstructedFromResult")
}
}
func TestChunkStrings_4_4(t *testing.T) {
testChunkStrings(t, 4, 4)
}
func TestChunkStrings_4_1(t *testing.T) {
testChunkStrings(t, 4, 1)
}
func TestChunkStrings_1_4(t *testing.T) {
testChunkStrings(t, 1, 4)
}
func TestChunkStrings_1000_8(t *testing.T) {
testChunkStrings(t, 1000, 8)
}
func TestChunkStrings_1000_9(t *testing.T) {
testChunkStrings(t, 1000, 9)
}
func testShuffleStrings(t *testing.T, inputLen int, seed int64) {
t.Logf("inputLen=%d, seed=%d", inputLen, seed)
x := generateInput(inputLen)
shuffleStrings(x, seed)
t.Logf("shuffled: %v", x)
}
func TestShuffleStrings_100(t *testing.T) {
testShuffleStrings(t, 100, time.Now().UnixNano())
}

View file

@ -0,0 +1,18 @@
package types
// Args is the type for funker args
type Args struct {
// ChunkID is an unique number of the chunk
ChunkID int `json:"chunk_id"`
// Tests is the set of the strings that are passed as `-check.f` filters
Tests []string `json:"tests"`
}
// Result is the type for funker result
type Result struct {
// ChunkID corresponds to Args.ChunkID
ChunkID int `json:"chunk_id"`
// Code is the exit code
Code int `json:"code"`
RawLog string `json:"raw_log"`
}

View file

@ -0,0 +1,2 @@
# dependencies specific to worker (i.e. github.com/docker/docker/...) are not vendored here
github.com/bfirsh/funker-go eaa0a2e06f30e72c9a0b7f858951e581e26ef773

View file

@ -0,0 +1,118 @@
package main
import (
"bytes"
"context"
"fmt"
"io"
"os"
"strings"
"github.com/docker/docker/api/types"
"github.com/docker/docker/api/types/container"
"github.com/docker/docker/api/types/mount"
"github.com/docker/docker/client"
"github.com/docker/docker/pkg/stdcopy"
)
// testChunkExecutor executes integration-cli binary.
// image needs to be the worker image itself. testFlags are OR-set of regexp for filtering tests.
type testChunkExecutor func(image string, tests []string) (int64, string, error)
func dryTestChunkExecutor() testChunkExecutor {
return func(image string, tests []string) (int64, string, error) {
return 0, fmt.Sprintf("DRY RUN (image=%q, tests=%v)", image, tests), nil
}
}
// privilegedTestChunkExecutor invokes a privileged container from the worker
// service via bind-mounted API socket so as to execute the test chunk
func privilegedTestChunkExecutor(autoRemove bool) testChunkExecutor {
return func(image string, tests []string) (int64, string, error) {
cli, err := client.NewEnvClient()
if err != nil {
return 0, "", err
}
// propagate variables from the host (needs to be defined in the compose file)
experimental := os.Getenv("DOCKER_EXPERIMENTAL")
graphdriver := os.Getenv("DOCKER_GRAPHDRIVER")
if graphdriver == "" {
info, err := cli.Info(context.Background())
if err != nil {
return 0, "", err
}
graphdriver = info.Driver
}
// `daemon_dest` is similar to `$DEST` (e.g. `bundles/VERSION/test-integration`)
// but it exists outside of `bundles` so as to make `$DOCKER_GRAPHDRIVER` work.
//
// Without this hack, `$DOCKER_GRAPHDRIVER` fails because of (e.g.) `overlay2 is not supported over overlayfs`
//
// see integration-cli/daemon/daemon.go
daemonDest := "/daemon_dest"
config := container.Config{
Image: image,
Env: []string{
"TESTFLAGS=-check.f " + strings.Join(tests, "|"),
"KEEPBUNDLE=1",
"DOCKER_INTEGRATION_TESTS_VERIFIED=1", // for avoiding rebuilding integration-cli
"DOCKER_EXPERIMENTAL=" + experimental,
"DOCKER_GRAPHDRIVER=" + graphdriver,
"DOCKER_INTEGRATION_DAEMON_DEST=" + daemonDest,
},
Labels: map[string]string{
"org.dockerproject.integration-cli-on-swarm": "",
"org.dockerproject.integration-cli-on-swarm.comment": "this non-service container is created for running privileged programs on Swarm. you can remove this container manually if the corresponding service is already stopped.",
},
Entrypoint: []string{"hack/dind"},
Cmd: []string{"hack/make.sh", "test-integration"},
}
hostConfig := container.HostConfig{
AutoRemove: autoRemove,
Privileged: true,
Mounts: []mount.Mount{
{
Type: mount.TypeVolume,
Target: daemonDest,
},
},
}
id, stream, err := runContainer(context.Background(), cli, config, hostConfig)
if err != nil {
return 0, "", err
}
var b bytes.Buffer
teeContainerStream(&b, os.Stdout, os.Stderr, stream)
resultC, errC := cli.ContainerWait(context.Background(), id, "")
select {
case err := <-errC:
return 0, "", err
case result := <-resultC:
return result.StatusCode, b.String(), nil
}
}
}
func runContainer(ctx context.Context, cli *client.Client, config container.Config, hostConfig container.HostConfig) (string, io.ReadCloser, error) {
created, err := cli.ContainerCreate(context.Background(),
&config, &hostConfig, nil, "")
if err != nil {
return "", nil, err
}
if err = cli.ContainerStart(ctx, created.ID, types.ContainerStartOptions{}); err != nil {
return "", nil, err
}
stream, err := cli.ContainerLogs(ctx,
created.ID,
types.ContainerLogsOptions{
ShowStdout: true,
ShowStderr: true,
Follow: true,
})
return created.ID, stream, err
}
func teeContainerStream(w, stdout, stderr io.Writer, stream io.ReadCloser) {
stdcopy.StdCopy(io.MultiWriter(w, stdout), io.MultiWriter(w, stderr), stream)
stream.Close()
}

View file

@ -0,0 +1,69 @@
package main
import (
"flag"
"fmt"
"log"
"time"
"github.com/bfirsh/funker-go"
"github.com/docker/distribution/reference"
"github.com/docker/docker/hack/integration-cli-on-swarm/agent/types"
)
func main() {
if err := xmain(); err != nil {
log.Fatalf("fatal error: %v", err)
}
}
func validImageDigest(s string) bool {
return reference.DigestRegexp.FindString(s) != ""
}
func xmain() error {
workerImageDigest := flag.String("worker-image-digest", "", "Needs to be the digest of this worker image itself")
dryRun := flag.Bool("dry-run", false, "Dry run")
keepExecutor := flag.Bool("keep-executor", false, "Do not auto-remove executor containers, which is used for running privileged programs on Swarm")
flag.Parse()
if !validImageDigest(*workerImageDigest) {
// Because of issue #29582.
// `docker service create localregistry.example.com/blahblah:latest` pulls the image data to local, but not a tag.
// So, `docker run localregistry.example.com/blahblah:latest` fails: `Unable to find image 'localregistry.example.com/blahblah:latest' locally`
return fmt.Errorf("worker-image-digest must be a digest, got %q", *workerImageDigest)
}
executor := privilegedTestChunkExecutor(!*keepExecutor)
if *dryRun {
executor = dryTestChunkExecutor()
}
return handle(*workerImageDigest, executor)
}
func handle(workerImageDigest string, executor testChunkExecutor) error {
log.Printf("Waiting for a funker request")
return funker.Handle(func(args *types.Args) types.Result {
log.Printf("Executing chunk %d, contains %d test filters",
args.ChunkID, len(args.Tests))
begin := time.Now()
code, rawLog, err := executor(workerImageDigest, args.Tests)
if err != nil {
log.Printf("Error while executing chunk %d: %v", args.ChunkID, err)
if code == 0 {
// Make sure this is a failure
code = 1
}
return types.Result{
ChunkID: args.ChunkID,
Code: int(code),
RawLog: rawLog,
}
}
elapsed := time.Since(begin)
log.Printf("Finished chunk %d, code=%d, elapsed=%v", args.ChunkID, code, elapsed)
return types.Result{
ChunkID: args.ChunkID,
Code: int(code),
RawLog: rawLog,
}
})
}

View file

@ -0,0 +1,122 @@
package main
import (
"context"
"io/ioutil"
"os"
"path/filepath"
"text/template"
"github.com/docker/docker/client"
)
const composeTemplate = `# generated by integration-cli-on-swarm
version: "3"
services:
worker:
image: "{{.WorkerImage}}"
command: ["-worker-image-digest={{.WorkerImageDigest}}", "-dry-run={{.DryRun}}", "-keep-executor={{.KeepExecutor}}"]
networks:
- net
volumes:
# Bind-mount the API socket so that we can invoke "docker run --privileged" within the service containers
- /var/run/docker.sock:/var/run/docker.sock
environment:
- DOCKER_GRAPHDRIVER={{.EnvDockerGraphDriver}}
- DOCKER_EXPERIMENTAL={{.EnvDockerExperimental}}
deploy:
mode: replicated
replicas: {{.Replicas}}
restart_policy:
# The restart condition needs to be any for funker function
condition: any
master:
image: "{{.MasterImage}}"
command: ["-worker-service=worker", "-input=/mnt/input", "-chunks={{.Chunks}}", "-shuffle={{.Shuffle}}", "-rand-seed={{.RandSeed}}"]
networks:
- net
volumes:
- {{.Volume}}:/mnt
deploy:
mode: replicated
replicas: 1
restart_policy:
condition: none
placement:
# Make sure the master can access the volume
constraints: [node.id == {{.SelfNodeID}}]
networks:
net:
volumes:
{{.Volume}}:
external: true
`
type composeOptions struct {
Replicas int
Chunks int
MasterImage string
WorkerImage string
Volume string
Shuffle bool
RandSeed int64
DryRun bool
KeepExecutor bool
}
type composeTemplateOptions struct {
composeOptions
WorkerImageDigest string
SelfNodeID string
EnvDockerGraphDriver string
EnvDockerExperimental string
}
// createCompose creates "dir/docker-compose.yml".
// If dir is empty, TempDir() is used.
func createCompose(dir string, cli *client.Client, opts composeOptions) (string, error) {
if dir == "" {
var err error
dir, err = ioutil.TempDir("", "integration-cli-on-swarm-")
if err != nil {
return "", err
}
}
resolved := composeTemplateOptions{}
resolved.composeOptions = opts
workerImageInspect, _, err := cli.ImageInspectWithRaw(context.Background(), defaultWorkerImageName)
if err != nil {
return "", err
}
if len(workerImageInspect.RepoDigests) > 0 {
resolved.WorkerImageDigest = workerImageInspect.RepoDigests[0]
} else {
// fall back for non-pushed image
resolved.WorkerImageDigest = workerImageInspect.ID
}
info, err := cli.Info(context.Background())
if err != nil {
return "", err
}
resolved.SelfNodeID = info.Swarm.NodeID
resolved.EnvDockerGraphDriver = os.Getenv("DOCKER_GRAPHDRIVER")
resolved.EnvDockerExperimental = os.Getenv("DOCKER_EXPERIMENTAL")
composeFilePath := filepath.Join(dir, "docker-compose.yml")
tmpl, err := template.New("").Parse(composeTemplate)
if err != nil {
return "", err
}
f, err := os.Create(composeFilePath)
if err != nil {
return "", err
}
defer f.Close()
if err = tmpl.Execute(f, resolved); err != nil {
return "", err
}
return composeFilePath, nil
}

View file

@ -0,0 +1,64 @@
package main
import (
"fmt"
"os"
"os/exec"
"strings"
"time"
"github.com/docker/docker/client"
)
func system(commands [][]string) error {
for _, c := range commands {
cmd := exec.Command(c[0], c[1:]...)
cmd.Stdout = os.Stdout
cmd.Stderr = os.Stderr
cmd.Env = os.Environ()
if err := cmd.Run(); err != nil {
return err
}
}
return nil
}
func pushImage(unusedCli *client.Client, remote, local string) error {
// FIXME: eliminate os/exec (but it is hard to pass auth without os/exec ...)
return system([][]string{
{"docker", "image", "tag", local, remote},
{"docker", "image", "push", remote},
})
}
func deployStack(unusedCli *client.Client, stackName, composeFilePath string) error {
// FIXME: eliminate os/exec (but stack is implemented in CLI ...)
return system([][]string{
{"docker", "stack", "deploy",
"--compose-file", composeFilePath,
"--with-registry-auth",
stackName},
})
}
func hasStack(unusedCli *client.Client, stackName string) bool {
// FIXME: eliminate os/exec (but stack is implemented in CLI ...)
out, err := exec.Command("docker", "stack", "ls").CombinedOutput()
if err != nil {
panic(fmt.Errorf("`docker stack ls` failed with: %s", string(out)))
}
// FIXME: not accurate
return strings.Contains(string(out), stackName)
}
func removeStack(unusedCli *client.Client, stackName string) error {
// FIXME: eliminate os/exec (but stack is implemented in CLI ...)
if err := system([][]string{
{"docker", "stack", "rm", stackName},
}); err != nil {
return err
}
// FIXME
time.Sleep(10 * time.Second)
return nil
}

View file

@ -0,0 +1,55 @@
package main
import (
"fmt"
"io/ioutil"
"path/filepath"
"regexp"
)
var testFuncRegexp *regexp.Regexp
func init() {
testFuncRegexp = regexp.MustCompile(`(?m)^\s*func\s+\(\w*\s*\*(\w+Suite)\)\s+(Test\w+)`)
}
func enumerateTestsForBytes(b []byte) ([]string, error) {
var tests []string
submatches := testFuncRegexp.FindAllSubmatch(b, -1)
for _, submatch := range submatches {
if len(submatch) == 3 {
tests = append(tests, fmt.Sprintf("%s.%s$", submatch[1], submatch[2]))
}
}
return tests, nil
}
// enumerateTests enumerates valid `-check.f` strings for all the test functions.
// Note that we use regexp rather than parsing Go files for performance reason.
// (Try `TESTFLAGS=-check.list make test-integration` to see the slowness of parsing)
// The files needs to be `gofmt`-ed
//
// The result will be as follows, but unsorted ('$' is appended because they are regexp for `-check.f`):
// "DockerAuthzSuite.TestAuthZPluginAPIDenyResponse$"
// "DockerAuthzSuite.TestAuthZPluginAllowEventStream$"
// ...
// "DockerTrustedSwarmSuite.TestTrustedServiceUpdate$"
func enumerateTests(wd string) ([]string, error) {
testGoFiles, err := filepath.Glob(filepath.Join(wd, "integration-cli", "*_test.go"))
if err != nil {
return nil, err
}
var allTests []string
for _, testGoFile := range testGoFiles {
b, err := ioutil.ReadFile(testGoFile)
if err != nil {
return nil, err
}
tests, err := enumerateTestsForBytes(b)
if err != nil {
return nil, err
}
allTests = append(allTests, tests...)
}
return allTests, nil
}

View file

@ -0,0 +1,84 @@
package main
import (
"os"
"path/filepath"
"reflect"
"sort"
"strings"
"testing"
)
func getRepoTopDir(t *testing.T) string {
wd, err := os.Getwd()
if err != nil {
t.Fatal(err)
}
wd = filepath.Clean(wd)
suffix := "hack/integration-cli-on-swarm/host"
if !strings.HasSuffix(wd, suffix) {
t.Skipf("cwd seems strange (needs to have suffix %s): %v", suffix, wd)
}
return filepath.Clean(filepath.Join(wd, "../../.."))
}
func TestEnumerateTests(t *testing.T) {
if testing.Short() {
t.Skip("skipping in short mode")
}
tests, err := enumerateTests(getRepoTopDir(t))
if err != nil {
t.Fatal(err)
}
sort.Strings(tests)
t.Logf("enumerated %d test filter strings:", len(tests))
for _, s := range tests {
t.Logf("- %q", s)
}
}
func TestEnumerateTestsForBytes(t *testing.T) {
b := []byte(`package main
import (
"github.com/go-check/check"
)
func (s *FooSuite) TestA(c *check.C) {
}
func (s *FooSuite) TestAAA(c *check.C) {
}
func (s *BarSuite) TestBar(c *check.C) {
}
func (x *FooSuite) TestC(c *check.C) {
}
func (*FooSuite) TestD(c *check.C) {
}
// should not be counted
func (s *FooSuite) testE(c *check.C) {
}
// counted, although we don't support ungofmt file
func (s *FooSuite) TestF (c *check.C){}
`)
expected := []string{
"FooSuite.TestA$",
"FooSuite.TestAAA$",
"BarSuite.TestBar$",
"FooSuite.TestC$",
"FooSuite.TestD$",
"FooSuite.TestF$",
}
actual, err := enumerateTestsForBytes(b)
if err != nil {
t.Fatal(err)
}
if !reflect.DeepEqual(expected, actual) {
t.Fatalf("expected %q, got %q", expected, actual)
}
}

View file

@ -0,0 +1,198 @@
package main
import (
"context"
"flag"
"fmt"
"io"
"io/ioutil"
"os"
"strings"
"time"
"github.com/docker/docker/api/types"
"github.com/docker/docker/api/types/filters"
"github.com/docker/docker/client"
"github.com/docker/docker/pkg/stdcopy"
"github.com/sirupsen/logrus"
)
const (
defaultStackName = "integration-cli-on-swarm"
defaultVolumeName = "integration-cli-on-swarm"
defaultMasterImageName = "integration-cli-master"
defaultWorkerImageName = "integration-cli-worker"
)
func main() {
rc, err := xmain()
if err != nil {
logrus.Fatalf("fatal error: %v", err)
}
os.Exit(rc)
}
func xmain() (int, error) {
// Should we use cobra maybe?
replicas := flag.Int("replicas", 1, "Number of worker service replica")
chunks := flag.Int("chunks", 0, "Number of test chunks executed in batch (0 == replicas)")
pushWorkerImage := flag.String("push-worker-image", "", "Push the worker image to the registry. Required for distributed execution. (empty == not to push)")
shuffle := flag.Bool("shuffle", false, "Shuffle the input so as to mitigate makespan nonuniformity")
// flags below are rarely used
randSeed := flag.Int64("rand-seed", int64(0), "Random seed used for shuffling (0 == current time)")
filtersFile := flag.String("filters-file", "", "Path to optional file composed of `-check.f` filter strings")
dryRun := flag.Bool("dry-run", false, "Dry run")
keepExecutor := flag.Bool("keep-executor", false, "Do not auto-remove executor containers, which is used for running privileged programs on Swarm")
flag.Parse()
if *chunks == 0 {
*chunks = *replicas
}
if *randSeed == int64(0) {
*randSeed = time.Now().UnixNano()
}
cli, err := client.NewEnvClient()
if err != nil {
return 1, err
}
if hasStack(cli, defaultStackName) {
logrus.Infof("Removing stack %s", defaultStackName)
removeStack(cli, defaultStackName)
}
if hasVolume(cli, defaultVolumeName) {
logrus.Infof("Removing volume %s", defaultVolumeName)
removeVolume(cli, defaultVolumeName)
}
if err = ensureImages(cli, []string{defaultWorkerImageName, defaultMasterImageName}); err != nil {
return 1, err
}
workerImageForStack := defaultWorkerImageName
if *pushWorkerImage != "" {
logrus.Infof("Pushing %s to %s", defaultWorkerImageName, *pushWorkerImage)
if err = pushImage(cli, *pushWorkerImage, defaultWorkerImageName); err != nil {
return 1, err
}
workerImageForStack = *pushWorkerImage
}
compose, err := createCompose("", cli, composeOptions{
Replicas: *replicas,
Chunks: *chunks,
MasterImage: defaultMasterImageName,
WorkerImage: workerImageForStack,
Volume: defaultVolumeName,
Shuffle: *shuffle,
RandSeed: *randSeed,
DryRun: *dryRun,
KeepExecutor: *keepExecutor,
})
if err != nil {
return 1, err
}
filters, err := filtersBytes(*filtersFile)
if err != nil {
return 1, err
}
logrus.Infof("Creating volume %s with input data", defaultVolumeName)
if err = createVolumeWithData(cli,
defaultVolumeName,
map[string][]byte{"/input": filters},
defaultMasterImageName); err != nil {
return 1, err
}
logrus.Infof("Deploying stack %s from %s", defaultStackName, compose)
defer func() {
logrus.Infof("NOTE: You may want to inspect or clean up following resources:")
logrus.Infof(" - Stack: %s", defaultStackName)
logrus.Infof(" - Volume: %s", defaultVolumeName)
logrus.Infof(" - Compose file: %s", compose)
logrus.Infof(" - Master image: %s", defaultMasterImageName)
logrus.Infof(" - Worker image: %s", workerImageForStack)
}()
if err = deployStack(cli, defaultStackName, compose); err != nil {
return 1, err
}
logrus.Infof("The log will be displayed here after some duration."+
"You can watch the live status via `docker service logs %s_worker`",
defaultStackName)
masterContainerID, err := waitForMasterUp(cli, defaultStackName)
if err != nil {
return 1, err
}
rc, err := waitForContainerCompletion(cli, os.Stdout, os.Stderr, masterContainerID)
if err != nil {
return 1, err
}
logrus.Infof("Exit status: %d", rc)
return int(rc), nil
}
func ensureImages(cli *client.Client, images []string) error {
for _, image := range images {
_, _, err := cli.ImageInspectWithRaw(context.Background(), image)
if err != nil {
return fmt.Errorf("could not find image %s, please run `make build-integration-cli-on-swarm`: %v",
image, err)
}
}
return nil
}
func filtersBytes(optionalFiltersFile string) ([]byte, error) {
var b []byte
if optionalFiltersFile == "" {
tests, err := enumerateTests(".")
if err != nil {
return b, err
}
b = []byte(strings.Join(tests, "\n") + "\n")
} else {
var err error
b, err = ioutil.ReadFile(optionalFiltersFile)
if err != nil {
return b, err
}
}
return b, nil
}
func waitForMasterUp(cli *client.Client, stackName string) (string, error) {
// FIXME(AkihiroSuda): it should retry until master is up, rather than pre-sleeping
time.Sleep(10 * time.Second)
fil := filters.NewArgs()
fil.Add("label", "com.docker.stack.namespace="+stackName)
// FIXME(AkihiroSuda): we should not rely on internal service naming convention
fil.Add("label", "com.docker.swarm.service.name="+stackName+"_master")
masters, err := cli.ContainerList(context.Background(), types.ContainerListOptions{
All: true,
Filters: fil,
})
if err != nil {
return "", err
}
if len(masters) == 0 {
return "", fmt.Errorf("master not running in stack %s?", stackName)
}
return masters[0].ID, nil
}
func waitForContainerCompletion(cli *client.Client, stdout, stderr io.Writer, containerID string) (int64, error) {
stream, err := cli.ContainerLogs(context.Background(),
containerID,
types.ContainerLogsOptions{
ShowStdout: true,
ShowStderr: true,
Follow: true,
})
if err != nil {
return 1, err
}
stdcopy.StdCopy(stdout, stderr, stream)
stream.Close()
resultC, errC := cli.ContainerWait(context.Background(), containerID, "")
select {
case err := <-errC:
return 1, err
case result := <-resultC:
return result.StatusCode, nil
}
}

View file

@ -0,0 +1,88 @@
package main
import (
"archive/tar"
"bytes"
"context"
"io"
"github.com/docker/docker/api/types"
"github.com/docker/docker/api/types/container"
"github.com/docker/docker/api/types/mount"
"github.com/docker/docker/api/types/volume"
"github.com/docker/docker/client"
)
func createTar(data map[string][]byte) (io.Reader, error) {
var b bytes.Buffer
tw := tar.NewWriter(&b)
for path, datum := range data {
hdr := tar.Header{
Name: path,
Mode: 0644,
Size: int64(len(datum)),
}
if err := tw.WriteHeader(&hdr); err != nil {
return nil, err
}
_, err := tw.Write(datum)
if err != nil {
return nil, err
}
}
if err := tw.Close(); err != nil {
return nil, err
}
return &b, nil
}
// createVolumeWithData creates a volume with the given data (e.g. data["/foo"] = []byte("bar"))
// Internally, a container is created from the image so as to provision the data to the volume,
// which is attached to the container.
func createVolumeWithData(cli *client.Client, volumeName string, data map[string][]byte, image string) error {
_, err := cli.VolumeCreate(context.Background(),
volume.VolumesCreateBody{
Driver: "local",
Name: volumeName,
})
if err != nil {
return err
}
mnt := "/mnt"
miniContainer, err := cli.ContainerCreate(context.Background(),
&container.Config{
Image: image,
},
&container.HostConfig{
Mounts: []mount.Mount{
{
Type: mount.TypeVolume,
Source: volumeName,
Target: mnt,
},
},
}, nil, "")
if err != nil {
return err
}
tr, err := createTar(data)
if err != nil {
return err
}
if cli.CopyToContainer(context.Background(),
miniContainer.ID, mnt, tr, types.CopyToContainerOptions{}); err != nil {
return err
}
return cli.ContainerRemove(context.Background(),
miniContainer.ID,
types.ContainerRemoveOptions{})
}
func hasVolume(cli *client.Client, volumeName string) bool {
_, err := cli.VolumeInspect(context.Background(), volumeName)
return err == nil
}
func removeVolume(cli *client.Client, volumeName string) error {
return cli.VolumeRemove(context.Background(), volumeName, true)
}

View file

@ -0,0 +1,457 @@
<#
.NOTES
Author: @jhowardmsft
Summary: Windows native build script. This is similar to functionality provided
by hack\make.sh, but uses native Windows PowerShell semantics. It does
not support the full set of options provided by the Linux counterpart.
For example:
- You can't cross-build Linux docker binaries on Windows
- Hashes aren't generated on binaries
- 'Releasing' isn't supported.
- Integration tests. This is because they currently cannot run inside a container,
and require significant external setup.
It does however provided the minimum necessary to support parts of local Windows
development and Windows to Windows CI.
Usage Examples (run from repo root):
"hack\make.ps1 -Client" to build docker.exe client 64-bit binary (remote repo)
"hack\make.ps1 -TestUnit" to run unit tests
"hack\make.ps1 -Daemon -TestUnit" to build the daemon and run unit tests
"hack\make.ps1 -All" to run everything this script knows about that can run in a container
"hack\make.ps1" to build the daemon binary (same as -Daemon)
"hack\make.ps1 -Binary" shortcut to -Client and -Daemon
.PARAMETER Client
Builds the client binaries.
.PARAMETER Daemon
Builds the daemon binary.
.PARAMETER Binary
Builds the client and daemon binaries. A convenient shortcut to `make.ps1 -Client -Daemon`.
.PARAMETER Race
Use -race in go build and go test.
.PARAMETER Noisy
Use -v in go build.
.PARAMETER ForceBuildAll
Use -a in go build.
.PARAMETER NoOpt
Use -gcflags -N -l in go build to disable optimisation (can aide debugging).
.PARAMETER CommitSuffix
Adds a custom string to be appended to the commit ID (spaces are stripped).
.PARAMETER DCO
Runs the DCO (Developer Certificate Of Origin) test (must be run outside a container).
.PARAMETER PkgImports
Runs the pkg\ directory imports test (must be run outside a container).
.PARAMETER GoFormat
Runs the Go formatting test (must be run outside a container).
.PARAMETER TestUnit
Runs unit tests.
.PARAMETER All
Runs everything this script knows about that can run in a container.
TODO
- Unify the head commit
- Add golint and other checks (swagger maybe?)
#>
param(
[Parameter(Mandatory=$False)][switch]$Client,
[Parameter(Mandatory=$False)][switch]$Daemon,
[Parameter(Mandatory=$False)][switch]$Binary,
[Parameter(Mandatory=$False)][switch]$Race,
[Parameter(Mandatory=$False)][switch]$Noisy,
[Parameter(Mandatory=$False)][switch]$ForceBuildAll,
[Parameter(Mandatory=$False)][switch]$NoOpt,
[Parameter(Mandatory=$False)][string]$CommitSuffix="",
[Parameter(Mandatory=$False)][switch]$DCO,
[Parameter(Mandatory=$False)][switch]$PkgImports,
[Parameter(Mandatory=$False)][switch]$GoFormat,
[Parameter(Mandatory=$False)][switch]$TestUnit,
[Parameter(Mandatory=$False)][switch]$All
)
$ErrorActionPreference = "Stop"
$ProgressPreference = "SilentlyContinue"
$pushed=$False # To restore the directory if we have temporarily pushed to one.
# Utility function to get the commit ID of the repository
Function Get-GitCommit() {
if (-not (Test-Path ".\.git")) {
# If we don't have a .git directory, but we do have the environment
# variable DOCKER_GITCOMMIT set, that can override it.
if ($env:DOCKER_GITCOMMIT.Length -eq 0) {
Throw ".git directory missing and DOCKER_GITCOMMIT environment variable not specified."
}
Write-Host "INFO: Git commit ($env:DOCKER_GITCOMMIT) assumed from DOCKER_GITCOMMIT environment variable"
return $env:DOCKER_GITCOMMIT
}
$gitCommit=$(git rev-parse --short HEAD)
if ($(git status --porcelain --untracked-files=no).Length -ne 0) {
$gitCommit="$gitCommit-unsupported"
Write-Host ""
Write-Warning "This version is unsupported because there are uncommitted file(s)."
Write-Warning "Either commit these changes, or add them to .gitignore."
git status --porcelain --untracked-files=no | Write-Warning
Write-Host ""
}
return $gitCommit
}
# Utility function to determine if we are running in a container or not.
# In Windows, we get this through an environment variable set in `Dockerfile.Windows`
Function Check-InContainer() {
if ($env:FROM_DOCKERFILE.Length -eq 0) {
Write-Host ""
Write-Warning "Not running in a container. The result might be an incorrect build."
Write-Host ""
return $False
}
return $True
}
# Utility function to warn if the version of go is correct. Used for local builds
# outside of a container where it may be out of date with master.
Function Verify-GoVersion() {
Try {
$goVersionDockerfile=(Get-Content ".\Dockerfile" | Select-String "ENV GO_VERSION").ToString().Split(" ")[2]
$goVersionInstalled=(go version).ToString().Split(" ")[2].SubString(2)
}
Catch [Exception] {
Throw "Failed to validate go version correctness: $_"
}
if (-not($goVersionInstalled -eq $goVersionDockerfile)) {
Write-Host ""
Write-Warning "Building with golang version $goVersionInstalled. You should update to $goVersionDockerfile"
Write-Host ""
}
}
# Utility function to get the commit for HEAD
Function Get-HeadCommit() {
$head = Invoke-Expression "git rev-parse --verify HEAD"
if ($LASTEXITCODE -ne 0) { Throw "Failed getting HEAD commit" }
return $head
}
# Utility function to get the commit for upstream
Function Get-UpstreamCommit() {
Invoke-Expression "git fetch -q https://github.com/docker/docker.git refs/heads/master"
if ($LASTEXITCODE -ne 0) { Throw "Failed fetching" }
$upstream = Invoke-Expression "git rev-parse --verify FETCH_HEAD"
if ($LASTEXITCODE -ne 0) { Throw "Failed getting upstream commit" }
return $upstream
}
# Build a binary (client or daemon)
Function Execute-Build($type, $additionalBuildTags, $directory) {
# Generate the build flags
$buildTags = "autogen"
if ($Noisy) { $verboseParm=" -v" }
if ($Race) { Write-Warning "Using race detector"; $raceParm=" -race"}
if ($ForceBuildAll) { $allParm=" -a" }
if ($NoOpt) { $optParm=" -gcflags "+""""+"-N -l"+"""" }
if ($additionalBuildTags -ne "") { $buildTags += $(" " + $additionalBuildTags) }
# Do the go build in the appropriate directory
# Note -linkmode=internal is required to be able to debug on Windows.
# https://github.com/golang/go/issues/14319#issuecomment-189576638
Write-Host "INFO: Building $type..."
Push-Location $root\cmd\$directory; $global:pushed=$True
$buildCommand = "go build" + `
$raceParm + `
$verboseParm + `
$allParm + `
$optParm + `
" -tags """ + $buildTags + """" + `
" -ldflags """ + "-linkmode=internal" + """" + `
" -o $root\bundles\"+$directory+".exe"
Invoke-Expression $buildCommand
if ($LASTEXITCODE -ne 0) { Throw "Failed to compile $type" }
Pop-Location; $global:pushed=$False
}
# Validates the DCO marker is present on each commit
Function Validate-DCO($headCommit, $upstreamCommit) {
Write-Host "INFO: Validating Developer Certificate of Origin..."
# Username may only contain alphanumeric characters or dashes and cannot begin with a dash
$usernameRegex='[a-zA-Z0-9][a-zA-Z0-9-]+'
$dcoPrefix="Signed-off-by:"
$dcoRegex="^(Docker-DCO-1.1-)?$dcoPrefix ([^<]+) <([^<>@]+@[^<>]+)>( \(github: ($usernameRegex)\))?$"
$counts = Invoke-Expression "git diff --numstat $upstreamCommit...$headCommit"
if ($LASTEXITCODE -ne 0) { Throw "Failed git diff --numstat" }
# Counts of adds and deletes after removing multiple white spaces. AWK anyone? :(
$adds=0; $dels=0; $($counts -replace '\s+', ' ') | %{
$a=$_.Split(" ");
if ($a[0] -ne "-") { $adds+=[int]$a[0] }
if ($a[1] -ne "-") { $dels+=[int]$a[1] }
}
if (($adds -eq 0) -and ($dels -eq 0)) {
Write-Warning "DCO validation - nothing to validate!"
return
}
$commits = Invoke-Expression "git log $upstreamCommit..$headCommit --format=format:%H%n"
if ($LASTEXITCODE -ne 0) { Throw "Failed git log --format" }
$commits = $($commits -split '\s+' -match '\S')
$badCommits=@()
$commits | %{
# Skip commits with no content such as merge commits etc
if ($(git log -1 --format=format: --name-status $_).Length -gt 0) {
# Ignore exit code on next call - always process regardless
$commitMessage = Invoke-Expression "git log -1 --format=format:%B --name-status $_"
if (($commitMessage -match $dcoRegex).Length -eq 0) { $badCommits+=$_ }
}
}
if ($badCommits.Length -eq 0) {
Write-Host "Congratulations! All commits are properly signed with the DCO!"
} else {
$e = "`nThese commits do not have a proper '$dcoPrefix' marker:`n"
$badCommits | %{ $e+=" - $_`n"}
$e += "`nPlease amend each commit to include a properly formatted DCO marker.`n`n"
$e += "Visit the following URL for information about the Docker DCO:`n"
$e += "https://github.com/docker/docker/blob/master/CONTRIBUTING.md#sign-your-work`n"
Throw $e
}
}
# Validates that .\pkg\... is safely isolated from internal code
Function Validate-PkgImports($headCommit, $upstreamCommit) {
Write-Host "INFO: Validating pkg import isolation..."
# Get a list of go source-code files which have changed under pkg\. Ignore exit code on next call - always process regardless
$files=@(); $files = Invoke-Expression "git diff $upstreamCommit...$headCommit --diff-filter=ACMR --name-only -- `'pkg\*.go`'"
$badFiles=@(); $files | %{
$file=$_
# For the current changed file, get its list of dependencies, sorted and uniqued.
$imports = Invoke-Expression "go list -e -f `'{{ .Deps }}`' $file"
if ($LASTEXITCODE -ne 0) { Throw "Failed go list for dependencies on $file" }
$imports = $imports -Replace "\[" -Replace "\]", "" -Split(" ") | Sort-Object | Get-Unique
# Filter out what we are looking for
$imports = $imports -NotMatch "^github.com/docker/docker/pkg/" `
-NotMatch "^github.com/docker/docker/vendor" `
-Match "^github.com/docker/docker" `
-Replace "`n", ""
$imports | % { $badFiles+="$file imports $_`n" }
}
if ($badFiles.Length -eq 0) {
Write-Host 'Congratulations! ".\pkg\*.go" is safely isolated from internal code.'
} else {
$e = "`nThese files import internal code: (either directly or indirectly)`n"
$badFiles | %{ $e+=" - $_"}
Throw $e
}
}
# Validates that changed files are correctly go-formatted
Function Validate-GoFormat($headCommit, $upstreamCommit) {
Write-Host "INFO: Validating go formatting on changed files..."
# Verify gofmt is installed
if ($(Get-Command gofmt -ErrorAction SilentlyContinue) -eq $nil) { Throw "gofmt does not appear to be installed" }
# Get a list of all go source-code files which have changed. Ignore exit code on next call - always process regardless
$files=@(); $files = Invoke-Expression "git diff $upstreamCommit...$headCommit --diff-filter=ACMR --name-only -- `'*.go`'"
$files = $files | Select-String -NotMatch "^vendor/"
$badFiles=@(); $files | %{
# Deliberately ignore error on next line - treat as failed
$content=Invoke-Expression "git show $headCommit`:$_"
# Next set of hoops are to ensure we have LF not CRLF semantics as otherwise gofmt on Windows will not succeed.
# Also note that gofmt on Windows does not appear to support stdin piping correctly. Hence go through a temporary file.
$content=$content -join "`n"
$content+="`n"
$outputFile=[System.IO.Path]::GetTempFileName()
if (Test-Path $outputFile) { Remove-Item $outputFile }
[System.IO.File]::WriteAllText($outputFile, $content, (New-Object System.Text.UTF8Encoding($False)))
$currentFile = $_ -Replace("/","\")
Write-Host Checking $currentFile
Invoke-Expression "gofmt -s -l $outputFile"
if ($LASTEXITCODE -ne 0) { $badFiles+=$currentFile }
if (Test-Path $outputFile) { Remove-Item $outputFile }
}
if ($badFiles.Length -eq 0) {
Write-Host 'Congratulations! All Go source files are properly formatted.'
} else {
$e = "`nThese files are not properly gofmt`'d:`n"
$badFiles | %{ $e+=" - $_`n"}
$e+= "`nPlease reformat the above files using `"gofmt -s -w`" and commit the result."
Throw $e
}
}
# Run the unit tests
Function Run-UnitTests() {
Write-Host "INFO: Running unit tests..."
$testPath="./..."
$goListCommand = "go list -e -f '{{if ne .Name """ + '\"github.com/docker/docker\"' + """}}{{.ImportPath}}{{end}}' $testPath"
$pkgList = $(Invoke-Expression $goListCommand)
if ($LASTEXITCODE -ne 0) { Throw "go list for unit tests failed" }
$pkgList = $pkgList | Select-String -Pattern "github.com/docker/docker"
$pkgList = $pkgList | Select-String -NotMatch "github.com/docker/docker/vendor"
$pkgList = $pkgList | Select-String -NotMatch "github.com/docker/docker/man"
$pkgList = $pkgList | Select-String -NotMatch "github.com/docker/docker/integration"
$pkgList = $pkgList -replace "`r`n", " "
$goTestCommand = "go test" + $raceParm + " -cover -ldflags -w -tags """ + "autogen daemon" + """ -a """ + "-test.timeout=10m" + """ $pkgList"
Invoke-Expression $goTestCommand
if ($LASTEXITCODE -ne 0) { Throw "Unit tests failed" }
}
# Start of main code.
Try {
Write-Host -ForegroundColor Cyan "INFO: make.ps1 starting at $(Get-Date)"
# Get to the root of the repo
$root = $(Split-Path $MyInvocation.MyCommand.Definition -Parent | Split-Path -Parent)
Push-Location $root
# Handle the "-All" shortcut to turn on all things we can handle.
# Note we expressly only include the items which can run in a container - the validations tests cannot
# as they require the .git directory which is excluded from the image by .dockerignore
if ($All) { $Client=$True; $Daemon=$True; $TestUnit=$True }
# Handle the "-Binary" shortcut to build both client and daemon.
if ($Binary) { $Client = $True; $Daemon = $True }
# Default to building the daemon if not asked for anything explicitly.
if (-not($Client) -and -not($Daemon) -and -not($DCO) -and -not($PkgImports) -and -not($GoFormat) -and -not($TestUnit)) { $Daemon=$True }
# Verify git is installed
if ($(Get-Command git -ErrorAction SilentlyContinue) -eq $nil) { Throw "Git does not appear to be installed" }
# Verify go is installed
if ($(Get-Command go -ErrorAction SilentlyContinue) -eq $nil) { Throw "GoLang does not appear to be installed" }
# Get the git commit. This will also verify if we are in a repo or not. Then add a custom string if supplied.
$gitCommit=Get-GitCommit
if ($CommitSuffix -ne "") { $gitCommit += "-"+$CommitSuffix -Replace ' ', '' }
# Get the version of docker (eg 17.04.0-dev)
$dockerVersion="0.0.0-dev"
# Give a warning if we are not running in a container and are building binaries or running unit tests.
# Not relevant for validation tests as these are fine to run outside of a container.
if ($Client -or $Daemon -or $TestUnit) { $inContainer=Check-InContainer }
# If we are not in a container, validate the version of GO that is installed.
if (-not $inContainer) { Verify-GoVersion }
# Verify GOPATH is set
if ($env:GOPATH.Length -eq 0) { Throw "Missing GOPATH environment variable. See https://golang.org/doc/code.html#GOPATH" }
# Run autogen if building binaries or running unit tests.
if ($Client -or $Daemon -or $TestUnit) {
Write-Host "INFO: Invoking autogen..."
Try { .\hack\make\.go-autogen.ps1 -CommitString $gitCommit -DockerVersion $dockerVersion -Platform "$env:PLATFORM" }
Catch [Exception] { Throw $_ }
}
# DCO, Package import and Go formatting tests.
if ($DCO -or $PkgImports -or $GoFormat) {
# We need the head and upstream commits for these
$headCommit=Get-HeadCommit
$upstreamCommit=Get-UpstreamCommit
# Run DCO validation
if ($DCO) { Validate-DCO $headCommit $upstreamCommit }
# Run `gofmt` validation
if ($GoFormat) { Validate-GoFormat $headCommit $upstreamCommit }
# Run pkg isolation validation
if ($PkgImports) { Validate-PkgImports $headCommit $upstreamCommit }
}
# Build the binaries
if ($Client -or $Daemon) {
# Create the bundles directory if it doesn't exist
if (-not (Test-Path ".\bundles")) { New-Item ".\bundles" -ItemType Directory | Out-Null }
# Perform the actual build
if ($Daemon) { Execute-Build "daemon" "daemon" "dockerd" }
if ($Client) {
# Get the Docker channel and version from the environment, or use the defaults.
if (-not ($channel = $env:DOCKERCLI_CHANNEL)) { $channel = "edge" }
if (-not ($version = $env:DOCKERCLI_VERSION)) { $version = "17.06.0-ce" }
# Download the zip file and extract the client executable.
Write-Host "INFO: Downloading docker/cli version $version from $channel..."
$url = "https://download.docker.com/win/static/$channel/x86_64/docker-$version.zip"
Invoke-WebRequest $url -OutFile "docker.zip"
Try {
Add-Type -AssemblyName System.IO.Compression.FileSystem
$zip = [System.IO.Compression.ZipFile]::OpenRead("$PWD\docker.zip")
Try {
if (-not ($entry = $zip.Entries | Where-Object { $_.Name -eq "docker.exe" })) {
Throw "Cannot find docker.exe in $url"
}
[System.IO.Compression.ZipFileExtensions]::ExtractToFile($entry, "$PWD\bundles\docker.exe", $true)
}
Finally {
$zip.Dispose()
}
}
Finally {
Remove-Item -Force "docker.zip"
}
}
}
# Run unit tests
if ($TestUnit) { Run-UnitTests }
# Gratuitous ASCII art.
if ($Daemon -or $Client) {
Write-Host
Write-Host -ForegroundColor Green " ________ ____ __."
Write-Host -ForegroundColor Green " \_____ \ `| `|/ _`|"
Write-Host -ForegroundColor Green " / `| \`| `<"
Write-Host -ForegroundColor Green " / `| \ `| \"
Write-Host -ForegroundColor Green " \_______ /____`|__ \"
Write-Host -ForegroundColor Green " \/ \/"
Write-Host
}
}
Catch [Exception] {
Write-Host -ForegroundColor Red ("`nERROR: make.ps1 failed:`n$_")
# More gratuitous ASCII art.
Write-Host
Write-Host -ForegroundColor Red "___________ .__.__ .___"
Write-Host -ForegroundColor Red "\_ _____/____ `|__`| `| ____ __`| _/"
Write-Host -ForegroundColor Red " `| __) \__ \ `| `| `| _/ __ \ / __ `| "
Write-Host -ForegroundColor Red " `| \ / __ \`| `| `|_\ ___// /_/ `| "
Write-Host -ForegroundColor Red " \___ / (____ /__`|____/\___ `>____ `| "
Write-Host -ForegroundColor Red " \/ \/ \/ \/ "
Write-Host
Throw $_
}
Finally {
Pop-Location # As we pushed to the root of the repo as the very first thing
if ($global:pushed) { Pop-Location }
Write-Host -ForegroundColor Cyan "INFO: make.ps1 ended at $(Get-Date)"
}

View file

@ -0,0 +1,216 @@
#!/usr/bin/env bash
set -e
# This script builds various binary artifacts from a checkout of the docker
# source code.
#
# Requirements:
# - The current directory should be a checkout of the docker source code
# (https://github.com/docker/docker). Whatever version is checked out
# will be built.
# - The VERSION file, at the root of the repository, should exist, and
# will be used as Docker binary version and package version.
# - The hash of the git commit will also be included in the Docker binary,
# with the suffix -unsupported if the repository isn't clean.
# - The script is intended to be run inside the docker container specified
# in the Dockerfile at the root of the source. In other words:
# DO NOT CALL THIS SCRIPT DIRECTLY.
# - The right way to call this script is to invoke "make" from
# your checkout of the Docker repository.
# the Makefile will do a "docker build -t docker ." and then
# "docker run hack/make.sh" in the resulting image.
#
set -o pipefail
export DOCKER_PKG='github.com/docker/docker'
export SCRIPTDIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
export MAKEDIR="$SCRIPTDIR/make"
export PKG_CONFIG=${PKG_CONFIG:-pkg-config}
# We're a nice, sexy, little shell script, and people might try to run us;
# but really, they shouldn't. We want to be in a container!
inContainer="AssumeSoInitially"
if [ "$(go env GOHOSTOS)" = 'windows' ]; then
if [ -z "$FROM_DOCKERFILE" ]; then
unset inContainer
fi
else
if [ "$PWD" != "/go/src/$DOCKER_PKG" ]; then
unset inContainer
fi
fi
if [ -z "$inContainer" ]; then
{
echo "# WARNING! I don't seem to be running in a Docker container."
echo "# The result of this command might be an incorrect build, and will not be"
echo "# officially supported."
echo "#"
echo "# Try this instead: make all"
echo "#"
} >&2
fi
echo
# List of bundles to create when no argument is passed
DEFAULT_BUNDLES=(
binary-daemon
dynbinary
test-integration
test-docker-py
cross
)
VERSION=${VERSION:-dev}
! BUILDTIME=$(date -u -d "@${SOURCE_DATE_EPOCH:-$(date +%s)}" --rfc-3339 ns 2> /dev/null | sed -e 's/ /T/')
if [ "$DOCKER_GITCOMMIT" ]; then
GITCOMMIT="$DOCKER_GITCOMMIT"
elif command -v git &> /dev/null && [ -e .git ] && git rev-parse &> /dev/null; then
GITCOMMIT=$(git rev-parse --short HEAD)
if [ -n "$(git status --porcelain --untracked-files=no)" ]; then
GITCOMMIT="$GITCOMMIT-unsupported"
echo "#~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~"
echo "# GITCOMMIT = $GITCOMMIT"
echo "# The version you are building is listed as unsupported because"
echo "# there are some files in the git repository that are in an uncommitted state."
echo "# Commit these changes, or add to .gitignore to remove the -unsupported from the version."
echo "# Here is the current list:"
git status --porcelain --untracked-files=no
echo "#~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~"
fi
else
echo >&2 'error: .git directory missing and DOCKER_GITCOMMIT not specified'
echo >&2 ' Please either build with the .git directory accessible, or specify the'
echo >&2 ' exact (--short) commit hash you are building using DOCKER_GITCOMMIT for'
echo >&2 ' future accountability in diagnosing build issues. Thanks!'
exit 1
fi
if [ "$AUTO_GOPATH" ]; then
rm -rf .gopath
mkdir -p .gopath/src/"$(dirname "${DOCKER_PKG}")"
ln -sf ../../../.. .gopath/src/"${DOCKER_PKG}"
export GOPATH="${PWD}/.gopath"
fi
if [ ! "$GOPATH" ]; then
echo >&2 'error: missing GOPATH; please see https://golang.org/doc/code.html#GOPATH'
echo >&2 ' alternatively, set AUTO_GOPATH=1'
exit 1
fi
if ${PKG_CONFIG} 'libsystemd >= 209' 2> /dev/null ; then
DOCKER_BUILDTAGS+=" journald"
elif ${PKG_CONFIG} 'libsystemd-journal' 2> /dev/null ; then
DOCKER_BUILDTAGS+=" journald journald_compat"
fi
# test whether "btrfs/version.h" exists and apply btrfs_noversion appropriately
if \
command -v gcc &> /dev/null \
&& ! gcc -E - -o /dev/null &> /dev/null <<<'#include <btrfs/version.h>' \
; then
DOCKER_BUILDTAGS+=' btrfs_noversion'
fi
# test whether "libdevmapper.h" is new enough to support deferred remove
# functionality.
if \
command -v gcc &> /dev/null \
&& ! ( echo -e '#include <libdevmapper.h>\nint main() { dm_task_deferred_remove(NULL); }'| gcc -xc - -o /dev/null $(pkg-config --libs devmapper) &> /dev/null ) \
; then
DOCKER_BUILDTAGS+=' libdm_no_deferred_remove'
fi
# Use these flags when compiling the tests and final binary
IAMSTATIC='true'
if [ -z "$DOCKER_DEBUG" ]; then
LDFLAGS='-w'
fi
LDFLAGS_STATIC=''
EXTLDFLAGS_STATIC='-static'
# ORIG_BUILDFLAGS is necessary for the cross target which cannot always build
# with options like -race.
ORIG_BUILDFLAGS=( -tags "autogen netgo static_build $DOCKER_BUILDTAGS" -installsuffix netgo )
# see https://github.com/golang/go/issues/9369#issuecomment-69864440 for why -installsuffix is necessary here
# When $DOCKER_INCREMENTAL_BINARY is set in the environment, enable incremental
# builds by installing dependent packages to the GOPATH.
REBUILD_FLAG="-a"
if [ "$DOCKER_INCREMENTAL_BINARY" == "1" ] || [ "$DOCKER_INCREMENTAL_BINARY" == "true" ]; then
REBUILD_FLAG="-i"
fi
ORIG_BUILDFLAGS+=( $REBUILD_FLAG )
BUILDFLAGS=( $BUILDFLAGS "${ORIG_BUILDFLAGS[@]}" )
# Test timeout.
if [ "${DOCKER_ENGINE_GOARCH}" == "arm" ]; then
: ${TIMEOUT:=10m}
elif [ "${DOCKER_ENGINE_GOARCH}" == "windows" ]; then
: ${TIMEOUT:=8m}
else
: ${TIMEOUT:=5m}
fi
LDFLAGS_STATIC_DOCKER="
$LDFLAGS_STATIC
-extldflags \"$EXTLDFLAGS_STATIC\"
"
if [ "$(uname -s)" = 'FreeBSD' ]; then
# Tell cgo the compiler is Clang, not GCC
# https://code.google.com/p/go/source/browse/src/cmd/cgo/gcc.go?spec=svne77e74371f2340ee08622ce602e9f7b15f29d8d3&r=e6794866ebeba2bf8818b9261b54e2eef1c9e588#752
export CC=clang
# "-extld clang" is a workaround for
# https://code.google.com/p/go/issues/detail?id=6845
LDFLAGS="$LDFLAGS -extld clang"
fi
bundle() {
local bundle="$1"; shift
echo "---> Making bundle: $(basename "$bundle") (in $DEST)"
source "$SCRIPTDIR/make/$bundle" "$@"
}
main() {
if [ -z "${KEEPBUNDLE-}" ]; then
echo "Removing bundles/"
rm -rf "bundles/*"
echo
fi
mkdir -p bundles
# Windows and symlinks don't get along well
if [ "$(go env GOHOSTOS)" != 'windows' ]; then
rm -f bundles/latest
# preserve latest symlink for backward compatibility
ln -sf . bundles/latest
fi
if [ $# -lt 1 ]; then
bundles=(${DEFAULT_BUNDLES[@]})
else
bundles=($@)
fi
for bundle in ${bundles[@]}; do
export DEST="bundles/$(basename "$bundle")"
# Cygdrive paths don't play well with go build -o.
if [[ "$(uname -s)" == CYGWIN* ]]; then
export DEST="$(cygpath -mw "$DEST")"
fi
mkdir -p "$DEST"
ABS_DEST="$(cd "$DEST" && pwd -P)"
bundle "$bundle"
echo
done
}
main "$@"

View file

@ -0,0 +1,73 @@
#!/usr/bin/env bash
set -e
# a helper to provide ".exe" when it's appropriate
binary_extension() {
if [ "$(go env GOOS)" = 'windows' ]; then
echo -n '.exe'
fi
}
GO_PACKAGE='github.com/docker/docker/cmd/dockerd'
BINARY_SHORT_NAME='dockerd'
BINARY_NAME="$BINARY_SHORT_NAME-$VERSION"
BINARY_EXTENSION="$(binary_extension)"
BINARY_FULLNAME="$BINARY_NAME$BINARY_EXTENSION"
source "${MAKEDIR}/.go-autogen"
hash_files() {
while [ $# -gt 0 ]; do
f="$1"
shift
dir="$(dirname "$f")"
base="$(basename "$f")"
for hashAlgo in md5 sha256; do
if command -v "${hashAlgo}sum" &> /dev/null; then
(
# subshell and cd so that we get output files like:
# $HASH docker-$VERSION
# instead of:
# $HASH /go/src/github.com/.../$VERSION/binary/docker-$VERSION
cd "$dir"
"${hashAlgo}sum" "$base" > "$base.$hashAlgo"
)
fi
done
done
}
(
export GOGC=${DOCKER_BUILD_GOGC:-1000}
if [ "$(go env GOOS)/$(go env GOARCH)" != "$(go env GOHOSTOS)/$(go env GOHOSTARCH)" ]; then
# must be cross-compiling!
case "$(go env GOOS)/$(go env GOARCH)" in
windows/amd64)
export CC=x86_64-w64-mingw32-gcc
export CGO_ENABLED=1
;;
esac
fi
# -buildmode=pie is not supported on Windows.
if [ "$(go env GOOS)" != "windows" ]; then
BUILDFLAGS+=( "-buildmode=pie" )
fi
echo "Building: $DEST/$BINARY_FULLNAME"
go build \
-o "$DEST/$BINARY_FULLNAME" \
"${BUILDFLAGS[@]}" \
-ldflags "
$LDFLAGS
$LDFLAGS_STATIC_DOCKER
$DOCKER_LDFLAGS
" \
$GO_PACKAGE
)
echo "Created binary: $DEST/$BINARY_FULLNAME"
ln -sf "$BINARY_FULLNAME" "$DEST/$BINARY_SHORT_NAME$BINARY_EXTENSION"
hash_files "$DEST/$BINARY_FULLNAME"

View file

@ -0,0 +1,9 @@
#!/usr/bin/env bash
DOCKER_DAEMON_BINARY_NAME='dockerd'
DOCKER_RUNC_BINARY_NAME='docker-runc'
DOCKER_CONTAINERD_BINARY_NAME='docker-containerd'
DOCKER_CONTAINERD_CTR_BINARY_NAME='docker-containerd-ctr'
DOCKER_CONTAINERD_SHIM_BINARY_NAME='docker-containerd-shim'
DOCKER_PROXY_BINARY_NAME='docker-proxy'
DOCKER_INIT_BINARY_NAME='docker-init'

View file

@ -0,0 +1,43 @@
#!/usr/bin/env bash
set -e
docker-version-osarch() {
if ! type docker &>/dev/null; then
# docker is not installed
return
fi
local target="$1" # "Client" or "Server"
local fmtStr="{{.${target}.Os}}/{{.${target}.Arch}}"
if docker version -f "$fmtStr" 2>/dev/null; then
# if "docker version -f" works, let's just use that!
return
fi
docker version | awk '
$1 ~ /^(Client|Server):$/ { section = 0 }
$1 == "'"$target"':" { section = 1; next }
section && $1 == "OS/Arch:" { print $2 }
# old versions of Docker
$1 == "OS/Arch" && $2 == "('"${target,,}"'):" { print $3 }
'
}
# Retrieve OS/ARCH of docker daemon, e.g. linux/amd64
export DOCKER_ENGINE_OSARCH="${DOCKER_ENGINE_OSARCH:=$(docker-version-osarch 'Server')}"
export DOCKER_ENGINE_GOOS="${DOCKER_ENGINE_OSARCH%/*}"
export DOCKER_ENGINE_GOARCH="${DOCKER_ENGINE_OSARCH##*/}"
DOCKER_ENGINE_GOARCH=${DOCKER_ENGINE_GOARCH:=amd64}
# and the client, just in case
export DOCKER_CLIENT_OSARCH="$(docker-version-osarch 'Client')"
export DOCKER_CLIENT_GOOS="${DOCKER_CLIENT_OSARCH%/*}"
export DOCKER_CLIENT_GOARCH="${DOCKER_CLIENT_OSARCH##*/}"
DOCKER_CLIENT_GOARCH=${DOCKER_CLIENT_GOARCH:=amd64}
DOCKERFILE='Dockerfile'
if [ "${DOCKER_ENGINE_GOOS:-$DOCKER_CLIENT_GOOS}" = "windows" ]; then
DOCKERFILE='Dockerfile.windows'
fi
export DOCKERFILE

View file

@ -0,0 +1,23 @@
#!/usr/bin/env bash
set -e
if ! docker image inspect emptyfs > /dev/null; then
# build a "docker save" tarball for "emptyfs"
# see https://github.com/docker/docker/pull/5262
# and also https://github.com/docker/docker/issues/4242
dir="$DEST/emptyfs"
uuid=511136ea3c5a64f264b78b5433614aec563103b4d4702f3ba7d4d2698e22c158
mkdir -p "$dir/$uuid"
(
echo '{"emptyfs":{"latest":"511136ea3c5a64f264b78b5433614aec563103b4d4702f3ba7d4d2698e22c158"}}' > "$dir/repositories"
cd "$dir/$uuid"
echo '{"id":"511136ea3c5a64f264b78b5433614aec563103b4d4702f3ba7d4d2698e22c158","comment":"Imported from -","created":"2013-06-13T14:03:50.821769-07:00","container_config":{"Hostname":"","Domainname":"","User":"","Memory":0,"MemorySwap":0,"CpuShares":0,"AttachStdin":false,"AttachStdout":false,"AttachStderr":false,"PortSpecs":null,"ExposedPorts":null,"Tty":false,"OpenStdin":false,"StdinOnce":false,"Env":null,"Cmd":null,"Image":"","Volumes":null,"WorkingDir":"","Entrypoint":null,"NetworkDisabled":false,"OnBuild":null},"docker_version":"0.4.0","architecture":"x86_64","Size":0}' > json
echo '1.0' > VERSION
tar -cf layer.tar --files-from /dev/null
)
(
[ -n "$TESTDEBUG" ] && set -x
tar -cC "$dir" . | docker load
)
rm -rf "$dir"
fi

View file

@ -0,0 +1,89 @@
#!/usr/bin/env bash
rm -rf autogen
source hack/dockerfile/install/runc.installer
source hack/dockerfile/install/tini.installer
source hack/dockerfile/install/containerd.installer
cat > dockerversion/version_autogen.go <<DVEOF
// +build autogen
// Package dockerversion is auto-generated at build-time
package dockerversion
// Default build-time variable for library-import.
// This file is overridden on build with build-time informations.
const (
GitCommit string = "$GITCOMMIT"
Version string = "$VERSION"
BuildTime string = "$BUILDTIME"
IAmStatic string = "${IAMSTATIC:-true}"
ContainerdCommitID string = "${CONTAINERD_COMMIT}"
PlatformName string = "${PLATFORM}"
)
// AUTOGENERATED FILE; see /go/src/github.com/docker/docker/hack/make/.go-autogen
DVEOF
cat > dockerversion/version_autogen_unix.go <<DVEOF
// +build autogen,!windows
// Package dockerversion is auto-generated at build-time
package dockerversion
// Default build-time variable for library-import.
// This file is overridden on build with build-time informations.
const (
RuncCommitID string = "${RUNC_COMMIT}"
InitCommitID string = "${TINI_COMMIT}"
)
// AUTOGENERATED FILE; see /go/src/github.com/docker/docker/hack/make/.go-autogen
DVEOF
# Compile the Windows resources into the sources
if [ "$(go env GOOS)" = "windows" ]; then
mkdir -p autogen/winresources/tmp autogen/winresources/docker autogen/winresources/dockerd
cp hack/make/.resources-windows/resources.go autogen/winresources/docker/
cp hack/make/.resources-windows/resources.go autogen/winresources/dockerd/
if [ "$(go env GOHOSTOS)" == "windows" ]; then
WINDRES=windres
WINDMC=windmc
else
# Cross compiling
WINDRES=x86_64-w64-mingw32-windres
WINDMC=x86_64-w64-mingw32-windmc
fi
# Generate a Windows file version of the form major,minor,patch,build (with any part optional)
VERSION_QUAD=$(echo -n $VERSION | sed -re 's/^([0-9.]*).*$/\1/' | tr . ,)
# Pass version and commit information into the resource compiler
defs=
[ ! -z $VERSION ] && defs="$defs -D DOCKER_VERSION=\"$VERSION\""
[ ! -z $VERSION_QUAD ] && defs="$defs -D DOCKER_VERSION_QUAD=$VERSION_QUAD"
[ ! -z $GITCOMMIT ] && defs="$defs -D DOCKER_COMMIT=\"$GITCOMMIT\""
function makeres {
$WINDRES \
-i hack/make/.resources-windows/$1 \
-o $3 \
-F $2 \
--use-temp-file \
-I autogen/winresources/tmp \
$defs
}
$WINDMC \
hack/make/.resources-windows/event_messages.mc \
-h autogen/winresources/tmp \
-r autogen/winresources/tmp
makeres docker.rc pe-x86-64 autogen/winresources/docker/rsrc_amd64.syso
makeres docker.rc pe-i386 autogen/winresources/docker/rsrc_386.syso
makeres dockerd.rc pe-x86-64 autogen/winresources/dockerd/rsrc_amd64.syso
rm -r autogen/winresources/tmp
fi

View file

@ -0,0 +1,93 @@
<#
.NOTES
Author: @jhowardmsft
Summary: Windows native version of .go-autogen which generates the
.go source code for building, and performs resource compilation.
.PARAMETER CommitString
The commit string. This is calculated externally to this script.
.PARAMETER DockerVersion
The version such as 17.04.0-dev. This is calculated externally to this script.
#>
param(
[Parameter(Mandatory=$true)][string]$CommitString,
[Parameter(Mandatory=$true)][string]$DockerVersion,
[Parameter(Mandatory=$false)][string]$Platform
)
$ErrorActionPreference = "Stop"
# Utility function to get the build date/time in UTC
Function Get-BuildDateTime() {
return $(Get-Date).ToUniversalTime()
}
try {
$buildDateTime=Get-BuildDateTime
if (Test-Path ".\autogen") {
Remove-Item ".\autogen" -Recurse -Force | Out-Null
}
$fileContents = '
// +build autogen
// Package dockerversion is auto-generated at build-time
package dockerversion
// Default build-time variable for library-import.
// This file is overridden on build with build-time informations.
const (
GitCommit string = "'+$CommitString+'"
Version string = "'+$DockerVersion+'"
BuildTime string = "'+$buildDateTime+'"
PlatformName string = "'+$Platform+'"
)
// AUTOGENERATED FILE; see hack\make\.go-autogen.ps1
'
# Write the file without BOM
$outputFile="$(pwd)\dockerversion\version_autogen.go"
if (Test-Path $outputFile) { Remove-Item $outputFile }
[System.IO.File]::WriteAllText($outputFile, $fileContents, (New-Object System.Text.UTF8Encoding($False)))
New-Item -ItemType Directory -Path "autogen\winresources\tmp" | Out-Null
New-Item -ItemType Directory -Path "autogen\winresources\docker" | Out-Null
New-Item -ItemType Directory -Path "autogen\winresources\dockerd" | Out-Null
Copy-Item "hack\make\.resources-windows\resources.go" "autogen\winresources\docker"
Copy-Item "hack\make\.resources-windows\resources.go" "autogen\winresources\dockerd"
# Generate a version in the form major,minor,patch,build
$versionQuad=$DockerVersion -replace "[^0-9.]*" -replace "\.", ","
# Compile the messages
windmc hack\make\.resources-windows\event_messages.mc -h autogen\winresources\tmp -r autogen\winresources\tmp
if ($LASTEXITCODE -ne 0) { Throw "Failed to compile event message resources" }
# If you really want to understand this madness below, search the Internet for powershell variables after verbatim arguments... Needed to get double-quotes passed through to the compiler options.
# Generate the .syso files containing all the resources and manifest needed to compile the final docker binaries. Both 32 and 64-bit clients.
$env:_ag_dockerVersion=$DockerVersion
$env:_ag_gitCommit=$CommitString
windres -i hack/make/.resources-windows/docker.rc -o autogen/winresources/docker/rsrc_amd64.syso -F pe-x86-64 --use-temp-file -I autogen/winresources/tmp -D DOCKER_VERSION_QUAD=$versionQuad --% -D DOCKER_VERSION=\"%_ag_dockerVersion%\" -D DOCKER_COMMIT=\"%_ag_gitCommit%\"
if ($LASTEXITCODE -ne 0) { Throw "Failed to compile client 64-bit resources" }
windres -i hack/make/.resources-windows/docker.rc -o autogen/winresources/docker/rsrc_386.syso -F pe-i386 --use-temp-file -I autogen/winresources/tmp -D DOCKER_VERSION_QUAD=$versionQuad --% -D DOCKER_VERSION=\"%_ag_dockerVersion%\" -D DOCKER_COMMIT=\"%_ag_gitCommit%\"
if ($LASTEXITCODE -ne 0) { Throw "Failed to compile client 32-bit resources" }
windres -i hack/make/.resources-windows/dockerd.rc -o autogen/winresources/dockerd/rsrc_amd64.syso -F pe-x86-64 --use-temp-file -I autogen/winresources/tmp -D DOCKER_VERSION_QUAD=$versionQuad --% -D DOCKER_VERSION=\"%_ag_dockerVersion%\" -D DOCKER_COMMIT=\"%_ag_gitCommit%\"
if ($LASTEXITCODE -ne 0) { Throw "Failed to compile daemon resources" }
}
Catch [Exception] {
# Throw the error onto the caller to display errors. We don't expect this script to be called directly
Throw ".go-autogen.ps1 failed with error $_"
}
Finally {
Remove-Item .\autogen\winresources\tmp -Recurse -Force -ErrorAction SilentlyContinue | Out-Null
$env:_ag_dockerVersion=""
$env:_ag_gitCommit=""
}

View file

@ -0,0 +1,7 @@
#!/usr/bin/env bash
set -e
source "$MAKEDIR/.detect-daemon-osarch"
if [ "$DOCKER_ENGINE_GOOS" != "windows" ]; then
bundle .ensure-emptyfs
fi

View file

@ -0,0 +1,126 @@
#!/usr/bin/env bash
# see test-integration for example usage of this script
base="$ABS_DEST/.."
export PATH="$base/binary-daemon:$base/dynbinary-daemon:$PATH"
export TEST_CLIENT_BINARY=docker
if [ -n "$DOCKER_CLI_PATH" ]; then
export TEST_CLIENT_BINARY=/usr/local/cli/$(basename "$DOCKER_CLI_PATH")
fi
echo "Using test binary $TEST_CLIENT_BINARY"
if ! command -v "$TEST_CLIENT_BINARY" &> /dev/null; then
echo >&2 'error: missing test client $TEST_CLIENT_BINARY'
false
fi
# This is a temporary hack for split-binary mode. It can be removed once
# https://github.com/docker/docker/pull/22134 is merged into docker master
if [ "$(go env GOOS)" = 'windows' ]; then
return
fi
if [ -z "$DOCKER_TEST_HOST" ]; then
if docker version &> /dev/null; then
echo >&2 'skipping daemon start, since daemon appears to be already started'
return
fi
fi
if ! command -v dockerd &> /dev/null; then
echo >&2 'error: binary-daemon or dynbinary-daemon must be run before .integration-daemon-start'
false
fi
# intentionally open a couple bogus file descriptors to help test that they get scrubbed in containers
exec 41>&1 42>&2
export DOCKER_GRAPHDRIVER=${DOCKER_GRAPHDRIVER:-vfs}
export DOCKER_USERLANDPROXY=${DOCKER_USERLANDPROXY:-true}
# example usage: DOCKER_STORAGE_OPTS="dm.basesize=20G,dm.loopdatasize=200G"
storage_params=""
if [ -n "$DOCKER_STORAGE_OPTS" ]; then
IFS=','
for i in ${DOCKER_STORAGE_OPTS}; do
storage_params="--storage-opt $i $storage_params"
done
unset IFS
fi
# example usage: DOCKER_REMAP_ROOT=default
extra_params=""
if [ "$DOCKER_REMAP_ROOT" ]; then
extra_params="--userns-remap $DOCKER_REMAP_ROOT"
fi
# example usage: DOCKER_EXPERIMENTAL=1
if [ "$DOCKER_EXPERIMENTAL" ]; then
echo >&2 '# DOCKER_EXPERIMENTAL is set: starting daemon with experimental features enabled! '
extra_params="$extra_params --experimental"
fi
if [ -z "$DOCKER_TEST_HOST" ]; then
# Start apparmor if it is enabled
if [ -e "/sys/module/apparmor/parameters/enabled" ] && [ "$(cat /sys/module/apparmor/parameters/enabled)" == "Y" ]; then
# reset container variable so apparmor profile is applied to process
# see https://github.com/docker/libcontainer/blob/master/apparmor/apparmor.go#L16
export container=""
(
[ -n "$TESTDEBUG" ] && set -x
/etc/init.d/apparmor start
)
fi
# "pwd" tricks to make sure $DEST is an absolute path, not a relative one
export DOCKER_HOST="unix://$(cd "$DEST" && pwd)/docker.sock"
(
echo "Starting dockerd"
[ -n "$TESTDEBUG" ] && set -x
exec \
dockerd --debug \
--host "$DOCKER_HOST" \
--storage-driver "$DOCKER_GRAPHDRIVER" \
--pidfile "$DEST/docker.pid" \
--userland-proxy="$DOCKER_USERLANDPROXY" \
$storage_params \
$extra_params \
&> "$DEST/docker.log"
) &
else
export DOCKER_HOST="$DOCKER_TEST_HOST"
fi
# give it a little time to come up so it's "ready"
tries=60
echo "INFO: Waiting for daemon to start..."
while ! $TEST_CLIENT_BINARY version &> /dev/null; do
(( tries-- ))
if [ $tries -le 0 ]; then
printf "\n"
if [ -z "$DOCKER_HOST" ]; then
echo >&2 "error: daemon failed to start"
echo >&2 " check $DEST/docker.log for details"
else
echo >&2 "error: daemon at $DOCKER_HOST fails to '$TEST_CLIENT_BINARY version':"
$TEST_CLIENT_BINARY version >&2 || true
# Additional Windows CI debugging as this is a common error as of
# January 2016
if [ "$(go env GOOS)" = 'windows' ]; then
echo >&2 "Container log below:"
echo >&2 "---"
# Important - use the docker on the CI host, not the one built locally
# which is currently in our path.
! /c/bin/docker -H=$MAIN_DOCKER_HOST logs docker-$COMMITHASH
echo >&2 "---"
fi
fi
false
fi
printf "."
sleep 2
done
printf "\n"

View file

@ -0,0 +1,28 @@
#!/usr/bin/env bash
if [ ! "$(go env GOOS)" = 'windows' ]; then
for pidFile in $(find "$DEST" -name docker.pid); do
pid=$([ -n "$TESTDEBUG" ] && set -x; cat "$pidFile")
(
[ -n "$TESTDEBUG" ] && set -x
kill "$pid"
)
if ! wait "$pid"; then
echo >&2 "warning: PID $pid from $pidFile had a nonzero exit code"
fi
done
if [ -z "$DOCKER_TEST_HOST" ]; then
# Stop apparmor if it is enabled
if [ -e "/sys/module/apparmor/parameters/enabled" ] && [ "$(cat /sys/module/apparmor/parameters/enabled)" == "Y" ]; then
(
[ -n "$TESTDEBUG" ] && set -x
/etc/init.d/apparmor stop
)
fi
fi
else
# Note this script is not actionable on Windows to Linux CI. Instead the
# DIND daemon under test is torn down by the Jenkins tear-down script
echo "INFO: Not stopping daemon on Windows CI"
fi

View file

@ -0,0 +1,121 @@
#!/usr/bin/env bash
#
# For integration-cli test, we use [gocheck](https://labix.org/gocheck), if you want
# to run certain tests on your local host, you should run with command:
#
# TESTFLAGS='-check.f DockerSuite.TestBuild*' ./hack/make.sh binary test-integration
#
if [ -z $MAKEDIR ]; then
export MAKEDIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
fi
source "$MAKEDIR/.go-autogen"
# Set defaults
: ${TEST_REPEAT:=1}
: ${TESTFLAGS:=}
: ${TESTDEBUG:=}
integration_api_dirs=${TEST_INTEGRATION_DIR:-"$(
find ./integration -type d |
grep -vE '(^./integration($|/internal)|/testdata)')"}
run_test_integration() {
[[ "$TESTFLAGS" != *-check.f* ]] && run_test_integration_suites
run_test_integration_legacy_suites
}
run_test_integration_suites() {
local flags="-test.v -test.timeout=${TIMEOUT} $TESTFLAGS"
for dir in $integration_api_dirs; do
if ! (
cd $dir
echo "Running $PWD"
test_env ./test.main $flags
); then exit 1; fi
done
}
run_test_integration_legacy_suites() {
(
flags="-check.v -check.timeout=${TIMEOUT} -test.timeout=360m $TESTFLAGS"
cd integration-cli
echo "Running $PWD"
test_env ./test.main $flags
)
}
build_test_suite_binaries() {
if [ ${DOCKER_INTEGRATION_TESTS_VERIFIED-} ]; then
echo "Skipping building test binaries; as DOCKER_INTEGRATION_TESTS_VERIFIED is set"
return
fi
build_test_suite_binary ./integration-cli "test.main"
for dir in $integration_api_dirs; do
build_test_suite_binary "$dir" "test.main"
done
}
# Build a binary for a test suite package
build_test_suite_binary() {
local dir="$1"
local out="$2"
echo Building test suite binary "$dir/$out"
go test -c -o "$dir/$out" -ldflags "$LDFLAGS" "${BUILDFLAGS[@]}" "$dir"
}
cleanup_test_suite_binaries() {
[ -n "$TESTDEBUG" ] && return
echo "Removing test suite binaries"
find integration* -name test.main | xargs -r rm
}
repeat() {
for i in $(seq 1 $TEST_REPEAT); do
echo "Running integration-test (iteration $i)"
$@
done
}
# use "env -i" to tightly control the environment variables that bleed into the tests
test_env() {
(
set -e
[ -n "$TESTDEBUG" ] && set -x
env -i \
DEST="$ABS_DEST" \
DOCKER_API_VERSION="$DOCKER_API_VERSION" \
DOCKER_INTEGRATION_DAEMON_DEST="$DOCKER_INTEGRATION_DAEMON_DEST" \
DOCKER_TLS_VERIFY="$DOCKER_TEST_TLS_VERIFY" \
DOCKER_CERT_PATH="$DOCKER_TEST_CERT_PATH" \
DOCKER_ENGINE_GOARCH="$DOCKER_ENGINE_GOARCH" \
DOCKER_GRAPHDRIVER="$DOCKER_GRAPHDRIVER" \
DOCKER_USERLANDPROXY="$DOCKER_USERLANDPROXY" \
DOCKER_HOST="$DOCKER_HOST" \
DOCKER_REMAP_ROOT="$DOCKER_REMAP_ROOT" \
DOCKER_REMOTE_DAEMON="$DOCKER_REMOTE_DAEMON" \
DOCKERFILE="$DOCKERFILE" \
GOPATH="$GOPATH" \
GOTRACEBACK=all \
HOME="$ABS_DEST/fake-HOME" \
PATH="$PATH" \
TEMP="$TEMP" \
TEST_CLIENT_BINARY="$TEST_CLIENT_BINARY" \
"$@"
)
}
error_on_leaked_containerd_shims() {
if [ "$(go env GOOS)" == 'windows' ]; then
return
fi
leftovers=$(ps -ax -o pid,cmd |
awk '$2 == "docker-containerd-shim" && $4 ~ /.*\/bundles\/.*\/test-integration/ { print $1 }')
if [ -n "$leftovers" ]; then
ps aux
kill -9 $leftovers 2> /dev/null
echo "!!!! WARNING you have left over shim(s), Cleanup your test !!!!"
exit 1
fi
}

View file

@ -0,0 +1,38 @@
// Application icon
1 ICON "docker.ico"
// Windows executable manifest
1 24 /* RT_MANIFEST */ "docker.exe.manifest"
// Version information
1 VERSIONINFO
#ifdef DOCKER_VERSION_QUAD
FILEVERSION DOCKER_VERSION_QUAD
PRODUCTVERSION DOCKER_VERSION_QUAD
#endif
BEGIN
BLOCK "StringFileInfo"
BEGIN
BLOCK "000004B0"
BEGIN
VALUE "ProductName", DOCKER_NAME
#ifdef DOCKER_VERSION
VALUE "FileVersion", DOCKER_VERSION
VALUE "ProductVersion", DOCKER_VERSION
#endif
#ifdef DOCKER_COMMIT
VALUE "OriginalFileName", DOCKER_COMMIT
#endif
END
END
BLOCK "VarFileInfo"
BEGIN
VALUE "Translation", 0x0000, 0x04B0
END
END

View file

@ -0,0 +1,18 @@
<?xml version="1.0" encoding="UTF-8" standalone="yes"?>
<assembly xmlns="urn:schemas-microsoft-com:asm.v1" manifestVersion="1.0">
<description>Docker</description>
<compatibility xmlns="urn:schemas-microsoft-com:compatibility.v1">
<application>
<!-- Windows 10 -->
<supportedOS Id="{8e0f7a12-bfb3-4fe8-b9a5-48fd50a15a9a}"/>
<!-- Windows 8.1 -->
<supportedOS Id="{1f676c76-80e1-4239-95bb-83d0f6d0da78}"/>
<!-- Windows Vista -->
<supportedOS Id="{e2011457-1546-43c5-a5fe-008deee3d3f0}"/>
<!-- Windows 7 -->
<supportedOS Id="{35138b9a-5d96-4fbd-8e2d-a2440225f93a}"/>
<!-- Windows 8 -->
<supportedOS Id="{4a2f28e3-53b9-4441-ba9c-d69d4a4a6e38}"/>
</application>
</compatibility>
</assembly>

Binary file not shown.

After

Width:  |  Height:  |  Size: 361 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 643 KiB

View file

@ -0,0 +1,3 @@
#define DOCKER_NAME "Docker Client"
#include "common.rc"

View file

@ -0,0 +1,4 @@
#define DOCKER_NAME "Docker Engine"
#include "common.rc"
#include "event_messages.rc"

View file

@ -0,0 +1,39 @@
MessageId=1
Language=English
%1
.
MessageId=2
Language=English
debug: %1
.
MessageId=3
Language=English
panic: %1
.
MessageId=4
Language=English
fatal: %1
.
MessageId=11
Language=English
%1 [%2]
.
MessageId=12
Language=English
debug: %1 [%2]
.
MessageId=13
Language=English
panic: %1 [%2]
.
MessageId=14
Language=English
fatal: %1 [%2]
.

View file

@ -0,0 +1,18 @@
/*
Package winresources is used to embed Windows resources into docker.exe.
These resources are used to provide
* Version information
* An icon
* A Windows manifest declaring Windows version support
The resource object files are generated in hack/make/.go-autogen from
source files in hack/make/.resources-windows. This occurs automatically
when you run hack/make.sh.
These object files are picked up automatically by go build when this package
is included.
*/
package winresources

View file

@ -0,0 +1,16 @@
This directory holds scripts called by `make.sh` in the parent directory.
Each script is named after the bundle it creates.
They should not be called directly - instead, pass it as argument to make.sh, for example:
```
./hack/make.sh binary ubuntu
# Or to run all default bundles:
./hack/make.sh
```
To add a bundle:
* Create a shell-compatible file here
* Add it to $DEFAULT_BUNDLES in make.sh

View file

@ -0,0 +1,10 @@
#!/usr/bin/env bash
set -e
rm -rf "$DEST"
# This script exists as backwards compatibility for CI
(
DEST="${DEST}-daemon"
ABS_DEST="${ABS_DEST}-daemon"
. hack/make/binary-daemon
)

View file

@ -0,0 +1,27 @@
#!/usr/bin/env bash
set -e
copy_binaries() {
local dir="$1"
local hash="$2"
# Add nested executables to bundle dir so we have complete set of
# them available, but only if the native OS/ARCH is the same as the
# OS/ARCH of the build target
if [ "$(go env GOOS)/$(go env GOARCH)" != "$(go env GOHOSTOS)/$(go env GOHOSTARCH)" ]; then
return
fi
if [ ! -x /usr/local/bin/docker-runc ]; then
return
fi
echo "Copying nested executables into $dir"
for file in containerd containerd-shim containerd-ctr runc init proxy; do
cp -f `which "docker-$file"` "$dir/"
if [ "$hash" == "hash" ]; then
hash_files "$dir/docker-$file"
fi
done
}
[ -z "$KEEPDEST" ] && rm -rf "$DEST"
source "${MAKEDIR}/.binary"
copy_binaries "$DEST" 'hash'

View file

@ -0,0 +1,7 @@
#!/usr/bin/env bash
# required by `make build-integration-cli-on-swarm`
set -e
source hack/make/.integration-test-helpers
build_test_suite_binaries

View file

@ -0,0 +1,29 @@
#!/usr/bin/env bash
set -e
# if we have our linux/amd64 version compiled, let's symlink it in
if [ -x "$DEST/../binary-daemon/dockerd-$VERSION" ]; then
arch=$(go env GOHOSTARCH)
mkdir -p "$DEST/linux/${arch}"
(
cd "$DEST/linux/${arch}"
ln -sf ../../../binary-daemon/* ./
)
echo "Created symlinks:" "$DEST/linux/${arch}/"*
fi
DOCKER_CROSSPLATFORMS=${DOCKER_CROSSPLATFORMS:-"linux/amd64 windows/amd64"}
for platform in $DOCKER_CROSSPLATFORMS; do
(
export KEEPDEST=1
export DEST="$DEST/$platform" # bundles/VERSION/cross/GOOS/GOARCH/docker-VERSION
export GOOS=${platform%/*}
export GOARCH=${platform##*/}
echo "Cross building: $DEST"
mkdir -p "$DEST"
ABS_DEST="$(cd "$DEST" && pwd -P)"
source "${MAKEDIR}/binary-daemon"
)
done

View file

@ -0,0 +1,10 @@
#!/usr/bin/env bash
set -e
# This script exists as backwards compatibility for CI
(
DEST="${DEST}-daemon"
ABS_DEST="${ABS_DEST}-daemon"
. hack/make/dynbinary-daemon
)

View file

@ -0,0 +1,10 @@
#!/usr/bin/env bash
set -e
(
export IAMSTATIC='false'
export LDFLAGS_STATIC_DOCKER=''
export BUILDFLAGS=( "${BUILDFLAGS[@]/netgo /}" ) # disable netgo, since we don't need it for a dynamic binary
export BUILDFLAGS=( "${BUILDFLAGS[@]/static_build /}" ) # we're not building a "static" binary here
source "${MAKEDIR}/.binary"
)

View file

@ -0,0 +1,29 @@
#!/usr/bin/env bash
set -e
rm -rf "$DEST"
install_binary() {
local file="$1"
local target="${DOCKER_MAKE_INSTALL_PREFIX:=/usr/local}/bin/"
if [ "$(go env GOOS)" == "linux" ]; then
echo "Installing $(basename $file) to ${target}"
mkdir -p "$target"
cp -f -L "$file" "$target"
else
echo "Install is only supported on linux"
return 1
fi
}
(
DEST="$(dirname $DEST)/binary-daemon"
source "${MAKEDIR}/.binary-setup"
install_binary "${DEST}/${DOCKER_DAEMON_BINARY_NAME}"
install_binary "${DEST}/${DOCKER_RUNC_BINARY_NAME}"
install_binary "${DEST}/${DOCKER_CONTAINERD_BINARY_NAME}"
install_binary "${DEST}/${DOCKER_CONTAINERD_CTR_BINARY_NAME}"
install_binary "${DEST}/${DOCKER_CONTAINERD_SHIM_BINARY_NAME}"
install_binary "${DEST}/${DOCKER_PROXY_BINARY_NAME}"
install_binary "${DEST}/${DOCKER_INIT_BINARY_NAME}"
)

View file

@ -0,0 +1,44 @@
#!/usr/bin/env bash
set -e
rm -rf "$DEST"
if ! command -v dockerd &> /dev/null; then
echo >&2 'error: binary-daemon or dynbinary-daemon must be run before run'
false
fi
DOCKER_GRAPHDRIVER=${DOCKER_GRAPHDRIVER:-vfs}
DOCKER_USERLANDPROXY=${DOCKER_USERLANDPROXY:-true}
# example usage: DOCKER_STORAGE_OPTS="dm.basesize=20G,dm.loopdatasize=200G"
storage_params=""
if [ -n "$DOCKER_STORAGE_OPTS" ]; then
IFS=','
for i in ${DOCKER_STORAGE_OPTS}; do
storage_params="--storage-opt $i $storage_params"
done
unset IFS
fi
listen_port=2375
if [ -n "$DOCKER_PORT" ]; then
IFS=':' read -r -a ports <<< "$DOCKER_PORT"
listen_port="${ports[-1]}"
fi
extra_params=""
if [ "$DOCKER_REMAP_ROOT" ]; then
extra_params="--userns-remap $DOCKER_REMAP_ROOT"
fi
args="--debug \
--host tcp://0.0.0.0:${listen_port} --host unix:///var/run/docker.sock \
--storage-driver "$DOCKER_GRAPHDRIVER" \
--userland-proxy="$DOCKER_USERLANDPROXY" \
$storage_params \
$extra_params"
echo dockerd $args
exec dockerd $args

View file

@ -0,0 +1,20 @@
#!/usr/bin/env bash
set -e
source hack/make/.integration-test-helpers
# subshell so that we can export PATH without breaking other things
(
bundle .integration-daemon-start
dockerPy='/docker-py'
[ -d "$dockerPy" ] || {
dockerPy="$DEST/docker-py"
git clone https://github.com/docker/docker-py.git "$dockerPy"
}
# exporting PYTHONPATH to import "docker" from our local docker-py
test_env PYTHONPATH="$dockerPy" py.test --junitxml="$DEST/results.xml" "$dockerPy/tests/integration"
bundle .integration-daemon-stop
) 2>&1 | tee -a "$DEST/test.log"

View file

@ -0,0 +1,21 @@
#!/usr/bin/env bash
set -e -o pipefail
source hack/make/.integration-test-helpers
(
build_test_suite_binaries
bundle .integration-daemon-start
bundle .integration-daemon-setup
local testexit=0
( repeat run_test_integration ) || testexit=$?
# Always run cleanup, even if the subshell fails
bundle .integration-daemon-stop
cleanup_test_suite_binaries
error_on_leaked_containerd_shims
exit $testexit
) 2>&1 | tee -a "$DEST/test.log"

View file

@ -0,0 +1,6 @@
#!/usr/bin/env bash
set -e
echo "WARNING: test-integration-cli is DEPRECATED. Use test-integration." >&2
# TODO: remove this and exit 1 once CI has changed to use test-integration
bundle test-integration

View file

@ -0,0 +1,9 @@
#!/usr/bin/env bash
bundle .integration-daemon-start
bundle .integration-daemon-setup
export ABS_DEST
bash +e
bundle .integration-daemon-stop

View file

@ -0,0 +1,71 @@
#!/usr/bin/env bash
set -e -u -o pipefail
ARCH=$(uname -m)
if [ "$ARCH" == "x86_64" ]; then
ARCH="amd64"
fi
export DOCKER_ENGINE_GOARCH=${DOCKER_ENGINE_GOARCH:-${ARCH}}
# Set defaults
: ${TESTFLAGS:=}
: ${TESTDEBUG:=}
integration_api_dirs=${TEST_INTEGRATION_DIR:-"$(
find ./integration -type d |
grep -vE '(^./integration($|/internal)|/testdata)')"}
run_test_integration() {
[[ "$TESTFLAGS" != *-check.f* ]] && run_test_integration_suites
run_test_integration_legacy_suites
}
run_test_integration_suites() {
local flags="-test.v -test.timeout=${TIMEOUT:=10m} $TESTFLAGS"
for dir in $integration_api_dirs; do
if ! (
cd $dir
echo "Running $PWD"
test_env ./test.main $flags
); then exit 1; fi
done
}
run_test_integration_legacy_suites() {
(
flags="-check.v -check.timeout=${TIMEOUT} -test.timeout=360m $TESTFLAGS"
cd test/integration-cli
echo "Running $PWD"
test_env ./test.main $flags
)
}
# use "env -i" to tightly control the environment variables that bleed into the tests
test_env() {
(
set -e +u
[ -n "$TESTDEBUG" ] && set -x
env -i \
DOCKER_API_VERSION="$DOCKER_API_VERSION" \
DOCKER_INTEGRATION_DAEMON_DEST="$DOCKER_INTEGRATION_DAEMON_DEST" \
DOCKER_TLS_VERIFY="$DOCKER_TEST_TLS_VERIFY" \
DOCKER_CERT_PATH="$DOCKER_TEST_CERT_PATH" \
DOCKER_ENGINE_GOARCH="$DOCKER_ENGINE_GOARCH" \
DOCKER_GRAPHDRIVER="$DOCKER_GRAPHDRIVER" \
DOCKER_USERLANDPROXY="$DOCKER_USERLANDPROXY" \
DOCKER_HOST="$DOCKER_HOST" \
DOCKER_REMAP_ROOT="$DOCKER_REMAP_ROOT" \
DOCKER_REMOTE_DAEMON="$DOCKER_REMOTE_DAEMON" \
DOCKERFILE="$DOCKERFILE" \
GOPATH="$GOPATH" \
GOTRACEBACK=all \
HOME="$ABS_DEST/fake-HOME" \
PATH="$PATH" \
TEMP="$TEMP" \
TEST_CLIENT_BINARY="$TEST_CLIENT_BINARY" \
"$@"
)
}
run_test_integration

View file

@ -0,0 +1,38 @@
#!/usr/bin/env bash
#
# Run unit tests
#
# TESTFLAGS - add additional test flags. Ex:
#
# TESTFLAGS="-v -run TestBuild" hack/test/unit
#
# TESTDIRS - run tests for specified packages. Ex:
#
# TESTDIRS="./pkg/term" hack/test/unit
#
set -eu -o pipefail
TESTFLAGS+=" -test.timeout=${TIMEOUT:-5m}"
BUILDFLAGS=( -tags "netgo seccomp libdm_no_deferred_remove" )
TESTDIRS="${TESTDIRS:-"./..."}"
exclude_paths="/vendor/|/integration"
pkg_list=$(go list $TESTDIRS | grep -vE "($exclude_paths)")
# install test dependencies once before running tests for each package. This
# significantly reduces the runtime.
go test -i "${BUILDFLAGS[@]}" $pkg_list
for pkg in $pkg_list; do
go test "${BUILDFLAGS[@]}" \
-cover \
-coverprofile=profile.out \
-covermode=atomic \
$TESTFLAGS \
"${pkg}"
if test -f profile.out; then
cat profile.out >> coverage.txt
rm profile.out
fi
done

View file

@ -0,0 +1,4 @@
extends: default
rules:
document-start: disable
line-length: disable

View file

@ -0,0 +1,30 @@
#!/usr/bin/env bash
set -e -o pipefail
if [ -z "$VALIDATE_UPSTREAM" ]; then
# this is kind of an expensive check, so let's not do this twice if we
# are running more than one validate bundlescript
VALIDATE_REPO='https://github.com/docker/docker.git'
VALIDATE_BRANCH='master'
VALIDATE_HEAD="$(git rev-parse --verify HEAD)"
git fetch -q "$VALIDATE_REPO" "refs/heads/$VALIDATE_BRANCH"
VALIDATE_UPSTREAM="$(git rev-parse --verify FETCH_HEAD)"
VALIDATE_COMMIT_LOG="$VALIDATE_UPSTREAM..$VALIDATE_HEAD"
VALIDATE_COMMIT_DIFF="$VALIDATE_UPSTREAM...$VALIDATE_HEAD"
validate_diff() {
if [ "$VALIDATE_UPSTREAM" != "$VALIDATE_HEAD" ]; then
git diff "$VALIDATE_COMMIT_DIFF" "$@"
fi
}
validate_log() {
if [ "$VALIDATE_UPSTREAM" != "$VALIDATE_HEAD" ]; then
git log "$VALIDATE_COMMIT_LOG" "$@"
fi
}
fi

View file

@ -0,0 +1,8 @@
#!/usr/bin/env bash
#
# Run all validation
export SCRIPTDIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
. $SCRIPTDIR/default
. $SCRIPTDIR/vendor

View file

@ -0,0 +1,12 @@
#!/usr/bin/env bash
changelogFile=${1:-CHANGELOG.md}
if [ ! -r "$changelogFile" ]; then
echo "Unable to read file $changelogFile" >&2
exit 1
fi
grep -e '^## ' "$changelogFile" | awk '{print$3}' | sort -c -r || exit 2
echo "Congratulations! Changelog $changelogFile dates are in descending order."

View file

@ -0,0 +1,25 @@
#!/usr/bin/env bash
changelogFile=${1:-CHANGELOG.md}
if [ ! -r "$changelogFile" ]; then
echo "Unable to read file $changelogFile" >&2
exit 1
fi
changelogWellFormed=1
# e.g. "## 1.12.3 (2016-10-26)"
VER_LINE_REGEX='^## [0-9]+\.[0-9]+\.[0-9]+(-ce)? \([0-9]+-[0-9]+-[0-9]+\)$'
while read -r line; do
if ! [[ "$line" =~ $VER_LINE_REGEX ]]; then
echo "Malformed changelog $changelogFile line \"$line\"" >&2
changelogWellFormed=0
fi
done < <(grep '^## ' $changelogFile)
if [[ "$changelogWellFormed" == "1" ]]; then
echo "Congratulations! Changelog $changelogFile is well-formed."
else
exit 2
fi

View file

@ -0,0 +1,55 @@
#!/usr/bin/env bash
export SCRIPTDIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
source "${SCRIPTDIR}/.validate"
adds=$(validate_diff --numstat | awk '{ s += $1 } END { print s }')
dels=$(validate_diff --numstat | awk '{ s += $2 } END { print s }')
#notDocs="$(validate_diff --numstat | awk '$3 !~ /^docs\// { print $3 }')"
: ${adds:=0}
: ${dels:=0}
# "Username may only contain alphanumeric characters or dashes and cannot begin with a dash"
githubUsernameRegex='[a-zA-Z0-9][a-zA-Z0-9-]+'
# https://github.com/docker/docker/blob/master/CONTRIBUTING.md#sign-your-work
dcoPrefix='Signed-off-by:'
dcoRegex="^(Docker-DCO-1.1-)?$dcoPrefix ([^<]+) <([^<>@]+@[^<>]+)>( \\(github: ($githubUsernameRegex)\\))?$"
check_dco() {
grep -qE "$dcoRegex"
}
if [ $adds -eq 0 -a $dels -eq 0 ]; then
echo '0 adds, 0 deletions; nothing to validate! :)'
else
commits=( $(validate_log --format='format:%H%n') )
badCommits=()
for commit in "${commits[@]}"; do
if [ -z "$(git log -1 --format='format:' --name-status "$commit")" ]; then
# no content (ie, Merge commit, etc)
continue
fi
if ! git log -1 --format='format:%B' "$commit" | check_dco; then
badCommits+=( "$commit" )
fi
done
if [ ${#badCommits[@]} -eq 0 ]; then
echo "Congratulations! All commits are properly signed with the DCO!"
else
{
echo "These commits do not have a proper '$dcoPrefix' marker:"
for commit in "${badCommits[@]}"; do
echo " - $commit"
done
echo
echo 'Please amend each commit to include a properly formatted DCO marker.'
echo
echo 'Visit the following URL for information about the Docker DCO:'
echo ' https://github.com/docker/docker/blob/master/CONTRIBUTING.md#sign-your-work'
echo
} >&2
false
fi
fi

View file

@ -0,0 +1,17 @@
#!/usr/bin/env bash
#
# Run default validation, exclude vendor because it's slow
export SCRIPTDIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
. $SCRIPTDIR/dco
. $SCRIPTDIR/default-seccomp
. $SCRIPTDIR/gometalinter
. $SCRIPTDIR/pkg-imports
. $SCRIPTDIR/swagger
. $SCRIPTDIR/swagger-gen
. $SCRIPTDIR/test-imports
. $SCRIPTDIR/toml
. $SCRIPTDIR/changelog-well-formed
. $SCRIPTDIR/changelog-date-descending
. $SCRIPTDIR/deprecate-integration-cli

View file

@ -0,0 +1,28 @@
#!/usr/bin/env bash
export SCRIPTDIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
source "${SCRIPTDIR}/.validate"
IFS=$'\n'
files=( $(validate_diff --diff-filter=ACMR --name-only -- 'profiles/seccomp' || true) )
unset IFS
if [ ${#files[@]} -gt 0 ]; then
# We run 'go generate' and see if we have a diff afterwards
go generate ./profiles/seccomp/ >/dev/null
# Let see if the working directory is clean
diffs="$(git status --porcelain -- profiles/seccomp 2>/dev/null)"
if [ "$diffs" ]; then
{
echo 'The result of go generate ./profiles/seccomp/ differs'
echo
echo "$diffs"
echo
echo 'Please re-run go generate ./profiles/seccomp/'
echo
} >&2
false
else
echo 'Congratulations! Seccomp profile generation is done correctly.'
fi
fi

View file

@ -0,0 +1,25 @@
#!/usr/bin/env bash
# Check that no new tests are being added to integration-cli
export SCRIPTDIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
source "${SCRIPTDIR}/.validate"
new_tests=$(
validate_diff --diff-filter=ACMR --unified=0 -- 'integration-cli/*_cli_*.go' |
grep -E '^\+func (.*) Test' || true
)
if [ -z "$new_tests" ]; then
echo 'Congratulations! No new tests added to integration-cli.'
exit
fi
echo "The following new tests were added to integration-cli:"
echo
echo "$new_tests"
echo
echo "integration-cli is deprecated. Please add an API integration test to"
echo "./integration/COMPONENT/. See ./TESTING.md for more details."
echo
exit 1

View file

@ -0,0 +1,13 @@
#!/usr/bin/env bash
set -e -o pipefail
SCRIPTDIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
# CI platforms differ, so per-platform GOMETALINTER_OPTS can be set
# from a platform-specific Dockerfile, otherwise let's just set
# (somewhat pessimistic) default of 10 minutes.
: ${GOMETALINTER_OPTS=--deadline=10m}
gometalinter \
${GOMETALINTER_OPTS} \
--config $SCRIPTDIR/gometalinter.json ./...

View file

@ -0,0 +1,26 @@
{
"Vendor": true,
"EnableGC": true,
"Sort": ["linter", "severity", "path"],
"Exclude": [
".*\\.pb\\.go",
"dockerversion/version_autogen.go",
"api/types/container/container_.*",
"integration-cli/"
],
"Skip": ["integration-cli/"],
"Enable": [
"deadcode",
"gofmt",
"goimports",
"golint",
"gosimple",
"ineffassign",
"interfacer",
"unconvert",
"vet"
],
"LineLength": 200
}

View file

@ -0,0 +1,33 @@
#!/usr/bin/env bash
set -e
export SCRIPTDIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
source "${SCRIPTDIR}/.validate"
IFS=$'\n'
files=( $(validate_diff --diff-filter=ACMR --name-only -- 'pkg/*.go' || true) )
unset IFS
badFiles=()
for f in "${files[@]}"; do
IFS=$'\n'
badImports=( $(go list -e -f '{{ join .Deps "\n" }}' "$f" | sort -u | grep -vE '^github.com/docker/docker/pkg/' | grep -vE '^github.com/docker/docker/vendor' | grep -E '^github.com/docker/docker' || true) )
unset IFS
for import in "${badImports[@]}"; do
badFiles+=( "$f imports $import" )
done
done
if [ ${#badFiles[@]} -eq 0 ]; then
echo 'Congratulations! "./pkg/..." is safely isolated from internal code.'
else
{
echo 'These files import internal code: (either directly or indirectly)'
for f in "${badFiles[@]}"; do
echo " - $f"
done
echo
} >&2
false
fi

View file

@ -0,0 +1,13 @@
#!/usr/bin/env bash
set -e
export SCRIPTDIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
source "${SCRIPTDIR}/.validate"
IFS=$'\n'
files=( $(validate_diff --diff-filter=ACMR --name-only -- 'api/swagger.yaml' || true) )
unset IFS
if [ ${#files[@]} -gt 0 ]; then
yamllint -c ${SCRIPTDIR}/.swagger-yamllint api/swagger.yaml
swagger validate api/swagger.yaml
fi

View file

@ -0,0 +1,29 @@
#!/usr/bin/env bash
export SCRIPTDIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
source "${SCRIPTDIR}/.validate"
IFS=$'\n'
files=( $(validate_diff --diff-filter=ACMR --name-only -- 'api/types/' 'api/swagger.yaml' || true) )
unset IFS
if [ ${#files[@]} -gt 0 ]; then
${SCRIPTDIR}/../generate-swagger-api.sh 2> /dev/null
# Let see if the working directory is clean
diffs="$(git diff -- api/types/)"
if [ "$diffs" ]; then
{
echo 'The result of hack/generate-swagger-api.sh differs'
echo
echo "$diffs"
echo
echo 'Please update api/swagger.yaml with any api changes, then '
echo 'run `hack/generate-swagger-api.sh`.'
} >&2
false
else
echo 'Congratulations! All api changes are done the right way.'
fi
else
echo 'No api/types/ or api/swagger.yaml changes in diff.'
fi

View file

@ -0,0 +1,38 @@
#!/usr/bin/env bash
# Make sure we're not using gos' Testing package any more in integration-cli
export SCRIPTDIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
source "${SCRIPTDIR}/.validate"
IFS=$'\n'
files=( $(validate_diff --diff-filter=ACMR --name-only -- 'integration-cli/*.go' || true) )
unset IFS
badFiles=()
for f in "${files[@]}"; do
# skip check_test.go since it *does* use the testing package
if [ "$f" = "integration-cli/check_test.go" ]; then
continue
fi
# we use "git show" here to validate that what's committed doesn't contain golang built-in testing
if git show "$VALIDATE_HEAD:$f" | grep -q testing.T; then
if [ "$(echo $f | grep '_test')" ]; then
# allow testing.T for non- _test files
badFiles+=( "$f" )
fi
fi
done
if [ ${#badFiles[@]} -eq 0 ]; then
echo 'Congratulations! No testing.T found.'
else
{
echo "These files use the wrong testing infrastructure:"
for f in "${badFiles[@]}"; do
echo " - $f"
done
echo
} >&2
false
fi

View file

@ -0,0 +1,31 @@
#!/usr/bin/env bash
export SCRIPTDIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
source "${SCRIPTDIR}/.validate"
IFS=$'\n'
files=( $(validate_diff --diff-filter=ACMR --name-only -- 'MAINTAINERS' || true) )
unset IFS
badFiles=()
for f in "${files[@]}"; do
# we use "git show" here to validate that what's committed has valid toml syntax
if ! git show "$VALIDATE_HEAD:$f" | tomlv /proc/self/fd/0 ; then
badFiles+=( "$f" )
fi
done
if [ ${#badFiles[@]} -eq 0 ]; then
echo 'Congratulations! All toml source files changed here have valid syntax.'
else
{
echo "These files are not valid toml:"
for f in "${badFiles[@]}"; do
echo " - $f"
done
echo
echo 'Please reformat the above files as valid toml'
echo
} >&2
false
fi

View file

@ -0,0 +1,55 @@
#!/usr/bin/env bash
export SCRIPTDIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
source "${SCRIPTDIR}/.validate"
validate_vendor_diff(){
IFS=$'\n'
files=( $(validate_diff --diff-filter=ACMR --name-only -- 'vendor.conf' 'vendor/' || true) )
unset IFS
if [ ${#files[@]} -gt 0 ]; then
# Remove vendor/ first so that anything not included in vendor.conf will
# cause the validation to fail. archive/tar is a special case, see vendor.conf
# for details.
ls -d vendor/* | grep -v vendor/archive | xargs rm -rf
# run vndr to recreate vendor/
vndr
# check if any files have changed
diffs="$(git status --porcelain -- vendor 2>/dev/null)"
if [ "$diffs" ]; then
{
echo 'The result of vndr differs'
echo
echo "$diffs"
echo
echo 'Please vendor your package with github.com/LK4D4/vndr.'
echo
} >&2
false
else
echo 'Congratulations! All vendoring changes are done the right way.'
fi
else
echo 'No vendor changes in diff.'
fi
}
# 1. make sure all the vendored packages are used
# 2. make sure all the packages contain license information (just warning, because it can cause false-positive)
validate_vendor_used() {
pkgs=$(mawk '/^[a-zA-Z0-9]/ { print $1 }' < vendor.conf)
for f in $pkgs; do
if ls -d vendor/$f > /dev/null 2>&1; then
found=$(find vendor/$f -iregex '.*LICENSE.*' -or -iregex '.*COPYRIGHT.*' -or -iregex '.*COPYING.*' | wc -l)
if [ $found -eq 0 ]; then
echo "WARNING: could not find copyright information for $f"
fi
else
echo "WARNING: $f is vendored but unused"
fi
done
}
validate_vendor_diff
validate_vendor_used

View file

@ -0,0 +1,15 @@
#!/usr/bin/env bash
# This file is just wrapper around vndr (github.com/LK4D4/vndr) tool.
# For updating dependencies you should change `vendor.conf` file in root of the
# project. Please refer to https://github.com/LK4D4/vndr/blob/master/README.md for
# vndr usage.
set -e
if ! hash vndr; then
echo "Please install vndr with \"go get github.com/LK4D4/vndr\" and put it in your \$GOPATH"
exit 1
fi
vndr "$@"