Add initial integration tests framework

Signed-off-by: Kenfe-Mickael Laventure <mickael.laventure@gmail.com>
This commit is contained in:
Kenfe-Mickael Laventure 2016-03-21 12:06:39 -07:00
parent 9992d2e1bd
commit cb69ab45ee
5 changed files with 662 additions and 3 deletions

View File

@ -1,5 +1,7 @@
BUILDTAGS=
PROJECT=github.com/docker/containerd
GIT_COMMIT := $(shell git rev-parse HEAD 2> /dev/null || true)
GIT_BRANCH := $(shell git rev-parse --abbrev-ref HEAD 2> /dev/null)
@ -13,8 +15,12 @@ ifeq ($(INTERACTIVE), 1)
DOCKER_FLAGS += -t
endif
TEST_ARTIFACTS_DIR := integration-test/test-artifacts
BUNDLE_ARCHIVES_DIR := $(TEST_ARTIFACTS_DIR)/archives
DOCKER_IMAGE := containerd-dev$(if $(GIT_BRANCH),:$(GIT_BRANCH))
DOCKER_RUN := docker run --rm -i $(DOCKER_FLAGS) "$(DOCKER_IMAGE)"
DOCKER_RUN := docker run --privileged --rm -i $(DOCKER_FLAGS) "$(DOCKER_IMAGE)"
export GOPATH:=$(CURDIR)/vendor:$(GOPATH)
@ -46,7 +52,13 @@ shim: bin
shim-static:
cd containerd-shim && go build -ldflags "-w -extldflags -static ${LDFLAGS}" -tags "$(BUILDTAGS)" -o ../bin/containerd-shim
dbuild:
$(BUNDLE_ARCHIVES_DIR)/busybox.tar:
@mkdir -p $(BUNDLE_ARCHIVES_DIR)
curl -sSL 'https://github.com/jpetazzo/docker-busybox/raw/buildroot-2014.11/rootfs.tar' -o $(BUNDLE_ARCHIVES_DIR)/busybox.tar
bundles-rootfs: $(BUNDLE_ARCHIVES_DIR)/busybox.tar
dbuild: $(BUNDLE_ARCHIVES_DIR)/busybox.tar
@docker build --rm --force-rm -t "$(DOCKER_IMAGE)" .
dtest: dbuild
@ -68,7 +80,12 @@ shell: dbuild
$(DOCKER_RUN) bash
test: all validate
go test -v $(shell go list ./... | grep -v /vendor)
go test -v $(shell go list ./... | grep -v /vendor | grep -v /integration-test)
ifneq ($(wildcard /.dockerenv), )
$(MAKE) install bundles-rootfs
cd integration-test ; \
go test -check.v $(TESTFLAGS) github.com/docker/containerd/integration-test
endif
validate: fmt

View File

@ -0,0 +1,107 @@
package main
import (
"encoding/json"
"fmt"
"os"
"os/exec"
"path/filepath"
"reflect"
ocs "github.com/opencontainers/specs/specs-go"
)
var (
bundlesDir = filepath.Join("test-artifacts", "oci-bundles")
refOciSpecsPath = filepath.Join(bundlesDir, "config.json")
)
type OciProcessArgs struct {
Cmd string
Args []string
}
type Bundle struct {
Source string
Name string
Spec ocs.Spec
Path string
}
var bundleMap map[string]Bundle
// untarRootfs untars the given `source` tarPath into `destination/rootfs`
func untarRootfs(source string, destination string) error {
destination = filepath.Join(destination, "rootfs")
if err := os.MkdirAll(destination, 0755); err != nil {
return nil
}
tar := exec.Command("tar", "-C", destination, "-xf", source)
return tar.Run()
}
// CreateBundleWithFilter generate a new oci-bundle named `name` from
// the provide `source` rootfs. It starts from the default spec
// generated by `runc spec`, overrides the `spec.Process.Args` value
// with `args` and set `spec.Process.Terminal` to false. It then apply
// `filter()` to the resulting spec if it is provided.
func CreateBundleWithFilter(source, name string, args []string, filter func(spec *ocs.Spec)) error {
// Generate the spec
var spec ocs.Spec
if f, err := os.Open(refOciSpecsPath); err != nil {
return fmt.Errorf("Failed to open default spec: %v", err)
} else {
if err := json.NewDecoder(f).Decode(&spec); err != nil {
return fmt.Errorf("Failed to load default spec: %v", err)
}
f.Close()
}
spec.Process.Args = args
spec.Process.Terminal = false
if filter != nil {
filter(&spec)
}
bundlePath := filepath.Join(bundlesDir, name)
nb := Bundle{source, name, spec, bundlePath}
// Check that we don't already have such a bundle
if b, ok := bundleMap[name]; ok {
if reflect.DeepEqual(b, nb) == false {
return fmt.Errorf("A bundle name named '%s' already exist but with different properties! %#v != %#v",
name, b, nb)
}
return nil
}
// Nothing should be there, but just in case
os.RemoveAll(bundlePath)
if err := untarRootfs(filepath.Join(archivesDir, source+".tar"), bundlePath); err != nil {
return fmt.Errorf("Failed to untar %s.tar: %v", source, err)
}
// create a place for the io fifo
if err := os.Mkdir(filepath.Join(bundlePath, "io"), 0755); err != nil {
return fmt.Errorf("Failed to create bundle io directory: %v", err)
}
// Write the updated spec to the right location
config, e := os.Create(filepath.Join(bundlePath, "config.json"))
if e != nil {
return fmt.Errorf("Failed to create oci spec: %v", e)
}
defer config.Close()
if err := json.NewEncoder(config).Encode(&spec); err != nil {
return fmt.Errorf("Failed to encore oci spec: %v", e)
}
bundleMap[name] = nb
return nil
}
func CreateBusyboxBundle(name string, args []string) error {
return CreateBundleWithFilter("busybox", name, args, nil)
}

View File

@ -0,0 +1,228 @@
package main
import (
"fmt"
"io/ioutil"
"log"
"net"
"os"
"os/exec"
"path/filepath"
"sync"
"testing"
"time"
"golang.org/x/net/context"
"google.golang.org/grpc"
"google.golang.org/grpc/grpclog"
"github.com/docker/containerd/api/grpc/types"
"github.com/go-check/check"
)
var (
outputDirFormat = filepath.Join("test-artifacts", "runs", "%s")
archivesDir = filepath.Join("test-artifacts", "archives")
)
func Test(t *testing.T) {
check.TestingT(t)
}
func init() {
check.Suite(&ContainerdSuite{})
}
type ContainerdSuite struct {
cwd string
outputDir string
logFile *os.File
cd *exec.Cmd
syncChild chan error
grpcClient types.APIClient
eventFiltersMutex sync.Mutex
eventFilters map[string]func(event *types.Event)
}
// getClient returns a connection to the Suite containerd
func (cs *ContainerdSuite) getClient(socket string) error {
// reset the logger for grpc to log to dev/null so that it does not mess with our stdio
grpclog.SetLogger(log.New(ioutil.Discard, "", log.LstdFlags))
dialOpts := []grpc.DialOption{grpc.WithInsecure()}
dialOpts = append(dialOpts,
grpc.WithDialer(func(addr string, timeout time.Duration) (net.Conn, error) {
return net.DialTimeout("unix", addr, timeout)
},
))
conn, err := grpc.Dial(socket, dialOpts...)
if err != nil {
return err
}
cs.grpcClient = types.NewAPIClient(conn)
return nil
}
// ContainerdEventsHandler will process all events coming from
// containerd. If a filter as been register for a given container id
// via `SetContainerEventFilter()`, it will be invoked every time an
// event for that id is received
func (cs *ContainerdSuite) ContainerdEventsHandler(events types.API_EventsClient) {
timestamp := uint64(time.Now().Unix())
for {
e, err := events.Recv()
if err != nil {
time.Sleep(1 * time.Second)
events, _ = cs.grpcClient.Events(context.Background(), &types.EventsRequest{Timestamp: timestamp})
continue
}
timestamp = e.Timestamp
cs.eventFiltersMutex.Lock()
if f, ok := cs.eventFilters[e.Id]; ok {
f(e)
if e.Type == "exit" && e.Pid == "init" {
delete(cs.eventFilters, e.Id)
}
}
cs.eventFiltersMutex.Unlock()
}
}
// generateReferencesSpecs invoke `runc spec` to produce the baseline
// specs from which all future bundle will be generated
func generateReferenceSpecs(destination string) error {
specs := exec.Command("runc", "spec")
specs.Dir = destination
return specs.Run()
}
func (cs *ContainerdSuite) SetUpSuite(c *check.C) {
bundleMap = make(map[string]Bundle)
cs.eventFilters = make(map[string]func(event *types.Event))
// Get our CWD
if cwd, err := os.Getwd(); err != nil {
c.Fatalf("Could not determine current working directory: %v", err)
} else {
cs.cwd = cwd
}
// Clean old bundles
os.RemoveAll(bundlesDir)
// Ensure the oci bundles directory exists
if err := os.MkdirAll(bundlesDir, 0755); err != nil {
c.Fatalf("Failed to create bundles directory: %v", err)
}
// Generate the reference spec
if err := generateReferenceSpecs(bundlesDir); err != nil {
c.Fatalf("Unable to generate OCI reference spec: %v", err)
}
// Create our output directory
od := fmt.Sprintf(outputDirFormat, time.Now().Format("2006-01-02_150405.000000"))
cdStateDir := fmt.Sprintf("%s/containerd-master", od)
if err := os.MkdirAll(cdStateDir, 0755); err != nil {
c.Fatalf("Unable to created output directory '%s': %v", cdStateDir, err)
}
cdGRPCSock := filepath.Join(od, "containerd-master", "containerd.sock")
cdLogFile := filepath.Join(od, "containerd-master", "containerd.log")
f, err := os.OpenFile(cdLogFile, os.O_CREATE|os.O_TRUNC|os.O_RDWR|os.O_SYNC, 0777)
if err != nil {
c.Fatalf("Failed to create master containerd log file: %v", err)
}
cs.logFile = f
cd := exec.Command("containerd", "--debug",
"--state-dir", cdStateDir,
"--listen", cdGRPCSock,
"--metrics-interval", "0m0s",
"--runtime-args", fmt.Sprintf("--root=%s", filepath.Join(cs.cwd, cdStateDir, "runc")),
)
cd.Stderr = f
cd.Stdout = f
if err := cd.Start(); err != nil {
c.Fatalf("Unable to start the master containerd: %v", err)
}
cs.outputDir = od
cs.cd = cd
cs.syncChild = make(chan error)
if err := cs.getClient(cdGRPCSock); err != nil {
// Kill the daemon
cs.cd.Process.Kill()
c.Fatalf("Failed to connect to daemon: %v", err)
}
// Monitor events
events, err := cs.grpcClient.Events(context.Background(), &types.EventsRequest{})
if err != nil {
c.Fatalf("Could not register containerd event handler: %v", err)
}
go cs.ContainerdEventsHandler(events)
go func() {
cs.syncChild <- cd.Wait()
}()
}
func (cs *ContainerdSuite) TearDownSuite(c *check.C) {
// tell containerd to stop
if cs.cd != nil {
cs.cd.Process.Signal(os.Interrupt)
done := false
for done == false {
select {
case err := <-cs.syncChild:
if err != nil {
c.Errorf("master containerd did not exit cleanly: %v", err)
}
done = true
case <-time.After(3 * time.Second):
fmt.Println("Timeout while waiting for containerd to exit, killing it!")
cs.cd.Process.Kill()
}
}
}
if cs.logFile != nil {
cs.logFile.Close()
}
}
func (cs *ContainerdSuite) SetContainerEventFilter(id string, filter func(event *types.Event)) {
cs.eventFiltersMutex.Lock()
cs.eventFilters[id] = filter
cs.eventFiltersMutex.Unlock()
}
func (cs *ContainerdSuite) TearDownTest(c *check.C) {
ctrs, err := cs.ListRunningContainers()
if err != nil {
c.Fatalf("Unable to retrieve running containers: %v", err)
}
// Kill all containers that survived
for _, ctr := range ctrs {
ch := make(chan interface{})
cs.SetContainerEventFilter(ctr.Id, func(e *types.Event) {
if e.Type == "exit" && e.Pid == "init" {
ch <- nil
}
})
if err := cs.KillContainer(ctr.Id); err != nil {
fmt.Fprintf(os.Stderr, "Failed to cleanup leftover test containers: %v", err)
}
<-ch
}
}

View File

@ -0,0 +1,249 @@
package main
import (
"bytes"
"fmt"
"io"
"os"
"path/filepath"
"syscall"
"github.com/docker/containerd/api/grpc/types"
"golang.org/x/net/context"
)
func (cs *ContainerdSuite) ListRunningContainers() ([]*types.Container, error) {
resp, err := cs.grpcClient.State(context.Background(), &types.StateRequest{})
if err != nil {
return nil, err
}
return resp.Containers, nil
}
func (cs *ContainerdSuite) SignalContainerProcess(id string, procId string, sig uint32) error {
_, err := cs.grpcClient.Signal(context.Background(), &types.SignalRequest{
Id: id,
Pid: procId,
Signal: sig,
})
return err
}
func (cs *ContainerdSuite) SignalContainer(id string, sig uint32) error {
return cs.SignalContainerProcess(id, "init", sig)
}
func (cs *ContainerdSuite) KillContainer(id string) error {
return cs.SignalContainerProcess(id, "init", uint32(syscall.SIGKILL))
}
func (cs *ContainerdSuite) PauseContainer(id string) error {
_, err := cs.grpcClient.UpdateContainer(context.Background(), &types.UpdateContainerRequest{
Id: id,
Pid: "init",
Status: "paused",
})
return err
}
func (cs *ContainerdSuite) ResumeContainer(id string) error {
_, err := cs.grpcClient.UpdateContainer(context.Background(), &types.UpdateContainerRequest{
Id: id,
Pid: "init",
Status: "running",
})
return err
}
func (cs *ContainerdSuite) GetContainerStats(id string) (*types.StatsResponse, error) {
stats, err := cs.grpcClient.Stats(context.Background(), &types.StatsRequest{
Id: id,
})
return stats, err
}
type stdio struct {
stdin string
stdout string
stderr string
stdinf *os.File
stdoutf *os.File
stderrf *os.File
stdoutBuffer bytes.Buffer
stderrBuffer bytes.Buffer
}
type containerProcess struct {
containerId string
pid string
bundle *Bundle
io stdio
eventsCh chan *types.Event
cs *ContainerdSuite
hasExited bool
}
func (c *containerProcess) openIo() (err error) {
defer func() {
if err != nil {
c.Cleanup()
}
}()
c.io.stdinf, err = os.OpenFile(c.io.stdin, os.O_RDWR, 0)
if err != nil {
return err
}
c.io.stdoutf, err = os.OpenFile(c.io.stdout, os.O_RDWR, 0)
if err != nil {
return err
}
go io.Copy(&c.io.stdoutBuffer, c.io.stdoutf)
c.io.stderrf, err = os.OpenFile(c.io.stderr, os.O_RDWR, 0)
if err != nil {
return err
}
go io.Copy(&c.io.stderrBuffer, c.io.stderrf)
return nil
}
func (c *containerProcess) GetNextEvent() *types.Event {
e := <-c.eventsCh
if e.Type == "exit" && e.Pid == c.pid {
c.Cleanup()
c.hasExited = true
}
return e
}
func (c *containerProcess) CloseStdin() error {
_, err := c.cs.grpcClient.UpdateProcess(context.Background(), &types.UpdateProcessRequest{
Id: c.containerId,
Pid: c.pid,
CloseStdin: true,
})
return err
}
func (c *containerProcess) Cleanup() {
for _, f := range []*os.File{
c.io.stdinf,
c.io.stdoutf,
c.io.stderrf,
} {
if f != nil {
f.Close()
f = nil
}
}
}
func NewContainerProcess(cs *ContainerdSuite, bundle *Bundle, cid, pid string) (c *containerProcess, err error) {
c = &containerProcess{
containerId: cid,
pid: "init",
bundle: bundle,
eventsCh: make(chan *types.Event, 8),
cs: cs,
hasExited: false,
}
for name, path := range map[string]*string{
"stdin": &c.io.stdin,
"stdout": &c.io.stdout,
"stderr": &c.io.stderr,
} {
*path = filepath.Join(bundle.Path, "io", cid+"-"+pid+"-"+name)
if err = syscall.Mkfifo(*path, 0755); err != nil && !os.IsExist(err) {
return nil, err
}
}
if err = c.openIo(); err != nil {
return nil, err
}
return c, nil
}
func (cs *ContainerdSuite) StartContainer(id, bundleName string) (c *containerProcess, err error) {
bundle, ok := bundleMap[bundleName]
if !ok {
return nil, fmt.Errorf("No such bundle '%s'", bundleName)
}
c, err = NewContainerProcess(cs, &bundle, id, "init")
if err != nil {
return nil, err
}
r := &types.CreateContainerRequest{
Id: id,
BundlePath: filepath.Join(cs.cwd, bundle.Path),
Stdin: filepath.Join(cs.cwd, c.io.stdin),
Stdout: filepath.Join(cs.cwd, c.io.stdout),
Stderr: filepath.Join(cs.cwd, c.io.stderr),
}
cs.SetContainerEventFilter(id, func(event *types.Event) {
c.eventsCh <- event
})
if _, err := cs.grpcClient.CreateContainer(context.Background(), r); err != nil {
c.Cleanup()
return nil, err
}
return c, nil
}
func (cs *ContainerdSuite) RunContainer(id, bundleName string) (c *containerProcess, err error) {
c, err = cs.StartContainer(id, bundleName)
if err != nil {
return nil, err
}
for {
e := c.GetNextEvent()
if e.Type == "exit" && e.Pid == "init" {
break
}
}
return c, err
}
func (cs *ContainerdSuite) AddProcessToContainer(init *containerProcess, pid, cwd string, env, args []string, uid, gid uint32) (c *containerProcess, err error) {
c, err = NewContainerProcess(cs, init.bundle, init.containerId, pid)
if err != nil {
return nil, err
}
pr := &types.AddProcessRequest{
Id: init.containerId,
Pid: pid,
Args: args,
Cwd: cwd,
Env: env,
User: &types.User{
Uid: uid,
Gid: gid,
},
Stdin: filepath.Join(cs.cwd, c.io.stdin),
Stdout: filepath.Join(cs.cwd, c.io.stdout),
Stderr: filepath.Join(cs.cwd, c.io.stderr),
}
_, err = cs.grpcClient.AddProcess(context.Background(), pr)
if err != nil {
c.Cleanup()
return nil, err
}
return c, nil
}

View File

@ -0,0 +1,58 @@
package main
import (
"github.com/docker/docker/pkg/integration/checker"
"github.com/go-check/check"
)
func (cs *ContainerdSuite) TestStartBusyboxLsSlash(t *check.C) {
expectedOutput := `bin
dev
etc
home
lib
lib64
linuxrc
media
mnt
opt
proc
root
run
sbin
sys
tmp
usr
var
`
if err := CreateBusyboxBundle("busybox-ls-slash", []string{"ls", "/"}); err != nil {
t.Fatal(err)
}
c, err := cs.RunContainer("myls", "busybox-ls-slash")
if err != nil {
t.Fatal(err)
}
t.Assert(c.io.stdoutBuffer.String(), checker.Equals, expectedOutput)
}
func (cs *ContainerdSuite) TestStartBusyboxNoSuchFile(t *check.C) {
expectedOutput := `oci runtime error: exec: \"NoSuchFile\": executable file not found in $PATH`
if err := CreateBusyboxBundle("busybox-NoSuchFile", []string{"NoSuchFile"}); err != nil {
t.Fatal(err)
}
_, err := cs.RunContainer("NoSuchFile", "busybox-NoSuchFile")
t.Assert(err.Error(), checker.Contains, expectedOutput)
}
func (cs *ContainerdSuite) TestStartBusyboxTop(t *check.C) {
if err := CreateBusyboxBundle("busybox-top", []string{"top"}); err != nil {
t.Fatal(err)
}
_, err := cs.StartContainer("top", "busybox-top")
t.Assert(err, checker.Equals, nil)
}