Add initial integration tests framework

Signed-off-by: Kenfe-Mickael Laventure <mickael.laventure@gmail.com>

Makefile

@@ -1,5 +1,7 @@
 BUILDTAGS=
 
+PROJECT=github.com/docker/containerd
+
 GIT_COMMIT := $(shell git rev-parse HEAD 2> /dev/null || true)
 GIT_BRANCH := $(shell git rev-parse --abbrev-ref HEAD 2> /dev/null)
 
@@ -13,8 +15,12 @@ ifeq ($(INTERACTIVE), 1)
 	DOCKER_FLAGS += -t
 endif
 
+TEST_ARTIFACTS_DIR := integration-test/test-artifacts
+BUNDLE_ARCHIVES_DIR := $(TEST_ARTIFACTS_DIR)/archives
+
 DOCKER_IMAGE := containerd-dev$(if $(GIT_BRANCH),:$(GIT_BRANCH))
-DOCKER_RUN := docker run --rm -i $(DOCKER_FLAGS) "$(DOCKER_IMAGE)"
+DOCKER_RUN := docker run --privileged --rm -i $(DOCKER_FLAGS) "$(DOCKER_IMAGE)"
+
 
 export GOPATH:=$(CURDIR)/vendor:$(GOPATH)
 
@@ -46,7 +52,13 @@ shim: bin
 shim-static:
 	cd containerd-shim && go build -ldflags "-w -extldflags -static ${LDFLAGS}" -tags "$(BUILDTAGS)" -o ../bin/containerd-shim
 
-dbuild:
+$(BUNDLE_ARCHIVES_DIR)/busybox.tar:
+	@mkdir -p $(BUNDLE_ARCHIVES_DIR)
+	curl -sSL 'https://github.com/jpetazzo/docker-busybox/raw/buildroot-2014.11/rootfs.tar' -o $(BUNDLE_ARCHIVES_DIR)/busybox.tar
+
+bundles-rootfs: $(BUNDLE_ARCHIVES_DIR)/busybox.tar
+
+dbuild: $(BUNDLE_ARCHIVES_DIR)/busybox.tar
 	@docker build --rm --force-rm -t "$(DOCKER_IMAGE)" .
 
 dtest: dbuild
@@ -68,7 +80,12 @@ shell: dbuild
 	$(DOCKER_RUN) bash
 
 test: all validate
-	go test -v $(shell go list ./... | grep -v /vendor)
+	go test -v $(shell go list ./... | grep -v /vendor | grep -v /integration-test)
+ifneq ($(wildcard /.dockerenv), )
+	$(MAKE) install bundles-rootfs
+	cd integration-test ; \
+	go test -check.v $(TESTFLAGS) github.com/docker/containerd/integration-test
+endif
 
 validate: fmt
 

integration-test/bundle_utils_test.go

@@ -0,0 +1,107 @@
+package main
+
+import (
+	"encoding/json"
+	"fmt"
+	"os"
+	"os/exec"
+	"path/filepath"
+	"reflect"
+
+	ocs "github.com/opencontainers/specs/specs-go"
+)
+
+var (
+	bundlesDir      = filepath.Join("test-artifacts", "oci-bundles")
+	refOciSpecsPath = filepath.Join(bundlesDir, "config.json")
+)
+
+type OciProcessArgs struct {
+	Cmd  string
+	Args []string
+}
+
+type Bundle struct {
+	Source string
+	Name   string
+	Spec   ocs.Spec
+	Path   string
+}
+
+var bundleMap map[string]Bundle
+
+// untarRootfs untars the given `source` tarPath into `destination/rootfs`
+func untarRootfs(source string, destination string) error {
+	destination = filepath.Join(destination, "rootfs")
+	if err := os.MkdirAll(destination, 0755); err != nil {
+		return nil
+	}
+	tar := exec.Command("tar", "-C", destination, "-xf", source)
+	return tar.Run()
+}
+
+// CreateBundleWithFilter generate a new oci-bundle named `name` from
+// the provide `source` rootfs. It starts from the default spec
+// generated by `runc spec`, overrides the `spec.Process.Args` value
+// with `args` and set `spec.Process.Terminal` to false. It then apply
+// `filter()` to the resulting spec if it is provided.
+func CreateBundleWithFilter(source, name string, args []string, filter func(spec *ocs.Spec)) error {
+	// Generate the spec
+	var spec ocs.Spec
+	if f, err := os.Open(refOciSpecsPath); err != nil {
+		return fmt.Errorf("Failed to open default spec: %v", err)
+	} else {
+		if err := json.NewDecoder(f).Decode(&spec); err != nil {
+			return fmt.Errorf("Failed to load default spec: %v", err)
+		}
+		f.Close()
+	}
+
+	spec.Process.Args = args
+	spec.Process.Terminal = false
+	if filter != nil {
+		filter(&spec)
+	}
+
+	bundlePath := filepath.Join(bundlesDir, name)
+	nb := Bundle{source, name, spec, bundlePath}
+
+	// Check that we don't already have such a bundle
+	if b, ok := bundleMap[name]; ok {
+		if reflect.DeepEqual(b, nb) == false {
+			return fmt.Errorf("A bundle name named '%s' already exist but with different properties! %#v != %#v",
+				name, b, nb)
+		}
+		return nil
+	}
+
+	// Nothing should be there, but just in case
+	os.RemoveAll(bundlePath)
+
+	if err := untarRootfs(filepath.Join(archivesDir, source+".tar"), bundlePath); err != nil {
+		return fmt.Errorf("Failed to untar %s.tar: %v", source, err)
+	}
+
+	// create a place for the io fifo
+	if err := os.Mkdir(filepath.Join(bundlePath, "io"), 0755); err != nil {
+		return fmt.Errorf("Failed to create bundle io directory: %v", err)
+	}
+
+	// Write the updated spec to the right location
+	config, e := os.Create(filepath.Join(bundlePath, "config.json"))
+	if e != nil {
+		return fmt.Errorf("Failed to create oci spec: %v", e)
+	}
+	defer config.Close()
+
+	if err := json.NewEncoder(config).Encode(&spec); err != nil {
+		return fmt.Errorf("Failed to encore oci spec: %v", e)
+	}
+
+	bundleMap[name] = nb
+	return nil
+}
+
+func CreateBusyboxBundle(name string, args []string) error {
+	return CreateBundleWithFilter("busybox", name, args, nil)
+}

integration-test/check_test.go

@@ -0,0 +1,228 @@
+package main
+
+import (
+	"fmt"
+	"io/ioutil"
+	"log"
+	"net"
+	"os"
+	"os/exec"
+	"path/filepath"
+	"sync"
+	"testing"
+	"time"
+
+	"golang.org/x/net/context"
+
+	"google.golang.org/grpc"
+	"google.golang.org/grpc/grpclog"
+
+	"github.com/docker/containerd/api/grpc/types"
+	"github.com/go-check/check"
+)
+
+var (
+	outputDirFormat = filepath.Join("test-artifacts", "runs", "%s")
+	archivesDir     = filepath.Join("test-artifacts", "archives")
+)
+
+func Test(t *testing.T) {
+	check.TestingT(t)
+}
+
+func init() {
+	check.Suite(&ContainerdSuite{})
+}
+
+type ContainerdSuite struct {
+	cwd               string
+	outputDir         string
+	logFile           *os.File
+	cd                *exec.Cmd
+	syncChild         chan error
+	grpcClient        types.APIClient
+	eventFiltersMutex sync.Mutex
+	eventFilters      map[string]func(event *types.Event)
+}
+
+// getClient returns a connection to the Suite containerd
+func (cs *ContainerdSuite) getClient(socket string) error {
+	// reset the logger for grpc to log to dev/null so that it does not mess with our stdio
+	grpclog.SetLogger(log.New(ioutil.Discard, "", log.LstdFlags))
+	dialOpts := []grpc.DialOption{grpc.WithInsecure()}
+	dialOpts = append(dialOpts,
+		grpc.WithDialer(func(addr string, timeout time.Duration) (net.Conn, error) {
+			return net.DialTimeout("unix", addr, timeout)
+		},
+		))
+	conn, err := grpc.Dial(socket, dialOpts...)
+	if err != nil {
+		return err
+	}
+	cs.grpcClient = types.NewAPIClient(conn)
+
+	return nil
+}
+
+// ContainerdEventsHandler will process all events coming from
+// containerd. If a filter as been register for a given container id
+// via `SetContainerEventFilter()`, it will be invoked every time an
+// event for that id is received
+func (cs *ContainerdSuite) ContainerdEventsHandler(events types.API_EventsClient) {
+	timestamp := uint64(time.Now().Unix())
+	for {
+		e, err := events.Recv()
+		if err != nil {
+			time.Sleep(1 * time.Second)
+			events, _ = cs.grpcClient.Events(context.Background(), &types.EventsRequest{Timestamp: timestamp})
+			continue
+		}
+		timestamp = e.Timestamp
+		cs.eventFiltersMutex.Lock()
+		if f, ok := cs.eventFilters[e.Id]; ok {
+			f(e)
+			if e.Type == "exit" && e.Pid == "init" {
+				delete(cs.eventFilters, e.Id)
+			}
+		}
+		cs.eventFiltersMutex.Unlock()
+	}
+}
+
+// generateReferencesSpecs invoke `runc spec` to produce the baseline
+// specs from which all future bundle will be generated
+func generateReferenceSpecs(destination string) error {
+	specs := exec.Command("runc", "spec")
+	specs.Dir = destination
+	return specs.Run()
+}
+
+func (cs *ContainerdSuite) SetUpSuite(c *check.C) {
+	bundleMap = make(map[string]Bundle)
+	cs.eventFilters = make(map[string]func(event *types.Event))
+
+	// Get our CWD
+	if cwd, err := os.Getwd(); err != nil {
+		c.Fatalf("Could not determine current working directory: %v", err)
+	} else {
+		cs.cwd = cwd
+	}
+
+	// Clean old bundles
+	os.RemoveAll(bundlesDir)
+
+	// Ensure the oci bundles directory exists
+	if err := os.MkdirAll(bundlesDir, 0755); err != nil {
+		c.Fatalf("Failed to create bundles directory: %v", err)
+	}
+
+	// Generate the reference spec
+	if err := generateReferenceSpecs(bundlesDir); err != nil {
+		c.Fatalf("Unable to generate OCI reference spec: %v", err)
+	}
+
+	// Create our output directory
+	od := fmt.Sprintf(outputDirFormat, time.Now().Format("2006-01-02_150405.000000"))
+	cdStateDir := fmt.Sprintf("%s/containerd-master", od)
+	if err := os.MkdirAll(cdStateDir, 0755); err != nil {
+		c.Fatalf("Unable to created output directory '%s': %v", cdStateDir, err)
+	}
+
+	cdGRPCSock := filepath.Join(od, "containerd-master", "containerd.sock")
+	cdLogFile := filepath.Join(od, "containerd-master", "containerd.log")
+
+	f, err := os.OpenFile(cdLogFile, os.O_CREATE|os.O_TRUNC|os.O_RDWR|os.O_SYNC, 0777)
+	if err != nil {
+		c.Fatalf("Failed to create master containerd log file: %v", err)
+	}
+	cs.logFile = f
+
+	cd := exec.Command("containerd", "--debug",
+		"--state-dir", cdStateDir,
+		"--listen", cdGRPCSock,
+		"--metrics-interval", "0m0s",
+		"--runtime-args", fmt.Sprintf("--root=%s", filepath.Join(cs.cwd, cdStateDir, "runc")),
+	)
+	cd.Stderr = f
+	cd.Stdout = f
+
+	if err := cd.Start(); err != nil {
+		c.Fatalf("Unable to start the master containerd: %v", err)
+	}
+
+	cs.outputDir = od
+	cs.cd = cd
+	cs.syncChild = make(chan error)
+	if err := cs.getClient(cdGRPCSock); err != nil {
+		// Kill the daemon
+		cs.cd.Process.Kill()
+		c.Fatalf("Failed to connect to daemon: %v", err)
+	}
+
+	// Monitor events
+	events, err := cs.grpcClient.Events(context.Background(), &types.EventsRequest{})
+	if err != nil {
+		c.Fatalf("Could not register containerd event handler: %v", err)
+	}
+
+	go cs.ContainerdEventsHandler(events)
+
+	go func() {
+		cs.syncChild <- cd.Wait()
+	}()
+}
+
+func (cs *ContainerdSuite) TearDownSuite(c *check.C) {
+
+	// tell containerd to stop
+	if cs.cd != nil {
+		cs.cd.Process.Signal(os.Interrupt)
+
+		done := false
+		for done == false {
+			select {
+			case err := <-cs.syncChild:
+				if err != nil {
+					c.Errorf("master containerd did not exit cleanly: %v", err)
+				}
+				done = true
+			case <-time.After(3 * time.Second):
+				fmt.Println("Timeout while waiting for containerd to exit, killing it!")
+				cs.cd.Process.Kill()
+			}
+		}
+	}
+
+	if cs.logFile != nil {
+		cs.logFile.Close()
+	}
+}
+
+func (cs *ContainerdSuite) SetContainerEventFilter(id string, filter func(event *types.Event)) {
+	cs.eventFiltersMutex.Lock()
+	cs.eventFilters[id] = filter
+	cs.eventFiltersMutex.Unlock()
+}
+
+func (cs *ContainerdSuite) TearDownTest(c *check.C) {
+	ctrs, err := cs.ListRunningContainers()
+	if err != nil {
+		c.Fatalf("Unable to retrieve running containers: %v", err)
+	}
+
+	// Kill all containers that survived
+	for _, ctr := range ctrs {
+		ch := make(chan interface{})
+		cs.SetContainerEventFilter(ctr.Id, func(e *types.Event) {
+			if e.Type == "exit" && e.Pid == "init" {
+				ch <- nil
+			}
+		})
+
+		if err := cs.KillContainer(ctr.Id); err != nil {
+			fmt.Fprintf(os.Stderr, "Failed to cleanup leftover test containers: %v", err)
+		}
+
+		<-ch
+	}
+}

integration-test/container_utils_test.go

@@ -0,0 +1,249 @@
+package main
+
+import (
+	"bytes"
+	"fmt"
+	"io"
+	"os"
+	"path/filepath"
+	"syscall"
+
+	"github.com/docker/containerd/api/grpc/types"
+	"golang.org/x/net/context"
+)
+
+func (cs *ContainerdSuite) ListRunningContainers() ([]*types.Container, error) {
+	resp, err := cs.grpcClient.State(context.Background(), &types.StateRequest{})
+	if err != nil {
+		return nil, err
+	}
+	return resp.Containers, nil
+}
+
+func (cs *ContainerdSuite) SignalContainerProcess(id string, procId string, sig uint32) error {
+	_, err := cs.grpcClient.Signal(context.Background(), &types.SignalRequest{
+		Id:     id,
+		Pid:    procId,
+		Signal: sig,
+	})
+	return err
+}
+
+func (cs *ContainerdSuite) SignalContainer(id string, sig uint32) error {
+	return cs.SignalContainerProcess(id, "init", sig)
+}
+
+func (cs *ContainerdSuite) KillContainer(id string) error {
+	return cs.SignalContainerProcess(id, "init", uint32(syscall.SIGKILL))
+}
+
+func (cs *ContainerdSuite) PauseContainer(id string) error {
+	_, err := cs.grpcClient.UpdateContainer(context.Background(), &types.UpdateContainerRequest{
+		Id:     id,
+		Pid:    "init",
+		Status: "paused",
+	})
+	return err
+}
+
+func (cs *ContainerdSuite) ResumeContainer(id string) error {
+	_, err := cs.grpcClient.UpdateContainer(context.Background(), &types.UpdateContainerRequest{
+		Id:     id,
+		Pid:    "init",
+		Status: "running",
+	})
+	return err
+}
+
+func (cs *ContainerdSuite) GetContainerStats(id string) (*types.StatsResponse, error) {
+	stats, err := cs.grpcClient.Stats(context.Background(), &types.StatsRequest{
+		Id: id,
+	})
+	return stats, err
+}
+
+type stdio struct {
+	stdin        string
+	stdout       string
+	stderr       string
+	stdinf       *os.File
+	stdoutf      *os.File
+	stderrf      *os.File
+	stdoutBuffer bytes.Buffer
+	stderrBuffer bytes.Buffer
+}
+
+type containerProcess struct {
+	containerId string
+	pid         string
+	bundle      *Bundle
+	io          stdio
+	eventsCh    chan *types.Event
+	cs          *ContainerdSuite
+	hasExited   bool
+}
+
+func (c *containerProcess) openIo() (err error) {
+	defer func() {
+		if err != nil {
+			c.Cleanup()
+		}
+	}()
+
+	c.io.stdinf, err = os.OpenFile(c.io.stdin, os.O_RDWR, 0)
+	if err != nil {
+		return err
+	}
+
+	c.io.stdoutf, err = os.OpenFile(c.io.stdout, os.O_RDWR, 0)
+	if err != nil {
+		return err
+	}
+	go io.Copy(&c.io.stdoutBuffer, c.io.stdoutf)
+
+	c.io.stderrf, err = os.OpenFile(c.io.stderr, os.O_RDWR, 0)
+	if err != nil {
+		return err
+	}
+	go io.Copy(&c.io.stderrBuffer, c.io.stderrf)
+
+	return nil
+}
+
+func (c *containerProcess) GetNextEvent() *types.Event {
+	e := <-c.eventsCh
+
+	if e.Type == "exit" && e.Pid == c.pid {
+		c.Cleanup()
+		c.hasExited = true
+	}
+
+	return e
+}
+
+func (c *containerProcess) CloseStdin() error {
+	_, err := c.cs.grpcClient.UpdateProcess(context.Background(), &types.UpdateProcessRequest{
+		Id:         c.containerId,
+		Pid:        c.pid,
+		CloseStdin: true,
+	})
+	return err
+}
+
+func (c *containerProcess) Cleanup() {
+	for _, f := range []*os.File{
+		c.io.stdinf,
+		c.io.stdoutf,
+		c.io.stderrf,
+	} {
+		if f != nil {
+			f.Close()
+			f = nil
+		}
+	}
+}
+
+func NewContainerProcess(cs *ContainerdSuite, bundle *Bundle, cid, pid string) (c *containerProcess, err error) {
+	c = &containerProcess{
+		containerId: cid,
+		pid:         "init",
+		bundle:      bundle,
+		eventsCh:    make(chan *types.Event, 8),
+		cs:          cs,
+		hasExited:   false,
+	}
+
+	for name, path := range map[string]*string{
+		"stdin":  &c.io.stdin,
+		"stdout": &c.io.stdout,
+		"stderr": &c.io.stderr,
+	} {
+		*path = filepath.Join(bundle.Path, "io", cid+"-"+pid+"-"+name)
+		if err = syscall.Mkfifo(*path, 0755); err != nil && !os.IsExist(err) {
+			return nil, err
+		}
+	}
+
+	if err = c.openIo(); err != nil {
+		return nil, err
+	}
+
+	return c, nil
+}
+
+func (cs *ContainerdSuite) StartContainer(id, bundleName string) (c *containerProcess, err error) {
+	bundle, ok := bundleMap[bundleName]
+	if !ok {
+		return nil, fmt.Errorf("No such bundle '%s'", bundleName)
+	}
+
+	c, err = NewContainerProcess(cs, &bundle, id, "init")
+	if err != nil {
+		return nil, err
+	}
+
+	r := &types.CreateContainerRequest{
+		Id:         id,
+		BundlePath: filepath.Join(cs.cwd, bundle.Path),
+		Stdin:      filepath.Join(cs.cwd, c.io.stdin),
+		Stdout:     filepath.Join(cs.cwd, c.io.stdout),
+		Stderr:     filepath.Join(cs.cwd, c.io.stderr),
+	}
+
+	cs.SetContainerEventFilter(id, func(event *types.Event) {
+		c.eventsCh <- event
+	})
+
+	if _, err := cs.grpcClient.CreateContainer(context.Background(), r); err != nil {
+		c.Cleanup()
+		return nil, err
+	}
+
+	return c, nil
+}
+
+func (cs *ContainerdSuite) RunContainer(id, bundleName string) (c *containerProcess, err error) {
+	c, err = cs.StartContainer(id, bundleName)
+	if err != nil {
+		return nil, err
+	}
+
+	for {
+		e := c.GetNextEvent()
+		if e.Type == "exit" && e.Pid == "init" {
+			break
+		}
+	}
+
+	return c, err
+}
+
+func (cs *ContainerdSuite) AddProcessToContainer(init *containerProcess, pid, cwd string, env, args []string, uid, gid uint32) (c *containerProcess, err error) {
+	c, err = NewContainerProcess(cs, init.bundle, init.containerId, pid)
+	if err != nil {
+		return nil, err
+	}
+
+	pr := &types.AddProcessRequest{
+		Id:   init.containerId,
+		Pid:  pid,
+		Args: args,
+		Cwd:  cwd,
+		Env:  env,
+		User: &types.User{
+			Uid: uid,
+			Gid: gid,
+		},
+		Stdin:  filepath.Join(cs.cwd, c.io.stdin),
+		Stdout: filepath.Join(cs.cwd, c.io.stdout),
+		Stderr: filepath.Join(cs.cwd, c.io.stderr),
+	}
+
+	_, err = cs.grpcClient.AddProcess(context.Background(), pr)
+	if err != nil {
+		c.Cleanup()
+		return nil, err
+	}
+
+	return c, nil
+}

integration-test/start_test.go

@@ -0,0 +1,58 @@
+package main
+
+import (
+	"github.com/docker/docker/pkg/integration/checker"
+	"github.com/go-check/check"
+)
+
+func (cs *ContainerdSuite) TestStartBusyboxLsSlash(t *check.C) {
+	expectedOutput := `bin
+dev
+etc
+home
+lib
+lib64
+linuxrc
+media
+mnt
+opt
+proc
+root
+run
+sbin
+sys
+tmp
+usr
+var
+`
+	if err := CreateBusyboxBundle("busybox-ls-slash", []string{"ls", "/"}); err != nil {
+		t.Fatal(err)
+	}
+
+	c, err := cs.RunContainer("myls", "busybox-ls-slash")
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	t.Assert(c.io.stdoutBuffer.String(), checker.Equals, expectedOutput)
+}
+
+func (cs *ContainerdSuite) TestStartBusyboxNoSuchFile(t *check.C) {
+	expectedOutput := `oci runtime error: exec: \"NoSuchFile\": executable file not found in $PATH`
+
+	if err := CreateBusyboxBundle("busybox-NoSuchFile", []string{"NoSuchFile"}); err != nil {
+		t.Fatal(err)
+	}
+
+	_, err := cs.RunContainer("NoSuchFile", "busybox-NoSuchFile")
+	t.Assert(err.Error(), checker.Contains, expectedOutput)
+}
+
+func (cs *ContainerdSuite) TestStartBusyboxTop(t *check.C) {
+	if err := CreateBusyboxBundle("busybox-top", []string{"top"}); err != nil {
+		t.Fatal(err)
+	}
+
+	_, err := cs.StartContainer("top", "busybox-top")
+	t.Assert(err, checker.Equals, nil)
+}