diff --git a/Makefile b/Makefile index 39b1928..2248e87 100644 --- a/Makefile +++ b/Makefile @@ -1,5 +1,7 @@ BUILDTAGS= +PROJECT=github.com/docker/containerd + GIT_COMMIT := $(shell git rev-parse HEAD 2> /dev/null || true) GIT_BRANCH := $(shell git rev-parse --abbrev-ref HEAD 2> /dev/null) @@ -13,8 +15,12 @@ ifeq ($(INTERACTIVE), 1) DOCKER_FLAGS += -t endif +TEST_ARTIFACTS_DIR := integration-test/test-artifacts +BUNDLE_ARCHIVES_DIR := $(TEST_ARTIFACTS_DIR)/archives + DOCKER_IMAGE := containerd-dev$(if $(GIT_BRANCH),:$(GIT_BRANCH)) -DOCKER_RUN := docker run --rm -i $(DOCKER_FLAGS) "$(DOCKER_IMAGE)" +DOCKER_RUN := docker run --privileged --rm -i $(DOCKER_FLAGS) "$(DOCKER_IMAGE)" + export GOPATH:=$(CURDIR)/vendor:$(GOPATH) @@ -46,7 +52,13 @@ shim: bin shim-static: cd containerd-shim && go build -ldflags "-w -extldflags -static ${LDFLAGS}" -tags "$(BUILDTAGS)" -o ../bin/containerd-shim -dbuild: +$(BUNDLE_ARCHIVES_DIR)/busybox.tar: + @mkdir -p $(BUNDLE_ARCHIVES_DIR) + curl -sSL 'https://github.com/jpetazzo/docker-busybox/raw/buildroot-2014.11/rootfs.tar' -o $(BUNDLE_ARCHIVES_DIR)/busybox.tar + +bundles-rootfs: $(BUNDLE_ARCHIVES_DIR)/busybox.tar + +dbuild: $(BUNDLE_ARCHIVES_DIR)/busybox.tar @docker build --rm --force-rm -t "$(DOCKER_IMAGE)" . dtest: dbuild @@ -68,7 +80,12 @@ shell: dbuild $(DOCKER_RUN) bash test: all validate - go test -v $(shell go list ./... | grep -v /vendor) + go test -v $(shell go list ./... | grep -v /vendor | grep -v /integration-test) +ifneq ($(wildcard /.dockerenv), ) + $(MAKE) install bundles-rootfs + cd integration-test ; \ + go test -check.v $(TESTFLAGS) github.com/docker/containerd/integration-test +endif validate: fmt diff --git a/integration-test/bundle_utils_test.go b/integration-test/bundle_utils_test.go new file mode 100644 index 0000000..3ed4232 --- /dev/null +++ b/integration-test/bundle_utils_test.go @@ -0,0 +1,107 @@ +package main + +import ( + "encoding/json" + "fmt" + "os" + "os/exec" + "path/filepath" + "reflect" + + ocs "github.com/opencontainers/specs/specs-go" +) + +var ( + bundlesDir = filepath.Join("test-artifacts", "oci-bundles") + refOciSpecsPath = filepath.Join(bundlesDir, "config.json") +) + +type OciProcessArgs struct { + Cmd string + Args []string +} + +type Bundle struct { + Source string + Name string + Spec ocs.Spec + Path string +} + +var bundleMap map[string]Bundle + +// untarRootfs untars the given `source` tarPath into `destination/rootfs` +func untarRootfs(source string, destination string) error { + destination = filepath.Join(destination, "rootfs") + if err := os.MkdirAll(destination, 0755); err != nil { + return nil + } + tar := exec.Command("tar", "-C", destination, "-xf", source) + return tar.Run() +} + +// CreateBundleWithFilter generate a new oci-bundle named `name` from +// the provide `source` rootfs. It starts from the default spec +// generated by `runc spec`, overrides the `spec.Process.Args` value +// with `args` and set `spec.Process.Terminal` to false. It then apply +// `filter()` to the resulting spec if it is provided. +func CreateBundleWithFilter(source, name string, args []string, filter func(spec *ocs.Spec)) error { + // Generate the spec + var spec ocs.Spec + if f, err := os.Open(refOciSpecsPath); err != nil { + return fmt.Errorf("Failed to open default spec: %v", err) + } else { + if err := json.NewDecoder(f).Decode(&spec); err != nil { + return fmt.Errorf("Failed to load default spec: %v", err) + } + f.Close() + } + + spec.Process.Args = args + spec.Process.Terminal = false + if filter != nil { + filter(&spec) + } + + bundlePath := filepath.Join(bundlesDir, name) + nb := Bundle{source, name, spec, bundlePath} + + // Check that we don't already have such a bundle + if b, ok := bundleMap[name]; ok { + if reflect.DeepEqual(b, nb) == false { + return fmt.Errorf("A bundle name named '%s' already exist but with different properties! %#v != %#v", + name, b, nb) + } + return nil + } + + // Nothing should be there, but just in case + os.RemoveAll(bundlePath) + + if err := untarRootfs(filepath.Join(archivesDir, source+".tar"), bundlePath); err != nil { + return fmt.Errorf("Failed to untar %s.tar: %v", source, err) + } + + // create a place for the io fifo + if err := os.Mkdir(filepath.Join(bundlePath, "io"), 0755); err != nil { + return fmt.Errorf("Failed to create bundle io directory: %v", err) + } + + // Write the updated spec to the right location + config, e := os.Create(filepath.Join(bundlePath, "config.json")) + if e != nil { + return fmt.Errorf("Failed to create oci spec: %v", e) + } + defer config.Close() + + if err := json.NewEncoder(config).Encode(&spec); err != nil { + return fmt.Errorf("Failed to encore oci spec: %v", e) + } + + bundleMap[name] = nb + return nil +} + +func CreateBusyboxBundle(name string, args []string) error { + return CreateBundleWithFilter("busybox", name, args, nil) +} diff --git a/integration-test/check_test.go b/integration-test/check_test.go new file mode 100644 index 0000000..23508e7 --- /dev/null +++ b/integration-test/check_test.go @@ -0,0 +1,228 @@ +package main + +import ( + "fmt" + "io/ioutil" + "log" + "net" + "os" + "os/exec" + "path/filepath" + "sync" + "testing" + "time" + + "golang.org/x/net/context" + + "google.golang.org/grpc" + "google.golang.org/grpc/grpclog" + + "github.com/docker/containerd/api/grpc/types" + "github.com/go-check/check" +) + +var ( + outputDirFormat = filepath.Join("test-artifacts", "runs", "%s") + archivesDir = filepath.Join("test-artifacts", "archives") +) + +func Test(t *testing.T) { + check.TestingT(t) +} + +func init() { + check.Suite(&ContainerdSuite{}) +} + +type ContainerdSuite struct { + cwd string + outputDir string + logFile *os.File + cd *exec.Cmd + syncChild chan error + grpcClient types.APIClient + eventFiltersMutex sync.Mutex + eventFilters map[string]func(event *types.Event) +} + +// getClient returns a connection to the Suite containerd +func (cs *ContainerdSuite) getClient(socket string) error { + // reset the logger for grpc to log to dev/null so that it does not mess with our stdio + grpclog.SetLogger(log.New(ioutil.Discard, "", log.LstdFlags)) + dialOpts := []grpc.DialOption{grpc.WithInsecure()} + dialOpts = append(dialOpts, + grpc.WithDialer(func(addr string, timeout time.Duration) (net.Conn, error) { + return net.DialTimeout("unix", addr, timeout) + }, + )) + conn, err := grpc.Dial(socket, dialOpts...) + if err != nil { + return err + } + cs.grpcClient = types.NewAPIClient(conn) + + return nil +} + +// ContainerdEventsHandler will process all events coming from +// containerd. If a filter as been register for a given container id +// via `SetContainerEventFilter()`, it will be invoked every time an +// event for that id is received +func (cs *ContainerdSuite) ContainerdEventsHandler(events types.API_EventsClient) { + timestamp := uint64(time.Now().Unix()) + for { + e, err := events.Recv() + if err != nil { + time.Sleep(1 * time.Second) + events, _ = cs.grpcClient.Events(context.Background(), &types.EventsRequest{Timestamp: timestamp}) + continue + } + timestamp = e.Timestamp + cs.eventFiltersMutex.Lock() + if f, ok := cs.eventFilters[e.Id]; ok { + f(e) + if e.Type == "exit" && e.Pid == "init" { + delete(cs.eventFilters, e.Id) + } + } + cs.eventFiltersMutex.Unlock() + } +} + +// generateReferencesSpecs invoke `runc spec` to produce the baseline +// specs from which all future bundle will be generated +func generateReferenceSpecs(destination string) error { + specs := exec.Command("runc", "spec") + specs.Dir = destination + return specs.Run() +} + +func (cs *ContainerdSuite) SetUpSuite(c *check.C) { + bundleMap = make(map[string]Bundle) + cs.eventFilters = make(map[string]func(event *types.Event)) + + // Get our CWD + if cwd, err := os.Getwd(); err != nil { + c.Fatalf("Could not determine current working directory: %v", err) + } else { + cs.cwd = cwd + } + + // Clean old bundles + os.RemoveAll(bundlesDir) + + // Ensure the oci bundles directory exists + if err := os.MkdirAll(bundlesDir, 0755); err != nil { + c.Fatalf("Failed to create bundles directory: %v", err) + } + + // Generate the reference spec + if err := generateReferenceSpecs(bundlesDir); err != nil { + c.Fatalf("Unable to generate OCI reference spec: %v", err) + } + + // Create our output directory + od := fmt.Sprintf(outputDirFormat, time.Now().Format("2006-01-02_150405.000000")) + cdStateDir := fmt.Sprintf("%s/containerd-master", od) + if err := os.MkdirAll(cdStateDir, 0755); err != nil { + c.Fatalf("Unable to created output directory '%s': %v", cdStateDir, err) + } + + cdGRPCSock := filepath.Join(od, "containerd-master", "containerd.sock") + cdLogFile := filepath.Join(od, "containerd-master", "containerd.log") + + f, err := os.OpenFile(cdLogFile, os.O_CREATE|os.O_TRUNC|os.O_RDWR|os.O_SYNC, 0777) + if err != nil { + c.Fatalf("Failed to create master containerd log file: %v", err) + } + cs.logFile = f + + cd := exec.Command("containerd", "--debug", + "--state-dir", cdStateDir, + "--listen", cdGRPCSock, + "--metrics-interval", "0m0s", + "--runtime-args", fmt.Sprintf("--root=%s", filepath.Join(cs.cwd, cdStateDir, "runc")), + ) + cd.Stderr = f + cd.Stdout = f + + if err := cd.Start(); err != nil { + c.Fatalf("Unable to start the master containerd: %v", err) + } + + cs.outputDir = od + cs.cd = cd + cs.syncChild = make(chan error) + if err := cs.getClient(cdGRPCSock); err != nil { + // Kill the daemon + cs.cd.Process.Kill() + c.Fatalf("Failed to connect to daemon: %v", err) + } + + // Monitor events + events, err := cs.grpcClient.Events(context.Background(), &types.EventsRequest{}) + if err != nil { + c.Fatalf("Could not register containerd event handler: %v", err) + } + + go cs.ContainerdEventsHandler(events) + + go func() { + cs.syncChild <- cd.Wait() + }() +} + +func (cs *ContainerdSuite) TearDownSuite(c *check.C) { + + // tell containerd to stop + if cs.cd != nil { + cs.cd.Process.Signal(os.Interrupt) + + done := false + for done == false { + select { + case err := <-cs.syncChild: + if err != nil { + c.Errorf("master containerd did not exit cleanly: %v", err) + } + done = true + case <-time.After(3 * time.Second): + fmt.Println("Timeout while waiting for containerd to exit, killing it!") + cs.cd.Process.Kill() + } + } + } + + if cs.logFile != nil { + cs.logFile.Close() + } +} + +func (cs *ContainerdSuite) SetContainerEventFilter(id string, filter func(event *types.Event)) { + cs.eventFiltersMutex.Lock() + cs.eventFilters[id] = filter + cs.eventFiltersMutex.Unlock() +} + +func (cs *ContainerdSuite) TearDownTest(c *check.C) { + ctrs, err := cs.ListRunningContainers() + if err != nil { + c.Fatalf("Unable to retrieve running containers: %v", err) + } + + // Kill all containers that survived + for _, ctr := range ctrs { + ch := make(chan interface{}) + cs.SetContainerEventFilter(ctr.Id, func(e *types.Event) { + if e.Type == "exit" && e.Pid == "init" { + ch <- nil + } + }) + + if err := cs.KillContainer(ctr.Id); err != nil { + fmt.Fprintf(os.Stderr, "Failed to cleanup leftover test containers: %v", err) + } + + <-ch + } +} diff --git a/integration-test/container_utils_test.go b/integration-test/container_utils_test.go new file mode 100644 index 0000000..b3429e3 --- /dev/null +++ b/integration-test/container_utils_test.go @@ -0,0 +1,249 @@ +package main + +import ( + "bytes" + "fmt" + "io" + "os" + "path/filepath" + "syscall" + + "github.com/docker/containerd/api/grpc/types" + "golang.org/x/net/context" +) + +func (cs *ContainerdSuite) ListRunningContainers() ([]*types.Container, error) { + resp, err := cs.grpcClient.State(context.Background(), &types.StateRequest{}) + if err != nil { + return nil, err + } + return resp.Containers, nil +} + +func (cs *ContainerdSuite) SignalContainerProcess(id string, procId string, sig uint32) error { + _, err := cs.grpcClient.Signal(context.Background(), &types.SignalRequest{ + Id: id, + Pid: procId, + Signal: sig, + }) + return err +} + +func (cs *ContainerdSuite) SignalContainer(id string, sig uint32) error { + return cs.SignalContainerProcess(id, "init", sig) +} + +func (cs *ContainerdSuite) KillContainer(id string) error { + return cs.SignalContainerProcess(id, "init", uint32(syscall.SIGKILL)) +} + +func (cs *ContainerdSuite) PauseContainer(id string) error { + _, err := cs.grpcClient.UpdateContainer(context.Background(), &types.UpdateContainerRequest{ + Id: id, + Pid: "init", + Status: "paused", + }) + return err +} + +func (cs *ContainerdSuite) ResumeContainer(id string) error { + _, err := cs.grpcClient.UpdateContainer(context.Background(), &types.UpdateContainerRequest{ + Id: id, + Pid: "init", + Status: "running", + }) + return err +} + +func (cs *ContainerdSuite) GetContainerStats(id string) (*types.StatsResponse, error) { + stats, err := cs.grpcClient.Stats(context.Background(), &types.StatsRequest{ + Id: id, + }) + return stats, err +} + +type stdio struct { + stdin string + stdout string + stderr string + stdinf *os.File + stdoutf *os.File + stderrf *os.File + stdoutBuffer bytes.Buffer + stderrBuffer bytes.Buffer +} + +type containerProcess struct { + containerId string + pid string + bundle *Bundle + io stdio + eventsCh chan *types.Event + cs *ContainerdSuite + hasExited bool +} + +func (c *containerProcess) openIo() (err error) { + defer func() { + if err != nil { + c.Cleanup() + } + }() + + c.io.stdinf, err = os.OpenFile(c.io.stdin, os.O_RDWR, 0) + if err != nil { + return err + } + + c.io.stdoutf, err = os.OpenFile(c.io.stdout, os.O_RDWR, 0) + if err != nil { + return err + } + go io.Copy(&c.io.stdoutBuffer, c.io.stdoutf) + + c.io.stderrf, err = os.OpenFile(c.io.stderr, os.O_RDWR, 0) + if err != nil { + return err + } + go io.Copy(&c.io.stderrBuffer, c.io.stderrf) + + return nil +} + +func (c *containerProcess) GetNextEvent() *types.Event { + e := <-c.eventsCh + + if e.Type == "exit" && e.Pid == c.pid { + c.Cleanup() + c.hasExited = true + } + + return e +} + +func (c *containerProcess) CloseStdin() error { + _, err := c.cs.grpcClient.UpdateProcess(context.Background(), &types.UpdateProcessRequest{ + Id: c.containerId, + Pid: c.pid, + CloseStdin: true, + }) + return err +} + +func (c *containerProcess) Cleanup() { + for _, f := range []*os.File{ + c.io.stdinf, + c.io.stdoutf, + c.io.stderrf, + } { + if f != nil { + f.Close() + f = nil + } + } +} + +func NewContainerProcess(cs *ContainerdSuite, bundle *Bundle, cid, pid string) (c *containerProcess, err error) { + c = &containerProcess{ + containerId: cid, + pid: "init", + bundle: bundle, + eventsCh: make(chan *types.Event, 8), + cs: cs, + hasExited: false, + } + + for name, path := range map[string]*string{ + "stdin": &c.io.stdin, + "stdout": &c.io.stdout, + "stderr": &c.io.stderr, + } { + *path = filepath.Join(bundle.Path, "io", cid+"-"+pid+"-"+name) + if err = syscall.Mkfifo(*path, 0755); err != nil && !os.IsExist(err) { + return nil, err + } + } + + if err = c.openIo(); err != nil { + return nil, err + } + + return c, nil +} + +func (cs *ContainerdSuite) StartContainer(id, bundleName string) (c *containerProcess, err error) { + bundle, ok := bundleMap[bundleName] + if !ok { + return nil, fmt.Errorf("No such bundle '%s'", bundleName) + } + + c, err = NewContainerProcess(cs, &bundle, id, "init") + if err != nil { + return nil, err + } + + r := &types.CreateContainerRequest{ + Id: id, + BundlePath: filepath.Join(cs.cwd, bundle.Path), + Stdin: filepath.Join(cs.cwd, c.io.stdin), + Stdout: filepath.Join(cs.cwd, c.io.stdout), + Stderr: filepath.Join(cs.cwd, c.io.stderr), + } + + cs.SetContainerEventFilter(id, func(event *types.Event) { + c.eventsCh <- event + }) + + if _, err := cs.grpcClient.CreateContainer(context.Background(), r); err != nil { + c.Cleanup() + return nil, err + } + + return c, nil +} + +func (cs *ContainerdSuite) RunContainer(id, bundleName string) (c *containerProcess, err error) { + c, err = cs.StartContainer(id, bundleName) + if err != nil { + return nil, err + } + + for { + e := c.GetNextEvent() + if e.Type == "exit" && e.Pid == "init" { + break + } + } + + return c, err +} + +func (cs *ContainerdSuite) AddProcessToContainer(init *containerProcess, pid, cwd string, env, args []string, uid, gid uint32) (c *containerProcess, err error) { + c, err = NewContainerProcess(cs, init.bundle, init.containerId, pid) + if err != nil { + return nil, err + } + + pr := &types.AddProcessRequest{ + Id: init.containerId, + Pid: pid, + Args: args, + Cwd: cwd, + Env: env, + User: &types.User{ + Uid: uid, + Gid: gid, + }, + Stdin: filepath.Join(cs.cwd, c.io.stdin), + Stdout: filepath.Join(cs.cwd, c.io.stdout), + Stderr: filepath.Join(cs.cwd, c.io.stderr), + } + + _, err = cs.grpcClient.AddProcess(context.Background(), pr) + if err != nil { + c.Cleanup() + return nil, err + } + + return c, nil +} diff --git a/integration-test/start_test.go b/integration-test/start_test.go new file mode 100644 index 0000000..8d1af08 --- /dev/null +++ b/integration-test/start_test.go @@ -0,0 +1,58 @@ +package main + +import ( + "github.com/docker/docker/pkg/integration/checker" + "github.com/go-check/check" +) + +func (cs *ContainerdSuite) TestStartBusyboxLsSlash(t *check.C) { + expectedOutput := `bin +dev +etc +home +lib +lib64 +linuxrc +media +mnt +opt +proc +root +run +sbin +sys +tmp +usr +var +` + if err := CreateBusyboxBundle("busybox-ls-slash", []string{"ls", "/"}); err != nil { + t.Fatal(err) + } + + c, err := cs.RunContainer("myls", "busybox-ls-slash") + if err != nil { + t.Fatal(err) + } + + t.Assert(c.io.stdoutBuffer.String(), checker.Equals, expectedOutput) +} + +func (cs *ContainerdSuite) TestStartBusyboxNoSuchFile(t *check.C) { + expectedOutput := `oci runtime error: exec: \"NoSuchFile\": executable file not found in $PATH` + + if err := CreateBusyboxBundle("busybox-NoSuchFile", []string{"NoSuchFile"}); err != nil { + t.Fatal(err) + } + + _, err := cs.RunContainer("NoSuchFile", "busybox-NoSuchFile") + t.Assert(err.Error(), checker.Contains, expectedOutput) +} + +func (cs *ContainerdSuite) TestStartBusyboxTop(t *check.C) { + if err := CreateBusyboxBundle("busybox-top", []string{"top"}); err != nil { + t.Fatal(err) + } + + _, err := cs.StartContainer("top", "busybox-top") + t.Assert(err, checker.Equals, nil) +}