Micro benchmarks for containerd. (#244)

This is the first in a series of micro benchmarks for containerd.
Performance measurement will use containerd objects and methods
that are not dependent on the grpc API and dont require the daemon
to the running. Test will require containerd-shim and runc.

The motivation is to understand the baseline performance at the lowest
containerd layer. A natural extension to this effort would be to write
macro benchmarks which would include API and daemon.

Note:
- Currently measures only one workload (busybox sh) start times. Will
add other bundles and args soon.
- Can use integration-test utils for bundle processing. However, json
marshal/unmarshal is currently timing out standard benchmark times. So
going with default spec for now.

Sample run:
BenchmarkBusyboxSh-4    / # / # / #        2     576013841 ns/op
ok      github.com/docker/containerd/runtime    1.800s

Signed-off-by: Anusha Ragunathan <anusha@docker.com>

Makefile

@@ -18,8 +18,8 @@ ifeq ($(INTERACTIVE), 1)
 	DOCKER_FLAGS += -t
 endif
 
-TEST_ARTIFACTS_DIR := integration-test/test-artifacts
+TESTBENCH_ARTIFACTS_DIR := output/test-artifacts
-BUNDLE_ARCHIVES_DIR := $(TEST_ARTIFACTS_DIR)/archives
+TESTBENCH_BUNDLE_DIR := $(TESTBENCH_ARTIFACTS_DIR)/archives
 
 DOCKER_IMAGE := containerd-dev$(if $(GIT_BRANCH),:$(GIT_BRANCH))
 DOCKER_RUN := docker run --privileged --rm -i $(DOCKER_FLAGS) "$(DOCKER_IMAGE)"
@@ -35,7 +35,7 @@ bin:
 	mkdir -p bin/
 
 clean:
-	rm -rf bin
+	rm -rf bin && rm -rf output
 
 client: bin
 	cd ctr && go build -ldflags "${LDFLAGS}" -o ../bin/ctr
@@ -55,18 +55,21 @@ shim: bin
 shim-static:
 	cd containerd-shim && go build -ldflags "-w -extldflags -static ${LDFLAGS}" -tags "$(BUILDTAGS)" -o ../bin/containerd-shim
 
-$(BUNDLE_ARCHIVES_DIR)/busybox.tar:
+$(TESTBENCH_BUNDLE_DIR)/busybox.tar:
-	@mkdir -p $(BUNDLE_ARCHIVES_DIR)
+	mkdir -p $(TESTBENCH_BUNDLE_DIR)
-	curl -sSL 'https://github.com/jpetazzo/docker-busybox/raw/buildroot-2014.11/rootfs.tar' -o $(BUNDLE_ARCHIVES_DIR)/busybox.tar
+	curl -sSL 'https://github.com/jpetazzo/docker-busybox/raw/buildroot-2014.11/rootfs.tar' -o $(TESTBENCH_BUNDLE_DIR)/busybox.tar
 
-bundles-rootfs: $(BUNDLE_ARCHIVES_DIR)/busybox.tar
+bundles-rootfs: $(TESTBENCH_BUNDLE_DIR)/busybox.tar
 
-dbuild: $(BUNDLE_ARCHIVES_DIR)/busybox.tar
+dbuild: $(TESTBENCH_BUNDLE_DIR)/busybox.tar
 	@docker build --rm --force-rm -t "$(DOCKER_IMAGE)" .
 
 dtest: dbuild
 	$(DOCKER_RUN) make test
 
+dbench: dbuild
+	$(DOCKER_RUN) make bench
+
 install:
 	cp bin/* /usr/local/bin/
 
@@ -82,14 +85,16 @@ lint:
 shell: dbuild
 	$(DOCKER_RUN) bash
 
-test: validate
+test: validate install bundles-rootfs
-	go test -v $(shell go list ./... | grep -v /vendor | grep -v /integration-test)
+	go test -bench=. -v $(shell go list ./... | grep -v /vendor | grep -v /integration-test)
 ifneq ($(wildcard /.dockerenv), )
-	$(MAKE) install bundles-rootfs
 	cd integration-test ; \
 	go test -check.v -check.timeout=$(TEST_TIMEOUT) timeout=$(TEST_SUITE_TIMEOUT) $(TESTFLAGS) github.com/docker/containerd/integration-test
 endif
 
+bench: shim validate install bundles-rootfs
+	go test -bench=. -v $(shell go list ./... | grep -v /vendor | grep -v /integration-test)
+
 validate: fmt
 
 uninstall:

integration-test/bundle_utils_test.go

@@ -8,14 +8,10 @@ import (
 	"path/filepath"
 	"reflect"
 
+	utils "github.com/docker/containerd/testutils"
 	ocs "github.com/opencontainers/runtime-spec/specs-go"
 )
 
-var (
-	bundlesDir      = filepath.Join("test-artifacts", "oci-bundles")
-	refOciSpecsPath = filepath.Join(bundlesDir, "config.json")
-)
-
 type OciProcessArgs struct {
 	Cmd  string
 	Args []string
@@ -48,7 +44,7 @@ func untarRootfs(source string, destination string) error {
 func CreateBundleWithFilter(source, name string, args []string, filter func(spec *ocs.Spec)) error {
 	// Generate the spec
 	var spec ocs.Spec
-	if f, err := os.Open(refOciSpecsPath); err != nil {
+	if f, err := os.Open(utils.RefOciSpecsPath); err != nil {
 		return fmt.Errorf("Failed to open default spec: %v", err)
 	} else {
 		if err := json.NewDecoder(f).Decode(&spec); err != nil {
@@ -63,7 +59,7 @@ func CreateBundleWithFilter(source, name string, args []string, filter func(spec
 		filter(&spec)
 	}
 
-	bundlePath := filepath.Join(bundlesDir, name)
+	bundlePath := filepath.Join(utils.BundlesRoot, name)
 	nb := Bundle{source, name, spec, bundlePath}
 
 	// Check that we don't already have such a bundle
@@ -78,7 +74,7 @@ func CreateBundleWithFilter(source, name string, args []string, filter func(spec
 	// Nothing should be there, but just in case
 	os.RemoveAll(bundlePath)
 
-	if err := untarRootfs(filepath.Join(archivesDir, source+".tar"), bundlePath); err != nil {
+	if err := untarRootfs(filepath.Join(utils.ArchivesDir, source+".tar"), bundlePath); err != nil {
 		return fmt.Errorf("Failed to untar %s.tar: %v", source, err)
 	}
 

integration-test/check_test.go

@@ -19,14 +19,10 @@ import (
 	"google.golang.org/grpc/grpclog"
 
 	"github.com/docker/containerd/api/grpc/types"
+	utils "github.com/docker/containerd/testutils"
 	"github.com/go-check/check"
 )
 
-var (
-	outputDirFormat = filepath.Join("test-artifacts", "runs", "%s")
-	archivesDir     = filepath.Join("test-artifacts", "archives")
-)
-
 func Test(t *testing.T) {
 	check.TestingT(t)
 }
@@ -102,14 +98,6 @@ func (cs *ContainerdSuite) ContainerdEventsHandler(events types.API_EventsClient
 	}
 }
 
-// generateReferencesSpecs invoke `runc spec` to produce the baseline
-// specs from which all future bundle will be generated
-func generateReferenceSpecs(destination string) error {
-	specs := exec.Command("runc", "spec")
-	specs.Dir = destination
-	return specs.Run()
-}
-
 func (cs *ContainerdSuite) StopDaemon(kill bool) {
 	if cs.cd == nil {
 		return
@@ -181,28 +169,29 @@ func (cs *ContainerdSuite) SetUpSuite(c *check.C) {
 	bundleMap = make(map[string]Bundle)
 	cs.eventFilters = make(map[string]func(event *types.Event))
 
-	// Get our CWD
+	// Get working directory for tests
-	if cwd, err := os.Getwd(); err != nil {
+	wd := utils.GetTestOutDir()
-		c.Fatalf("Could not determine current working directory: %v", err)
+	if err := os.Chdir(wd); err != nil {
-	} else {
+		c.Fatalf("Could not change working directory: %v", err)
-		cs.cwd = cwd
 	}
+	cs.cwd = wd
 
 	// Clean old bundles
-	os.RemoveAll(bundlesDir)
+	os.RemoveAll(utils.BundlesRoot)
 
 	// Ensure the oci bundles directory exists
-	if err := os.MkdirAll(bundlesDir, 0755); err != nil {
+	if err := os.MkdirAll(utils.BundlesRoot, 0755); err != nil {
 		c.Fatalf("Failed to create bundles directory: %v", err)
 	}
 
 	// Generate the reference spec
-	if err := generateReferenceSpecs(bundlesDir); err != nil {
+	if err := utils.GenerateReferenceSpecs(utils.BundlesRoot); err != nil {
 		c.Fatalf("Unable to generate OCI reference spec: %v", err)
 	}
 
 	// Create our output directory
-	cs.outputDir = fmt.Sprintf(outputDirFormat, time.Now().Format("2006-01-02_150405.000000"))
+	cs.outputDir = fmt.Sprintf(utils.OutputDirFormat, time.Now().Format("2006-01-02_150405.000000"))
+
 	cs.stateDir = filepath.Join(cs.outputDir, "containerd-master")
 	if err := os.MkdirAll(cs.stateDir, 0755); err != nil {
 		c.Fatalf("Unable to created output directory '%s': %v", cs.stateDir, err)

runtime/runtime_test.go

@@ -0,0 +1,174 @@
+package runtime
+
+import (
+	"fmt"
+	"io"
+	"os"
+	"os/exec"
+	"path/filepath"
+	"syscall"
+	"testing"
+	"time"
+
+	utils "github.com/docker/containerd/testutils"
+)
+
+var (
+	devNull = "/dev/null"
+	stdin   io.WriteCloser
+)
+
+// Create containerd state and oci bundles directory
+func setup() error {
+	if err := os.MkdirAll(utils.StateDir, 0755); err != nil {
+		return err
+	}
+
+	if err := os.MkdirAll(utils.BundlesRoot, 0755); err != nil {
+		return err
+	}
+	return nil
+}
+
+// Creates the bundleDir with rootfs, io fifo dir and a default spec.
+// On success, returns the bundlePath
+func setupBundle(bundleName string) (string, error) {
+	bundlePath := filepath.Join(utils.BundlesRoot, bundleName)
+	if err := os.MkdirAll(bundlePath, 0755); err != nil {
+		fmt.Println("Unable to create bundlePath due to ", err)
+		return "", err
+	}
+
+	io := filepath.Join(bundlePath, "io")
+	if err := os.MkdirAll(io, 0755); err != nil {
+		fmt.Println("Unable to create io dir due to ", err)
+		return "", err
+	}
+
+	if err := utils.GenerateReferenceSpecs(bundlePath); err != nil {
+		fmt.Println("Unable to generate OCI reference spec: ", err)
+		return "", err
+	}
+
+	if err := utils.CreateBusyboxBundle(bundleName); err != nil {
+		fmt.Println("CreateBusyboxBundle error: ", err)
+		return "", err
+	}
+
+	return bundlePath, nil
+}
+
+func setupStdio(cwd string, bundlePath string, bundleName string) (Stdio, error) {
+	s := NewStdio(devNull, devNull, devNull)
+
+	pid := "init"
+	for stdName, stdPath := range map[string]*string{
+		"stdin":  &s.Stdin,
+		"stdout": &s.Stdout,
+		"stderr": &s.Stderr,
+	} {
+		*stdPath = filepath.Join(cwd, bundlePath, "io", bundleName+"-"+pid+"-"+stdName)
+		if err := syscall.Mkfifo(*stdPath, 0755); err != nil && !os.IsExist(err) {
+			fmt.Println("Mkfifo error: ", err)
+			return s, err
+		}
+	}
+
+	err := attachStdio(s)
+	if err != nil {
+		fmt.Println("attachStdio error: ", err)
+		return s, err
+	}
+
+	return s, nil
+}
+
+func attachStdio(s Stdio) error {
+	stdinf, err := os.OpenFile(s.Stdin, syscall.O_RDWR, 0)
+	if err != nil {
+		return err
+	}
+	stdin = stdinf
+	stdoutf, err := os.OpenFile(s.Stdout, syscall.O_RDWR, 0)
+	if err != nil {
+		return err
+	}
+	go io.Copy(os.Stdout, stdoutf)
+	stderrf, err := os.OpenFile(s.Stderr, syscall.O_RDWR, 0)
+	if err != nil {
+		return err
+	}
+	go io.Copy(os.Stderr, stderrf)
+	return nil
+}
+
+func teardownBundle(bundleName string) {
+	containerRoot := filepath.Join(utils.StateDir, bundleName)
+	os.RemoveAll(containerRoot)
+
+	bundlePath := filepath.Join(utils.BundlesRoot, bundleName)
+	os.RemoveAll(bundlePath)
+	return
+}
+
+// Remove containerd state and oci bundles directory
+func teardown() {
+	os.RemoveAll(utils.StateDir)
+	os.RemoveAll(utils.BundlesRoot)
+}
+
+func BenchmarkBusyboxSh(b *testing.B) {
+	bundleName := "busybox-sh"
+
+	wd := utils.GetTestOutDir()
+	if err := os.Chdir(wd); err != nil {
+		b.Fatalf("Could not change working directory: %v", err)
+	}
+
+	if err := setup(); err != nil {
+		b.Fatalf("Error setting up test: %v", err)
+	}
+	defer teardown()
+
+	for n := 0; n < b.N; n++ {
+		bundlePath, err := setupBundle(bundleName)
+		if err != nil {
+			return
+		}
+
+		s, err := setupStdio(wd, bundlePath, bundleName)
+		if err != nil {
+			return
+		}
+
+		c, err := New(ContainerOpts{
+			Root:    utils.StateDir,
+			ID:      bundleName,
+			Bundle:  filepath.Join(wd, bundlePath),
+			Runtime: "runc",
+			Shim:    "containerd-shim",
+			Timeout: 15 * time.Second,
+		})
+
+		if err != nil {
+			b.Fatalf("Error creating a New container: ", err)
+		}
+
+		benchmarkStartContainer(b, c, s, bundleName)
+
+		teardownBundle(bundleName)
+	}
+}
+
+func benchmarkStartContainer(b *testing.B, c Container, s Stdio, bundleName string) {
+	if _, err := c.Start("", s); err != nil {
+		b.Fatalf("Error starting container %v", err)
+	}
+
+	kill := exec.Command("runc", "kill", bundleName, "KILL")
+	kill.Run()
+
+	// wait for kill to finish. selected wait time is arbitrary
+	time.Sleep(500 * time.Millisecond)
+
+}

testutils/testutils.go

@@ -0,0 +1,60 @@
+package testutils
+
+import (
+	"fmt"
+	"os"
+	"os/exec"
+	"path/filepath"
+	"strings"
+)
+
+// Output directory for testing and benchmark artifacts
+func GetTestOutDir() string {
+	out, _ := exec.Command("git", "rev-parse", "--show-toplevel").CombinedOutput()
+	repoRoot := string(out)
+	prefix := filepath.Join(strings.TrimSpace(repoRoot), "output")
+	return prefix
+}
+
+var (
+	ArchivesDir     = filepath.Join("test-artifacts", "archives")
+	BundlesRoot     = filepath.Join("test-artifacts", "oci-bundles")
+	OutputDirFormat = filepath.Join("test-artifacts", "runs", "%s")
+	RefOciSpecsPath = filepath.Join(BundlesRoot, "config.json")
+	StateDir        = "/run/containerd-bench-test"
+)
+
+// untarRootfs untars the given `source` tarPath into `destination/rootfs`
+func untarRootfs(source string, destination string) error {
+	rootfs := filepath.Join(destination, "rootfs")
+
+	if err := os.MkdirAll(rootfs, 0755); err != nil {
+		fmt.Println("untarRootfs os.MkdirAll failed with err %v", err)
+		return nil
+	}
+	tar := exec.Command("tar", "-C", rootfs, "-xf", source)
+	return tar.Run()
+}
+
+func GenerateReferenceSpecs(destination string) error {
+	if _, err := os.Stat(filepath.Join(destination, "config.json")); err == nil {
+		return nil
+	}
+	specs := exec.Command("runc", "spec")
+	specs.Dir = destination
+	return specs.Run()
+}
+
+func CreateBundle(source, name string) error {
+	bundlePath := filepath.Join(BundlesRoot, name)
+
+	if err := untarRootfs(filepath.Join(ArchivesDir, source+".tar"), bundlePath); err != nil {
+		return fmt.Errorf("Failed to untar %s.tar: %v", source, err)
+	}
+
+	return nil
+}
+
+func CreateBusyboxBundle(name string) error {
+	return CreateBundle("busybox", name)
+}