From 24144682a0520dc111f266e2cbd1829a4d5b7165 Mon Sep 17 00:00:00 2001 From: Anusha Ragunathan Date: Fri, 27 May 2016 10:35:42 -0700 Subject: [PATCH] Micro benchmarks for containerd. (#244) This is the first in a series of micro benchmarks for containerd. Performance measurement will use containerd objects and methods that are not dependent on the grpc API and dont require the daemon to the running. Test will require containerd-shim and runc. The motivation is to understand the baseline performance at the lowest containerd layer. A natural extension to this effort would be to write macro benchmarks which would include API and daemon. Note: - Currently measures only one workload (busybox sh) start times. Will add other bundles and args soon. - Can use integration-test utils for bundle processing. However, json marshal/unmarshal is currently timing out standard benchmark times. So going with default spec for now. Sample run: BenchmarkBusyboxSh-4 / # / # / # 2 576013841 ns/op ok github.com/docker/containerd/runtime 1.800s Signed-off-by: Anusha Ragunathan --- Makefile | 27 ++-- integration-test/bundle_utils_test.go | 12 +- integration-test/check_test.go | 33 ++--- runtime/runtime_test.go | 174 ++++++++++++++++++++++++++ testutils/testutils.go | 60 +++++++++ 5 files changed, 265 insertions(+), 41 deletions(-) create mode 100644 runtime/runtime_test.go create mode 100644 testutils/testutils.go diff --git a/Makefile b/Makefile index 95a0671..88c1028 100644 --- a/Makefile +++ b/Makefile @@ -18,8 +18,8 @@ ifeq ($(INTERACTIVE), 1) DOCKER_FLAGS += -t endif -TEST_ARTIFACTS_DIR := integration-test/test-artifacts -BUNDLE_ARCHIVES_DIR := $(TEST_ARTIFACTS_DIR)/archives +TESTBENCH_ARTIFACTS_DIR := output/test-artifacts +TESTBENCH_BUNDLE_DIR := $(TESTBENCH_ARTIFACTS_DIR)/archives DOCKER_IMAGE := containerd-dev$(if $(GIT_BRANCH),:$(GIT_BRANCH)) DOCKER_RUN := docker run --privileged --rm -i $(DOCKER_FLAGS) "$(DOCKER_IMAGE)" @@ -35,7 +35,7 @@ bin: mkdir -p bin/ clean: - rm -rf bin + rm -rf bin && rm -rf output client: bin cd ctr && go build -ldflags "${LDFLAGS}" -o ../bin/ctr @@ -55,18 +55,21 @@ shim: bin shim-static: cd containerd-shim && go build -ldflags "-w -extldflags -static ${LDFLAGS}" -tags "$(BUILDTAGS)" -o ../bin/containerd-shim -$(BUNDLE_ARCHIVES_DIR)/busybox.tar: - @mkdir -p $(BUNDLE_ARCHIVES_DIR) - curl -sSL 'https://github.com/jpetazzo/docker-busybox/raw/buildroot-2014.11/rootfs.tar' -o $(BUNDLE_ARCHIVES_DIR)/busybox.tar +$(TESTBENCH_BUNDLE_DIR)/busybox.tar: + mkdir -p $(TESTBENCH_BUNDLE_DIR) + curl -sSL 'https://github.com/jpetazzo/docker-busybox/raw/buildroot-2014.11/rootfs.tar' -o $(TESTBENCH_BUNDLE_DIR)/busybox.tar -bundles-rootfs: $(BUNDLE_ARCHIVES_DIR)/busybox.tar +bundles-rootfs: $(TESTBENCH_BUNDLE_DIR)/busybox.tar -dbuild: $(BUNDLE_ARCHIVES_DIR)/busybox.tar +dbuild: $(TESTBENCH_BUNDLE_DIR)/busybox.tar @docker build --rm --force-rm -t "$(DOCKER_IMAGE)" . dtest: dbuild $(DOCKER_RUN) make test +dbench: dbuild + $(DOCKER_RUN) make bench + install: cp bin/* /usr/local/bin/ @@ -82,14 +85,16 @@ lint: shell: dbuild $(DOCKER_RUN) bash -test: validate - go test -v $(shell go list ./... | grep -v /vendor | grep -v /integration-test) +test: validate install bundles-rootfs + go test -bench=. -v $(shell go list ./... | grep -v /vendor | grep -v /integration-test) ifneq ($(wildcard /.dockerenv), ) - $(MAKE) install bundles-rootfs cd integration-test ; \ go test -check.v -check.timeout=$(TEST_TIMEOUT) timeout=$(TEST_SUITE_TIMEOUT) $(TESTFLAGS) github.com/docker/containerd/integration-test endif +bench: shim validate install bundles-rootfs + go test -bench=. -v $(shell go list ./... | grep -v /vendor | grep -v /integration-test) + validate: fmt uninstall: diff --git a/integration-test/bundle_utils_test.go b/integration-test/bundle_utils_test.go index 3ffad2e..1ddba27 100644 --- a/integration-test/bundle_utils_test.go +++ b/integration-test/bundle_utils_test.go @@ -8,14 +8,10 @@ import ( "path/filepath" "reflect" + utils "github.com/docker/containerd/testutils" ocs "github.com/opencontainers/runtime-spec/specs-go" ) -var ( - bundlesDir = filepath.Join("test-artifacts", "oci-bundles") - refOciSpecsPath = filepath.Join(bundlesDir, "config.json") -) - type OciProcessArgs struct { Cmd string Args []string @@ -48,7 +44,7 @@ func untarRootfs(source string, destination string) error { func CreateBundleWithFilter(source, name string, args []string, filter func(spec *ocs.Spec)) error { // Generate the spec var spec ocs.Spec - if f, err := os.Open(refOciSpecsPath); err != nil { + if f, err := os.Open(utils.RefOciSpecsPath); err != nil { return fmt.Errorf("Failed to open default spec: %v", err) } else { if err := json.NewDecoder(f).Decode(&spec); err != nil { @@ -63,7 +59,7 @@ func CreateBundleWithFilter(source, name string, args []string, filter func(spec filter(&spec) } - bundlePath := filepath.Join(bundlesDir, name) + bundlePath := filepath.Join(utils.BundlesRoot, name) nb := Bundle{source, name, spec, bundlePath} // Check that we don't already have such a bundle @@ -78,7 +74,7 @@ func CreateBundleWithFilter(source, name string, args []string, filter func(spec // Nothing should be there, but just in case os.RemoveAll(bundlePath) - if err := untarRootfs(filepath.Join(archivesDir, source+".tar"), bundlePath); err != nil { + if err := untarRootfs(filepath.Join(utils.ArchivesDir, source+".tar"), bundlePath); err != nil { return fmt.Errorf("Failed to untar %s.tar: %v", source, err) } diff --git a/integration-test/check_test.go b/integration-test/check_test.go index 4bb97ab..d64a12f 100644 --- a/integration-test/check_test.go +++ b/integration-test/check_test.go @@ -19,14 +19,10 @@ import ( "google.golang.org/grpc/grpclog" "github.com/docker/containerd/api/grpc/types" + utils "github.com/docker/containerd/testutils" "github.com/go-check/check" ) -var ( - outputDirFormat = filepath.Join("test-artifacts", "runs", "%s") - archivesDir = filepath.Join("test-artifacts", "archives") -) - func Test(t *testing.T) { check.TestingT(t) } @@ -102,14 +98,6 @@ func (cs *ContainerdSuite) ContainerdEventsHandler(events types.API_EventsClient } } -// generateReferencesSpecs invoke `runc spec` to produce the baseline -// specs from which all future bundle will be generated -func generateReferenceSpecs(destination string) error { - specs := exec.Command("runc", "spec") - specs.Dir = destination - return specs.Run() -} - func (cs *ContainerdSuite) StopDaemon(kill bool) { if cs.cd == nil { return @@ -181,28 +169,29 @@ func (cs *ContainerdSuite) SetUpSuite(c *check.C) { bundleMap = make(map[string]Bundle) cs.eventFilters = make(map[string]func(event *types.Event)) - // Get our CWD - if cwd, err := os.Getwd(); err != nil { - c.Fatalf("Could not determine current working directory: %v", err) - } else { - cs.cwd = cwd + // Get working directory for tests + wd := utils.GetTestOutDir() + if err := os.Chdir(wd); err != nil { + c.Fatalf("Could not change working directory: %v", err) } + cs.cwd = wd // Clean old bundles - os.RemoveAll(bundlesDir) + os.RemoveAll(utils.BundlesRoot) // Ensure the oci bundles directory exists - if err := os.MkdirAll(bundlesDir, 0755); err != nil { + if err := os.MkdirAll(utils.BundlesRoot, 0755); err != nil { c.Fatalf("Failed to create bundles directory: %v", err) } // Generate the reference spec - if err := generateReferenceSpecs(bundlesDir); err != nil { + if err := utils.GenerateReferenceSpecs(utils.BundlesRoot); err != nil { c.Fatalf("Unable to generate OCI reference spec: %v", err) } // Create our output directory - cs.outputDir = fmt.Sprintf(outputDirFormat, time.Now().Format("2006-01-02_150405.000000")) + cs.outputDir = fmt.Sprintf(utils.OutputDirFormat, time.Now().Format("2006-01-02_150405.000000")) + cs.stateDir = filepath.Join(cs.outputDir, "containerd-master") if err := os.MkdirAll(cs.stateDir, 0755); err != nil { c.Fatalf("Unable to created output directory '%s': %v", cs.stateDir, err) diff --git a/runtime/runtime_test.go b/runtime/runtime_test.go new file mode 100644 index 0000000..49b4711 --- /dev/null +++ b/runtime/runtime_test.go @@ -0,0 +1,174 @@ +package runtime + +import ( + "fmt" + "io" + "os" + "os/exec" + "path/filepath" + "syscall" + "testing" + "time" + + utils "github.com/docker/containerd/testutils" +) + +var ( + devNull = "/dev/null" + stdin io.WriteCloser +) + +// Create containerd state and oci bundles directory +func setup() error { + if err := os.MkdirAll(utils.StateDir, 0755); err != nil { + return err + } + + if err := os.MkdirAll(utils.BundlesRoot, 0755); err != nil { + return err + } + return nil +} + +// Creates the bundleDir with rootfs, io fifo dir and a default spec. +// On success, returns the bundlePath +func setupBundle(bundleName string) (string, error) { + bundlePath := filepath.Join(utils.BundlesRoot, bundleName) + if err := os.MkdirAll(bundlePath, 0755); err != nil { + fmt.Println("Unable to create bundlePath due to ", err) + return "", err + } + + io := filepath.Join(bundlePath, "io") + if err := os.MkdirAll(io, 0755); err != nil { + fmt.Println("Unable to create io dir due to ", err) + return "", err + } + + if err := utils.GenerateReferenceSpecs(bundlePath); err != nil { + fmt.Println("Unable to generate OCI reference spec: ", err) + return "", err + } + + if err := utils.CreateBusyboxBundle(bundleName); err != nil { + fmt.Println("CreateBusyboxBundle error: ", err) + return "", err + } + + return bundlePath, nil +} + +func setupStdio(cwd string, bundlePath string, bundleName string) (Stdio, error) { + s := NewStdio(devNull, devNull, devNull) + + pid := "init" + for stdName, stdPath := range map[string]*string{ + "stdin": &s.Stdin, + "stdout": &s.Stdout, + "stderr": &s.Stderr, + } { + *stdPath = filepath.Join(cwd, bundlePath, "io", bundleName+"-"+pid+"-"+stdName) + if err := syscall.Mkfifo(*stdPath, 0755); err != nil && !os.IsExist(err) { + fmt.Println("Mkfifo error: ", err) + return s, err + } + } + + err := attachStdio(s) + if err != nil { + fmt.Println("attachStdio error: ", err) + return s, err + } + + return s, nil +} + +func attachStdio(s Stdio) error { + stdinf, err := os.OpenFile(s.Stdin, syscall.O_RDWR, 0) + if err != nil { + return err + } + stdin = stdinf + stdoutf, err := os.OpenFile(s.Stdout, syscall.O_RDWR, 0) + if err != nil { + return err + } + go io.Copy(os.Stdout, stdoutf) + stderrf, err := os.OpenFile(s.Stderr, syscall.O_RDWR, 0) + if err != nil { + return err + } + go io.Copy(os.Stderr, stderrf) + return nil +} + +func teardownBundle(bundleName string) { + containerRoot := filepath.Join(utils.StateDir, bundleName) + os.RemoveAll(containerRoot) + + bundlePath := filepath.Join(utils.BundlesRoot, bundleName) + os.RemoveAll(bundlePath) + return +} + +// Remove containerd state and oci bundles directory +func teardown() { + os.RemoveAll(utils.StateDir) + os.RemoveAll(utils.BundlesRoot) +} + +func BenchmarkBusyboxSh(b *testing.B) { + bundleName := "busybox-sh" + + wd := utils.GetTestOutDir() + if err := os.Chdir(wd); err != nil { + b.Fatalf("Could not change working directory: %v", err) + } + + if err := setup(); err != nil { + b.Fatalf("Error setting up test: %v", err) + } + defer teardown() + + for n := 0; n < b.N; n++ { + bundlePath, err := setupBundle(bundleName) + if err != nil { + return + } + + s, err := setupStdio(wd, bundlePath, bundleName) + if err != nil { + return + } + + c, err := New(ContainerOpts{ + Root: utils.StateDir, + ID: bundleName, + Bundle: filepath.Join(wd, bundlePath), + Runtime: "runc", + Shim: "containerd-shim", + Timeout: 15 * time.Second, + }) + + if err != nil { + b.Fatalf("Error creating a New container: ", err) + } + + benchmarkStartContainer(b, c, s, bundleName) + + teardownBundle(bundleName) + } +} + +func benchmarkStartContainer(b *testing.B, c Container, s Stdio, bundleName string) { + if _, err := c.Start("", s); err != nil { + b.Fatalf("Error starting container %v", err) + } + + kill := exec.Command("runc", "kill", bundleName, "KILL") + kill.Run() + + // wait for kill to finish. selected wait time is arbitrary + time.Sleep(500 * time.Millisecond) + +} diff --git a/testutils/testutils.go b/testutils/testutils.go new file mode 100644 index 0000000..6ea21cd --- /dev/null +++ b/testutils/testutils.go @@ -0,0 +1,60 @@ +package testutils + +import ( + "fmt" + "os" + "os/exec" + "path/filepath" + "strings" +) + +// Output directory for testing and benchmark artifacts +func GetTestOutDir() string { + out, _ := exec.Command("git", "rev-parse", "--show-toplevel").CombinedOutput() + repoRoot := string(out) + prefix := filepath.Join(strings.TrimSpace(repoRoot), "output") + return prefix +} + +var ( + ArchivesDir = filepath.Join("test-artifacts", "archives") + BundlesRoot = filepath.Join("test-artifacts", "oci-bundles") + OutputDirFormat = filepath.Join("test-artifacts", "runs", "%s") + RefOciSpecsPath = filepath.Join(BundlesRoot, "config.json") + StateDir = "/run/containerd-bench-test" +) + +// untarRootfs untars the given `source` tarPath into `destination/rootfs` +func untarRootfs(source string, destination string) error { + rootfs := filepath.Join(destination, "rootfs") + + if err := os.MkdirAll(rootfs, 0755); err != nil { + fmt.Println("untarRootfs os.MkdirAll failed with err %v", err) + return nil + } + tar := exec.Command("tar", "-C", rootfs, "-xf", source) + return tar.Run() +} + +func GenerateReferenceSpecs(destination string) error { + if _, err := os.Stat(filepath.Join(destination, "config.json")); err == nil { + return nil + } + specs := exec.Command("runc", "spec") + specs.Dir = destination + return specs.Run() +} + +func CreateBundle(source, name string) error { + bundlePath := filepath.Join(BundlesRoot, name) + + if err := untarRootfs(filepath.Join(ArchivesDir, source+".tar"), bundlePath); err != nil { + return fmt.Errorf("Failed to untar %s.tar: %v", source, err) + } + + return nil +} + +func CreateBusyboxBundle(name string) error { + return CreateBundle("busybox", name) +}