From 925806b8fab7e70c67be239e6c6bdf7b9d85f79f Mon Sep 17 00:00:00 2001 From: Nalin Dahyabhai Date: Tue, 10 Jan 2017 17:57:22 -0500 Subject: [PATCH] Add and use copyimg for caching images for tests Add a basic tool for copying images from one location to another, optionally adding a name if it's to local storage. Ideally we could use skopeo for this, but we don't want to build it. Use it to initially populate the test/testdata/redis-image directory, if it's not been cleaned out, with a copy of "docker://redis:latest", and to copy it in to the storage that ocid is using before we start up ocid. Signed-off-by: Nalin Dahyabhai --- .gitignore | 2 + Makefile | 8 +- test/copyimg/Makefile | 6 ++ test/copyimg/copyimg.go | 198 ++++++++++++++++++++++++++++++++++++++++ test/helpers.bash | 15 ++- 5 files changed, 227 insertions(+), 2 deletions(-) create mode 100644 test/copyimg/Makefile create mode 100644 test/copyimg/copyimg.go diff --git a/.gitignore b/.gitignore index d8ae2bad..d073fa34 100644 --- a/.gitignore +++ b/.gitignore @@ -8,3 +8,5 @@ ocid.conf *.orig *.rej test/bin2img/bin2img +test/copyimg/copyimg +test/testdata/redis-image diff --git a/Makefile b/Makefile index 616723ac..9a37318c 100644 --- a/Makefile +++ b/Makefile @@ -47,6 +47,9 @@ pause: bin2img: make -C test/$@ +copyimg: + make -C test/$@ + ocid: ifndef GOPATH $(error GOPATH is not set) @@ -72,11 +75,13 @@ ocid.conf: ocid clean: rm -f docs/*.1 docs/*.5 docs/*.8 + rm -fr test/testdata/redis-image find . -name \*~ -delete find . -name \#\* -delete make -C conmon clean make -C pause clean make -C test/bin2img clean + make -C test/copyimg clean ocidimage: docker build -t ${OCID_IMAGE} . @@ -90,7 +95,7 @@ integration: ocidimage localintegration: binaries ./test/test_runner.sh ${TESTFLAGS} -binaries: ocid ocic kpod conmon pause bin2img +binaries: ocid ocic kpod conmon pause bin2img copyimg MANPAGES_MD := $(wildcard docs/*.md) MANPAGES := $(MANPAGES_MD:%.md=%) @@ -188,6 +193,7 @@ install.tools: .install.gitvalidation .install.gometalinter .install.md2man binaries \ clean \ conmon \ + copyimg \ default \ docs \ gofmt \ diff --git a/test/copyimg/Makefile b/test/copyimg/Makefile new file mode 100644 index 00000000..cb1d7f10 --- /dev/null +++ b/test/copyimg/Makefile @@ -0,0 +1,6 @@ +copyimg: $(wildcard *.go) + go build -o $@ + +.PHONY: clean +clean: + rm -f copyimg diff --git a/test/copyimg/copyimg.go b/test/copyimg/copyimg.go new file mode 100644 index 00000000..ba8a7849 --- /dev/null +++ b/test/copyimg/copyimg.go @@ -0,0 +1,198 @@ +package main + +import ( + "os" + + "github.com/Sirupsen/logrus" + "github.com/containers/image/copy" + "github.com/containers/image/signature" + "github.com/containers/image/storage" + "github.com/containers/image/transports" + "github.com/containers/image/types" + "github.com/containers/storage/pkg/reexec" + sstorage "github.com/containers/storage/storage" + "github.com/urfave/cli" +) + +func main() { + if reexec.Init() { + return + } + + app := cli.NewApp() + app.Name = "copyimg" + app.Usage = "barebones image copier" + app.Version = "0.0.1" + + app.Flags = []cli.Flag{ + cli.BoolFlag{ + Name: "debug", + Usage: "turn on debug logging", + }, + cli.StringFlag{ + Name: "root", + Usage: "graph root directory", + }, + cli.StringFlag{ + Name: "runroot", + Usage: "run root directory", + }, + cli.StringFlag{ + Name: "storage-driver", + Usage: "storage driver", + }, + cli.StringSliceFlag{ + Name: "storage-option", + Usage: "storage option", + }, + cli.StringFlag{ + Name: "signature-policy", + Usage: "signature policy", + }, + cli.StringFlag{ + Name: "image-name", + Usage: "set image name", + }, + cli.StringFlag{ + Name: "add-name", + Usage: "name to add to image", + }, + cli.StringFlag{ + Name: "import-from", + Usage: "import source", + }, + cli.StringFlag{ + Name: "export-to", + Usage: "export target", + }, + } + + app.Action = func(c *cli.Context) error { + var store sstorage.Store + var ref, importRef, exportRef types.ImageReference + var err error + + debug := c.GlobalBool("debug") + rootDir := c.GlobalString("root") + runrootDir := c.GlobalString("runroot") + storageDriver := c.GlobalString("storage-driver") + storageOptions := c.GlobalStringSlice("storage-option") + signaturePolicy := c.GlobalString("signature-policy") + imageName := c.GlobalString("image-name") + addName := c.GlobalString("add-name") + importFrom := c.GlobalString("import-from") + exportTo := c.GlobalString("export-to") + + if debug { + logrus.SetLevel(logrus.DebugLevel) + } else { + logrus.SetLevel(logrus.ErrorLevel) + } + + if imageName != "" { + if rootDir == "" && runrootDir != "" { + logrus.Errorf("must set --root and --runroot, or neither") + os.Exit(1) + } + if rootDir != "" && runrootDir == "" { + logrus.Errorf("must set --root and --runroot, or neither") + os.Exit(1) + } + storeOptions := sstorage.DefaultStoreOptions + if rootDir != "" && runrootDir != "" { + storeOptions.GraphDriverName = storageDriver + storeOptions.GraphDriverOptions = storageOptions + storeOptions.GraphRoot = rootDir + storeOptions.RunRoot = runrootDir + } + store, err = sstorage.GetStore(storeOptions) + if err != nil { + logrus.Errorf("error opening storage: %v", err) + os.Exit(1) + } + defer store.Shutdown(false) + + storage.Transport.SetStore(store) + ref, err = storage.Transport.ParseStoreReference(store, imageName) + if err != nil { + logrus.Errorf("error parsing image name: %v", err) + os.Exit(1) + } + } + + systemContext := types.SystemContext{ + SignaturePolicyPath: signaturePolicy, + } + policy, err := signature.DefaultPolicy(&systemContext) + if err != nil { + logrus.Errorf("error loading signature policy: %v", err) + os.Exit(1) + } + policyContext, err := signature.NewPolicyContext(policy) + if err != nil { + logrus.Errorf("error loading signature policy: %v", err) + os.Exit(1) + } + defer policyContext.Destroy() + options := ©.Options{} + + if importFrom != "" { + importRef, err = transports.ParseImageName(importFrom) + if err != nil { + logrus.Errorf("error parsing image name %v: %v", importFrom, err) + os.Exit(1) + } + } + + if exportTo != "" { + exportRef, err = transports.ParseImageName(exportTo) + if err != nil { + logrus.Errorf("error parsing image name %v: %v", exportTo, err) + os.Exit(1) + } + } + + if imageName != "" { + if importFrom != "" { + err = copy.Image(policyContext, ref, importRef, options) + if err != nil { + logrus.Errorf("error importing %s: %v", importFrom, err) + os.Exit(1) + } + } + if addName != "" { + destImage, err := storage.Transport.GetStoreImage(store, ref) + if err != nil { + logrus.Errorf("error finding image: %v", err) + os.Exit(1) + } + names := append(destImage.Names, imageName, addName) + err = store.SetNames(destImage.ID, names) + if err != nil { + logrus.Errorf("error adding name to %s: %v", imageName, err) + os.Exit(1) + } + } + if exportTo != "" { + err = copy.Image(policyContext, exportRef, ref, options) + if err != nil { + logrus.Errorf("error exporting %s: %v", exportTo, err) + os.Exit(1) + } + } + } else { + if importFrom != "" && exportTo != "" { + err = copy.Image(policyContext, exportRef, importRef, options) + if err != nil { + logrus.Errorf("error copying %s to %s: %v", importFrom, exportTo, err) + os.Exit(1) + } + } + } + return nil + } + + if err := app.Run(os.Args); err != nil { + logrus.Fatal(err) + } +} diff --git a/test/helpers.bash b/test/helpers.bash index 226fef8e..58eaee24 100644 --- a/test/helpers.bash +++ b/test/helpers.bash @@ -38,6 +38,8 @@ BOOT_CONFIG_FILE_PATH=${BOOT_CONFIG_FILE_PATH:-/boot/config-`uname -r`} APPARMOR_PARAMETERS_FILE_PATH=${APPARMOR_PARAMETERS_FILE_PATH:-/sys/module/apparmor/parameters/enabled} # Path of the bin2img binary. BIN2IMG_BINARY=${BIN2IMG_BINARY:-${OCID_ROOT}/cri-o/test/bin2img/bin2img} +# Path of the copyimg binary. +COPYIMG_BINARY=${COPYIMG_BINARY:-${OCID_ROOT}/cri-o/test/copyimg/copyimg} TESTDIR=$(mktemp -d) if [ -e /usr/sbin/selinuxenabled ] && /usr/sbin/selinuxenabled; then @@ -58,6 +60,16 @@ mkdir -p $OCID_CNI_CONFIG PATH=$PATH:$TESTDIR +# Make sure we have a copy of the redis:latest image. +if ! [ -d "$TESTDATA"/redis-image ]; then + mkdir -p "$TESTDATA"/redis-image + if ! "$COPYIMG_BINARY" --import-from=docker://redis --export-to=dir:"$TESTDATA"/redis-image --signature-policy="$INTEGRATION_ROOT"/policy.json ; then + echo "Error pulling docker://redis" + rm -fr "$TESTDATA"/redis-image + exit 1 + fi +fi + # Run ocid using the binary specified by $OCID_BINARY. # This must ONLY be run on engines created with `start_ocid`. function ocid() { @@ -114,10 +126,11 @@ function start_ocid() { apparmor="$APPARMOR_PROFILE" fi - # Don't forget: bin2img and ocid have their own default drivers, so if you override either, you probably need to override both + # Don't forget: bin2img, copyimg, and ocid have their own default drivers, so if you override any, you probably need to override them all if ! [ "$3" = "--no-pause-image" ] ; then "$BIN2IMG_BINARY" --root "$TESTDIR/ocid" --runroot "$TESTDIR/ocid-run" --source-binary "$PAUSE_BINARY" fi + "$COPYIMG_BINARY" --root "$TESTDIR/ocid" --runroot "$TESTDIR/ocid-run" --image-name=redis --import-from=dir:"$TESTDATA"/redis-image --add-name=docker://docker.io/library/redis:latest "$OCID_BINARY" --conmon "$CONMON_BINARY" --listen "$OCID_SOCKET" --runtime "$RUNC_BINARY" --root "$TESTDIR/ocid" --runroot "$TESTDIR/ocid-run" --seccomp-profile "$seccomp" --apparmor-profile "$apparmor" --cni-config-dir "$OCID_CNI_CONFIG" --signature-policy "$INTEGRATION_ROOT"/policy.json config >$OCID_CONFIG "$OCID_BINARY" --debug --config "$OCID_CONFIG" & OCID_PID=$! wait_until_reachable