Add and use copyimg for caching images for tests

Add a basic tool for copying images from one location to another,
optionally adding a name if it's to local storage.  Ideally we could use
skopeo for this, but we don't want to build it.

Use it to initially populate the test/testdata/redis-image directory, if
it's not been cleaned out, with a copy of "docker://redis:latest", and
to copy it in to the storage that ocid is using before we start up ocid.

Signed-off-by: Nalin Dahyabhai <nalin@redhat.com>

.gitignore

@@ -8,3 +8,5 @@ ocid.conf
 *.orig
 *.rej
 test/bin2img/bin2img
+test/copyimg/copyimg
+test/testdata/redis-image

Makefile

@@ -47,6 +47,9 @@ pause:
 bin2img:
 	make -C test/$@
 
+copyimg:
+	make -C test/$@
+
 ocid:
 ifndef GOPATH
 	$(error GOPATH is not set)
@@ -72,11 +75,13 @@ ocid.conf: ocid
 
 clean:
 	rm -f docs/*.1 docs/*.5 docs/*.8
+	rm -fr test/testdata/redis-image
 	find . -name \*~ -delete
 	find . -name \#\* -delete
 	make -C conmon clean
 	make -C pause clean
 	make -C test/bin2img clean
+	make -C test/copyimg clean
 
 ocidimage:
 	docker build -t ${OCID_IMAGE} .
@@ -90,7 +95,7 @@ integration: ocidimage
 localintegration: binaries
 	./test/test_runner.sh ${TESTFLAGS}
 
-binaries: ocid ocic kpod conmon pause bin2img
+binaries: ocid ocic kpod conmon pause bin2img copyimg
 
 MANPAGES_MD := $(wildcard docs/*.md)
 MANPAGES    := $(MANPAGES_MD:%.md=%)
@@ -188,6 +193,7 @@ install.tools: .install.gitvalidation .install.gometalinter .install.md2man
 	binaries \
 	clean \
 	conmon \
+	copyimg \
 	default \
 	docs \
 	gofmt \

test/copyimg/Makefile

@@ -0,0 +1,6 @@
+copyimg: $(wildcard *.go)
+	go build -o $@
+
+.PHONY: clean
+clean:
+	rm -f copyimg

test/copyimg/copyimg.go

@@ -0,0 +1,198 @@
+package main
+
+import (
+	"os"
+
+	"github.com/Sirupsen/logrus"
+	"github.com/containers/image/copy"
+	"github.com/containers/image/signature"
+	"github.com/containers/image/storage"
+	"github.com/containers/image/transports"
+	"github.com/containers/image/types"
+	"github.com/containers/storage/pkg/reexec"
+	sstorage "github.com/containers/storage/storage"
+	"github.com/urfave/cli"
+)
+
+func main() {
+	if reexec.Init() {
+		return
+	}
+
+	app := cli.NewApp()
+	app.Name = "copyimg"
+	app.Usage = "barebones image copier"
+	app.Version = "0.0.1"
+
+	app.Flags = []cli.Flag{
+		cli.BoolFlag{
+			Name:  "debug",
+			Usage: "turn on debug logging",
+		},
+		cli.StringFlag{
+			Name:  "root",
+			Usage: "graph root directory",
+		},
+		cli.StringFlag{
+			Name:  "runroot",
+			Usage: "run root directory",
+		},
+		cli.StringFlag{
+			Name:  "storage-driver",
+			Usage: "storage driver",
+		},
+		cli.StringSliceFlag{
+			Name:  "storage-option",
+			Usage: "storage option",
+		},
+		cli.StringFlag{
+			Name:  "signature-policy",
+			Usage: "signature policy",
+		},
+		cli.StringFlag{
+			Name:  "image-name",
+			Usage: "set image name",
+		},
+		cli.StringFlag{
+			Name:  "add-name",
+			Usage: "name to add to image",
+		},
+		cli.StringFlag{
+			Name:  "import-from",
+			Usage: "import source",
+		},
+		cli.StringFlag{
+			Name:  "export-to",
+			Usage: "export target",
+		},
+	}
+
+	app.Action = func(c *cli.Context) error {
+		var store sstorage.Store
+		var ref, importRef, exportRef types.ImageReference
+		var err error
+
+		debug := c.GlobalBool("debug")
+		rootDir := c.GlobalString("root")
+		runrootDir := c.GlobalString("runroot")
+		storageDriver := c.GlobalString("storage-driver")
+		storageOptions := c.GlobalStringSlice("storage-option")
+		signaturePolicy := c.GlobalString("signature-policy")
+		imageName := c.GlobalString("image-name")
+		addName := c.GlobalString("add-name")
+		importFrom := c.GlobalString("import-from")
+		exportTo := c.GlobalString("export-to")
+
+		if debug {
+			logrus.SetLevel(logrus.DebugLevel)
+		} else {
+			logrus.SetLevel(logrus.ErrorLevel)
+		}
+
+		if imageName != "" {
+			if rootDir == "" && runrootDir != "" {
+				logrus.Errorf("must set --root and --runroot, or neither")
+				os.Exit(1)
+			}
+			if rootDir != "" && runrootDir == "" {
+				logrus.Errorf("must set --root and --runroot, or neither")
+				os.Exit(1)
+			}
+			storeOptions := sstorage.DefaultStoreOptions
+			if rootDir != "" && runrootDir != "" {
+				storeOptions.GraphDriverName = storageDriver
+				storeOptions.GraphDriverOptions = storageOptions
+				storeOptions.GraphRoot = rootDir
+				storeOptions.RunRoot = runrootDir
+			}
+			store, err = sstorage.GetStore(storeOptions)
+			if err != nil {
+				logrus.Errorf("error opening storage: %v", err)
+				os.Exit(1)
+			}
+			defer store.Shutdown(false)
+
+			storage.Transport.SetStore(store)
+			ref, err = storage.Transport.ParseStoreReference(store, imageName)
+			if err != nil {
+				logrus.Errorf("error parsing image name: %v", err)
+				os.Exit(1)
+			}
+		}
+
+		systemContext := types.SystemContext{
+			SignaturePolicyPath: signaturePolicy,
+		}
+		policy, err := signature.DefaultPolicy(&systemContext)
+		if err != nil {
+			logrus.Errorf("error loading signature policy: %v", err)
+			os.Exit(1)
+		}
+		policyContext, err := signature.NewPolicyContext(policy)
+		if err != nil {
+			logrus.Errorf("error loading signature policy: %v", err)
+			os.Exit(1)
+		}
+		defer policyContext.Destroy()
+		options := &copy.Options{}
+
+		if importFrom != "" {
+			importRef, err = transports.ParseImageName(importFrom)
+			if err != nil {
+				logrus.Errorf("error parsing image name %v: %v", importFrom, err)
+				os.Exit(1)
+			}
+		}
+
+		if exportTo != "" {
+			exportRef, err = transports.ParseImageName(exportTo)
+			if err != nil {
+				logrus.Errorf("error parsing image name %v: %v", exportTo, err)
+				os.Exit(1)
+			}
+		}
+
+		if imageName != "" {
+			if importFrom != "" {
+				err = copy.Image(policyContext, ref, importRef, options)
+				if err != nil {
+					logrus.Errorf("error importing %s: %v", importFrom, err)
+					os.Exit(1)
+				}
+			}
+			if addName != "" {
+				destImage, err := storage.Transport.GetStoreImage(store, ref)
+				if err != nil {
+					logrus.Errorf("error finding image: %v", err)
+					os.Exit(1)
+				}
+				names := append(destImage.Names, imageName, addName)
+				err = store.SetNames(destImage.ID, names)
+				if err != nil {
+					logrus.Errorf("error adding name to %s: %v", imageName, err)
+					os.Exit(1)
+				}
+			}
+			if exportTo != "" {
+				err = copy.Image(policyContext, exportRef, ref, options)
+				if err != nil {
+					logrus.Errorf("error exporting %s: %v", exportTo, err)
+					os.Exit(1)
+				}
+			}
+		} else {
+			if importFrom != "" && exportTo != "" {
+				err = copy.Image(policyContext, exportRef, importRef, options)
+				if err != nil {
+					logrus.Errorf("error copying %s to %s: %v", importFrom, exportTo, err)
+					os.Exit(1)
+				}
+			}
+		}
+		return nil
+	}
+
+	if err := app.Run(os.Args); err != nil {
+		logrus.Fatal(err)
+	}
+}

test/helpers.bash

@@ -38,6 +38,8 @@ BOOT_CONFIG_FILE_PATH=${BOOT_CONFIG_FILE_PATH:-/boot/config-`uname -r`}
 APPARMOR_PARAMETERS_FILE_PATH=${APPARMOR_PARAMETERS_FILE_PATH:-/sys/module/apparmor/parameters/enabled}
 # Path of the bin2img binary.
 BIN2IMG_BINARY=${BIN2IMG_BINARY:-${OCID_ROOT}/cri-o/test/bin2img/bin2img}
+# Path of the copyimg binary.
+COPYIMG_BINARY=${COPYIMG_BINARY:-${OCID_ROOT}/cri-o/test/copyimg/copyimg}
 
 TESTDIR=$(mktemp -d)
 if [ -e /usr/sbin/selinuxenabled ] && /usr/sbin/selinuxenabled; then
@@ -58,6 +60,16 @@ mkdir -p $OCID_CNI_CONFIG
 
 PATH=$PATH:$TESTDIR
 
+# Make sure we have a copy of the redis:latest image.
+if ! [ -d "$TESTDATA"/redis-image ]; then
+    mkdir -p "$TESTDATA"/redis-image
+    if ! "$COPYIMG_BINARY" --import-from=docker://redis --export-to=dir:"$TESTDATA"/redis-image --signature-policy="$INTEGRATION_ROOT"/policy.json ; then
+        echo "Error pulling docker://redis"
+        rm -fr "$TESTDATA"/redis-image
+        exit 1
+    fi
+fi
+
 # Run ocid using the binary specified by $OCID_BINARY.
 # This must ONLY be run on engines created with `start_ocid`.
 function ocid() {
@@ -114,10 +126,11 @@ function start_ocid() {
 		apparmor="$APPARMOR_PROFILE"
 	fi
 
-	# Don't forget: bin2img and ocid have their own default drivers, so if you override either, you probably need to override both
+	# Don't forget: bin2img, copyimg, and ocid have their own default drivers, so if you override any, you probably need to override them all
 	if ! [ "$3" = "--no-pause-image" ] ; then
 		"$BIN2IMG_BINARY" --root "$TESTDIR/ocid" --runroot "$TESTDIR/ocid-run" --source-binary "$PAUSE_BINARY"
 	fi
+	"$COPYIMG_BINARY" --root "$TESTDIR/ocid" --runroot "$TESTDIR/ocid-run" --image-name=redis --import-from=dir:"$TESTDATA"/redis-image --add-name=docker://docker.io/library/redis:latest
 	"$OCID_BINARY" --conmon "$CONMON_BINARY" --listen "$OCID_SOCKET" --runtime "$RUNC_BINARY" --root "$TESTDIR/ocid" --runroot "$TESTDIR/ocid-run" --seccomp-profile "$seccomp" --apparmor-profile "$apparmor" --cni-config-dir "$OCID_CNI_CONFIG" --signature-policy "$INTEGRATION_ROOT"/policy.json config >$OCID_CONFIG
 	"$OCID_BINARY" --debug --config "$OCID_CONFIG" & OCID_PID=$!
 	wait_until_reachable