From 9c44933b5831ca7ec248f4d90c92848a6a292210 Mon Sep 17 00:00:00 2001 From: Dan Williams Date: Mon, 27 Mar 2017 15:53:33 -0500 Subject: [PATCH 1/3] build: create a local GOPATH if none specified Instead of requiring the developer to set up their own GOPATH somewhere, do like Kubernetes and OpenShift Origin do: git clone xxxxx cd xxxxx make by creating an _output/ directory and linking the local source tree into it, and setting that to be the GOPATH. Signed-off-by: Dan Williams --- .gitignore | 1 + Makefile | 85 ++++++++++++++++++++++++-------------- test/bin2img/Makefile | 6 --- test/checkseccomp/Makefile | 6 --- test/copyimg/Makefile | 6 --- 5 files changed, 54 insertions(+), 50 deletions(-) delete mode 100644 test/bin2img/Makefile delete mode 100644 test/checkseccomp/Makefile delete mode 100644 test/copyimg/Makefile diff --git a/.gitignore b/.gitignore index be7505ca..e3b27e0f 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,5 @@ /.artifacts/ +/_output/ /conmon/conmon /conmon/conmon.o /docs/*.[158] diff --git a/Makefile b/Makefile index c197e9e6..5ea6d260 100644 --- a/Makefile +++ b/Makefile @@ -14,6 +14,17 @@ ETCDIR_OCID ?= ${ETCDIR}/ocid BUILDTAGS := selinux seccomp $(shell hack/btrfs_tag.sh) $(shell hack/libdm_tag.sh) BASHINSTALLDIR=${PREFIX}/share/bash-completion/completions +# If GOPATH not specified, use one in the local directory +ifeq ($(GOPATH),) +export GOPATH := $(CURDIR)/_output +unexport GOBIN +endif +GOPKGDIR := $(GOPATH)/src/$(PROJECT) +GOPKGBASEDIR := $(shell dirname "$(GOPKGDIR)") + +# Update VPATH so make finds .gopathok +VPATH := $(VPATH):$(GOPATH) + all: binaries ocid.conf docs default: help @@ -28,14 +39,14 @@ help: @echo " * 'lint' - Execute the source code linter" @echo " * 'gofmt' - Verify the source code gofmt" -.PHONY: check-gopath - -check-gopath: -ifndef GOPATH - $(error GOPATH is not set) +.gopathok: +ifeq ("$(wildcard $(GOPKGDIR))","") + mkdir -p "$(GOPKGBASEDIR)" + ln -s "$(CURDIR)" "$(GOPKGBASEDIR)" endif + touch "$(GOPATH)/.gopathok" -lint: check-gopath +lint: .gopathok @echo "checking lint" @./.tool/lint @@ -48,30 +59,34 @@ conmon: pause: $(MAKE) -C $@ -bin2img: - $(MAKE) -C test/$@ BUILDTAGS="$(BUILDTAGS)" +bin2img: .gopathok $(wildcard test/bin2img/*.go) + go build -tags "$(BUILDTAGS)" -o test/bin2img/$@ $(PROJECT)/test/bin2img -copyimg: - $(MAKE) -C test/$@ BUILDTAGS="$(BUILDTAGS)" +copyimg: .gopathok $(wildcard test/copyimg/*.go) + go build -tags "$(BUILDTAGS)" -o test/copyimg/$@ $(PROJECT)/test/copyimg -checkseccomp: check-gopath - $(MAKE) -C test/$@ +checkseccomp: .gopathok $(wildcard test/checkseccomp/*.go) + go build -o test/checkseccomp/$@ $(PROJECT)/test/checkseccomp -ocid: check-gopath +ocid: .gopathok $(GO) build -o $@ \ -tags "$(BUILDTAGS)" \ $(PROJECT)/cmd/ocid -ocic: check-gopath +ocic: .gopathok $(GO) build -o $@ $(PROJECT)/cmd/ocic -kpod: check-gopath +kpod: .gopathok $(GO) build -o $@ $(PROJECT)/cmd/kpod ocid.conf: ocid ./ocid --config="" config --default > ocid.conf clean: +ifneq ($(GOPATH),) + rm -f "$(GOPATH)/.gopathok" +endif + rm -rf _output rm -f docs/*.1 docs/*.5 docs/*.8 rm -fr test/testdata/redis-image find . -name \*~ -delete @@ -79,9 +94,9 @@ clean: rm -f ocic ocid kpod make -C conmon clean make -C pause clean - make -C test/bin2img clean - make -C test/copyimg clean - make -C test/checkseccomp clean + rm -f test/bin2img/bin2img + rm -f test/copyimg/copyimg + rm -f test/checkseccomp/checkseccomp ocidimage: docker build -t ${OCID_IMAGE} . @@ -100,18 +115,18 @@ binaries: ocid ocic kpod conmon pause bin2img copyimg checkseccomp MANPAGES_MD := $(wildcard docs/*.md) MANPAGES := $(MANPAGES_MD:%.md=%) -docs/%.1: docs/%.1.md check-gopath +docs/%.1: docs/%.1.md .gopathok $(GOPATH)/bin/go-md2man -in $< -out $@.tmp && touch $@.tmp && mv $@.tmp $@ -docs/%.5: docs/%.5.md check-gopath +docs/%.5: docs/%.5.md .gopathok $(GOPATH)/bin/go-md2man -in $< -out $@.tmp && touch $@.tmp && mv $@.tmp $@ -docs/%.8: docs/%.8.md check-gopath +docs/%.8: docs/%.8.md .gopathok $(GOPATH)/bin/go-md2man -in $< -out $@.tmp && touch $@.tmp && mv $@.tmp $@ docs: $(MANPAGES) -install: check-gopath +install: .gopathok install -D -m 755 ocid $(BINDIR)/ocid install -D -m 755 ocic $(BINDIR)/ocic install -D -m 755 kpod $(BINDIR)/kpod @@ -152,26 +167,32 @@ uninstall: .PHONY: .gitvalidation # When this is running in travis, it will only check the travis commit range -.gitvalidation: check-gopath +.gitvalidation: .gopathok ifeq ($(TRAVIS),true) - git-validation -q -run DCO,short-subject + $(GOPATH)/bin/git-validation -q -run DCO,short-subject else - git-validation -v -run DCO,short-subject -range $(EPOCH_TEST_COMMIT)..HEAD + $(GOPATH)/bin/git-validation -v -run DCO,short-subject -range $(EPOCH_TEST_COMMIT)..HEAD endif .PHONY: install.tools install.tools: .install.gitvalidation .install.gometalinter .install.md2man -.install.gitvalidation: check-gopath - go get -u github.com/vbatts/git-validation +.install.gitvalidation: .gopathok + if [ ! -x "$(GOPATH)/bin/git-validation" ]; then \ + go get -u github.com/vbatts/git-validation; \ + fi -.install.gometalinter: check-gopath - go get -u github.com/alecthomas/gometalinter - $(GOPATH)/bin/gometalinter --install +.install.gometalinter: .gopathok + if [ ! -x "$(GOPATH)/bin/gometalinter" ]; then \ + go get -u github.com/alecthomas/gometalinter; \ + $(GOPATH)/bin/gometalinter --install; \ + fi -.install.md2man: check-gopath - go get -u github.com/cpuguy83/go-md2man +.install.md2man: .gopathok + if [ ! -x "$(GOPATH)/bin/go-md2man" ]; then \ + go get -u github.com/cpuguy83/go-md2man; \ + fi .PHONY: \ bin2img \ diff --git a/test/bin2img/Makefile b/test/bin2img/Makefile deleted file mode 100644 index 67344382..00000000 --- a/test/bin2img/Makefile +++ /dev/null @@ -1,6 +0,0 @@ -bin2img: $(wildcard *.go) - go build -tags "$(BUILDTAGS)" -o $@ - -.PHONY: clean -clean: - rm -f bin2img diff --git a/test/checkseccomp/Makefile b/test/checkseccomp/Makefile deleted file mode 100644 index fc3ba4fb..00000000 --- a/test/checkseccomp/Makefile +++ /dev/null @@ -1,6 +0,0 @@ -checkseccomp: $(wildcard *.go) - go build -o $@ - -.PHONY: clean - clean: - rm -f checkseccomp diff --git a/test/copyimg/Makefile b/test/copyimg/Makefile deleted file mode 100644 index 55f60944..00000000 --- a/test/copyimg/Makefile +++ /dev/null @@ -1,6 +0,0 @@ -copyimg: $(wildcard *.go) - go build -tags "$(BUILDTAGS)" -o $@ - -.PHONY: clean -clean: - rm -f copyimg From 95846211c968077b88ec90001c54cf51e98000b2 Mon Sep 17 00:00:00 2001 From: Dan Williams Date: Fri, 31 Mar 2017 12:57:34 -0500 Subject: [PATCH 2/3] build: find dependencies for Go executables To ensure that changing a dependency source file actually triggers a rebuild of the core binaries when you type 'make', find their dependencies and add them to the makefile's target dependencies. Signed-off-by: Dan Williams --- Makefile | 6 +++--- hack/find-godeps.sh | 41 +++++++++++++++++++++++++++++++++++++++++ 2 files changed, 44 insertions(+), 3 deletions(-) create mode 100755 hack/find-godeps.sh diff --git a/Makefile b/Makefile index 5ea6d260..1936cc24 100644 --- a/Makefile +++ b/Makefile @@ -68,15 +68,15 @@ copyimg: .gopathok $(wildcard test/copyimg/*.go) checkseccomp: .gopathok $(wildcard test/checkseccomp/*.go) go build -o test/checkseccomp/$@ $(PROJECT)/test/checkseccomp -ocid: .gopathok +ocid: .gopathok $(shell hack/find-godeps.sh $(GOPKGDIR) cmd/ocid $(PROJECT)) $(GO) build -o $@ \ -tags "$(BUILDTAGS)" \ $(PROJECT)/cmd/ocid -ocic: .gopathok +ocic: .gopathok $(shell hack/find-godeps.sh $(GOPKGDIR) cmd/ocic $(PROJECT)) $(GO) build -o $@ $(PROJECT)/cmd/ocic -kpod: .gopathok +kpod: .gopathok $(shell hack/find-godeps.sh $(GOPKGDIR) cmd/kpod $(PROJECT)) $(GO) build -o $@ $(PROJECT)/cmd/kpod ocid.conf: ocid diff --git a/hack/find-godeps.sh b/hack/find-godeps.sh new file mode 100755 index 00000000..59689e6e --- /dev/null +++ b/hack/find-godeps.sh @@ -0,0 +1,41 @@ +#!/bin/bash +# +# $1 - base path of the source tree +# $2 - subpath under $1 to find *.go dependencies for +# $3 - package name (eg, github.com/organization/project) + +set -o errexit +set -o nounset +set -o pipefail + +# might be called from makefile before basepath is set up; just return +# empty deps. The make target will then ensure that GOPATH is set up +# correctly, and go build will build everything the first time around +# anyway. Next time we get here everything will be fine. +if [ ! -d "$1/$2" ]; then + exit 0 +fi + +function find-deps() { + local basepath=$1 + local srcdir=$2 + local pkgname=$3 + local deps= + + # gather imports from cri-o + pkgs=$(cd ${basepath}/${srcdir} && go list -f "{{.Imports}}" . | tr ' ' '\n' | grep -v "/vendor/" | grep ${pkgname} | sed -e "s|${pkgname}/||g") + + # add each Go import's sources to the deps list, + # and recursively get that imports's imports too + for dep in ${pkgs}; do + deps+="$(ls ${basepath}/${dep}/*.go | sed -e "s|${basepath}/||g") " + # add deps of this package too + deps+="$(find-deps ${basepath} ${dep} ${pkgname}) " + done + + echo "${deps}" | sort | uniq +} + +# add Go sources from the current package at the end +echo "$(find-deps "$1" "$2" "$3" | xargs) $(cd $1 && ls $2/*.go | xargs)" + From aa24e0554ddc3866ea6638cf91696d8b15224139 Mon Sep 17 00:00:00 2001 From: Vincent Batts Date: Mon, 17 Apr 2017 19:32:00 -0400 Subject: [PATCH 3/3] pkg/storage: switch to searchable fieldname `image` as a variable/field name becomes too redundant and difficult to grep for. Switching to `imageServer` makes for more readable code. Signed-off-by: Vincent Batts --- pkg/storage/runtime.go | 62 +++++++++++++++++++++--------------------- 1 file changed, 31 insertions(+), 31 deletions(-) diff --git a/pkg/storage/runtime.go b/pkg/storage/runtime.go index b910649a..8e45bcc8 100644 --- a/pkg/storage/runtime.go +++ b/pkg/storage/runtime.go @@ -39,8 +39,8 @@ var ( ) type runtimeService struct { - image ImageServer - pauseImage string + imageServer ImageServer + pauseImage string } // ContainerInfo wraps a subset of information about a container: its ID and @@ -159,22 +159,22 @@ func (r *runtimeService) createContainerOrPodSandbox(systemContext *types.System } // Check if we have the specified image. - ref, err := istorage.Transport.ParseStoreReference(r.image.GetStore(), imageName) + ref, err := istorage.Transport.ParseStoreReference(r.imageServer.GetStore(), imageName) if err != nil { // Maybe it's some other transport's copy of the image? otherRef, err2 := alltransports.ParseImageName(imageName) if err2 == nil && otherRef.DockerReference() != nil { - ref, err = istorage.Transport.ParseStoreReference(r.image.GetStore(), otherRef.DockerReference().Name()) + ref, err = istorage.Transport.ParseStoreReference(r.imageServer.GetStore(), otherRef.DockerReference().Name()) } if err != nil { // Maybe the image ID is sufficient? - ref, err = istorage.Transport.ParseStoreReference(r.image.GetStore(), "@"+imageID) + ref, err = istorage.Transport.ParseStoreReference(r.imageServer.GetStore(), "@"+imageID) if err != nil { return ContainerInfo{}, err } } } - img, err := istorage.Transport.GetStoreImage(r.image.GetStore(), ref) + img, err := istorage.Transport.GetStoreImage(r.imageServer.GetStore(), ref) if img == nil && err == storage.ErrImageUnknown && imageName == r.pauseImage { image := imageID if imageName != "" { @@ -184,11 +184,11 @@ func (r *runtimeService) createContainerOrPodSandbox(systemContext *types.System return ContainerInfo{}, ErrInvalidImageName } logrus.Debugf("couldn't find image %q, retrieving it", image) - ref, err = r.image.PullImage(systemContext, image, options) + ref, err = r.imageServer.PullImage(systemContext, image, options) if err != nil { return ContainerInfo{}, err } - img, err = istorage.Transport.GetStoreImage(r.image.GetStore(), ref) + img, err = istorage.Transport.GetStoreImage(r.imageServer.GetStore(), ref) if err != nil { return ContainerInfo{}, err } @@ -247,7 +247,7 @@ func (r *runtimeService) createContainerOrPodSandbox(systemContext *types.System if metadata.Pod { names = append(names, metadata.PodName) } - container, err := r.image.GetStore().CreateContainer(containerID, names, img.ID, "", string(mdata), nil) + container, err := r.imageServer.GetStore().CreateContainer(containerID, names, img.ID, "", string(mdata), nil) if err != nil { if metadata.Pod { logrus.Debugf("failed to create pod sandbox %s(%s): %v", metadata.PodName, metadata.PodID, err) @@ -266,7 +266,7 @@ func (r *runtimeService) createContainerOrPodSandbox(systemContext *types.System // container before returning. defer func() { if err != nil { - if err2 := r.image.GetStore().DeleteContainer(container.ID); err2 != nil { + if err2 := r.imageServer.GetStore().DeleteContainer(container.ID); err2 != nil { if metadata.Pod { logrus.Infof("%v deleting partially-created pod sandbox %q", err2, container.ID) } else { @@ -281,18 +281,18 @@ func (r *runtimeService) createContainerOrPodSandbox(systemContext *types.System // Add a name to the container's layer so that it's easier to follow // what's going on if we're just looking at the storage-eye view of things. layerName := metadata.ContainerName + "-layer" - names, err = r.image.GetStore().GetNames(container.LayerID) + names, err = r.imageServer.GetStore().GetNames(container.LayerID) if err != nil { return ContainerInfo{}, err } names = append(names, layerName) - err = r.image.GetStore().SetNames(container.LayerID, names) + err = r.imageServer.GetStore().SetNames(container.LayerID, names) if err != nil { return ContainerInfo{}, err } // Find out where the container work directories are, so that we can return them. - containerDir, err := r.image.GetStore().GetContainerDirectory(container.ID) + containerDir, err := r.imageServer.GetStore().GetContainerDirectory(container.ID) if err != nil { return ContainerInfo{}, err } @@ -302,7 +302,7 @@ func (r *runtimeService) createContainerOrPodSandbox(systemContext *types.System logrus.Debugf("container %q has work directory %q", container.ID, containerDir) } - containerRunDir, err := r.image.GetStore().GetContainerRunDirectory(container.ID) + containerRunDir, err := r.imageServer.GetStore().GetContainerRunDirectory(container.ID) if err != nil { return ContainerInfo{}, err } @@ -329,14 +329,14 @@ func (r *runtimeService) CreateContainer(systemContext *types.SystemContext, pod } func (r *runtimeService) RemovePodSandbox(idOrName string) error { - container, err := r.image.GetStore().GetContainer(idOrName) + container, err := r.imageServer.GetStore().GetContainer(idOrName) if err != nil { if err == storage.ErrContainerUnknown { return ErrInvalidSandboxID } return err } - err = r.image.GetStore().DeleteContainer(container.ID) + err = r.imageServer.GetStore().DeleteContainer(container.ID) if err != nil { logrus.Debugf("failed to delete pod sandbox %q: %v", container.ID, err) return err @@ -345,14 +345,14 @@ func (r *runtimeService) RemovePodSandbox(idOrName string) error { } func (r *runtimeService) DeleteContainer(idOrName string) error { - container, err := r.image.GetStore().GetContainer(idOrName) + container, err := r.imageServer.GetStore().GetContainer(idOrName) if err != nil { if err == storage.ErrContainerUnknown { return ErrInvalidContainerID } return err } - err = r.image.GetStore().DeleteContainer(container.ID) + err = r.imageServer.GetStore().DeleteContainer(container.ID) if err != nil { logrus.Debugf("failed to delete container %q: %v", container.ID, err) return err @@ -366,12 +366,12 @@ func (r *runtimeService) SetContainerMetadata(idOrName string, metadata RuntimeC logrus.Debugf("failed to encode metadata for %q: %v", idOrName, err) return err } - return r.image.GetStore().SetMetadata(idOrName, string(mdata)) + return r.imageServer.GetStore().SetMetadata(idOrName, string(mdata)) } func (r *runtimeService) GetContainerMetadata(idOrName string) (RuntimeContainerMetadata, error) { metadata := RuntimeContainerMetadata{} - mdata, err := r.image.GetStore().GetMetadata(idOrName) + mdata, err := r.imageServer.GetStore().GetMetadata(idOrName) if err != nil { return metadata, err } @@ -382,7 +382,7 @@ func (r *runtimeService) GetContainerMetadata(idOrName string) (RuntimeContainer } func (r *runtimeService) StartContainer(idOrName string) (string, error) { - container, err := r.image.GetStore().GetContainer(idOrName) + container, err := r.imageServer.GetStore().GetContainer(idOrName) if err != nil { if err == storage.ErrContainerUnknown { return "", ErrInvalidContainerID @@ -393,7 +393,7 @@ func (r *runtimeService) StartContainer(idOrName string) (string, error) { if err = json.Unmarshal([]byte(container.Metadata), &metadata); err != nil { return "", err } - mountPoint, err := r.image.GetStore().Mount(container.ID, metadata.MountLabel) + mountPoint, err := r.imageServer.GetStore().Mount(container.ID, metadata.MountLabel) if err != nil { logrus.Debugf("failed to mount container %q: %v", container.ID, err) return "", err @@ -403,14 +403,14 @@ func (r *runtimeService) StartContainer(idOrName string) (string, error) { } func (r *runtimeService) StopContainer(idOrName string) error { - container, err := r.image.GetStore().GetContainer(idOrName) + container, err := r.imageServer.GetStore().GetContainer(idOrName) if err != nil { if err == storage.ErrContainerUnknown { return ErrInvalidContainerID } return err } - err = r.image.GetStore().Unmount(container.ID) + err = r.imageServer.GetStore().Unmount(container.ID) if err != nil { logrus.Debugf("failed to unmount container %q: %v", container.ID, err) return err @@ -420,33 +420,33 @@ func (r *runtimeService) StopContainer(idOrName string) error { } func (r *runtimeService) GetWorkDir(id string) (string, error) { - container, err := r.image.GetStore().GetContainer(id) + container, err := r.imageServer.GetStore().GetContainer(id) if err != nil { if err == storage.ErrContainerUnknown { return "", ErrInvalidContainerID } return "", err } - return r.image.GetStore().GetContainerDirectory(container.ID) + return r.imageServer.GetStore().GetContainerDirectory(container.ID) } func (r *runtimeService) GetRunDir(id string) (string, error) { - container, err := r.image.GetStore().GetContainer(id) + container, err := r.imageServer.GetStore().GetContainer(id) if err != nil { if err == storage.ErrContainerUnknown { return "", ErrInvalidContainerID } return "", err } - return r.image.GetStore().GetContainerRunDirectory(container.ID) + return r.imageServer.GetStore().GetContainerRunDirectory(container.ID) } // GetRuntimeService returns a RuntimeServer that uses the passed-in image // service to pull and manage images, and its store to manage containers based // on those images. -func GetRuntimeService(image ImageServer, pauseImage string) RuntimeServer { +func GetRuntimeService(imageServer ImageServer, pauseImage string) RuntimeServer { return &runtimeService{ - image: image, - pauseImage: pauseImage, + imageServer: imageServer, + pauseImage: pauseImage, } }