diff --git a/.mailmap b/.mailmap new file mode 100644 index 00000000..29d8c860 --- /dev/null +++ b/.mailmap @@ -0,0 +1,10 @@ +Aleksa Sarai +Antonio Murdaca +CuiHaozhi +Daniel J Walsh +Haiyan Meng +Lorenzo Fontana +Mrunal Patel +Mrunal Patel +Pengfei Ni +Tobias Klauser diff --git a/Dockerfile b/Dockerfile index 6f170b26..1fb5e569 100644 --- a/Dockerfile +++ b/Dockerfile @@ -38,6 +38,7 @@ RUN apt-get update && apt-get install -y \ netcat \ socat \ --no-install-recommends \ + bsdmainutils \ && apt-get clean # install bats @@ -56,7 +57,7 @@ RUN mkdir -p /usr/src/criu \ && rm -rf /usr/src/criu # Install runc -ENV RUNC_COMMIT 84a082bfef6f932de921437815355186db37aeb1 +ENV RUNC_COMMIT c6e4a1ebeb1a72b529c6f1b6ee2b1ae5b868b14f RUN set -x \ && export GOPATH="$(mktemp -d)" \ && git clone https://github.com/opencontainers/runc.git "$GOPATH/src/github.com/opencontainers/runc" \ diff --git a/Makefile b/Makefile index d8508822..7cc1a4c1 100644 --- a/Makefile +++ b/Makefile @@ -11,7 +11,7 @@ LIBEXECDIR ?= ${PREFIX}/libexec MANDIR ?= ${PREFIX}/share/man ETCDIR ?= ${DESTDIR}/etc ETCDIR_CRIO ?= ${ETCDIR}/crio -BUILDTAGS ?= seccomp $(shell hack/btrfs_tag.sh) $(shell hack/libdm_tag.sh) $(shell hack/btrfs_installed_tag.sh) $(shell hack/ostree_tag.sh) $(shell hack/selinux_tag.sh) +BUILDTAGS ?= seccomp $(shell hack/btrfs_tag.sh) $(shell hack/libdm_installed.sh) $(shell hack/libdm_no_deferred_remove_tag.sh) $(shell hack/btrfs_installed_tag.sh) $(shell hack/ostree_tag.sh) $(shell hack/selinux_tag.sh) CRICTL_CONFIG_DIR=${DESTDIR}/etc BASHINSTALLDIR=${PREFIX}/share/bash-completion/completions @@ -64,7 +64,8 @@ lint: .gopathok @./.tool/lint gofmt: - @./hack/verify-gofmt.sh + find . -name '*.go' ! -path './vendor/*' -exec gofmt -s -w {} \+ + git diff --exit-code conmon: $(MAKE) -C $@ @@ -73,16 +74,16 @@ pause: $(MAKE) -C $@ test/bin2img/bin2img: .gopathok $(wildcard test/bin2img/*.go) - $(GO) build $(LDFLAGS) -tags "$(BUILDTAGS) containers_image_ostree_stub" -o $@ $(PROJECT)/test/bin2img + $(GO) build -i $(LDFLAGS) -tags "$(BUILDTAGS) containers_image_ostree_stub" -o $@ $(PROJECT)/test/bin2img test/copyimg/copyimg: .gopathok $(wildcard test/copyimg/*.go) - $(GO) build $(LDFLAGS) -tags "$(BUILDTAGS) containers_image_ostree_stub" -o $@ $(PROJECT)/test/copyimg + $(GO) build -i $(LDFLAGS) -tags "$(BUILDTAGS) containers_image_ostree_stub" -o $@ $(PROJECT)/test/copyimg test/checkseccomp/checkseccomp: .gopathok $(wildcard test/checkseccomp/*.go) - $(GO) build $(LDFLAGS) -tags "$(BUILDTAGS) containers_image_ostree_stub" -o $@ $(PROJECT)/test/checkseccomp + $(GO) build -i $(LDFLAGS) -tags "$(BUILDTAGS) containers_image_ostree_stub" -o $@ $(PROJECT)/test/checkseccomp crio: .gopathok $(shell hack/find-godeps.sh $(GOPKGDIR) cmd/crio $(PROJECT)) - $(GO) build $(LDFLAGS) -tags "$(BUILDTAGS) containers_image_ostree_stub" -o bin/$@ $(PROJECT)/cmd/crio + $(GO) build -i $(LDFLAGS) -tags "$(BUILDTAGS) containers_image_ostree_stub" -o bin/$@ $(PROJECT)/cmd/crio crio.conf: crio ./bin/crio --config="" config --default > crio.conf @@ -145,7 +146,7 @@ install.man: install ${SELINUXOPT} -m 644 $(filter %.5,$(MANPAGES)) -t $(MANDIR)/man5 install ${SELINUXOPT} -m 644 $(filter %.8,$(MANPAGES)) -t $(MANDIR)/man8 -install.config: +install.config: crio.conf install ${SELINUXOPT} -D -m 644 crio.conf $(ETCDIR_CRIO)/crio.conf install ${SELINUXOPT} -D -m 644 seccomp.json $(ETCDIR_CRIO)/seccomp.json install ${SELINUXOPT} -D -m 644 crio-umount.conf $(OCIUMOUNTINSTALLDIR)/crio-umount.conf diff --git a/README.md b/README.md index 14cc3014..69feaa34 100644 --- a/README.md +++ b/README.md @@ -12,6 +12,7 @@ |----------------------------|-------------------------------|--------------------| | CRI-O 1.0.x - release-1.0 | Kubernetes 1.7 branch, v1.7.x | = | | CRI-O 1.8.x - release-1.8 | Kubernetes 1.8 branch, v1.8.x | = | +| CRI-O 1.9.x - release-1.9 | Kubernetes 1.9 branch, v1.9.x | = | | CRI-O HEAD - master | Kubernetes master branch | ✓ | Key: diff --git a/cmd/crio/main.go b/cmd/crio/main.go index f2cd32c5..a058f296 100644 --- a/cmd/crio/main.go +++ b/cmd/crio/main.go @@ -509,7 +509,7 @@ func main() { if graceful && strings.Contains(strings.ToLower(err.Error()), "use of closed network connection") { err = nil } else { - logrus.Errorf("Failed to serve grpc grpc request: %v", err) + logrus.Errorf("Failed to serve grpc request: %v", err) } } }() diff --git a/code-of-conduct.md b/code-of-conduct.md index 215ce7ac..0d15c00c 100644 --- a/code-of-conduct.md +++ b/code-of-conduct.md @@ -1,55 +1,3 @@ -## Kubernetes Community Code of Conduct +# Kubernetes Community Code of Conduct -### Contributor Code of Conduct - -As contributors and maintainers of this project, and in the interest of fostering -an open and welcoming community, we pledge to respect all people who contribute -through reporting issues, posting feature requests, updating documentation, -submitting pull requests or patches, and other activities. - -We are committed to making participation in this project a harassment-free experience for -everyone, regardless of level of experience, gender, gender identity and expression, -sexual orientation, disability, personal appearance, body size, race, ethnicity, age, -religion, or nationality. - -Examples of unacceptable behavior by participants include: - -* The use of sexualized language or imagery. -* Personal attacks. -* Trolling or insulting/derogatory comments. -* Public or private harassment. -* Publishing other's private information, such as physical or electronic addresses, - without explicit permission. -* Other unethical or unprofessional conduct. - -Project maintainers have the right and responsibility to remove, edit, or reject -comments, commits, code, wiki edits, issues, and other contributions that are not -aligned to this Code of Conduct. By adopting this Code of Conduct, project maintainers -commit themselves to fairly and consistently applying these principles to every aspect -of managing this project. Project maintainers who do not follow or enforce the Code of -Conduct may be permanently removed from the project team. - -This code of conduct applies both within project spaces and in public spaces -when an individual is representing the project or its community. - -Instances of abusive, harassing, or otherwise unacceptable behavior may be reported by contacting a Kubernetes maintainer, Sarah Novotny , and/or Dan Kohn . - -This Code of Conduct is adapted from the Contributor Covenant -(http://contributor-covenant.org), version 1.2.0, available at -http://contributor-covenant.org/version/1/2/0/ - -### Kubernetes Events Code of Conduct - -Kubernetes events are working conferences intended for professional networking and collaboration in the -Kubernetes community. Attendees are expected to behave according to professional standards and in accordance -with their employer's policies on appropriate workplace behavior. - -While at Kubernetes events or related social networking opportunities, attendees should not engage in -discriminatory or offensive speech or actions regarding gender, sexuality, race, or religion. Speakers should -be especially aware of these concerns. - -The Kubernetes team does not condone any statements by speakers contrary to these standards. The Kubernetes -team reserves the right to deny entrance and/or eject from an event (without refund) any individual found to -be engaging in discriminatory or offensive speech or actions. - -Please bring any concerns to the immediate attention of the Kubernetes event staff. +Please refer to our [Kubernetes Community Code of Conduct](https://git.k8s.io/community/code-of-conduct.md) diff --git a/conmon/conmon.c b/conmon/conmon.c index b00cb0cd..477b98bf 100644 --- a/conmon/conmon.c +++ b/conmon/conmon.c @@ -12,7 +12,6 @@ #include #include #include -#include #include #include #include @@ -350,7 +349,7 @@ static int write_k8s_log(int fd, stdpipe_t pipe, const char *buf, ssize_t buflen /* Open the log path file again */ log_fd = open(opt_log_path, O_WRONLY | O_APPEND | O_CREAT | O_CLOEXEC, 0600); if (log_fd < 0) - pexit("Failed to open log file"); + pexit("Failed to open log file %s: %s", opt_log_path, strerror(errno)); fd = log_fd; } @@ -1121,6 +1120,8 @@ int main(int argc, char *argv[]) if (opt_runtime_path == NULL) nexit("Runtime path not provided. Use --runtime"); + if (access(opt_runtime_path, X_OK) < 0) + pexit("Runtime path %s is not valid: %s", opt_runtime_path, strerror(errno)); if (!opt_exec && opt_exit_dir == NULL) nexit("Container exit directory not provided. Use --exit-dir"); diff --git a/contrib/test/integration/build/kubernetes.yml b/contrib/test/integration/build/kubernetes.yml index 3a18321c..63d907f1 100644 --- a/contrib/test/integration/build/kubernetes.yml +++ b/contrib/test/integration/build/kubernetes.yml @@ -43,10 +43,12 @@ export CONTAINER_RUNTIME_ENDPOINT='{{ crio_socket }} --runtime-request-timeout=5m' export ALLOW_SECURITY_CONTEXT="," export ALLOW_PRIVILEGED=1 - export DNS_SERVER_IP={{ ansible_eth0.ipv4.address }} - export API_HOST={{ ansible_eth0.ipv4.address }} - export API_HOST_IP={{ ansible_eth0.ipv4.address }} + export DNS_SERVER_IP={{ ansible_default_ipv4.address }} + export API_HOST={{ ansible_default_ipv4.address }} + export API_HOST_IP={{ ansible_default_ipv4.address }} export KUBE_ENABLE_CLUSTER_DNS=true + export ENABLE_HOSTPATH_PROVISIONER=true + export KUBE_ENABLE_CLUSTER_DASHBOARD=true ./hack/local-up-cluster.sh mode: "u=rwx,g=rwx,o=x" diff --git a/contrib/test/integration/build/runc.yml b/contrib/test/integration/build/runc.yml index 7bb0491d..f3221f4a 100644 --- a/contrib/test/integration/build/runc.yml +++ b/contrib/test/integration/build/runc.yml @@ -4,7 +4,7 @@ git: repo: "https://github.com/opencontainers/runc.git" dest: "{{ ansible_env.GOPATH }}/src/github.com/opencontainers/runc" - version: "84a082bfef6f932de921437815355186db37aeb1" + version: "c6e4a1ebeb1a72b529c6f1b6ee2b1ae5b868b14f" - name: build runc make: diff --git a/contrib/test/integration/critest.yml b/contrib/test/integration/critest.yml index d3a38103..377ab59d 100644 --- a/contrib/test/integration/critest.yml +++ b/contrib/test/integration/critest.yml @@ -16,24 +16,21 @@ - name: Add masquerade for localhost command: iptables -t nat -I POSTROUTING -s 127.0.0.1 ! -d 127.0.0.1 -j MASQUERADE - # TODO(runcom): enable skipped tests once we fix them (image list related) - # https://github.com/kubernetes-incubator/cri-o/issues/1048 - name: run critest validation - shell: "critest -c --runtime-endpoint /var/run/crio/crio.sock --image-endpoint /var/run/crio/crio.sock -s 'listImage should get exactly 2 repoTags in the result image' v" + shell: "critest -c --runtime-endpoint /var/run/crio/crio.sock --image-endpoint /var/run/crio/crio.sock v" args: chdir: "{{ ansible_env.GOPATH }}/src/github.com/kubernetes-incubator/cri-o" async: 5400 poll: 30 when: ansible_distribution not in ['RedHat', 'CentOS'] - # XXX: RHEL has an additional test which fails beacuse of selinux but disabling + # XXX: RHEL has an additional test which fails because of selinux but disabling # it doesn't solve the issue. - # TODO(runcom): enable skipped tests once we fix them (image list related and selinux) - # https://github.com/kubernetes-incubator/cri-o/issues/1048 + # TODO(runcom): enable skipped tests once we fix them (selinux) # https://bugzilla.redhat.com/show_bug.cgi?id=1414236 # https://access.redhat.com/solutions/2897781 - name: run critest validation - shell: "critest -c --runtime-endpoint /var/run/crio/crio.sock --image-endpoint /var/run/crio/crio.sock -s 'listImage should get exactly 2 repoTags in the result image|should not allow privilege escalation when true' v" + shell: "critest -c --runtime-endpoint /var/run/crio/crio.sock --image-endpoint /var/run/crio/crio.sock -s 'should not allow privilege escalation when true' v" args: chdir: "{{ ansible_env.GOPATH }}/src/github.com/kubernetes-incubator/cri-o" async: 5400 diff --git a/contrib/test/integration/e2e.yml b/contrib/test/integration/e2e.yml index 5c4d656e..17982d1d 100644 --- a/contrib/test/integration/e2e.yml +++ b/contrib/test/integration/e2e.yml @@ -10,7 +10,7 @@ - name: update the server address for the custom cluster lineinfile: dest: /usr/local/bin/createcluster.sh - line: "export {{ item }}={{ ansible_eth0.ipv4.address }}" + line: "export {{ item }}={{ ansible_default_ipv4.address }}" regexp: "^export {{ item }}=" state: present with_items: @@ -37,13 +37,14 @@ path: "{{ artifacts }}" state: directory +# TODO remove the last test skipped once https://github.com/kubernetes-incubator/cri-o/pull/1217 is merged - name: Buffer the e2e testing command to workaround Ansible YAML folding "feature" set_fact: e2e_shell_cmd: > /usr/bin/go run hack/e2e.go --test --test_args="-host=https://{{ ansible_default_ipv4.address }}:6443 - --ginkgo.focus=\[Conformance\] + --ginkgo.skip=\[Slow\]|\[Serial\]|\[Disruptive\]|\[Flaky\]|\[Feature:.+\]|PersistentVolumes|\[HPA\]|should.support.building.a.client.with.a.CSR|should.support.inline.execution.and.attach --report-dir={{ artifacts }}" &> {{ artifacts }}/e2e.log # Fix vim syntax hilighting: " diff --git a/contrib/test/integration/main.yml b/contrib/test/integration/main.yml index b2cc9abe..1f6448f5 100644 --- a/contrib/test/integration/main.yml +++ b/contrib/test/integration/main.yml @@ -41,6 +41,8 @@ tags: - integration - e2e + - node-e2e + - critest tasks: - name: clone build and install cri-o include: "build/cri-o.yml" @@ -65,7 +67,7 @@ vars_files: - "{{ playbook_dir }}/vars.yml" tags: - - e2e + - critest tasks: - name: install Golang tools include: golang.yml @@ -78,12 +80,46 @@ cri_tools_git_version: "a9e38a4a000bc1a4052fb33de1c967b8cfe9ad40" - name: run critest validation and benchmarks include: critest.yml + +- hosts: all + remote_user: root + vars_files: + - "{{ playbook_dir }}/vars.yml" + tags: + - node-e2e + tasks: + - name: install Golang tools + include: golang.yml + vars: + version: "1.9.2" - name: clone build and install kubernetes include: "build/kubernetes.yml" vars: force_clone: True - k8s_git_version: "release-1.9" + k8s_git_version: "master" k8s_github_fork: "kubernetes" crio_socket: "/var/run/crio/crio.sock" + - name: run k8s node-e2e tests + include: node-e2e.yml + +- hosts: all + remote_user: root + vars_files: + - "{{ playbook_dir }}/vars.yml" + tags: + - e2e + tasks: + - name: install Golang tools + include: golang.yml + vars: + version: "1.9.2" + - name: clone build and install kubernetes + include: "build/kubernetes.yml" + vars: + force_clone: True + # master as of 12/11/2017 + k8s_git_version: "master-nfs-fix" + k8s_github_fork: "runcom" + crio_socket: "/var/run/crio/crio.sock" - name: run k8s e2e tests include: e2e.yml diff --git a/contrib/test/integration/node-e2e.yml b/contrib/test/integration/node-e2e.yml new file mode 100644 index 00000000..6ea8ac2c --- /dev/null +++ b/contrib/test/integration/node-e2e.yml @@ -0,0 +1,26 @@ +--- + +- name: enable and start CRI-O + systemd: + name: crio + state: started + enabled: yes + daemon_reload: yes + +- name: disable SELinux + command: setenforce 0 + +- name: Flush the iptables + command: iptables -F + +- name: run node-e2e tests + shell: | + # parametrize crio socket + # cgroup-driver??? + # TODO(runcom): remove conformance focus, we want everything for testgrid + make test-e2e-node PARALLELISM=1 RUNTIME=remote CONTAINER_RUNTIME_ENDPOINT=/var/run/crio.sock IMAGE_SERVICE_ENDPOINT=/var/run/crio/crio.sock TEST_ARGS='--prepull-images=true --kubelet-flags="--cgroup-driver=systemd"' FOCUS="\[Conformance\]" &> {{ artifacts }}/node-e2e.log + args: + chdir: "{{ ansible_env.GOPATH }}/src/k8s.io/kubernetes" + async: 7200 + poll: 10 + ignore_errors: true diff --git a/contrib/test/integration/system.yml b/contrib/test/integration/system.yml index d07ae0c8..453551fa 100644 --- a/contrib/test/integration/system.yml +++ b/contrib/test/integration/system.yml @@ -5,6 +5,7 @@ name: "{{ item }}" state: present with_items: + - atomic-registries - container-selinux - curl - device-mapper-devel @@ -41,9 +42,9 @@ - ostree-devel - pkgconfig - python - - python2-boto - python2-crypto - python-devel + - python-rhsm-certificates - python-virtualenv - PyYAML - redhat-rpm-config @@ -57,6 +58,22 @@ async: 600 poll: 10 +- name: Add python2-boto for Fedora + package: + name: "{{ item }}" + state: present + with_items: + - python2-boto + when: ansible_distribution in ['Fedora'] + +- name: Add python-boto for RHEL and CentOS + package: + name: "{{ item }}" + state: present + with_items: + - python-boto + when: ansible_distribution in ['RedHat', 'CentOS'] + - name: Add Btrfs for Fedora package: name: "{{ item }}" diff --git a/docs/play.png b/docs/play.png deleted file mode 100644 index 9be2868f..00000000 Binary files a/docs/play.png and /dev/null differ diff --git a/hack/libdm_installed.sh b/hack/libdm_installed.sh new file mode 100755 index 00000000..f48c7e27 --- /dev/null +++ b/hack/libdm_installed.sh @@ -0,0 +1,7 @@ +#!/bin/bash +cc -E - > /dev/null 2> /dev/null << EOF +#include +EOF +if test $? -ne 0 ; then + echo exclude_graphdriver_devicemapper +fi diff --git a/hack/libdm_tag.sh b/hack/libdm_no_deferred_remove_tag.sh similarity index 100% rename from hack/libdm_tag.sh rename to hack/libdm_no_deferred_remove_tag.sh diff --git a/hack/verify-gofmt.sh b/hack/verify-gofmt.sh deleted file mode 100755 index 5577d1b9..00000000 --- a/hack/verify-gofmt.sh +++ /dev/null @@ -1,21 +0,0 @@ -#!/bin/bash - -set -o errexit -set -o nounset -set -o pipefail - -find_files() { - find . -not \( \ - \( \ - -wholename '*/vendor/*' \ - \) -prune \ - \) -name '*.go' -} - -GOFMT="gofmt -s" -bad_files=$(find_files | xargs $GOFMT -l) -if [[ -n "${bad_files}" ]]; then - echo "!!! '$GOFMT' needs to be run on the following files: " - echo "${bad_files}" - exit 1 -fi diff --git a/hooks.md b/hooks.md index 809dbdc2..cd3d0a40 100644 --- a/hooks.md +++ b/hooks.md @@ -53,6 +53,7 @@ type HookParams struct { Cmds []string `json:"cmds"` Annotations []string `json:"annotations"` HasBindMounts bool `json:"hasbindmounts"` + Arguments []string `json:"arguments"` } ``` @@ -63,6 +64,7 @@ type HookParams struct { | cmds | List of regular expressions to match the command for running the container. If the command matches a regex, the hook will be run | Optional | | annotations | List of regular expressions to match against the Annotations in the container runtime spec, if an Annotation matches the hook will be run|optional | | hasbindmounts | Tells CRI-O to run the hook if the container has bind mounts from the host into the container | Optional | +| arguments | Additional arguments to append to the hook command when executing it. For example --debug | Optional | ### Example @@ -85,6 +87,7 @@ cat /etc/containers/oci/hooks.d/oci-systemd-hook.json "hasbindmounts": true, "hook": "/usr/libexec/oci/hooks.d/oci-umount", "stages": [ "prestart" ] + "arguments": [ "--debug" ] } ``` -In this example the oci-umount will only be run during the prestart phase if the container has volume/bind mounts from the host into the container. +In this example the oci-umount will only be run during the prestart phase if the container has volume/bind mounts from the host into the container, it will also execute oci-umount with the --debug argument. diff --git a/kubernetes.md b/kubernetes.md index 96ef5d89..e95694ac 100644 --- a/kubernetes.md +++ b/kubernetes.md @@ -13,17 +13,15 @@ Below, you can find an instruction how to switch one or more nodes on running ku ### Preparing crio -You must prepare and install `crio` on each node you would like to switch. Here's the list of files that must be provided: +You must prepare and install `crio` on each node you would like to switch. +Besides the files installed by `make install install.config`, here's the list of files that must be provided: -| File path | Description | Location | -|--------------------------------------------|----------------------------|-----------------------------------------------------| -| `/etc/crio/crio.conf` | crio configuration | Generated on cri-o `make install` | -| `/etc/crio/seccomp.conf` | seccomp config | Example stored in cri-o repository | -| `/etc/containers/policy.json` | containers policy | Example stored in cri-o repository | -| `/bin/{crio, runc}` | `crio` and `runc` binaries | Built from cri-o repository | -| `/usr/local/libexec/crio/conmon` | `conmon` binary | Built from cri-o repository | -| `/opt/cni/bin/{flannel, bridge,...}` | CNI plugins binaries | Can be built from sources `containernetworking/cni` | -| `/etc/cni/net.d/10-mynet.conf` | Network config | Example stored in [README file](README.md) | +| File path | Description | Location | +|--------------------------------------------|-----------------------------|---------------------------------------------------------| +| `/etc/containers/policy.json` | containers policy | [Example](test/policy.json) stored in cri-o repository | +| `/bin/runc` | `runc` or other OCI runtime | Can be build from sources `opencontainers/runc` | +| `/opt/cni/bin/{flannel, bridge,...}` | CNI plugins binaries | Can be built from sources `containernetworking/plugins` | +| `/etc/cni/net.d/...` | CNI network config | Example [here](contrib/cni) | `crio` binary can be executed directly on host, inside the container or in any way. However, recommended way is to set it as a systemd service. diff --git a/lib/hooks.go b/lib/hooks.go index 074ce7cc..fab563f0 100644 --- a/lib/hooks.go +++ b/lib/hooks.go @@ -27,6 +27,7 @@ type HookParams struct { Cmds []string `json:"cmd"` Annotations []string `json:"annotation"` HasBindMounts bool `json:"hasbindmounts"` + Arguments []string `json:"arguments"` } // readHook reads hooks json files, verifies it and returns the json config diff --git a/pkg/storage/image.go b/pkg/storage/image.go index 99b15f3a..5994d952 100644 --- a/pkg/storage/image.go +++ b/pkg/storage/image.go @@ -3,12 +3,13 @@ package storage import ( "errors" "net" - "path/filepath" + "path" "strings" "github.com/containers/image/copy" "github.com/containers/image/docker/reference" "github.com/containers/image/image" + "github.com/containers/image/manifest" "github.com/containers/image/signature" istorage "github.com/containers/image/storage" "github.com/containers/image/transports/alltransports" @@ -17,20 +18,26 @@ import ( digest "github.com/opencontainers/go-digest" ) +const ( + minimumTruncatedIDLength = 3 +) + var ( // ErrCannotParseImageID is returned when we try to ResolveNames for an image ID ErrCannotParseImageID = errors.New("cannot parse an image ID") + // ErrImageMultiplyTagged is returned when we try to remove an image that still has multiple names + ErrImageMultiplyTagged = errors.New("image still has multiple names applied") ) // ImageResult wraps a subset of information about an image: its ID, its names, // and the size, if known, or nil if it isn't. type ImageResult struct { - ID string - Names []string - Size *uint64 - // TODO(runcom): this is an hack for https://github.com/kubernetes-incubator/cri-o/pull/1136 - // drop this when we have proper image IDs (as in, image IDs should be just - // the config blog digest which is stable across same images). + ID string + Name string + RepoTags []string + RepoDigests []string + Size *uint64 + Digest digest.Digest ConfigDigest digest.Digest } @@ -47,6 +54,11 @@ type imageService struct { registries []string } +// sizer knows its size. +type sizer interface { + Size() (int64, error) +} + // ImageServer wraps up various CRI-related activities into a reusable // implementation. type ImageServer interface { @@ -59,6 +71,9 @@ type ImageServer interface { PrepareImage(systemContext *types.SystemContext, imageName string, options *copy.Options) (types.Image, error) // PullImage imports an image from the specified location. PullImage(systemContext *types.SystemContext, imageName string, options *copy.Options) (types.ImageReference, error) + // UntagImage removes a name from the specified image, and if it was + // the only name the image had, removes the image. + UntagImage(systemContext *types.SystemContext, imageName string) error // RemoveImage deletes the specified image. RemoveImage(systemContext *types.SystemContext, imageName string) error // GetStore returns the reference to the storage library Store which @@ -88,6 +103,66 @@ func (svc *imageService) getRef(name string) (types.ImageReference, error) { return ref, nil } +func sortNamesByType(names []string) (bestName string, tags, digests []string) { + for _, name := range names { + if len(name) > 72 && name[len(name)-72:len(name)-64] == "@sha256:" { + digests = append(digests, name) + } else { + tags = append(tags, name) + } + } + if len(digests) > 0 { + bestName = digests[0] + } + if len(tags) > 0 { + bestName = tags[0] + } + return bestName, tags, digests +} + +func (svc *imageService) makeRepoDigests(knownRepoDigests, tags []string, imageID string) (imageDigest digest.Digest, repoDigests []string) { + // Look up the image's digest. + img, err := svc.store.Image(imageID) + if err != nil { + return "", knownRepoDigests + } + imageDigest = img.Digest + if imageDigest == "" { + imgDigest, err := svc.store.ImageBigDataDigest(imageID, storage.ImageDigestBigDataKey) + if err != nil || imgDigest == "" { + return "", knownRepoDigests + } + imageDigest = imgDigest + } + // If there are no names to convert to canonical references, we're done. + if len(tags) == 0 { + return imageDigest, knownRepoDigests + } + // We only want to supplement what's already explicitly in the list, so keep track of values + // that we already know. + digestMap := make(map[string]struct{}) + repoDigests = knownRepoDigests + for _, repoDigest := range knownRepoDigests { + digestMap[repoDigest] = struct{}{} + } + // For each tagged name, parse the name, and if we can extract a named reference, convert + // it into a canonical reference using the digest and add it to the list. + for _, tag := range tags { + if ref, err2 := reference.ParseAnyReference(tag); err2 == nil { + if name, ok := ref.(reference.Named); ok { + trimmed := reference.TrimNamed(name) + if imageRef, err3 := reference.WithDigest(trimmed, imageDigest); err3 == nil { + if _, ok := digestMap[imageRef.String()]; !ok { + repoDigests = append(repoDigests, imageRef.String()) + digestMap[imageRef.String()] = struct{}{} + } + } + } + } + } + return imageDigest, repoDigests +} + func (svc *imageService) ListImages(systemContext *types.SystemContext, filter string) ([]ImageResult, error) { results := []ImageResult{} if filter != "" { @@ -96,16 +171,26 @@ func (svc *imageService) ListImages(systemContext *types.SystemContext, filter s return nil, err } if image, err := istorage.Transport.GetStoreImage(svc.store, ref); err == nil { - img, err := ref.NewImage(systemContext) + img, err := ref.NewImageSource(systemContext) if err != nil { return nil, err } size := imageSize(img) + configDigest, err := imageConfigDigest(img, nil) img.Close() + if err != nil { + return nil, err + } + name, tags, digests := sortNamesByType(image.Names) + imageDigest, repoDigests := svc.makeRepoDigests(digests, tags, image.ID) results = append(results, ImageResult{ - ID: image.ID, - Names: image.Names, - Size: size, + ID: image.ID, + Name: name, + RepoTags: tags, + RepoDigests: repoDigests, + Size: size, + Digest: imageDigest, + ConfigDigest: configDigest, }) } } else { @@ -118,16 +203,26 @@ func (svc *imageService) ListImages(systemContext *types.SystemContext, filter s if err != nil { return nil, err } - img, err := ref.NewImage(systemContext) + img, err := ref.NewImageSource(systemContext) if err != nil { return nil, err } size := imageSize(img) + configDigest, err := imageConfigDigest(img, nil) img.Close() + if err != nil { + return nil, err + } + name, tags, digests := sortNamesByType(image.Names) + imageDigest, repoDigests := svc.makeRepoDigests(digests, tags, image.ID) results = append(results, ImageResult{ - ID: image.ID, - Names: image.Names, - Size: size, + ID: image.ID, + Name: name, + RepoTags: tags, + RepoDigests: repoDigests, + Size: size, + Digest: imageDigest, + ConfigDigest: configDigest, }) } } @@ -152,29 +247,54 @@ func (svc *imageService) ImageStatus(systemContext *types.SystemContext, nameOrI return nil, err } - img, err := ref.NewImage(systemContext) + img, err := ref.NewImageSource(systemContext) if err != nil { return nil, err } defer img.Close() size := imageSize(img) + configDigest, err := imageConfigDigest(img, nil) + if err != nil { + return nil, err + } - return &ImageResult{ + name, tags, digests := sortNamesByType(image.Names) + imageDigest, repoDigests := svc.makeRepoDigests(digests, tags, image.ID) + result := ImageResult{ ID: image.ID, - Names: image.Names, + Name: name, + RepoTags: tags, + RepoDigests: repoDigests, Size: size, - ConfigDigest: img.ConfigInfo().Digest, - }, nil + Digest: imageDigest, + ConfigDigest: configDigest, + } + + return &result, nil } -func imageSize(img types.Image) *uint64 { - if sum, err := img.Size(); err == nil { - usum := uint64(sum) - return &usum +func imageSize(img types.ImageSource) *uint64 { + if s, ok := img.(sizer); ok { + if sum, err := s.Size(); err == nil { + usum := uint64(sum) + return &usum + } } return nil } +func imageConfigDigest(img types.ImageSource, instanceDigest *digest.Digest) (digest.Digest, error) { + manifestBytes, manifestType, err := img.GetManifest(instanceDigest) + if err != nil { + return "", err + } + imgManifest, err := manifest.FromBlob(manifestBytes, manifestType) + if err != nil { + return "", err + } + return imgManifest.ConfigInfo().Digest, nil +} + func (svc *imageService) CanPull(imageName string, options *copy.Options) (bool, error) { srcRef, err := svc.prepareReference(imageName, options) if err != nil { @@ -184,7 +304,11 @@ func (svc *imageService) CanPull(imageName string, options *copy.Options) (bool, if err != nil { return false, err } - src, err := image.FromSource(rawSource) + sourceCtx := &types.SystemContext{} + if options.SourceCtx != nil { + sourceCtx = options.SourceCtx + } + src, err := image.FromSource(sourceCtx, rawSource) if err != nil { rawSource.Close() return false, err @@ -274,6 +398,57 @@ func (svc *imageService) PullImage(systemContext *types.SystemContext, imageName return destRef, nil } +func (svc *imageService) UntagImage(systemContext *types.SystemContext, nameOrID string) error { + ref, err := alltransports.ParseImageName(nameOrID) + if err != nil { + ref2, err2 := istorage.Transport.ParseStoreReference(svc.store, "@"+nameOrID) + if err2 != nil { + ref3, err3 := istorage.Transport.ParseStoreReference(svc.store, nameOrID) + if err3 != nil { + return err + } + ref2 = ref3 + } + ref = ref2 + } + + img, err := istorage.Transport.GetStoreImage(svc.store, ref) + if err != nil { + return err + } + + if !strings.HasPrefix(img.ID, nameOrID) { + namedRef, err := svc.prepareReference(nameOrID, ©.Options{}) + if err != nil { + return err + } + + name := nameOrID + if namedRef.DockerReference() != nil { + name = namedRef.DockerReference().Name() + if tagged, ok := namedRef.DockerReference().(reference.NamedTagged); ok { + name = name + ":" + tagged.Tag() + } + if canonical, ok := namedRef.DockerReference().(reference.Canonical); ok { + name = name + "@" + canonical.Digest().String() + } + } + + prunedNames := make([]string, 0, len(img.Names)) + for _, imgName := range img.Names { + if imgName != name && imgName != nameOrID { + prunedNames = append(prunedNames, imgName) + } + } + + if len(prunedNames) > 0 { + return svc.store.SetNames(img.ID, prunedNames) + } + } + + return ref.DeleteImage(systemContext) +} + func (svc *imageService) RemoveImage(systemContext *types.SystemContext, nameOrID string) error { ref, err := alltransports.ParseImageName(nameOrID) if err != nil { @@ -341,6 +516,14 @@ func splitDockerDomain(name string) (domain, remainder string) { } func (svc *imageService) ResolveNames(imageName string) ([]string, error) { + // _Maybe_ it's a truncated image ID. Don't prepend a registry name, then. + if len(imageName) >= minimumTruncatedIDLength && svc.store != nil { + if img, err := svc.store.Image(imageName); err == nil && img != nil && strings.HasPrefix(img.ID, imageName) { + // It's a truncated version of the ID of an image that's present in local storage; + // we need to expand it. + return []string{img.ID}, nil + } + } // This to prevent any image ID to go through this routine _, err := reference.ParseNormalizedNamed(imageName) if err != nil { @@ -368,7 +551,7 @@ func (svc *imageService) ResolveNames(imageName string) ([]string, error) { if r == "docker.io" && !strings.ContainsRune(remainder, '/') { rem = "library/" + rem } - images = append(images, filepath.Join(r, rem)) + images = append(images, path.Join(r, rem)) } return images, nil } diff --git a/server/container_create.go b/server/container_create.go index 78c7e6c9..a4652cf3 100644 --- a/server/container_create.go +++ b/server/container_create.go @@ -14,7 +14,6 @@ import ( "strings" "time" - "github.com/docker/distribution/reference" dockermounts "github.com/docker/docker/pkg/mount" "github.com/docker/docker/pkg/stringid" "github.com/docker/docker/pkg/symlink" @@ -427,15 +426,18 @@ func buildOCIProcessArgs(containerKubeConfig *pb.ContainerConfig, imageOCIConfig func addOCIHook(specgen *generate.Generator, hook lib.HookParams) error { logrus.Debugf("AddOCIHook", hook) for _, stage := range hook.Stage { + h := rspec.Hook{ + Path: hook.Hook, + Args: append([]string{hook.Hook}, hook.Arguments...), + Env: []string{fmt.Sprintf("stage=%s", stage)}, + } switch stage { case "prestart": - specgen.AddPreStartHook(hook.Hook, []string{hook.Hook, "prestart"}) - + specgen.AddPreStartHook(h) case "poststart": - specgen.AddPostStartHook(hook.Hook, []string{hook.Hook, "poststart"}) - + specgen.AddPostStartHook(h) case "poststop": - specgen.AddPostStopHook(hook.Hook, []string{hook.Hook, "poststop"}) + specgen.AddPostStopHook(h) } } return nil @@ -488,6 +490,110 @@ func setupContainerUser(specgen *generate.Generator, rootfs string, sc *pb.Linux return nil } +// setupCapabilities sets process.capabilities in the OCI runtime config. +func setupCapabilities(specgen *generate.Generator, capabilities *pb.Capability) error { + if capabilities == nil { + return nil + } + + toCAPPrefixed := func(cap string) string { + if !strings.HasPrefix(strings.ToLower(cap), "cap_") { + return "CAP_" + strings.ToUpper(cap) + } + return cap + } + + // Add/drop all capabilities if "all" is specified, so that + // following individual add/drop could still work. E.g. + // AddCapabilities: []string{"ALL"}, DropCapabilities: []string{"CHOWN"} + // will be all capabilities without `CAP_CHOWN`. + // see https://github.com/kubernetes/kubernetes/issues/51980 + if inStringSlice(capabilities.GetAddCapabilities(), "ALL") { + for _, c := range getOCICapabilitiesList() { + if err := specgen.AddProcessCapabilityAmbient(c); err != nil { + return err + } + if err := specgen.AddProcessCapabilityBounding(c); err != nil { + return err + } + if err := specgen.AddProcessCapabilityEffective(c); err != nil { + return err + } + if err := specgen.AddProcessCapabilityInheritable(c); err != nil { + return err + } + if err := specgen.AddProcessCapabilityPermitted(c); err != nil { + return err + } + } + } + if inStringSlice(capabilities.GetDropCapabilities(), "ALL") { + for _, c := range getOCICapabilitiesList() { + if err := specgen.DropProcessCapabilityAmbient(c); err != nil { + return err + } + if err := specgen.DropProcessCapabilityBounding(c); err != nil { + return err + } + if err := specgen.DropProcessCapabilityEffective(c); err != nil { + return err + } + if err := specgen.DropProcessCapabilityInheritable(c); err != nil { + return err + } + if err := specgen.DropProcessCapabilityPermitted(c); err != nil { + return err + } + } + } + + for _, cap := range capabilities.GetAddCapabilities() { + if strings.ToUpper(cap) == "ALL" { + continue + } + capPrefixed := toCAPPrefixed(cap) + if err := specgen.AddProcessCapabilityAmbient(capPrefixed); err != nil { + return err + } + if err := specgen.AddProcessCapabilityBounding(capPrefixed); err != nil { + return err + } + if err := specgen.AddProcessCapabilityEffective(capPrefixed); err != nil { + return err + } + if err := specgen.AddProcessCapabilityInheritable(capPrefixed); err != nil { + return err + } + if err := specgen.AddProcessCapabilityPermitted(capPrefixed); err != nil { + return err + } + } + + for _, cap := range capabilities.GetDropCapabilities() { + if strings.ToUpper(cap) == "ALL" { + continue + } + capPrefixed := toCAPPrefixed(cap) + if err := specgen.DropProcessCapabilityAmbient(capPrefixed); err != nil { + return fmt.Errorf("failed to drop cap %s %v", capPrefixed, err) + } + if err := specgen.DropProcessCapabilityBounding(capPrefixed); err != nil { + return fmt.Errorf("failed to drop cap %s %v", capPrefixed, err) + } + if err := specgen.DropProcessCapabilityEffective(capPrefixed); err != nil { + return fmt.Errorf("failed to drop cap %s %v", capPrefixed, err) + } + if err := specgen.DropProcessCapabilityInheritable(capPrefixed); err != nil { + return fmt.Errorf("failed to drop cap %s %v", capPrefixed, err) + } + if err := specgen.DropProcessCapabilityPermitted(capPrefixed); err != nil { + return fmt.Errorf("failed to drop cap %s %v", capPrefixed, err) + } + } + + return nil +} + func hostNetwork(containerConfig *pb.ContainerConfig) bool { securityContext := containerConfig.GetLinux().GetSecurityContext() if securityContext == nil || securityContext.GetNamespaceOptions() == nil { @@ -563,7 +669,11 @@ func (s *Server) CreateContainer(ctx context.Context, req *pb.CreateContainerReq return nil, fmt.Errorf("CreateContainerRequest.ContainerConfig is nil") } - name := containerConfig.GetMetadata().Name + if containerConfig.GetMetadata() == nil { + return nil, fmt.Errorf("CreateContainerRequest.ContainerConfig.Metadata is nil") + } + + name := containerConfig.GetMetadata().GetName() if name == "" { return nil, fmt.Errorf("CreateContainerRequest.ContainerConfig.Name is empty") } @@ -711,8 +821,14 @@ func (s *Server) createSandboxContainer(ctx context.Context, containerID string, } specgen.AddAnnotation(annotations.Volumes, string(volumesJSON)) + mnt := rspec.Mount{ + Destination: "/sys/fs/cgroup", + Type: "cgroup", + Source: "cgroup", + Options: []string{"nosuid", "noexec", "nodev", "relatime", "ro"}, + } // Add cgroup mount so container process can introspect its own limits - specgen.AddCgroupsMount("ro") + specgen.AddMount(mnt) if err := addDevices(sb, containerConfig, &specgen); err != nil { return nil, err @@ -786,28 +902,13 @@ func (s *Server) createSandboxContainer(ctx context.Context, containerID string, if linux != nil { resources := linux.GetResources() if resources != nil { - cpuPeriod := resources.CpuPeriod - if cpuPeriod != 0 { - specgen.SetLinuxResourcesCPUPeriod(uint64(cpuPeriod)) - } - - cpuQuota := resources.CpuQuota - if cpuQuota != 0 { - specgen.SetLinuxResourcesCPUQuota(cpuQuota) - } - - cpuShares := resources.CpuShares - if cpuShares != 0 { - specgen.SetLinuxResourcesCPUShares(uint64(cpuShares)) - } - - memoryLimit := resources.MemoryLimitInBytes - if memoryLimit != 0 { - specgen.SetLinuxResourcesMemoryLimit(memoryLimit) - } - - oomScoreAdj := resources.OomScoreAdj - specgen.SetProcessOOMScoreAdj(int(oomScoreAdj)) + specgen.SetLinuxResourcesCPUPeriod(uint64(resources.GetCpuPeriod())) + specgen.SetLinuxResourcesCPUQuota(resources.GetCpuQuota()) + specgen.SetLinuxResourcesCPUShares(uint64(resources.GetCpuShares())) + specgen.SetLinuxResourcesMemoryLimit(resources.GetMemoryLimitInBytes()) + specgen.SetProcessOOMScoreAdj(int(resources.GetOomScoreAdj())) + specgen.SetLinuxResourcesCPUCpus(resources.GetCpusetCpus()) + specgen.SetLinuxResourcesCPUMems(resources.GetCpusetMems()) } var cgPath string @@ -826,57 +927,13 @@ func (s *Server) createSandboxContainer(ctx context.Context, containerID string, } specgen.SetLinuxCgroupsPath(cgPath) - capabilities := linux.GetSecurityContext().GetCapabilities() if privileged { - // this is setting correct capabilities as well for privileged mode specgen.SetupPrivileged(true) setOCIBindMountsPrivileged(&specgen) } else { - toCAPPrefixed := func(cap string) string { - if !strings.HasPrefix(strings.ToLower(cap), "cap_") { - return "CAP_" + strings.ToUpper(cap) - } - return cap - } - - // Add/drop all capabilities if "all" is specified, so that - // following individual add/drop could still work. E.g. - // AddCapabilities: []string{"ALL"}, DropCapabilities: []string{"CHOWN"} - // will be all capabilities without `CAP_CHOWN`. - // see https://github.com/kubernetes/kubernetes/issues/51980 - if inStringSlice(capabilities.GetAddCapabilities(), "ALL") { - for _, c := range getOCICapabilitiesList() { - if err := specgen.AddProcessCapability(c); err != nil { - return nil, err - } - } - } - if inStringSlice(capabilities.GetDropCapabilities(), "ALL") { - for _, c := range getOCICapabilitiesList() { - if err := specgen.DropProcessCapability(c); err != nil { - return nil, err - } - } - } - - if capabilities != nil { - for _, cap := range capabilities.GetAddCapabilities() { - if strings.ToUpper(cap) == "ALL" { - continue - } - if err := specgen.AddProcessCapability(toCAPPrefixed(cap)); err != nil { - return nil, err - } - } - - for _, cap := range capabilities.GetDropCapabilities() { - if strings.ToUpper(cap) == "ALL" { - continue - } - if err := specgen.DropProcessCapability(toCAPPrefixed(cap)); err != nil { - return nil, fmt.Errorf("failed to drop cap %s %v", toCAPPrefixed(cap), err) - } - } + err = setupCapabilities(&specgen, linux.GetSecurityContext().GetCapabilities()) + if err != nil { + return nil, err } } specgen.SetProcessSelinuxLabel(processLabel) @@ -963,46 +1020,31 @@ func (s *Server) createSandboxContainer(ctx context.Context, containerID string, return nil, err } } - image = images[0] - // Get imageName and imageRef that are requested in container status - imageName := image - status, err := s.StorageImageServer().ImageStatus(s.ImageContext(), image) + // Get imageName and imageRef that are later requested in container status + status, err := s.StorageImageServer().ImageStatus(s.ImageContext(), images[0]) if err != nil { return nil, err } - + imageName := status.Name imageRef := status.ID - // - // TODO: https://github.com/kubernetes-incubator/cri-o/issues/531 - // - //for _, n := range status.Names { - //r, err := reference.ParseNormalizedNamed(n) - //if err != nil { - //return nil, fmt.Errorf("failed to normalize image name for ImageRef: %v", err) - //} - //if digested, isDigested := r.(reference.Canonical); isDigested { - //imageRef = reference.FamiliarString(digested) - //break - //} - //} - for _, n := range status.Names { - r, err := reference.ParseNormalizedNamed(n) - if err != nil { - return nil, fmt.Errorf("failed to normalize image name for Image: %v", err) - } - if tagged, isTagged := r.(reference.Tagged); isTagged { - imageName = reference.FamiliarString(tagged) - break - } + if len(status.RepoDigests) > 0 { + imageRef = status.RepoDigests[0] } + specgen.AddAnnotation(annotations.Image, image) specgen.AddAnnotation(annotations.ImageName, imageName) specgen.AddAnnotation(annotations.ImageRef, imageRef) specgen.AddAnnotation(annotations.IP, sb.IP()) + mnt = rspec.Mount{ + Type: "bind", + Source: sb.ShmPath(), + Destination: "/etc/shm", + Options: []string{"rw", "bind"}, + } // bind mount the pod shm - specgen.AddBindMount(sb.ShmPath(), "/dev/shm", []string{"rw"}) + specgen.AddMount(mnt) options := []string{"rw"} if readOnlyRootfs { @@ -1013,8 +1055,14 @@ func (s *Server) createSandboxContainer(ctx context.Context, containerID string, return nil, err } + mnt = rspec.Mount{ + Type: "bind", + Source: sb.ResolvPath(), + Destination: "/etc/resolv.conf", + Options: append(options, "bind"), + } // bind mount the pod resolver file - specgen.AddBindMount(sb.ResolvPath(), "/etc/resolv.conf", options) + specgen.AddMount(mnt) } if sb.HostnamePath() != "" { @@ -1022,12 +1070,24 @@ func (s *Server) createSandboxContainer(ctx context.Context, containerID string, return nil, err } - specgen.AddBindMount(sb.HostnamePath(), "/etc/hostname", options) + mnt = rspec.Mount{ + Type: "bind", + Source: sb.HostnamePath(), + Destination: "/etc/hostname", + Options: append(options, "bind"), + } + specgen.AddMount(mnt) } // Bind mount /etc/hosts for host networking containers if hostNetwork(containerConfig) { - specgen.AddBindMount("/etc/hosts", "/etc/hosts", options) + mnt = rspec.Mount{ + Type: "bind", + Source: "/etc/hosts", + Destination: "/etc/hosts", + Options: append(options, "bind"), + } + specgen.AddMount(mnt) } // Set hostname and add env for hostname @@ -1043,7 +1103,6 @@ func (s *Server) createSandboxContainer(ctx context.Context, containerID string, specgen.AddAnnotation(annotations.TTY, fmt.Sprintf("%v", containerConfig.Tty)) specgen.AddAnnotation(annotations.Stdin, fmt.Sprintf("%v", containerConfig.Stdin)) specgen.AddAnnotation(annotations.StdinOnce, fmt.Sprintf("%v", containerConfig.StdinOnce)) - specgen.AddAnnotation(annotations.Image, image) specgen.AddAnnotation(annotations.ResolvPath, sb.InfraContainer().CrioAnnotations()[annotations.ResolvPath]) created := time.Now() @@ -1079,7 +1138,7 @@ func (s *Server) createSandboxContainer(ctx context.Context, containerID string, attempt := metadata.Attempt containerInfo, err := s.StorageRuntimeServer().CreateContainer(s.ImageContext(), sb.Name(), sb.ID(), - image, image, + image, status.ID, containerName, containerID, metaname, attempt, @@ -1088,6 +1147,14 @@ func (s *Server) createSandboxContainer(ctx context.Context, containerID string, if err != nil { return nil, err } + defer func() { + if err != nil { + err2 := s.StorageRuntimeServer().DeleteContainer(containerInfo.ID) + if err2 != nil { + logrus.Warnf("Failed to cleanup container directory: %v", err2) + } + } + }() mountPoint, err := s.StorageRuntimeServer().StartContainer(containerID) if err != nil { @@ -1097,7 +1164,8 @@ func (s *Server) createSandboxContainer(ctx context.Context, containerID string, containerImageConfig := containerInfo.Config if containerImageConfig == nil { - return nil, fmt.Errorf("empty image config for %s", image) + err = fmt.Errorf("empty image config for %s", image) + return nil, err } if containerImageConfig.Config.StopSignal != "" { @@ -1161,7 +1229,13 @@ func (s *Server) createSandboxContainer(ctx context.Context, containerID string, sort.Sort(orderedMounts(mounts)) for _, m := range mounts { - specgen.AddBindMount(m.Source, m.Destination, m.Options) + mnt = rspec.Mount{ + Type: "bind", + Source: m.Source, + Destination: m.Destination, + Options: append(m.Options, "bind"), + } + specgen.AddMount(mnt) } if err := s.setupOCIHooks(&specgen, sb, containerConfig, processArgs[0]); err != nil { diff --git a/server/container_list.go b/server/container_list.go index d32eea2d..060fa2af 100644 --- a/server/container_list.go +++ b/server/container_list.go @@ -97,6 +97,7 @@ func (s *Server) ListContainers(ctx context.Context, req *pb.ListContainersReque Metadata: ctr.Metadata(), Annotations: ctr.Annotations(), Image: img, + ImageRef: ctr.ImageRef(), } switch cState.Status { diff --git a/server/container_status.go b/server/container_status.go index f81be56f..3b84468f 100644 --- a/server/container_status.go +++ b/server/container_status.go @@ -3,6 +3,7 @@ package server import ( "time" + "github.com/containers/image/types" "github.com/kubernetes-incubator/cri-o/oci" "github.com/sirupsen/logrus" "golang.org/x/net/context" @@ -38,7 +39,10 @@ func (s *Server) ContainerStatus(ctx context.Context, req *pb.ContainerStatusReq ImageRef: c.ImageRef(), }, } - resp.Status.Image = &pb.ImageSpec{Image: c.ImageName()} + resp.Status.Image = &pb.ImageSpec{Image: c.Image()} + if status, err := s.StorageImageServer().ImageStatus(&types.SystemContext{}, c.ImageRef()); err == nil { + resp.Status.Image.Image = status.Name + } mounts := []*pb.Mount{} for _, cv := range c.Volumes() { diff --git a/server/image_list.go b/server/image_list.go index cbbd0d83..bcdc1036 100644 --- a/server/image_list.go +++ b/server/image_list.go @@ -33,14 +33,16 @@ func (s *Server) ListImages(ctx context.Context, req *pb.ListImagesRequest) (res for _, result := range results { if result.Size != nil { resp.Images = append(resp.Images, &pb.Image{ - Id: result.ID, - RepoTags: result.Names, - Size_: *result.Size, + Id: result.ID, + RepoTags: result.RepoTags, + RepoDigests: result.RepoDigests, + Size_: *result.Size, }) } else { resp.Images = append(resp.Images, &pb.Image{ - Id: result.ID, - RepoTags: result.Names, + Id: result.ID, + RepoTags: result.RepoTags, + RepoDigests: result.RepoDigests, }) } } diff --git a/server/image_pull.go b/server/image_pull.go index 2c7e8b2c..67dfc469 100644 --- a/server/image_pull.go +++ b/server/image_pull.go @@ -104,8 +104,16 @@ func (s *Server) PullImage(ctx context.Context, req *pb.PullImageRequest) (resp if pulled == "" && err != nil { return nil, err } + status, err := s.StorageImageServer().ImageStatus(s.ImageContext(), pulled) + if err != nil { + return nil, err + } + imageRef := status.ID + if len(status.RepoDigests) > 0 { + imageRef = status.RepoDigests[0] + } resp = &pb.PullImageResponse{ - ImageRef: pulled, + ImageRef: imageRef, } logrus.Debugf("PullImageResponse: %+v", resp) return resp, nil diff --git a/server/image_remove.go b/server/image_remove.go index da744490..d1f1e884 100644 --- a/server/image_remove.go +++ b/server/image_remove.go @@ -40,7 +40,7 @@ func (s *Server) RemoveImage(ctx context.Context, req *pb.RemoveImageRequest) (r } } for _, img := range images { - err = s.StorageImageServer().RemoveImage(s.ImageContext(), img) + err = s.StorageImageServer().UntagImage(s.ImageContext(), img) if err != nil { logrus.Debugf("error deleting image %s: %v", img, err) continue diff --git a/server/image_status.go b/server/image_status.go index c01a350c..4e2e6a0e 100644 --- a/server/image_status.go +++ b/server/image_status.go @@ -48,10 +48,10 @@ func (s *Server) ImageStatus(ctx context.Context, req *pb.ImageStatusRequest) (r } resp = &pb.ImageStatusResponse{ Image: &pb.Image{ - Id: status.ID, - RepoTags: status.Names, - Size_: *status.Size, - // TODO: https://github.com/kubernetes-incubator/cri-o/issues/531 + Id: status.ID, + RepoTags: status.RepoTags, + RepoDigests: status.RepoDigests, + Size_: *status.Size, }, } logrus.Debugf("ImageStatusResponse: %+v", resp) diff --git a/server/inspect.go b/server/inspect.go index 2aaac841..d1fe6abe 100644 --- a/server/inspect.go +++ b/server/inspect.go @@ -6,6 +6,7 @@ import ( "fmt" "net/http" + cimage "github.com/containers/image/types" "github.com/go-zoo/bone" "github.com/kubernetes-incubator/cri-o/lib/sandbox" "github.com/kubernetes-incubator/cri-o/oci" @@ -45,10 +46,17 @@ func (s *Server) getContainerInfo(id string, getContainerFunc func(id string) *o logrus.Debugf("can't find sandbox %s for container %s", ctr.Sandbox(), id) return types.ContainerInfo{}, errSandboxNotFound } + image := ctr.Image() + if s.ContainerServer != nil && s.ContainerServer.StorageImageServer() != nil { + if status, err := s.ContainerServer.StorageImageServer().ImageStatus(&cimage.SystemContext{}, ctr.ImageRef()); err == nil { + image = status.Name + } + } return types.ContainerInfo{ Name: ctr.Name(), Pid: ctrState.Pid, - Image: ctr.ImageName(), + Image: image, + ImageRef: ctr.ImageRef(), CreatedTime: ctrState.Created.UnixNano(), Labels: ctr.Labels(), Annotations: ctr.Annotations(), diff --git a/server/inspect_test.go b/server/inspect_test.go index c970fcd0..7246ef86 100644 --- a/server/inspect_test.go +++ b/server/inspect_test.go @@ -67,7 +67,7 @@ func TestGetContainerInfo(t *testing.T) { "io.kubernetes.test1": "value1", } getContainerFunc := func(id string) *oci.Container { - container, err := oci.NewContainer("testid", "testname", "", "/container/logs", mockNetNS{}, labels, annotations, annotations, "imageName", "imageName", "imageRef", &runtime.ContainerMetadata{}, "testsandboxid", false, false, false, false, false, "/root/for/container", created, "SIGKILL") + container, err := oci.NewContainer("testid", "testname", "", "/container/logs", mockNetNS{}, labels, annotations, annotations, "image", "imageName", "imageRef", &runtime.ContainerMetadata{}, "testsandboxid", false, false, false, false, false, "/root/for/container", created, "SIGKILL") if err != nil { t.Fatal(err) } @@ -101,8 +101,11 @@ func TestGetContainerInfo(t *testing.T) { if ci.Name != "testname" { t.Fatalf("expected name testname, got %s", ci.Name) } - if ci.Image != "imageName" { - t.Fatalf("expected image name imageName, got %s", ci.Image) + if ci.Image != "image" { + t.Fatalf("expected image name image, got %s", ci.Image) + } + if ci.ImageRef != "imageRef" { + t.Fatalf("expected image ref imageRef, got %s", ci.ImageRef) } if ci.Root != "/var/foo/container" { t.Fatalf("expected root to be /var/foo/container, got %s", ci.Root) diff --git a/server/sandbox_run.go b/server/sandbox_run.go index 9e9ff487..5ba007c2 100644 --- a/server/sandbox_run.go +++ b/server/sandbox_run.go @@ -101,16 +101,20 @@ func (s *Server) RunPodSandbox(ctx context.Context, req *pb.RunPodSandboxRequest s.updateLock.RLock() defer s.updateLock.RUnlock() + if req.GetConfig().GetMetadata() == nil { + return nil, fmt.Errorf("CreateContainerRequest.ContainerConfig.Metadata is nil") + } + logrus.Debugf("RunPodSandboxRequest %+v", req) var processLabel, mountLabel, resolvPath string // process req.Name - kubeName := req.GetConfig().GetMetadata().Name + kubeName := req.GetConfig().GetMetadata().GetName() if kubeName == "" { return nil, fmt.Errorf("PodSandboxConfig.Name should not be empty") } - namespace := req.GetConfig().GetMetadata().Namespace - attempt := req.GetConfig().GetMetadata().Attempt + namespace := req.GetConfig().GetMetadata().GetNamespace() + attempt := req.GetConfig().GetMetadata().GetAttempt() id, name, err := s.generatePodIDandName(req.GetConfig()) if err != nil { @@ -156,8 +160,8 @@ func (s *Server) RunPodSandbox(ctx context.Context, req *pb.RunPodSandboxRequest name, id, s.config.PauseImage, "", containerName, - req.GetConfig().GetMetadata().Name, - req.GetConfig().GetMetadata().Uid, + req.GetConfig().GetMetadata().GetName(), + req.GetConfig().GetMetadata().GetUid(), namespace, attempt, nil) @@ -210,8 +214,13 @@ func (s *Server) RunPodSandbox(ctx context.Context, req *pb.RunPodSandboxRequest if err := label.Relabel(resolvPath, mountLabel, true); err != nil && err != unix.ENOTSUP { return nil, err } - - g.AddBindMount(resolvPath, "/etc/resolv.conf", []string{"ro"}) + mnt := runtimespec.Mount{ + Type: "bind", + Source: resolvPath, + Destination: "/etc/resolv.conf", + Options: []string{"ro", "bind"}, + } + g.AddMount(mnt) } // add metadata @@ -480,7 +489,13 @@ func (s *Server) RunPodSandbox(ctx context.Context, req *pb.RunPodSandboxRequest if err := label.Relabel(hostnamePath, mountLabel, true); err != nil && err != unix.ENOTSUP { return nil, err } - g.AddBindMount(hostnamePath, "/etc/hostname", []string{"ro"}) + mnt := runtimespec.Mount{ + Type: "bind", + Source: hostnamePath, + Destination: "/etc/hostname", + Options: []string{"ro", "bind"}, + } + g.AddMount(mnt) g.AddAnnotation(annotations.HostnamePath, hostnamePath) sb.AddHostnamePath(hostnamePath) diff --git a/test/README.md b/test/README.md index eb314e8d..1d1742b0 100644 --- a/test/README.md +++ b/test/README.md @@ -41,11 +41,12 @@ You will also need to install the [CNI](https://github.com/containernetworking/c the the default pod test template runs without host networking: ``` -$ go get github.com/containernetworking/cni -$ cd "$GOPATH/src/github.com/containernetworking/cni" -$ git checkout -q d4bbce1865270cd2d2be558d6a23e63d314fe769 -$ ./build.sh \ -$ mkdir -p /opt/cni/bin \ +$ cd "$GOPATH/src/github.com/containernetworking" +$ git clone https://github.com/containernetworking/plugins.git +$ cd plugins +$ git checkout -q dcf7368eeab15e2affc6256f0bb1e84dd46a34de +$ ./build.sh +$ mkdir -p /opt/cni/bin $ cp bin/* /opt/cni/bin/ ``` @@ -69,11 +70,11 @@ Tests on the host will run with `runc` as the default runtime. However you can select other OCI compatible runtimes by setting the `RUNTIME` environment variable. -For example one could use the [Clear Containers](https://github.com/01org/cc-oci-runtime/wiki/Installation) +For example one could use the [Clear Containers](https://github.com/clearcontainers/runtime) runtime instead of `runc`: ``` -make localintegration RUNTIME=cc-oci-runtime +make localintegration RUNTIME=cc-runtime ``` ## Writing integration tests diff --git a/test/ctr.bats b/test/ctr.bats index 1bf4f8b9..5f37c708 100644 --- a/test/ctr.bats +++ b/test/ctr.bats @@ -1061,3 +1061,31 @@ function teardown() { cleanup_pods stop_crio } + +@test "ctr resources" { + start_crio + run crictl runs "$TESTDATA"/sandbox_config.json + echo "$output" + [ "$status" -eq 0 ] + pod_id="$output" + run crictl create "$pod_id" "$TESTDATA"/container_redis.json "$TESTDATA"/sandbox_config.json + echo "$output" + [ "$status" -eq 0 ] + ctr_id="$output" + run crictl start "$ctr_id" + echo "$output" + [ "$status" -eq 0 ] + + run crictl exec --sync "$ctr_id" sh -c "cat /sys/fs/cgroup/cpuset/cpuset.cpus" + echo "$output" + [ "$status" -eq 0 ] + [[ "$output" =~ "0-1" ]] + run crictl exec --sync "$ctr_id" sh -c "cat /sys/fs/cgroup/cpuset/cpuset.mems" + echo "$output" + [ "$status" -eq 0 ] + [[ "$output" =~ "0" ]] + + cleanup_ctrs + cleanup_pods + stop_crio +} diff --git a/test/helpers.bash b/test/helpers.bash index c698687a..a0c715e1 100644 --- a/test/helpers.bash +++ b/test/helpers.bash @@ -103,7 +103,7 @@ cp "$CONMON_BINARY" "$TESTDIR/conmon" PATH=$PATH:$TESTDIR -# Make sure we have a copy of the redis:latest image. +# Make sure we have a copy of the redis:alpine image. if ! [ -d "$ARTIFACTS_PATH"/redis-image ]; then mkdir -p "$ARTIFACTS_PATH"/redis-image if ! "$COPYIMG_BINARY" --import-from=docker://redis:alpine --export-to=dir:"$ARTIFACTS_PATH"/redis-image --signature-policy="$INTEGRATION_ROOT"/policy.json ; then @@ -113,19 +113,6 @@ if ! [ -d "$ARTIFACTS_PATH"/redis-image ]; then fi fi -# TODO: remove the code below for redis digested image id when -# https://github.com/kubernetes-incubator/cri-o/issues/531 is complete -# as the digested reference will be auto-stored when pulling the tag -# above -if ! [ -d "$ARTIFACTS_PATH"/redis-image-digest ]; then - mkdir -p "$ARTIFACTS_PATH"/redis-image-digest - if ! "$COPYIMG_BINARY" --import-from=docker://redis@sha256:03789f402b2ecfb98184bf128d180f398f81c63364948ff1454583b02442f73b --export-to=dir:"$ARTIFACTS_PATH"/redis-image-digest --signature-policy="$INTEGRATION_ROOT"/policy.json ; then - echo "Error pulling docker://redis@sha256:03789f402b2ecfb98184bf128d180f398f81c63364948ff1454583b02442f73b" - rm -fr "$ARTIFACTS_PATH"/redis-image-digest - exit 1 - fi -fi - # Make sure we have a copy of the runcom/stderr-test image. if ! [ -d "$ARTIFACTS_PATH"/stderr-test ]; then mkdir -p "$ARTIFACTS_PATH"/stderr-test @@ -225,16 +212,11 @@ function start_crio() { if ! [ "$3" = "--no-pause-image" ] ; then "$BIN2IMG_BINARY" --root "$TESTDIR/crio" $STORAGE_OPTIONS --runroot "$TESTDIR/crio-run" --source-binary "$PAUSE_BINARY" fi - "$COPYIMG_BINARY" --root "$TESTDIR/crio" $STORAGE_OPTIONS --runroot "$TESTDIR/crio-run" --image-name=redis:alpine --import-from=dir:"$ARTIFACTS_PATH"/redis-image --add-name=docker.io/library/redis:alpine --signature-policy="$INTEGRATION_ROOT"/policy.json -# TODO: remove the code below for redis:alpine digested image id when -# https://github.com/kubernetes-incubator/cri-o/issues/531 is complete -# as the digested reference will be auto-stored when pulling the tag -# above - "$COPYIMG_BINARY" --root "$TESTDIR/crio" $STORAGE_OPTIONS --runroot "$TESTDIR/crio-run" --image-name=redis@sha256:03789f402b2ecfb98184bf128d180f398f81c63364948ff1454583b02442f73b --import-from=dir:"$ARTIFACTS_PATH"/redis-image-digest --add-name=docker.io/library/redis@sha256:03789f402b2ecfb98184bf128d180f398f81c63364948ff1454583b02442f73b --signature-policy="$INTEGRATION_ROOT"/policy.json - "$COPYIMG_BINARY" --root "$TESTDIR/crio" $STORAGE_OPTIONS --runroot "$TESTDIR/crio-run" --image-name=mrunalp/oom --import-from=dir:"$ARTIFACTS_PATH"/oom-image --add-name=docker.io/library/mrunalp/oom --signature-policy="$INTEGRATION_ROOT"/policy.json - "$COPYIMG_BINARY" --root "$TESTDIR/crio" $STORAGE_OPTIONS --runroot "$TESTDIR/crio-run" --image-name=mrunalp/image-volume-test --import-from=dir:"$ARTIFACTS_PATH"/image-volume-test-image --add-name=docker.io/library/mrunalp/image-volume-test --signature-policy="$INTEGRATION_ROOT"/policy.json - "$COPYIMG_BINARY" --root "$TESTDIR/crio" $STORAGE_OPTIONS --runroot "$TESTDIR/crio-run" --image-name=busybox:latest --import-from=dir:"$ARTIFACTS_PATH"/busybox-image --add-name=docker.io/library/busybox:latest --signature-policy="$INTEGRATION_ROOT"/policy.json - "$COPYIMG_BINARY" --root "$TESTDIR/crio" $STORAGE_OPTIONS --runroot "$TESTDIR/crio-run" --image-name=runcom/stderr-test:latest --import-from=dir:"$ARTIFACTS_PATH"/stderr-test --add-name=docker.io/runcom/stderr-test:latest --signature-policy="$INTEGRATION_ROOT"/policy.json + "$COPYIMG_BINARY" --root "$TESTDIR/crio" $STORAGE_OPTIONS --runroot "$TESTDIR/crio-run" --image-name=docker.io/library/redis:alpine --import-from=dir:"$ARTIFACTS_PATH"/redis-image --signature-policy="$INTEGRATION_ROOT"/policy.json + "$COPYIMG_BINARY" --root "$TESTDIR/crio" $STORAGE_OPTIONS --runroot "$TESTDIR/crio-run" --image-name=docker.io/mrunalp/oom:latest --import-from=dir:"$ARTIFACTS_PATH"/oom-image --signature-policy="$INTEGRATION_ROOT"/policy.json + "$COPYIMG_BINARY" --root "$TESTDIR/crio" $STORAGE_OPTIONS --runroot "$TESTDIR/crio-run" --image-name=docker.io/mrunalp/image-volume-test:latest --import-from=dir:"$ARTIFACTS_PATH"/image-volume-test-image --signature-policy="$INTEGRATION_ROOT"/policy.json + "$COPYIMG_BINARY" --root "$TESTDIR/crio" $STORAGE_OPTIONS --runroot "$TESTDIR/crio-run" --image-name=docker.io/library/busybox:latest --import-from=dir:"$ARTIFACTS_PATH"/busybox-image --signature-policy="$INTEGRATION_ROOT"/policy.json + "$COPYIMG_BINARY" --root "$TESTDIR/crio" $STORAGE_OPTIONS --runroot "$TESTDIR/crio-run" --image-name=docker.io/runcom/stderr-test:latest --import-from=dir:"$ARTIFACTS_PATH"/stderr-test --signature-policy="$INTEGRATION_ROOT"/policy.json "$CRIO_BINARY" ${DEFAULT_MOUNTS_OPTS} ${HOOKS_OPTS} --conmon "$CONMON_BINARY" --listen "$CRIO_SOCKET" --cgroup-manager "$CGROUP_MANAGER" --registry "docker.io" --runtime "$RUNTIME_BINARY" --root "$TESTDIR/crio" --runroot "$TESTDIR/crio-run" $STORAGE_OPTIONS --seccomp-profile "$seccomp" --apparmor-profile "$apparmor" --cni-config-dir "$CRIO_CNI_CONFIG" --cni-plugin-dir "$CRIO_CNI_PLUGIN" --signature-policy "$INTEGRATION_ROOT"/policy.json --image-volumes "$IMAGE_VOLUMES" --pids-limit "$PIDS_LIMIT" --enable-shared-pid-namespace=${ENABLE_SHARED_PID_NAMESPACE} --log-size-max "$LOG_SIZE_MAX_LIMIT" --config /dev/null config >$CRIO_CONFIG # Prepare the CNI configuration files, we're running with non host networking by default @@ -252,44 +234,28 @@ function start_crio() { if [ "$status" -ne 0 ] ; then crictl pull redis:alpine fi - REDIS_IMAGEID=$(crictl inspecti redis:alpine | head -1 | sed -e "s/ID: //g") + REDIS_IMAGEID=$(crictl inspecti redis:alpine | grep ^ID: | head -n 1 | sed -e "s/ID: //g") + REDIS_IMAGEREF=$(crictl inspecti redis:alpine | grep ^Digest: | head -n 1 | sed -e "s/Digest: //g") run crictl inspecti mrunalp/oom if [ "$status" -ne 0 ] ; then crictl pull mrunalp/oom fi - # - # - # - # TODO: remove the code below for redis digested image id when - # https://github.com/kubernetes-incubator/cri-o/issues/531 is complete - # as the digested reference will be auto-stored when pulling the tag - # above - # - # - # - REDIS_IMAGEID_DIGESTED="redis@sha256:03789f402b2ecfb98184bf128d180f398f81c63364948ff1454583b02442f73b" - run crictl inspecti $REDIS_IMAGEID_DIGESTED - if [ "$status" -ne 0 ]; then - crictl pull $REDIS_IMAGEID_DIGESTED - fi - # - # - # - run crictl inspecti runcom/stderr-test + OOM_IMAGEID=$(crictl inspecti mrunalp/oom | grep ^ID: | head -n 1 | sed -e "s/ID: //g") + run crioctl image status --id=runcom/stderr-test if [ "$status" -ne 0 ] ; then crictl pull runcom/stderr-test:latest fi - STDERR_IMAGEID=$(crictl inspecti runcom/stderr-test | head -1 | sed -e "s/ID: //g") + STDERR_IMAGEID=$(crictl inspecti runcom/stderr-test | grep ^ID: | head -n 1 | sed -e "s/ID: //g") run crictl inspecti busybox if [ "$status" -ne 0 ] ; then crictl pull busybox:latest fi - BUSYBOX_IMAGEID=$(crictl inspecti busybox | head -1 | sed -e "s/ID: //g") + BUSYBOX_IMAGEID=$(crictl inspecti busybox | grep ^ID: | head -n 1 | sed -e "s/ID: //g") run crictl inspecti mrunalp/image-volume-test if [ "$status" -ne 0 ] ; then crictl pull mrunalp/image-volume-test:latest fi - VOLUME_IMAGEID=$(crictl inspecti mrunalp/image-volume-test | head -1 | sed -e "s/ID: //g") + VOLUME_IMAGEID=$(crictl inspecti mrunalp/image-volume-test | grep ^ID: | head -n 1 | sed -e "s/ID: //g") } function cleanup_ctrs() { diff --git a/test/image.bats b/test/image.bats index 65a047f6..52336641 100644 --- a/test/image.bats +++ b/test/image.bats @@ -20,12 +20,16 @@ function teardown() { run crictl create "$pod_id" "$TESTDIR"/ctr_by_imageid.json "$TESTDATA"/sandbox_config.json echo "$output" [ "$status" -eq 0 ] + ctr_id="$output" + run crictl start "$ctr_id" + echo "$output" + [ "$status" -eq 0 ] cleanup_ctrs cleanup_pods stop_crio } -@test "container status return image:tag if created by image ID" { +@test "container status when created by image ID" { start_crio run crictl runs "$TESTDATA"/sandbox_config.json @@ -43,16 +47,15 @@ function teardown() { run crictl inspect "$ctr_id" --output yaml echo "$output" [ "$status" -eq 0 ] - [[ "$output" =~ "image: redis:alpine" ]] + [[ "$output" =~ "image: docker.io/library/redis:alpine" ]] + [[ "$output" =~ "imageRef: $REDIS_IMAGEREF" ]] cleanup_ctrs cleanup_pods stop_crio } -@test "container status return image@digest if created by image ID and digest available" { - skip "depends on https://github.com/kubernetes-incubator/cri-o/issues/531" - +@test "container status when created by image tagged reference" { start_crio run crictl runs "$TESTDATA"/sandbox_config.json @@ -60,9 +63,9 @@ function teardown() { [ "$status" -eq 0 ] pod_id="$output" - sed -e "s/%VALUE%/$REDIS_IMAGEID_DIGESTED/g" "$TESTDATA"/container_config_by_imageid.json > "$TESTDIR"/ctr_by_imageid.json + sed -e "s/%VALUE%/redis:alpine/g" "$TESTDATA"/container_config_by_imageid.json > "$TESTDIR"/ctr_by_imagetag.json - run crictl create "$pod_id" "$TESTDIR"/ctr_by_imageid.json "$TESTDATA"/sandbox_config.json + run crictl create "$pod_id" "$TESTDIR"/ctr_by_imagetag.json "$TESTDATA"/sandbox_config.json echo "$output" [ "$status" -eq 0 ] ctr_id="$output" @@ -70,22 +73,64 @@ function teardown() { run crictl inspect "$ctr_id" --output yaml echo "$output" [ "$status" -eq 0 ] - [[ "$output" =~ "image_ref: redis@sha256:03789f402b2ecfb98184bf128d180f398f81c63364948ff1454583b02442f73b" ]] + [[ "$output" =~ "image: docker.io/library/redis:alpine" ]] + [[ "$output" =~ "imageRef: $REDIS_IMAGEREF" ]] cleanup_ctrs cleanup_pods stop_crio } -@test "image pull" { +@test "container status when created by image canonical reference" { + start_crio + + run crictl runs "$TESTDATA"/sandbox_config.json + echo "$output" + [ "$status" -eq 0 ] + pod_id="$output" + + sed -e "s|%VALUE%|$REDIS_IMAGEREF|g" "$TESTDATA"/container_config_by_imageid.json > "$TESTDIR"/ctr_by_imageref.json + + run crictl create "$pod_id" "$TESTDIR"/ctr_by_imageref.json "$TESTDATA"/sandbox_config.json + echo "$output" + [ "$status" -eq 0 ] + ctr_id="$output" + + run crictl start "$ctr_id" + echo "$output" + [ "$status" -eq 0 ] + + run crictl inspect "$ctr_id" --output yaml + echo "$output" + [ "$status" -eq 0 ] + [[ "$output" =~ "image: docker.io/library/redis:alpine" ]] + [[ "$output" =~ "imageRef: $REDIS_IMAGEREF" ]] + + cleanup_ctrs + cleanup_pods + stop_crio +} + +@test "image pull and list" { start_crio "" "" --no-pause-image run crictl pull "$IMAGE" echo "$output" [ "$status" -eq 0 ] - run crictl inspecti "$IMAGE" + + run crictl images --quiet "$IMAGE" + [ "$status" -eq 0 ] echo "$output" + [ "$output" != "" ] + imageid="$output" + + run crictl images @"$imageid" [ "$status" -eq 0 ] [[ "$output" =~ "$IMAGE" ]] + + run crictl images --quiet "$imageid" + [ "$status" -eq 0 ] + echo "$output" + [ "$output" != "" ] cleanup_images stop_crio } @@ -108,7 +153,33 @@ function teardown() { stop_crio } -@test "image pull and list by digest" { +@test "image pull and list by tag and ID" { + start_crio "" "" --no-pause-image + run crictl pull "$IMAGE:go" + echo "$output" + [ "$status" -eq 0 ] + + run crictl images --quiet "$IMAGE:go" + [ "$status" -eq 0 ] + echo "$output" + [ "$output" != "" ] + imageid="$output" + + run crictl images --quiet @"$imageid" + [ "$status" -eq 0 ] + echo "$output" + [ "$output" != "" ] + + run crictl images --quiet "$imageid" + [ "$status" -eq 0 ] + echo "$output" + [ "$output" != "" ] + + cleanup_images + stop_crio +} + +@test "image pull and list by digest and ID" { start_crio "" "" --no-pause-image run crictl pull nginx@sha256:33eb1ed1e802d4f71e52421f56af028cdf12bb3bfff5affeaf5bf0e328ffa1bc echo "$output" @@ -118,18 +189,14 @@ function teardown() { [ "$status" -eq 0 ] echo "$output" [ "$output" != "" ] + imageid="$output" - run crictl images --quiet nginx@33eb1ed1e802d4f71e52421f56af028cdf12bb3bfff5affeaf5bf0e328ffa1bc + run crictl images --quiet @"$imageid" [ "$status" -eq 0 ] echo "$output" [ "$output" != "" ] - run crictl images --quiet @33eb1ed1e802d4f71e52421f56af028cdf12bb3bfff5affeaf5bf0e328ffa1bc - [ "$status" -eq 0 ] - echo "$output" - [ "$output" != "" ] - - run crictl images --quiet 33eb1ed1e802d4f71e52421f56af028cdf12bb3bfff5affeaf5bf0e328ffa1bc + run crictl images --quiet "$imageid" [ "$status" -eq 0 ] echo "$output" [ "$output" != "" ] @@ -198,7 +265,7 @@ function teardown() { [ "$status" -eq 0 ] [ "$output" != "" ] printf '%s\n' "$output" | while IFS= read -r id; do - run crictl inspecti "$id" + run crictl images -v "$id" echo "$output" [ "$status" -eq 0 ] [ "$output" != "" ] diff --git a/test/image_remove.bats b/test/image_remove.bats new file mode 100644 index 00000000..54b06c05 --- /dev/null +++ b/test/image_remove.bats @@ -0,0 +1,75 @@ +#!/usr/bin/env bats + +load helpers + +IMAGE=docker.io/kubernetes/pause + +function teardown() { + cleanup_test +} + +@test "image remove with multiple names, by name" { + start_crio "" "" --no-pause-image + # Pull the image, giving it one name. + run crictl pull "$IMAGE" + echo "$output" + [ "$status" -eq 0 ] + # Add a second name to the image. + run "$COPYIMG_BINARY" --root "$TESTDIR/crio" $STORAGE_OPTIONS --runroot "$TESTDIR/crio-run" --image-name="$IMAGE":latest --add-name="$IMAGE":othertag --signature-policy="$INTEGRATION_ROOT"/policy.json + echo "$output" + [ "$status" -eq 0 ] + # Get the list of image names and IDs. + run crictl images -v + echo "$output" + [ "$status" -eq 0 ] + [ "$output" != "" ] + # Cycle through each name, removing it by name. The image that we assigned a second + # name to should still be around when we get to removing its second name. + grep ^RepoTags: <<< "$output" | while read -r header tag ignored ; do + run crictl rmi "$tag" + echo "$output" + [ "$status" -eq 0 ] + done + # List all images and their names. There should be none now. + run crictl images --quiet + echo "$output" + [ "$status" -eq 0 ] + [ "$output" = "" ] + printf '%s\n' "$output" | while IFS= read -r id; do + echo "$id" + done + # All done. + cleanup_images + stop_crio +} + +@test "image remove with multiple names, by ID" { + start_crio "" "" --no-pause-image + # Pull the image, giving it one name. + run crictl pull "$IMAGE" + echo "$output" + [ "$status" -eq 0 ] + # Add a second name to the image. + run "$COPYIMG_BINARY" --root "$TESTDIR/crio" $STORAGE_OPTIONS --runroot "$TESTDIR/crio-run" --image-name="$IMAGE":latest --add-name="$IMAGE":othertag --signature-policy="$INTEGRATION_ROOT"/policy.json + echo "$output" + [ "$status" -eq 0 ] + # Get the list of the image's names and its ID. + run crictl images -v "$IMAGE":latest + echo "$output" + [ "$status" -eq 0 ] + [ "$output" != "" ] + # Try to remove the image using its ID. That should succeed. + grep ^ID: <<< "$output" | while read -r header id ; do + run crictl rmi "$id" + echo "$output" + [ "$status" -eq 0 ] + done + # The image should be gone now. + run crictl images -v "$IMAGE" + echo "$output" + [ "$status" -eq 0 ] + [ "$output" = "" ] + # All done. + cleanup_images + stop_crio +} diff --git a/test/image_volume.bats b/test/image_volume.bats index 1b2d967b..f5b39401 100644 --- a/test/image_volume.bats +++ b/test/image_volume.bats @@ -55,7 +55,7 @@ function teardown() { run crictl exec --sync "$ctr_id" touch /imagevolume/test_file echo "$output" [ "$status" -eq 0 ] - + [ "$output" = "" ] run crictl stops "$pod_id" echo "$output" [ "$status" -eq 0 ] diff --git a/test/inspect.bats b/test/inspect.bats index bfbf82a0..bb7977e5 100644 --- a/test/inspect.bats +++ b/test/inspect.bats @@ -30,13 +30,15 @@ function teardown() { out=`echo -e "GET /containers/$ctr_id HTTP/1.1\r\nHost: crio\r\n" | socat - UNIX-CONNECT:$CRIO_SOCKET` echo "$out" [[ "$out" =~ "\"sandbox\":\"$pod_id\"" ]] - [[ "$out" =~ "\"image\":\"redis:alpine\"" ]] + [[ "$out" =~ "\"image\":\"docker.io/library/redis:alpine\"" ]] + [[ "$out" =~ "\"image_ref\":\"$REDIS_IMAGEREF\"" ]] run crictl inspect --output json "$ctr_id" echo "$output" [ "$status" -eq 0 ] [[ "$output" =~ "\"id\": \"$ctr_id\"" ]] - [[ "$output" =~ "\"image\": \"redis:alpine\"" ]] + [[ "$output" =~ "\"image\": \"docker.io/library/redis:alpine\"" ]] + [[ "$output" =~ "\"imageRef\": \"$REDIS_IMAGEREF\"" ]] run crictl inspects --output json "$pod_id" echo "$output" diff --git a/test/namespaces.bats b/test/namespaces.bats index ff9787a9..033cbab2 100644 --- a/test/namespaces.bats +++ b/test/namespaces.bats @@ -6,8 +6,8 @@ function teardown() { cleanup_test } -@test "pod disable shared pid namespace" { - ENABLE_SHARED_PID_NAMESPACE="false" start_crio +function pid_namespace_test() { + start_crio run crictl runs "$TESTDATA"/sandbox_config.json echo "$output" @@ -23,7 +23,7 @@ function teardown() { run crictl exec --sync "$ctr_id" cat /proc/1/cmdline echo "$output" [ "$status" -eq 0 ] - [[ "$output" =~ "redis" ]] + [[ "$output" =~ "${EXPECTED_INIT:-redis}" ]] run crictl stops "$pod_id" echo "$output" @@ -36,32 +36,10 @@ function teardown() { stop_crio } +@test "pod disable shared pid namespace" { + ENABLE_SHARED_PID_NAMESPACE=false pid_namespace_test +} + @test "pod enable shared pid namespace" { - ENABLE_SHARED_PID_NAMESPACE="true" start_crio - - run crictl runs "$TESTDATA"/sandbox_config.json - echo "$output" - [ "$status" -eq 0 ] - pod_id="$output" - run crictl create "$pod_id" "$TESTDATA"/container_redis.json "$TESTDATA"/sandbox_config.json - echo "$output" - [ "$status" -eq 0 ] - ctr_id="$output" - run crictl start "$ctr_id" - [ "$status" -eq 0 ] - - run crictl exec --sync "$ctr_id" cat /proc/1/cmdline - echo "$output" - [ "$status" -eq 0 ] - [[ "$output" =~ "pause" ]] - - run crictl stops "$pod_id" - echo "$output" - [ "$status" -eq 0 ] - run crictl rms "$pod_id" - echo "$output" - [ "$status" -eq 0 ] - cleanup_ctrs - cleanup_pods - stop_crio + ENABLE_SHARED_PID_NAMESPACE=true EXPECTED_INIT=pause pid_namespace_test } diff --git a/test/testdata/container_redis.json b/test/testdata/container_redis.json index 20f1cbe5..39a0865b 100644 --- a/test/testdata/container_redis.json +++ b/test/testdata/container_redis.json @@ -49,7 +49,9 @@ "cpu_period": 10000, "cpu_quota": 20000, "cpu_shares": 512, - "oom_score_adj": 30 + "oom_score_adj": 30, + "cpuset_cpus": "0-1", + "cpuset_mems": "0" }, "security_context": { "capabilities": { diff --git a/tutorial.md b/tutorial.md index 656a706f..5f89ccb8 100644 --- a/tutorial.md +++ b/tutorial.md @@ -138,32 +138,12 @@ make sudo make install ``` -Output: - -``` -install -D -m 755 crio /usr/local/bin/crio -install -D -m 755 conmon/conmon /usr/local/libexec/crio/conmon -install -D -m 755 pause/pause /usr/local/libexec/crio/pause -install -d -m 755 /usr/local/share/man/man{1,5,8} -install -m 644 docs/crio.conf.5 -t /usr/local/share/man/man5 -install -m 644 docs/crio.8 -t /usr/local/share/man/man8 -install -D -m 644 crio.conf /etc/crio/crio.conf -install -D -m 644 seccomp.json /etc/crio/seccomp.json -``` - -If you are installing for the first time, generate config as follows: +If you are installing for the first time, generate and install configuration files with: ``` sudo make install.config ``` -Output: - -``` -install -D -m 644 crio.conf /etc/crio/crio.conf -install -D -m 644 seccomp.json /etc/crio/seccomp.json -``` - #### Start the crio system daemon ``` @@ -320,15 +300,15 @@ cd $GOPATH/src/github.com/kubernetes-incubator/cri-o Next create the Pod and capture the Pod ID for later use: ``` -POD_ID=$(sudo crictl runs test/testdata/sandbox_config.json) +POD_ID=$(sudo crictl runp test/testdata/sandbox_config.json) ``` -> sudo crictl runs test/testdata/sandbox_config.json +> sudo crictl runp test/testdata/sandbox_config.json Use the `crictl` command to get the status of the Pod: ``` -sudo crictl inspects --output table $POD_ID +sudo crictl inspectp --output table $POD_ID ``` Output: diff --git a/types/types.go b/types/types.go index 63780143..cedc3abd 100644 --- a/types/types.go +++ b/types/types.go @@ -5,6 +5,7 @@ type ContainerInfo struct { Name string `json:"name"` Pid int `json:"pid"` Image string `json:"image"` + ImageRef string `json:"image_ref"` CreatedTime int64 `json:"created_time"` Labels map[string]string `json:"labels"` Annotations map[string]string `json:"annotations"` diff --git a/vendor.conf b/vendor.conf index e7529a8c..acd066aa 100644 --- a/vendor.conf +++ b/vendor.conf @@ -1,26 +1,26 @@ -k8s.io/kubernetes release-1.9 https://github.com/kubernetes/kubernetes -k8s.io/client-go release-6.0 https://github.com/kubernetes/client-go -k8s.io/apimachinery release-1.9 https://github.com/kubernetes/apimachinery -k8s.io/apiserver release-1.9 https://github.com/kubernetes/apiserver +k8s.io/kubernetes a48f11c2257d84b0bec89864025508b0ef626b4f https://github.com/kubernetes/kubernetes +k8s.io/client-go master https://github.com/kubernetes/client-go +k8s.io/apimachinery master https://github.com/kubernetes/apimachinery +k8s.io/apiserver master https://github.com/kubernetes/apiserver k8s.io/utils 4fe312863be2155a7b68acd2aff1c9221b24e68c https://github.com/kubernetes/utils -k8s.io/api release-1.9 https://github.com/kubernetes/api +k8s.io/api master https://github.com/kubernetes/api k8s.io/kube-openapi 39a7bf85c140f972372c2a0d1ee40adbf0c8bfe1 https://github.com/kubernetes/kube-openapi -k8s.io/apiextensions-apiserver release-1.9 https://github.com/kubernetes/apiextensions-apiserver +k8s.io/apiextensions-apiserver master https://github.com/kubernetes/apiextensions-apiserver # github.com/googleapis/gnostic 0c5108395e2debce0d731cf0287ddf7242066aba github.com/gregjones/httpcache 787624de3eb7bd915c329cba748687a3b22666a6 github.com/json-iterator/go 1.0.0 github.com/peterbourgon/diskv v2.0.1 github.com/sirupsen/logrus v1.0.0 -github.com/containers/image 57b257d128d6075ea3287991ee408d24c7bd2758 +github.com/containers/image 3d0304a02154dddc8f97cc833aa0861cea5e9ade github.com/docker/docker-credential-helpers d68f9aeca33f5fd3f08eeae5e9d175edf4e731d1 github.com/ostreedev/ostree-go master -github.com/containers/storage d7921c6facc516358070a1306689eda18adaa20a +github.com/containers/storage 0d32dfce498e06c132c60dac945081bf44c22464 github.com/containernetworking/cni v0.4.0 google.golang.org/grpc v1.0.4 https://github.com/grpc/grpc-go github.com/opencontainers/selinux b29023b86e4a69d1b46b7e7b4e2b6fda03f0b9cd github.com/opencontainers/go-digest v1.0.0-rc0 -github.com/opencontainers/runtime-tools d3f7e9e9e631c7e87552d67dc7c86de33c3fb68a +github.com/opencontainers/runtime-tools 625e2322645b151a7cbb93a8b42920933e72167f github.com/opencontainers/runc 45bde006ca8c90e089894508708bcf0e2cdf9e13 github.com/mrunalp/fileutils master github.com/vishvananda/netlink master @@ -113,3 +113,6 @@ github.com/hashicorp/errwrap 7554cd9344cec97297fa6649b055a8c98c2a1e55 github.com/pquerna/ffjson d49c2bc1aa135aad0c6f4fc2056623ec78f5d5ac github.com/stretchr/testify 4d4bfba8f1d1027c4fdbe371823030df51419987 github.com/pmezard/go-difflib v1.0.0 +github.com/xeipuuv/gojsonreference master +github.com/xeipuuv/gojsonschema master +github.com/xeipuuv/gojsonpointer master diff --git a/vendor/github.com/containers/image/copy/copy.go b/vendor/github.com/containers/image/copy/copy.go index 590b3787..29065e03 100644 --- a/vendor/github.com/containers/image/copy/copy.go +++ b/vendor/github.com/containers/image/copy/copy.go @@ -12,8 +12,6 @@ import ( "strings" "time" - pb "gopkg.in/cheggaaa/pb.v1" - "github.com/containers/image/image" "github.com/containers/image/pkg/compression" "github.com/containers/image/signature" @@ -22,6 +20,7 @@ import ( "github.com/opencontainers/go-digest" "github.com/pkg/errors" "github.com/sirupsen/logrus" + pb "gopkg.in/cheggaaa/pb.v1" ) type digestingReader struct { @@ -31,23 +30,6 @@ type digestingReader struct { validationFailed bool } -// imageCopier allows us to keep track of diffID values for blobs, and other -// data, that we're copying between images, and cache other information that -// might allow us to take some shortcuts -type imageCopier struct { - copiedBlobs map[digest.Digest]digest.Digest - cachedDiffIDs map[digest.Digest]digest.Digest - manifestUpdates *types.ManifestUpdateOptions - dest types.ImageDestination - src types.Image - rawSource types.ImageSource - diffIDsAreNeeded bool - canModifyManifest bool - reportWriter io.Writer - progressInterval time.Duration - progress chan types.ProgressProperties -} - // newDigestingReader returns an io.Reader implementation with contents of source, which will eventually return a non-EOF error // and set validationFailed to true if the source stream does not match expectedDigest. func newDigestingReader(source io.Reader, expectedDigest digest.Digest) (*digestingReader, error) { @@ -86,6 +68,27 @@ func (d *digestingReader) Read(p []byte) (int, error) { return n, err } +// copier allows us to keep track of diffID values for blobs, and other +// data shared across one or more images in a possible manifest list. +type copier struct { + copiedBlobs map[digest.Digest]digest.Digest + cachedDiffIDs map[digest.Digest]digest.Digest + dest types.ImageDestination + rawSource types.ImageSource + reportWriter io.Writer + progressInterval time.Duration + progress chan types.ProgressProperties +} + +// imageCopier tracks state specific to a single image (possibly an item of a manifest list) +type imageCopier struct { + c *copier + manifestUpdates *types.ManifestUpdateOptions + src types.Image + diffIDsAreNeeded bool + canModifyManifest bool +} + // Options allows supplying non-default configuration modifying the behavior of CopyImage. type Options struct { RemoveSignatures bool // Remove any pre-existing signatures. SignBy will still add a new signature. @@ -95,6 +98,8 @@ type Options struct { DestinationCtx *types.SystemContext ProgressInterval time.Duration // time to wait between reports to signal the progress channel Progress chan types.ProgressProperties // Reported to when ProgressInterval has arrived for a single artifact+offset. + // manifest MIME type of image set by user. "" is default and means use the autodetection to the the manifest MIME type + ForceManifestMIMEType string } // Image copies image from srcRef to destRef, using policyContext to validate @@ -115,10 +120,6 @@ func Image(policyContext *signature.PolicyContext, destRef, srcRef types.ImageRe reportWriter = options.ReportWriter } - writeReport := func(f string, a ...interface{}) { - fmt.Fprintf(reportWriter, f, a...) - } - dest, err := destRef.NewImageDestination(options.DestinationCtx) if err != nil { return errors.Wrapf(err, "Error initializing destination %s", transports.ImageName(destRef)) @@ -133,43 +134,89 @@ func Image(policyContext *signature.PolicyContext, destRef, srcRef types.ImageRe if err != nil { return errors.Wrapf(err, "Error initializing source %s", transports.ImageName(srcRef)) } - unparsedImage := image.UnparsedFromSource(rawSource) defer func() { - if unparsedImage != nil { - if err := unparsedImage.Close(); err != nil { - retErr = errors.Wrapf(retErr, " (unparsed: %v)", err) - } + if err := rawSource.Close(); err != nil { + retErr = errors.Wrapf(retErr, " (src: %v)", err) } }() + c := &copier{ + copiedBlobs: make(map[digest.Digest]digest.Digest), + cachedDiffIDs: make(map[digest.Digest]digest.Digest), + dest: dest, + rawSource: rawSource, + reportWriter: reportWriter, + progressInterval: options.ProgressInterval, + progress: options.Progress, + } + + unparsedToplevel := image.UnparsedInstance(rawSource, nil) + multiImage, err := isMultiImage(unparsedToplevel) + if err != nil { + return errors.Wrapf(err, "Error determining manifest MIME type for %s", transports.ImageName(srcRef)) + } + + if !multiImage { + // The simple case: Just copy a single image. + if err := c.copyOneImage(policyContext, options, unparsedToplevel); err != nil { + return err + } + } else { + // This is a manifest list. Choose a single image and copy it. + // FIXME: Copy to destinations which support manifest lists, one image at a time. + instanceDigest, err := image.ChooseManifestInstanceFromManifestList(options.SourceCtx, unparsedToplevel) + if err != nil { + return errors.Wrapf(err, "Error choosing an image from manifest list %s", transports.ImageName(srcRef)) + } + logrus.Debugf("Source is a manifest list; copying (only) instance %s", instanceDigest) + unparsedInstance := image.UnparsedInstance(rawSource, &instanceDigest) + + if err := c.copyOneImage(policyContext, options, unparsedInstance); err != nil { + return err + } + } + + if err := c.dest.Commit(); err != nil { + return errors.Wrap(err, "Error committing the finished image") + } + + return nil +} + +// Image copies a single (on-manifest-list) image unparsedImage, using policyContext to validate +// source image admissibility. +func (c *copier) copyOneImage(policyContext *signature.PolicyContext, options *Options, unparsedImage *image.UnparsedImage) (retErr error) { + // The caller is handling manifest lists; this could happen only if a manifest list contains a manifest list. + // Make sure we fail cleanly in such cases. + multiImage, err := isMultiImage(unparsedImage) + if err != nil { + // FIXME FIXME: How to name a reference for the sub-image? + return errors.Wrapf(err, "Error determining manifest MIME type for %s", transports.ImageName(unparsedImage.Reference())) + } + if multiImage { + return fmt.Errorf("Unexpectedly received a manifest list instead of a manifest for a single image") + } + // Please keep this policy check BEFORE reading any other information about the image. + // (the multiImage check above only matches the MIME type, which we have received anyway. + // Actual parsing of anything should be deferred.) if allowed, err := policyContext.IsRunningImageAllowed(unparsedImage); !allowed || err != nil { // Be paranoid and fail if either return value indicates so. return errors.Wrap(err, "Source image rejected") } - src, err := image.FromUnparsedImage(unparsedImage) + src, err := image.FromUnparsedImage(options.SourceCtx, unparsedImage) if err != nil { - return errors.Wrapf(err, "Error initializing image from source %s", transports.ImageName(srcRef)) + return errors.Wrapf(err, "Error initializing image from source %s", transports.ImageName(c.rawSource.Reference())) } - unparsedImage = nil - defer func() { - if err := src.Close(); err != nil { - retErr = errors.Wrapf(retErr, " (source: %v)", err) - } - }() - if err := checkImageDestinationForCurrentRuntimeOS(src, dest); err != nil { + if err := checkImageDestinationForCurrentRuntimeOS(options.DestinationCtx, src, c.dest); err != nil { return err } - if src.IsMultiImage() { - return errors.Errorf("can not copy %s: manifest contains multiple images", transports.ImageName(srcRef)) - } - var sigs [][]byte if options.RemoveSignatures { sigs = [][]byte{} } else { - writeReport("Getting image source signatures\n") + c.Printf("Getting image source signatures\n") s, err := src.Signatures(context.TODO()) if err != nil { return errors.Wrap(err, "Error reading signatures") @@ -177,41 +224,33 @@ func Image(policyContext *signature.PolicyContext, destRef, srcRef types.ImageRe sigs = s } if len(sigs) != 0 { - writeReport("Checking if image destination supports signatures\n") - if err := dest.SupportsSignatures(); err != nil { + c.Printf("Checking if image destination supports signatures\n") + if err := c.dest.SupportsSignatures(); err != nil { return errors.Wrap(err, "Can not copy signatures") } } - canModifyManifest := len(sigs) == 0 - manifestUpdates := types.ManifestUpdateOptions{} - manifestUpdates.InformationOnly.Destination = dest + ic := imageCopier{ + c: c, + manifestUpdates: &types.ManifestUpdateOptions{InformationOnly: types.ManifestUpdateInformation{Destination: c.dest}}, + src: src, + // diffIDsAreNeeded is computed later + canModifyManifest: len(sigs) == 0, + } - if err := updateEmbeddedDockerReference(&manifestUpdates, dest, src, canModifyManifest); err != nil { + if err := ic.updateEmbeddedDockerReference(); err != nil { return err } // We compute preferredManifestMIMEType only to show it in error messages. // Without having to add this context in an error message, we would be happy enough to know only that no conversion is needed. - preferredManifestMIMEType, otherManifestMIMETypeCandidates, err := determineManifestConversion(&manifestUpdates, src, dest.SupportedManifestMIMETypes(), canModifyManifest) + preferredManifestMIMEType, otherManifestMIMETypeCandidates, err := ic.determineManifestConversion(c.dest.SupportedManifestMIMETypes(), options.ForceManifestMIMEType) if err != nil { return err } - // If src.UpdatedImageNeedsLayerDiffIDs(manifestUpdates) will be true, it needs to be true by the time we get here. - ic := imageCopier{ - copiedBlobs: make(map[digest.Digest]digest.Digest), - cachedDiffIDs: make(map[digest.Digest]digest.Digest), - manifestUpdates: &manifestUpdates, - dest: dest, - src: src, - rawSource: rawSource, - diffIDsAreNeeded: src.UpdatedImageNeedsLayerDiffIDs(manifestUpdates), - canModifyManifest: canModifyManifest, - reportWriter: reportWriter, - progressInterval: options.ProgressInterval, - progress: options.Progress, - } + // If src.UpdatedImageNeedsLayerDiffIDs(ic.manifestUpdates) will be true, it needs to be true by the time we get here. + ic.diffIDsAreNeeded = src.UpdatedImageNeedsLayerDiffIDs(*ic.manifestUpdates) if err := ic.copyLayers(); err != nil { return err @@ -233,9 +272,9 @@ func Image(policyContext *signature.PolicyContext, destRef, srcRef types.ImageRe } // If the original MIME type is acceptable, determineManifestConversion always uses it as preferredManifestMIMEType. // So if we are here, we will definitely be trying to convert the manifest. - // With !canModifyManifest, that would just be a string of repeated failures for the same reason, + // With !ic.canModifyManifest, that would just be a string of repeated failures for the same reason, // so let’s bail out early and with a better error message. - if !canModifyManifest { + if !ic.canModifyManifest { return errors.Wrap(err, "Writing manifest failed (and converting it is not possible)") } @@ -243,7 +282,7 @@ func Image(policyContext *signature.PolicyContext, destRef, srcRef types.ImageRe errs := []string{fmt.Sprintf("%s(%v)", preferredManifestMIMEType, err)} for _, manifestMIMEType := range otherManifestMIMETypeCandidates { logrus.Debugf("Trying to use manifest type %s…", manifestMIMEType) - manifestUpdates.ManifestMIMEType = manifestMIMEType + ic.manifestUpdates.ManifestMIMEType = manifestMIMEType attemptedManifest, err := ic.copyUpdatedConfigAndManifest() if err != nil { logrus.Debugf("Upload of manifest type %s failed: %v", manifestMIMEType, err) @@ -262,35 +301,44 @@ func Image(policyContext *signature.PolicyContext, destRef, srcRef types.ImageRe } if options.SignBy != "" { - newSig, err := createSignature(dest, manifest, options.SignBy, reportWriter) + newSig, err := c.createSignature(manifest, options.SignBy) if err != nil { return err } sigs = append(sigs, newSig) } - writeReport("Storing signatures\n") - if err := dest.PutSignatures(sigs); err != nil { + c.Printf("Storing signatures\n") + if err := c.dest.PutSignatures(sigs); err != nil { return errors.Wrap(err, "Error writing signatures") } - if err := dest.Commit(); err != nil { - return errors.Wrap(err, "Error committing the finished image") - } - return nil } -func checkImageDestinationForCurrentRuntimeOS(src types.Image, dest types.ImageDestination) error { +// Printf writes a formatted string to c.reportWriter. +// Note that the method name Printf is not entirely arbitrary: (go tool vet) +// has a built-in list of functions/methods (whatever object they are for) +// which have their format strings checked; for other names we would have +// to pass a parameter to every (go tool vet) invocation. +func (c *copier) Printf(format string, a ...interface{}) { + fmt.Fprintf(c.reportWriter, format, a...) +} + +func checkImageDestinationForCurrentRuntimeOS(ctx *types.SystemContext, src types.Image, dest types.ImageDestination) error { if dest.MustMatchRuntimeOS() { + wantedOS := runtime.GOOS + if ctx != nil && ctx.OSChoice != "" { + wantedOS = ctx.OSChoice + } c, err := src.OCIConfig() if err != nil { return errors.Wrapf(err, "Error parsing image configuration") } - osErr := fmt.Errorf("image operating system %q cannot be used on %q", c.OS, runtime.GOOS) - if runtime.GOOS == "windows" && c.OS == "linux" { + osErr := fmt.Errorf("image operating system %q cannot be used on %q", c.OS, wantedOS) + if wantedOS == "windows" && c.OS == "linux" { return osErr - } else if runtime.GOOS != "windows" && c.OS == "windows" { + } else if wantedOS != "windows" && c.OS == "windows" { return osErr } } @@ -298,35 +346,44 @@ func checkImageDestinationForCurrentRuntimeOS(src types.Image, dest types.ImageD } // updateEmbeddedDockerReference handles the Docker reference embedded in Docker schema1 manifests. -func updateEmbeddedDockerReference(manifestUpdates *types.ManifestUpdateOptions, dest types.ImageDestination, src types.Image, canModifyManifest bool) error { - destRef := dest.Reference().DockerReference() +func (ic *imageCopier) updateEmbeddedDockerReference() error { + destRef := ic.c.dest.Reference().DockerReference() if destRef == nil { return nil // Destination does not care about Docker references } - if !src.EmbeddedDockerReferenceConflicts(destRef) { + if !ic.src.EmbeddedDockerReferenceConflicts(destRef) { return nil // No reference embedded in the manifest, or it matches destRef already. } - if !canModifyManifest { + if !ic.canModifyManifest { return errors.Errorf("Copying a schema1 image with an embedded Docker reference to %s (Docker reference %s) would invalidate existing signatures. Explicitly enable signature removal to proceed anyway", - transports.ImageName(dest.Reference()), destRef.String()) + transports.ImageName(ic.c.dest.Reference()), destRef.String()) } - manifestUpdates.EmbeddedDockerReference = destRef + ic.manifestUpdates.EmbeddedDockerReference = destRef return nil } -// copyLayers copies layers from src/rawSource to dest, using and updating ic.manifestUpdates if necessary and ic.canModifyManifest. +// copyLayers copies layers from ic.src/ic.c.rawSource to dest, using and updating ic.manifestUpdates if necessary and ic.canModifyManifest. func (ic *imageCopier) copyLayers() error { srcInfos := ic.src.LayerInfos() destInfos := []types.BlobInfo{} diffIDs := []digest.Digest{} + updatedSrcInfos := ic.src.LayerInfosForCopy() + srcInfosUpdated := false + if updatedSrcInfos != nil && !reflect.DeepEqual(srcInfos, updatedSrcInfos) { + if !ic.canModifyManifest { + return errors.Errorf("Internal error: copyLayers() needs to use an updated manifest but that was known to be forbidden") + } + srcInfos = updatedSrcInfos + srcInfosUpdated = true + } for _, srcLayer := range srcInfos { var ( destInfo types.BlobInfo diffID digest.Digest err error ) - if ic.dest.AcceptsForeignLayerURLs() && len(srcLayer.URLs) != 0 { + if ic.c.dest.AcceptsForeignLayerURLs() && len(srcLayer.URLs) != 0 { // DiffIDs are, currently, needed only when converting from schema1. // In which case src.LayerInfos will not have URLs because schema1 // does not support them. @@ -334,7 +391,7 @@ func (ic *imageCopier) copyLayers() error { return errors.New("getting DiffID for foreign layers is unimplemented") } destInfo = srcLayer - fmt.Fprintf(ic.reportWriter, "Skipping foreign layer %q copy to %s\n", destInfo.Digest, ic.dest.Reference().Transport().Name()) + ic.c.Printf("Skipping foreign layer %q copy to %s\n", destInfo.Digest, ic.c.dest.Reference().Transport().Name()) } else { destInfo, diffID, err = ic.copyLayer(srcLayer) if err != nil { @@ -348,7 +405,7 @@ func (ic *imageCopier) copyLayers() error { if ic.diffIDsAreNeeded { ic.manifestUpdates.InformationOnly.LayerDiffIDs = diffIDs } - if layerDigestsDiffer(srcInfos, destInfos) { + if srcInfosUpdated || layerDigestsDiffer(srcInfos, destInfos) { ic.manifestUpdates.LayerInfos = destInfos } return nil @@ -379,7 +436,7 @@ func (ic *imageCopier) copyUpdatedConfigAndManifest() ([]byte, error) { // We have set ic.diffIDsAreNeeded based on the preferred MIME type returned by determineManifestConversion. // So, this can only happen if we are trying to upload using one of the other MIME type candidates. // Because UpdatedImageNeedsLayerDiffIDs is true only when converting from s1 to s2, this case should only arise - // when ic.dest.SupportedManifestMIMETypes() includes both s1 and s2, the upload using s1 failed, and we are now trying s2. + // when ic.c.dest.SupportedManifestMIMETypes() includes both s1 and s2, the upload using s1 failed, and we are now trying s2. // Supposedly s2-only registries do not exist or are extremely rare, so failing with this error message is good enough for now. // If handling such registries turns out to be necessary, we could compute ic.diffIDsAreNeeded based on the full list of manifest MIME type candidates. return nil, errors.Errorf("Can not convert image to %s, preparing DiffIDs for this case is not supported", ic.manifestUpdates.ManifestMIMEType) @@ -395,27 +452,27 @@ func (ic *imageCopier) copyUpdatedConfigAndManifest() ([]byte, error) { return nil, errors.Wrap(err, "Error reading manifest") } - if err := ic.copyConfig(pendingImage); err != nil { + if err := ic.c.copyConfig(pendingImage); err != nil { return nil, err } - fmt.Fprintf(ic.reportWriter, "Writing manifest to image destination\n") - if err := ic.dest.PutManifest(manifest); err != nil { + ic.c.Printf("Writing manifest to image destination\n") + if err := ic.c.dest.PutManifest(manifest); err != nil { return nil, errors.Wrap(err, "Error writing manifest") } return manifest, nil } // copyConfig copies config.json, if any, from src to dest. -func (ic *imageCopier) copyConfig(src types.Image) error { +func (c *copier) copyConfig(src types.Image) error { srcInfo := src.ConfigInfo() if srcInfo.Digest != "" { - fmt.Fprintf(ic.reportWriter, "Copying config %s\n", srcInfo.Digest) + c.Printf("Copying config %s\n", srcInfo.Digest) configBlob, err := src.ConfigBlob() if err != nil { return errors.Wrapf(err, "Error reading config blob %s", srcInfo.Digest) } - destInfo, err := ic.copyBlobFromStream(bytes.NewReader(configBlob), srcInfo, nil, false) + destInfo, err := c.copyBlobFromStream(bytes.NewReader(configBlob), srcInfo, nil, false) if err != nil { return err } @@ -437,12 +494,12 @@ type diffIDResult struct { // and returns a complete blobInfo of the copied layer, and a value for LayerDiffIDs if diffIDIsNeeded func (ic *imageCopier) copyLayer(srcInfo types.BlobInfo) (types.BlobInfo, digest.Digest, error) { // Check if we already have a blob with this digest - haveBlob, extantBlobSize, err := ic.dest.HasBlob(srcInfo) + haveBlob, extantBlobSize, err := ic.c.dest.HasBlob(srcInfo) if err != nil { return types.BlobInfo{}, "", errors.Wrapf(err, "Error checking for blob %s at destination", srcInfo.Digest) } // If we already have a cached diffID for this blob, we don't need to compute it - diffIDIsNeeded := ic.diffIDsAreNeeded && (ic.cachedDiffIDs[srcInfo.Digest] == "") + diffIDIsNeeded := ic.diffIDsAreNeeded && (ic.c.cachedDiffIDs[srcInfo.Digest] == "") // If we already have the blob, and we don't need to recompute the diffID, then we might be able to avoid reading it again if haveBlob && !diffIDIsNeeded { // Check the blob sizes match, if we were given a size this time @@ -451,17 +508,17 @@ func (ic *imageCopier) copyLayer(srcInfo types.BlobInfo) (types.BlobInfo, digest } srcInfo.Size = extantBlobSize // Tell the image destination that this blob's delta is being applied again. For some image destinations, this can be faster than using GetBlob/PutBlob - blobinfo, err := ic.dest.ReapplyBlob(srcInfo) + blobinfo, err := ic.c.dest.ReapplyBlob(srcInfo) if err != nil { return types.BlobInfo{}, "", errors.Wrapf(err, "Error reapplying blob %s at destination", srcInfo.Digest) } - fmt.Fprintf(ic.reportWriter, "Skipping fetch of repeat blob %s\n", srcInfo.Digest) - return blobinfo, ic.cachedDiffIDs[srcInfo.Digest], err + ic.c.Printf("Skipping fetch of repeat blob %s\n", srcInfo.Digest) + return blobinfo, ic.c.cachedDiffIDs[srcInfo.Digest], err } // Fallback: copy the layer, computing the diffID if we need to do so - fmt.Fprintf(ic.reportWriter, "Copying blob %s\n", srcInfo.Digest) - srcStream, srcBlobSize, err := ic.rawSource.GetBlob(srcInfo) + ic.c.Printf("Copying blob %s\n", srcInfo.Digest) + srcStream, srcBlobSize, err := ic.c.rawSource.GetBlob(srcInfo) if err != nil { return types.BlobInfo{}, "", errors.Wrapf(err, "Error reading blob %s", srcInfo.Digest) } @@ -479,7 +536,7 @@ func (ic *imageCopier) copyLayer(srcInfo types.BlobInfo) (types.BlobInfo, digest return types.BlobInfo{}, "", errors.Wrap(diffIDResult.err, "Error computing layer DiffID") } logrus.Debugf("Computed DiffID %s for layer %s", diffIDResult.digest, srcInfo.Digest) - ic.cachedDiffIDs[srcInfo.Digest] = diffIDResult.digest + ic.c.cachedDiffIDs[srcInfo.Digest] = diffIDResult.digest } return blobInfo, diffIDResult.digest, nil } @@ -513,7 +570,7 @@ func (ic *imageCopier) copyLayerFromStream(srcStream io.Reader, srcInfo types.Bl return pipeWriter } } - blobInfo, err := ic.copyBlobFromStream(srcStream, srcInfo, getDiffIDRecorder, ic.canModifyManifest) // Sets err to nil on success + blobInfo, err := ic.c.copyBlobFromStream(srcStream, srcInfo, getDiffIDRecorder, ic.canModifyManifest) // Sets err to nil on success return blobInfo, diffIDChan, err // We need the defer … pipeWriter.CloseWithError() to happen HERE so that the caller can block on reading from diffIDChan } @@ -547,7 +604,7 @@ func computeDiffID(stream io.Reader, decompressor compression.DecompressorFunc) // perhaps sending a copy to an io.Writer if getOriginalLayerCopyWriter != nil, // perhaps compressing it if canCompress, // and returns a complete blobInfo of the copied blob. -func (ic *imageCopier) copyBlobFromStream(srcStream io.Reader, srcInfo types.BlobInfo, +func (c *copier) copyBlobFromStream(srcStream io.Reader, srcInfo types.BlobInfo, getOriginalLayerCopyWriter func(decompressor compression.DecompressorFunc) io.Writer, canCompress bool) (types.BlobInfo, error) { // The copying happens through a pipeline of connected io.Readers. @@ -575,7 +632,7 @@ func (ic *imageCopier) copyBlobFromStream(srcStream io.Reader, srcInfo types.Blo // === Report progress using a pb.Reader. bar := pb.New(int(srcInfo.Size)).SetUnits(pb.U_BYTES) - bar.Output = ic.reportWriter + bar.Output = c.reportWriter bar.SetMaxWidth(80) bar.ShowTimeLeft = false bar.ShowPercent = false @@ -592,7 +649,7 @@ func (ic *imageCopier) copyBlobFromStream(srcStream io.Reader, srcInfo types.Blo // === Compress the layer if it is uncompressed and compression is desired var inputInfo types.BlobInfo - if !canCompress || isCompressed || !ic.dest.ShouldCompressLayers() { + if !canCompress || isCompressed || !c.dest.ShouldCompressLayers() { logrus.Debugf("Using original blob without modification") inputInfo = srcInfo } else { @@ -609,19 +666,19 @@ func (ic *imageCopier) copyBlobFromStream(srcStream io.Reader, srcInfo types.Blo inputInfo.Size = -1 } - // === Report progress using the ic.progress channel, if required. - if ic.progress != nil && ic.progressInterval > 0 { + // === Report progress using the c.progress channel, if required. + if c.progress != nil && c.progressInterval > 0 { destStream = &progressReader{ source: destStream, - channel: ic.progress, - interval: ic.progressInterval, + channel: c.progress, + interval: c.progressInterval, artifact: srcInfo, lastTime: time.Now(), } } // === Finally, send the layer stream to dest. - uploadedInfo, err := ic.dest.PutBlob(destStream, inputInfo) + uploadedInfo, err := c.dest.PutBlob(destStream, inputInfo) if err != nil { return types.BlobInfo{}, errors.Wrap(err, "Error writing blob") } diff --git a/vendor/github.com/containers/image/copy/manifest.go b/vendor/github.com/containers/image/copy/manifest.go index e3b294dd..7e4cd10e 100644 --- a/vendor/github.com/containers/image/copy/manifest.go +++ b/vendor/github.com/containers/image/copy/manifest.go @@ -37,16 +37,20 @@ func (os *orderedSet) append(s string) { } } -// determineManifestConversion updates manifestUpdates to convert manifest to a supported MIME type, if necessary and canModifyManifest. -// Note that the conversion will only happen later, through src.UpdatedImage +// determineManifestConversion updates ic.manifestUpdates to convert manifest to a supported MIME type, if necessary and ic.canModifyManifest. +// Note that the conversion will only happen later, through ic.src.UpdatedImage // Returns the preferred manifest MIME type (whether we are converting to it or using it unmodified), // and a list of other possible alternatives, in order. -func determineManifestConversion(manifestUpdates *types.ManifestUpdateOptions, src types.Image, destSupportedManifestMIMETypes []string, canModifyManifest bool) (string, []string, error) { - _, srcType, err := src.Manifest() +func (ic *imageCopier) determineManifestConversion(destSupportedManifestMIMETypes []string, forceManifestMIMEType string) (string, []string, error) { + _, srcType, err := ic.src.Manifest() if err != nil { // This should have been cached?! return "", nil, errors.Wrap(err, "Error reading manifest") } + if forceManifestMIMEType != "" { + destSupportedManifestMIMETypes = []string{forceManifestMIMEType} + } + if len(destSupportedManifestMIMETypes) == 0 { return srcType, []string{}, nil // Anything goes; just use the original as is, do not try any conversions. } @@ -67,10 +71,10 @@ func determineManifestConversion(manifestUpdates *types.ManifestUpdateOptions, s if _, ok := supportedByDest[srcType]; ok { prioritizedTypes.append(srcType) } - if !canModifyManifest { - // We could also drop the !canModifyManifest parameter and have the caller + if !ic.canModifyManifest { + // We could also drop the !ic.canModifyManifest check and have the caller // make the choice; it is already doing that to an extent, to improve error - // messages. But it is nice to hide the “if !canModifyManifest, do no conversion” + // messages. But it is nice to hide the “if !ic.canModifyManifest, do no conversion” // special case in here; the caller can then worry (or not) only about a good UI. logrus.Debugf("We can't modify the manifest, hoping for the best...") return srcType, []string{}, nil // Take our chances - FIXME? Or should we fail without trying? @@ -94,9 +98,18 @@ func determineManifestConversion(manifestUpdates *types.ManifestUpdateOptions, s } preferredType := prioritizedTypes.list[0] if preferredType != srcType { - manifestUpdates.ManifestMIMEType = preferredType + ic.manifestUpdates.ManifestMIMEType = preferredType } else { logrus.Debugf("... will first try using the original manifest unmodified") } return preferredType, prioritizedTypes.list[1:], nil } + +// isMultiImage returns true if img is a list of images +func isMultiImage(img types.UnparsedImage) (bool, error) { + _, mt, err := img.Manifest() + if err != nil { + return false, err + } + return manifest.MIMETypeIsMultiImage(mt), nil +} diff --git a/vendor/github.com/containers/image/copy/sign.go b/vendor/github.com/containers/image/copy/sign.go index 9187d70b..91394d2b 100644 --- a/vendor/github.com/containers/image/copy/sign.go +++ b/vendor/github.com/containers/image/copy/sign.go @@ -1,17 +1,13 @@ package copy import ( - "fmt" - "io" - "github.com/containers/image/signature" "github.com/containers/image/transports" - "github.com/containers/image/types" "github.com/pkg/errors" ) -// createSignature creates a new signature of manifest at (identified by) dest using keyIdentity. -func createSignature(dest types.ImageDestination, manifest []byte, keyIdentity string, reportWriter io.Writer) ([]byte, error) { +// createSignature creates a new signature of manifest using keyIdentity. +func (c *copier) createSignature(manifest []byte, keyIdentity string) ([]byte, error) { mech, err := signature.NewGPGSigningMechanism() if err != nil { return nil, errors.Wrap(err, "Error initializing GPG") @@ -21,12 +17,12 @@ func createSignature(dest types.ImageDestination, manifest []byte, keyIdentity s return nil, errors.Wrap(err, "Signing not supported") } - dockerReference := dest.Reference().DockerReference() + dockerReference := c.dest.Reference().DockerReference() if dockerReference == nil { - return nil, errors.Errorf("Cannot determine canonical Docker reference for destination %s", transports.ImageName(dest.Reference())) + return nil, errors.Errorf("Cannot determine canonical Docker reference for destination %s", transports.ImageName(c.dest.Reference())) } - fmt.Fprintf(reportWriter, "Signing manifest\n") + c.Printf("Signing manifest\n") newSig, err := signature.SignDockerManifest(manifest, dockerReference.String(), mech, keyIdentity) if err != nil { return nil, errors.Wrap(err, "Error creating signature") diff --git a/vendor/github.com/containers/image/directory/directory_dest.go b/vendor/github.com/containers/image/directory/directory_dest.go index ea46a27e..47d59d9f 100644 --- a/vendor/github.com/containers/image/directory/directory_dest.go +++ b/vendor/github.com/containers/image/directory/directory_dest.go @@ -4,19 +4,77 @@ import ( "io" "io/ioutil" "os" + "path/filepath" "github.com/containers/image/types" "github.com/opencontainers/go-digest" "github.com/pkg/errors" + "github.com/sirupsen/logrus" ) +const version = "Directory Transport Version: 1.0\n" + +// ErrNotContainerImageDir indicates that the directory doesn't match the expected contents of a directory created +// using the 'dir' transport +var ErrNotContainerImageDir = errors.New("not a containers image directory, don't want to overwrite important data") + type dirImageDestination struct { - ref dirReference + ref dirReference + compress bool } -// newImageDestination returns an ImageDestination for writing to an existing directory. -func newImageDestination(ref dirReference) types.ImageDestination { - return &dirImageDestination{ref} +// newImageDestination returns an ImageDestination for writing to a directory. +func newImageDestination(ref dirReference, compress bool) (types.ImageDestination, error) { + d := &dirImageDestination{ref: ref, compress: compress} + + // If directory exists check if it is empty + // if not empty, check whether the contents match that of a container image directory and overwrite the contents + // if the contents don't match throw an error + dirExists, err := pathExists(d.ref.resolvedPath) + if err != nil { + return nil, errors.Wrapf(err, "error checking for path %q", d.ref.resolvedPath) + } + if dirExists { + isEmpty, err := isDirEmpty(d.ref.resolvedPath) + if err != nil { + return nil, err + } + + if !isEmpty { + versionExists, err := pathExists(d.ref.versionPath()) + if err != nil { + return nil, errors.Wrapf(err, "error checking if path exists %q", d.ref.versionPath()) + } + if versionExists { + contents, err := ioutil.ReadFile(d.ref.versionPath()) + if err != nil { + return nil, err + } + // check if contents of version file is what we expect it to be + if string(contents) != version { + return nil, ErrNotContainerImageDir + } + } else { + return nil, ErrNotContainerImageDir + } + // delete directory contents so that only one image is in the directory at a time + if err = removeDirContents(d.ref.resolvedPath); err != nil { + return nil, errors.Wrapf(err, "error erasing contents in %q", d.ref.resolvedPath) + } + logrus.Debugf("overwriting existing container image directory %q", d.ref.resolvedPath) + } + } else { + // create directory if it doesn't exist + if err := os.MkdirAll(d.ref.resolvedPath, 0755); err != nil { + return nil, errors.Wrapf(err, "unable to create directory %q", d.ref.resolvedPath) + } + } + // create version file + err = ioutil.WriteFile(d.ref.versionPath(), []byte(version), 0755) + if err != nil { + return nil, errors.Wrapf(err, "error creating version file %q", d.ref.versionPath()) + } + return d, nil } // Reference returns the reference used to set up this destination. Note that this should directly correspond to user's intent, @@ -42,7 +100,7 @@ func (d *dirImageDestination) SupportsSignatures() error { // ShouldCompressLayers returns true iff it is desirable to compress layer blobs written to this destination. func (d *dirImageDestination) ShouldCompressLayers() bool { - return false + return d.compress } // AcceptsForeignLayerURLs returns false iff foreign layers in manifest should be actually @@ -147,3 +205,39 @@ func (d *dirImageDestination) PutSignatures(signatures [][]byte) error { func (d *dirImageDestination) Commit() error { return nil } + +// returns true if path exists +func pathExists(path string) (bool, error) { + _, err := os.Stat(path) + if err == nil { + return true, nil + } + if err != nil && os.IsNotExist(err) { + return false, nil + } + return false, err +} + +// returns true if directory is empty +func isDirEmpty(path string) (bool, error) { + files, err := ioutil.ReadDir(path) + if err != nil { + return false, err + } + return len(files) == 0, nil +} + +// deletes the contents of a directory +func removeDirContents(path string) error { + files, err := ioutil.ReadDir(path) + if err != nil { + return err + } + + for _, file := range files { + if err := os.RemoveAll(filepath.Join(path, file.Name())); err != nil { + return err + } + } + return nil +} diff --git a/vendor/github.com/containers/image/directory/directory_src.go b/vendor/github.com/containers/image/directory/directory_src.go index fddc1c52..0a8acf6b 100644 --- a/vendor/github.com/containers/image/directory/directory_src.go +++ b/vendor/github.com/containers/image/directory/directory_src.go @@ -35,7 +35,12 @@ func (s *dirImageSource) Close() error { // GetManifest returns the image's manifest along with its MIME type (which may be empty when it can't be determined but the manifest is available). // It may use a remote (= slow) service. -func (s *dirImageSource) GetManifest() ([]byte, string, error) { +// If instanceDigest is not nil, it contains a digest of the specific manifest instance to retrieve (when the primary manifest is a manifest list); +// this never happens if the primary manifest is not a manifest list (e.g. if the source never returns manifest lists). +func (s *dirImageSource) GetManifest(instanceDigest *digest.Digest) ([]byte, string, error) { + if instanceDigest != nil { + return nil, "", errors.Errorf(`Getting target manifest not supported by "dir:"`) + } m, err := ioutil.ReadFile(s.ref.manifestPath()) if err != nil { return nil, "", err @@ -43,10 +48,6 @@ func (s *dirImageSource) GetManifest() ([]byte, string, error) { return m, manifest.GuessMIMEType(m), err } -func (s *dirImageSource) GetTargetManifest(digest digest.Digest) ([]byte, string, error) { - return nil, "", errors.Errorf(`Getting target manifest not supported by "dir:"`) -} - // GetBlob returns a stream for the specified blob, and the blob’s size (or -1 if unknown). func (s *dirImageSource) GetBlob(info types.BlobInfo) (io.ReadCloser, int64, error) { r, err := os.Open(s.ref.layerPath(info.Digest)) @@ -60,7 +61,14 @@ func (s *dirImageSource) GetBlob(info types.BlobInfo) (io.ReadCloser, int64, err return r, fi.Size(), nil } -func (s *dirImageSource) GetSignatures(ctx context.Context) ([][]byte, error) { +// GetSignatures returns the image's signatures. It may use a remote (= slow) service. +// If instanceDigest is not nil, it contains a digest of the specific manifest instance to retrieve signatures for +// (when the primary manifest is a manifest list); this never happens if the primary manifest is not a manifest list +// (e.g. if the source never returns manifest lists). +func (s *dirImageSource) GetSignatures(ctx context.Context, instanceDigest *digest.Digest) ([][]byte, error) { + if instanceDigest != nil { + return nil, errors.Errorf(`Manifests lists are not supported by "dir:"`) + } signatures := [][]byte{} for i := 0; ; i++ { signature, err := ioutil.ReadFile(s.ref.signaturePath(i)) @@ -74,3 +82,8 @@ func (s *dirImageSource) GetSignatures(ctx context.Context) ([][]byte, error) { } return signatures, nil } + +// LayerInfosForCopy() returns updated layer info that should be used when copying, in preference to values in the manifest, if specified. +func (s *dirImageSource) LayerInfosForCopy() []types.BlobInfo { + return nil +} diff --git a/vendor/github.com/containers/image/directory/directory_transport.go b/vendor/github.com/containers/image/directory/directory_transport.go index b9ce01a2..c3875308 100644 --- a/vendor/github.com/containers/image/directory/directory_transport.go +++ b/vendor/github.com/containers/image/directory/directory_transport.go @@ -134,13 +134,14 @@ func (ref dirReference) PolicyConfigurationNamespaces() []string { return res } -// NewImage returns a types.Image for this reference, possibly specialized for this ImageTransport. -// The caller must call .Close() on the returned Image. +// NewImage returns a types.ImageCloser for this reference, possibly specialized for this ImageTransport. +// The caller must call .Close() on the returned ImageCloser. // NOTE: If any kind of signature verification should happen, build an UnparsedImage from the value returned by NewImageSource, // verify that UnparsedImage, and convert it into a real Image via image.FromUnparsedImage. -func (ref dirReference) NewImage(ctx *types.SystemContext) (types.Image, error) { +// WARNING: This may not do the right thing for a manifest list, see image.FromSource for details. +func (ref dirReference) NewImage(ctx *types.SystemContext) (types.ImageCloser, error) { src := newImageSource(ref) - return image.FromSource(src) + return image.FromSource(ctx, src) } // NewImageSource returns a types.ImageSource for this reference. @@ -152,7 +153,11 @@ func (ref dirReference) NewImageSource(ctx *types.SystemContext) (types.ImageSou // NewImageDestination returns a types.ImageDestination for this reference. // The caller must call .Close() on the returned ImageDestination. func (ref dirReference) NewImageDestination(ctx *types.SystemContext) (types.ImageDestination, error) { - return newImageDestination(ref), nil + compress := false + if ctx != nil { + compress = ctx.DirForceCompress + } + return newImageDestination(ref, compress) } // DeleteImage deletes the named image from the registry, if supported. @@ -175,3 +180,8 @@ func (ref dirReference) layerPath(digest digest.Digest) string { func (ref dirReference) signaturePath(index int) string { return filepath.Join(ref.path, fmt.Sprintf("signature-%d", index+1)) } + +// versionPath returns a path for the version file within a directory using our conventions. +func (ref dirReference) versionPath() string { + return filepath.Join(ref.path, "version") +} diff --git a/vendor/github.com/containers/image/docker/archive/src.go b/vendor/github.com/containers/image/docker/archive/src.go index aebcaa82..b2ffd965 100644 --- a/vendor/github.com/containers/image/docker/archive/src.go +++ b/vendor/github.com/containers/image/docker/archive/src.go @@ -34,3 +34,8 @@ func (s *archiveImageSource) Reference() types.ImageReference { func (s *archiveImageSource) Close() error { return nil } + +// LayerInfosForCopy() returns updated layer info that should be used when reading, in preference to values in the manifest, if specified. +func (s *archiveImageSource) LayerInfosForCopy() []types.BlobInfo { + return nil +} diff --git a/vendor/github.com/containers/image/docker/archive/transport.go b/vendor/github.com/containers/image/docker/archive/transport.go index f38d4ace..047df73d 100644 --- a/vendor/github.com/containers/image/docker/archive/transport.go +++ b/vendor/github.com/containers/image/docker/archive/transport.go @@ -125,13 +125,14 @@ func (ref archiveReference) PolicyConfigurationNamespaces() []string { return []string{} } -// NewImage returns a types.Image for this reference, possibly specialized for this ImageTransport. -// The caller must call .Close() on the returned Image. +// NewImage returns a types.ImageCloser for this reference, possibly specialized for this ImageTransport. +// The caller must call .Close() on the returned ImageCloser. // NOTE: If any kind of signature verification should happen, build an UnparsedImage from the value returned by NewImageSource, // verify that UnparsedImage, and convert it into a real Image via image.FromUnparsedImage. -func (ref archiveReference) NewImage(ctx *types.SystemContext) (types.Image, error) { +// WARNING: This may not do the right thing for a manifest list, see image.FromSource for details. +func (ref archiveReference) NewImage(ctx *types.SystemContext) (types.ImageCloser, error) { src := newImageSource(ctx, ref) - return ctrImage.FromSource(src) + return ctrImage.FromSource(ctx, src) } // NewImageSource returns a types.ImageSource for this reference. diff --git a/vendor/github.com/containers/image/docker/daemon/client.go b/vendor/github.com/containers/image/docker/daemon/client.go new file mode 100644 index 00000000..82fab4b1 --- /dev/null +++ b/vendor/github.com/containers/image/docker/daemon/client.go @@ -0,0 +1,69 @@ +package daemon + +import ( + "net/http" + "path/filepath" + + "github.com/containers/image/types" + dockerclient "github.com/docker/docker/client" + "github.com/docker/go-connections/tlsconfig" +) + +const ( + // The default API version to be used in case none is explicitly specified + defaultAPIVersion = "1.22" +) + +// NewDockerClient initializes a new API client based on the passed SystemContext. +func newDockerClient(ctx *types.SystemContext) (*dockerclient.Client, error) { + host := dockerclient.DefaultDockerHost + if ctx != nil && ctx.DockerDaemonHost != "" { + host = ctx.DockerDaemonHost + } + + // Sadly, unix:// sockets don't work transparently with dockerclient.NewClient. + // They work fine with a nil httpClient; with a non-nil httpClient, the transport’s + // TLSClientConfig must be nil (or the client will try using HTTPS over the PF_UNIX socket + // regardless of the values in the *tls.Config), and we would have to call sockets.ConfigureTransport. + // + // We don't really want to configure anything for unix:// sockets, so just pass a nil *http.Client. + proto, _, _, err := dockerclient.ParseHost(host) + if err != nil { + return nil, err + } + var httpClient *http.Client + if proto != "unix" { + hc, err := tlsConfig(ctx) + if err != nil { + return nil, err + } + httpClient = hc + } + + return dockerclient.NewClient(host, defaultAPIVersion, httpClient, nil) +} + +func tlsConfig(ctx *types.SystemContext) (*http.Client, error) { + options := tlsconfig.Options{} + if ctx != nil && ctx.DockerDaemonInsecureSkipTLSVerify { + options.InsecureSkipVerify = true + } + + if ctx != nil && ctx.DockerDaemonCertPath != "" { + options.CAFile = filepath.Join(ctx.DockerDaemonCertPath, "ca.pem") + options.CertFile = filepath.Join(ctx.DockerDaemonCertPath, "cert.pem") + options.KeyFile = filepath.Join(ctx.DockerDaemonCertPath, "key.pem") + } + + tlsc, err := tlsconfig.Client(options) + if err != nil { + return nil, err + } + + return &http.Client{ + Transport: &http.Transport{ + TLSClientConfig: tlsc, + }, + CheckRedirect: dockerclient.CheckRedirect, + }, nil +} diff --git a/vendor/github.com/containers/image/docker/daemon/daemon_dest.go b/vendor/github.com/containers/image/docker/daemon/daemon_dest.go index 559e5c71..f73ac233 100644 --- a/vendor/github.com/containers/image/docker/daemon/daemon_dest.go +++ b/vendor/github.com/containers/image/docker/daemon/daemon_dest.go @@ -14,6 +14,7 @@ import ( type daemonImageDestination struct { ref daemonReference + mustMatchRuntimeOS bool *tarfile.Destination // Implements most of types.ImageDestination // For talking to imageLoadGoroutine goroutineCancel context.CancelFunc @@ -24,7 +25,7 @@ type daemonImageDestination struct { } // newImageDestination returns a types.ImageDestination for the specified image reference. -func newImageDestination(systemCtx *types.SystemContext, ref daemonReference) (types.ImageDestination, error) { +func newImageDestination(ctx *types.SystemContext, ref daemonReference) (types.ImageDestination, error) { if ref.ref == nil { return nil, errors.Errorf("Invalid destination docker-daemon:%s: a destination must be a name:tag", ref.StringWithinTransport()) } @@ -33,7 +34,12 @@ func newImageDestination(systemCtx *types.SystemContext, ref daemonReference) (t return nil, errors.Errorf("Invalid destination docker-daemon:%s: a destination must be a name:tag", ref.StringWithinTransport()) } - c, err := client.NewClient(client.DefaultDockerHost, "1.22", nil, nil) // FIXME: overridable host + var mustMatchRuntimeOS = true + if ctx != nil && ctx.DockerDaemonHost != client.DefaultDockerHost { + mustMatchRuntimeOS = false + } + + c, err := newDockerClient(ctx) if err != nil { return nil, errors.Wrap(err, "Error initializing docker engine client") } @@ -42,16 +48,17 @@ func newImageDestination(systemCtx *types.SystemContext, ref daemonReference) (t // Commit() may never be called, so we may never read from this channel; so, make this buffered to allow imageLoadGoroutine to write status and terminate even if we never read it. statusChannel := make(chan error, 1) - ctx, goroutineCancel := context.WithCancel(context.Background()) - go imageLoadGoroutine(ctx, c, reader, statusChannel) + goroutineContext, goroutineCancel := context.WithCancel(context.Background()) + go imageLoadGoroutine(goroutineContext, c, reader, statusChannel) return &daemonImageDestination{ - ref: ref, - Destination: tarfile.NewDestination(writer, namedTaggedRef), - goroutineCancel: goroutineCancel, - statusChannel: statusChannel, - writer: writer, - committed: false, + ref: ref, + mustMatchRuntimeOS: mustMatchRuntimeOS, + Destination: tarfile.NewDestination(writer, namedTaggedRef), + goroutineCancel: goroutineCancel, + statusChannel: statusChannel, + writer: writer, + committed: false, }, nil } @@ -80,7 +87,7 @@ func imageLoadGoroutine(ctx context.Context, c *client.Client, reader *io.PipeRe // MustMatchRuntimeOS returns true iff the destination can store only images targeted for the current runtime OS. False otherwise. func (d *daemonImageDestination) MustMatchRuntimeOS() bool { - return true + return d.mustMatchRuntimeOS } // Close removes resources associated with an initialized ImageDestination, if any. diff --git a/vendor/github.com/containers/image/docker/daemon/daemon_src.go b/vendor/github.com/containers/image/docker/daemon/daemon_src.go index 644dbeec..5cf7679b 100644 --- a/vendor/github.com/containers/image/docker/daemon/daemon_src.go +++ b/vendor/github.com/containers/image/docker/daemon/daemon_src.go @@ -6,14 +6,12 @@ import ( "os" "github.com/containers/image/docker/tarfile" + "github.com/containers/image/internal/tmpdir" "github.com/containers/image/types" - "github.com/docker/docker/client" "github.com/pkg/errors" "golang.org/x/net/context" ) -const temporaryDirectoryForBigFiles = "/var/tmp" // Do not use the system default of os.TempDir(), usually /tmp, because with systemd it could be a tmpfs. - type daemonImageSource struct { ref daemonReference *tarfile.Source // Implements most of types.ImageSource @@ -35,7 +33,7 @@ type layerInfo struct { // is the config, and that the following len(RootFS) files are the layers, but that feels // way too brittle.) func newImageSource(ctx *types.SystemContext, ref daemonReference) (types.ImageSource, error) { - c, err := client.NewClient(client.DefaultDockerHost, "1.22", nil, nil) // FIXME: overridable host + c, err := newDockerClient(ctx) if err != nil { return nil, errors.Wrap(err, "Error initializing docker engine client") } @@ -48,7 +46,7 @@ func newImageSource(ctx *types.SystemContext, ref daemonReference) (types.ImageS defer inputStream.Close() // FIXME: use SystemContext here. - tarCopyFile, err := ioutil.TempFile(temporaryDirectoryForBigFiles, "docker-daemon-tar") + tarCopyFile, err := ioutil.TempFile(tmpdir.TemporaryDirectoryForBigFiles(), "docker-daemon-tar") if err != nil { return nil, err } @@ -83,3 +81,8 @@ func (s *daemonImageSource) Reference() types.ImageReference { func (s *daemonImageSource) Close() error { return os.Remove(s.tarCopyPath) } + +// LayerInfosForCopy() returns updated layer info that should be used when reading, in preference to values in the manifest, if specified. +func (s *daemonImageSource) LayerInfosForCopy() []types.BlobInfo { + return nil +} diff --git a/vendor/github.com/containers/image/docker/daemon/daemon_transport.go b/vendor/github.com/containers/image/docker/daemon/daemon_transport.go index 41be1b2d..8ad6b521 100644 --- a/vendor/github.com/containers/image/docker/daemon/daemon_transport.go +++ b/vendor/github.com/containers/image/docker/daemon/daemon_transport.go @@ -151,14 +151,17 @@ func (ref daemonReference) PolicyConfigurationNamespaces() []string { return []string{} } -// NewImage returns a types.Image for this reference. -// The caller must call .Close() on the returned Image. -func (ref daemonReference) NewImage(ctx *types.SystemContext) (types.Image, error) { +// NewImage returns a types.ImageCloser for this reference, possibly specialized for this ImageTransport. +// The caller must call .Close() on the returned ImageCloser. +// NOTE: If any kind of signature verification should happen, build an UnparsedImage from the value returned by NewImageSource, +// verify that UnparsedImage, and convert it into a real Image via image.FromUnparsedImage. +// WARNING: This may not do the right thing for a manifest list, see image.FromSource for details. +func (ref daemonReference) NewImage(ctx *types.SystemContext) (types.ImageCloser, error) { src, err := newImageSource(ctx, ref) if err != nil { return nil, err } - return image.FromSource(src) + return image.FromSource(ctx, src) } // NewImageSource returns a types.ImageSource for this reference. diff --git a/vendor/github.com/containers/image/docker/docker_client.go b/vendor/github.com/containers/image/docker/docker_client.go index 24b82d6f..217e9dcb 100644 --- a/vendor/github.com/containers/image/docker/docker_client.go +++ b/vendor/github.com/containers/image/docker/docker_client.go @@ -8,7 +8,6 @@ import ( "io" "io/ioutil" "net/http" - "os" "path/filepath" "strings" "time" @@ -125,69 +124,6 @@ func dockerCertDir(ctx *types.SystemContext, hostPort string) string { return filepath.Join(hostCertDir, hostPort) } -func setupCertificates(dir string, tlsc *tls.Config) error { - logrus.Debugf("Looking for TLS certificates and private keys in %s", dir) - fs, err := ioutil.ReadDir(dir) - if err != nil { - if os.IsNotExist(err) { - return nil - } - if os.IsPermission(err) { - logrus.Debugf("Skipping scan of %s due to permission error: %v", dir, err) - return nil - } - return err - } - - for _, f := range fs { - fullPath := filepath.Join(dir, f.Name()) - if strings.HasSuffix(f.Name(), ".crt") { - systemPool, err := tlsconfig.SystemCertPool() - if err != nil { - return errors.Wrap(err, "unable to get system cert pool") - } - tlsc.RootCAs = systemPool - logrus.Debugf(" crt: %s", fullPath) - data, err := ioutil.ReadFile(fullPath) - if err != nil { - return err - } - tlsc.RootCAs.AppendCertsFromPEM(data) - } - if strings.HasSuffix(f.Name(), ".cert") { - certName := f.Name() - keyName := certName[:len(certName)-5] + ".key" - logrus.Debugf(" cert: %s", fullPath) - if !hasFile(fs, keyName) { - return errors.Errorf("missing key %s for client certificate %s. Note that CA certificates should use the extension .crt", keyName, certName) - } - cert, err := tls.LoadX509KeyPair(filepath.Join(dir, certName), filepath.Join(dir, keyName)) - if err != nil { - return err - } - tlsc.Certificates = append(tlsc.Certificates, cert) - } - if strings.HasSuffix(f.Name(), ".key") { - keyName := f.Name() - certName := keyName[:len(keyName)-4] + ".cert" - logrus.Debugf(" key: %s", fullPath) - if !hasFile(fs, certName) { - return errors.Errorf("missing client certificate %s for key %s", certName, keyName) - } - } - } - return nil -} - -func hasFile(files []os.FileInfo, name string) bool { - for _, f := range files { - if f.Name() == name { - return true - } - } - return false -} - // newDockerClientFromRef returns a new dockerClient instance for refHostname (a host a specified in the Docker image reference, not canonicalized to dockerRegistry) // “write” specifies whether the client will be used for "write" access (in particular passed to lookaside.go:toplevelFromSection) func newDockerClientFromRef(ctx *types.SystemContext, ref dockerReference, write bool, actions string) (*dockerClient, error) { diff --git a/vendor/github.com/containers/image/docker/docker_image.go b/vendor/github.com/containers/image/docker/docker_image.go index 8be35b73..2148ed8b 100644 --- a/vendor/github.com/containers/image/docker/docker_image.go +++ b/vendor/github.com/containers/image/docker/docker_image.go @@ -12,26 +12,26 @@ import ( "github.com/pkg/errors" ) -// Image is a Docker-specific implementation of types.Image with a few extra methods +// Image is a Docker-specific implementation of types.ImageCloser with a few extra methods // which are specific to Docker. type Image struct { - types.Image + types.ImageCloser src *dockerImageSource } // newImage returns a new Image interface type after setting up // a client to the registry hosting the given image. // The caller must call .Close() on the returned Image. -func newImage(ctx *types.SystemContext, ref dockerReference) (types.Image, error) { +func newImage(ctx *types.SystemContext, ref dockerReference) (types.ImageCloser, error) { s, err := newImageSource(ctx, ref) if err != nil { return nil, err } - img, err := image.FromSource(s) + img, err := image.FromSource(ctx, s) if err != nil { return nil, err } - return &Image{Image: img, src: s}, nil + return &Image{ImageCloser: img, src: s}, nil } // SourceRefFullName returns a fully expanded name for the repository this image is in. diff --git a/vendor/github.com/containers/image/docker/docker_image_dest.go b/vendor/github.com/containers/image/docker/docker_image_dest.go index 32d5a18b..79c38622 100644 --- a/vendor/github.com/containers/image/docker/docker_image_dest.go +++ b/vendor/github.com/containers/image/docker/docker_image_dest.go @@ -236,7 +236,7 @@ func (d *dockerImageDestination) PutManifest(m []byte) error { return err } defer res.Body.Close() - if res.StatusCode != http.StatusCreated { + if !successStatus(res.StatusCode) { err = errors.Wrapf(client.HandleErrorResponse(res), "Error uploading manifest to %s", path) if isManifestInvalidError(errors.Cause(err)) { err = types.ManifestTypeRejectedError{Err: err} @@ -246,6 +246,12 @@ func (d *dockerImageDestination) PutManifest(m []byte) error { return nil } +// successStatus returns true if the argument is a successful HTTP response +// code (in the range 200 - 399 inclusive). +func successStatus(status int) bool { + return status >= 200 && status <= 399 +} + // isManifestInvalidError returns true iff err from client.HandleErrorReponse is a “manifest invalid” error. func isManifestInvalidError(err error) bool { errors, ok := err.(errcode.Errors) diff --git a/vendor/github.com/containers/image/docker/docker_image_src.go b/vendor/github.com/containers/image/docker/docker_image_src.go index 232c3cf9..63bfe8aa 100644 --- a/vendor/github.com/containers/image/docker/docker_image_src.go +++ b/vendor/github.com/containers/image/docker/docker_image_src.go @@ -52,6 +52,11 @@ func (s *dockerImageSource) Close() error { return nil } +// LayerInfosForCopy() returns updated layer info that should be used when reading, in preference to values in the manifest, if specified. +func (s *dockerImageSource) LayerInfosForCopy() []types.BlobInfo { + return nil +} + // simplifyContentType drops parameters from a HTTP media type (see https://tools.ietf.org/html/rfc7231#section-3.1.1.1) // Alternatively, an empty string is returned unchanged, and invalid values are "simplified" to an empty string. func simplifyContentType(contentType string) string { @@ -67,7 +72,12 @@ func simplifyContentType(contentType string) string { // GetManifest returns the image's manifest along with its MIME type (which may be empty when it can't be determined but the manifest is available). // It may use a remote (= slow) service. -func (s *dockerImageSource) GetManifest() ([]byte, string, error) { +// If instanceDigest is not nil, it contains a digest of the specific manifest instance to retrieve (when the primary manifest is a manifest list); +// this never happens if the primary manifest is not a manifest list (e.g. if the source never returns manifest lists). +func (s *dockerImageSource) GetManifest(instanceDigest *digest.Digest) ([]byte, string, error) { + if instanceDigest != nil { + return s.fetchManifest(context.TODO(), instanceDigest.String()) + } err := s.ensureManifestIsLoaded(context.TODO()) if err != nil { return nil, "", err @@ -94,18 +104,12 @@ func (s *dockerImageSource) fetchManifest(ctx context.Context, tagOrDigest strin return manblob, simplifyContentType(res.Header.Get("Content-Type")), nil } -// GetTargetManifest returns an image's manifest given a digest. -// This is mainly used to retrieve a single image's manifest out of a manifest list. -func (s *dockerImageSource) GetTargetManifest(digest digest.Digest) ([]byte, string, error) { - return s.fetchManifest(context.TODO(), digest.String()) -} - // ensureManifestIsLoaded sets s.cachedManifest and s.cachedManifestMIMEType // // ImageSource implementations are not required or expected to do any caching, // but because our signatures are “attached” to the manifest digest, -// we need to ensure that the digest of the manifest returned by GetManifest -// and used by GetSignatures are consistent, otherwise we would get spurious +// we need to ensure that the digest of the manifest returned by GetManifest(nil) +// and used by GetSignatures(ctx, nil) are consistent, otherwise we would get spurious // signature verification failures when pulling while a tag is being updated. func (s *dockerImageSource) ensureManifestIsLoaded(ctx context.Context) error { if s.cachedManifest != nil { @@ -176,22 +180,30 @@ func (s *dockerImageSource) GetBlob(info types.BlobInfo) (io.ReadCloser, int64, return res.Body, getBlobSize(res), nil } -func (s *dockerImageSource) GetSignatures(ctx context.Context) ([][]byte, error) { +// GetSignatures returns the image's signatures. It may use a remote (= slow) service. +// If instanceDigest is not nil, it contains a digest of the specific manifest instance to retrieve signatures for +// (when the primary manifest is a manifest list); this never happens if the primary manifest is not a manifest list +// (e.g. if the source never returns manifest lists). +func (s *dockerImageSource) GetSignatures(ctx context.Context, instanceDigest *digest.Digest) ([][]byte, error) { if err := s.c.detectProperties(ctx); err != nil { return nil, err } switch { case s.c.signatureBase != nil: - return s.getSignaturesFromLookaside(ctx) + return s.getSignaturesFromLookaside(ctx, instanceDigest) case s.c.supportsSignatures: - return s.getSignaturesFromAPIExtension(ctx) + return s.getSignaturesFromAPIExtension(ctx, instanceDigest) default: return [][]byte{}, nil } } -// manifestDigest returns a digest of the manifest, either from the supplied reference or from a fetched manifest. -func (s *dockerImageSource) manifestDigest(ctx context.Context) (digest.Digest, error) { +// manifestDigest returns a digest of the manifest, from instanceDigest if non-nil; or from the supplied reference, +// or finally, from a fetched manifest. +func (s *dockerImageSource) manifestDigest(ctx context.Context, instanceDigest *digest.Digest) (digest.Digest, error) { + if instanceDigest != nil { + return *instanceDigest, nil + } if digested, ok := s.ref.ref.(reference.Digested); ok { d := digested.Digest() if d.Algorithm() == digest.Canonical { @@ -206,8 +218,8 @@ func (s *dockerImageSource) manifestDigest(ctx context.Context) (digest.Digest, // getSignaturesFromLookaside implements GetSignatures() from the lookaside location configured in s.c.signatureBase, // which is not nil. -func (s *dockerImageSource) getSignaturesFromLookaside(ctx context.Context) ([][]byte, error) { - manifestDigest, err := s.manifestDigest(ctx) +func (s *dockerImageSource) getSignaturesFromLookaside(ctx context.Context, instanceDigest *digest.Digest) ([][]byte, error) { + manifestDigest, err := s.manifestDigest(ctx, instanceDigest) if err != nil { return nil, err } @@ -276,8 +288,8 @@ func (s *dockerImageSource) getOneSignature(ctx context.Context, url *url.URL) ( } // getSignaturesFromAPIExtension implements GetSignatures() using the X-Registry-Supports-Signatures API extension. -func (s *dockerImageSource) getSignaturesFromAPIExtension(ctx context.Context) ([][]byte, error) { - manifestDigest, err := s.manifestDigest(ctx) +func (s *dockerImageSource) getSignaturesFromAPIExtension(ctx context.Context, instanceDigest *digest.Digest) ([][]byte, error) { + manifestDigest, err := s.manifestDigest(ctx, instanceDigest) if err != nil { return nil, err } diff --git a/vendor/github.com/containers/image/docker/docker_transport.go b/vendor/github.com/containers/image/docker/docker_transport.go index 1d67cc4f..cc0aa298 100644 --- a/vendor/github.com/containers/image/docker/docker_transport.go +++ b/vendor/github.com/containers/image/docker/docker_transport.go @@ -122,11 +122,12 @@ func (ref dockerReference) PolicyConfigurationNamespaces() []string { return policyconfiguration.DockerReferenceNamespaces(ref.ref) } -// NewImage returns a types.Image for this reference, possibly specialized for this ImageTransport. -// The caller must call .Close() on the returned Image. +// NewImage returns a types.ImageCloser for this reference, possibly specialized for this ImageTransport. +// The caller must call .Close() on the returned ImageCloser. // NOTE: If any kind of signature verification should happen, build an UnparsedImage from the value returned by NewImageSource, // verify that UnparsedImage, and convert it into a real Image via image.FromUnparsedImage. -func (ref dockerReference) NewImage(ctx *types.SystemContext) (types.Image, error) { +// WARNING: This may not do the right thing for a manifest list, see image.FromSource for details. +func (ref dockerReference) NewImage(ctx *types.SystemContext) (types.ImageCloser, error) { return newImage(ctx, ref) } diff --git a/vendor/github.com/containers/image/docker/tarfile/dest.go b/vendor/github.com/containers/image/docker/tarfile/dest.go index 72c85c70..eb11ca86 100644 --- a/vendor/github.com/containers/image/docker/tarfile/dest.go +++ b/vendor/github.com/containers/image/docker/tarfile/dest.go @@ -11,6 +11,7 @@ import ( "time" "github.com/containers/image/docker/reference" + "github.com/containers/image/internal/tmpdir" "github.com/containers/image/manifest" "github.com/containers/image/types" "github.com/opencontainers/go-digest" @@ -18,8 +19,6 @@ import ( "github.com/sirupsen/logrus" ) -const temporaryDirectoryForBigFiles = "/var/tmp" // Do not use the system default of os.TempDir(), usually /tmp, because with systemd it could be a tmpfs. - // Destination is a partial implementation of types.ImageDestination for writing to an io.Writer. type Destination struct { writer io.Writer @@ -107,7 +106,7 @@ func (d *Destination) PutBlob(stream io.Reader, inputInfo types.BlobInfo) (types if inputInfo.Size == -1 { // Ouch, we need to stream the blob into a temporary file just to determine the size. logrus.Debugf("docker tarfile: input with unknown size, streaming to disk first ...") - streamCopy, err := ioutil.TempFile(temporaryDirectoryForBigFiles, "docker-tarfile-blob") + streamCopy, err := ioutil.TempFile(tmpdir.TemporaryDirectoryForBigFiles(), "docker-tarfile-blob") if err != nil { return types.BlobInfo{}, err } @@ -168,7 +167,7 @@ func (d *Destination) ReapplyBlob(info types.BlobInfo) (types.BlobInfo, error) { func (d *Destination) PutManifest(m []byte) error { // We do not bother with types.ManifestTypeRejectedError; our .SupportedManifestMIMETypes() above is already providing only one alternative, // so the caller trying a different manifest kind would be pointless. - var man schema2Manifest + var man manifest.Schema2 if err := json.Unmarshal(m, &man); err != nil { return errors.Wrap(err, "Error parsing manifest") } @@ -177,12 +176,12 @@ func (d *Destination) PutManifest(m []byte) error { } layerPaths := []string{} - for _, l := range man.Layers { + for _, l := range man.LayersDescriptors { layerPaths = append(layerPaths, l.Digest.String()) } items := []ManifestItem{{ - Config: man.Config.Digest.String(), + Config: man.ConfigDescriptor.Digest.String(), RepoTags: []string{d.repoTag}, Layers: layerPaths, Parent: "", diff --git a/vendor/github.com/containers/image/docker/tarfile/src.go b/vendor/github.com/containers/image/docker/tarfile/src.go index f77cb713..a18e2105 100644 --- a/vendor/github.com/containers/image/docker/tarfile/src.go +++ b/vendor/github.com/containers/image/docker/tarfile/src.go @@ -24,8 +24,8 @@ type Source struct { tarManifest *ManifestItem // nil if not available yet. configBytes []byte configDigest digest.Digest - orderedDiffIDList []diffID - knownLayers map[diffID]*layerInfo + orderedDiffIDList []digest.Digest + knownLayers map[digest.Digest]*layerInfo // Other state generatedManifest []byte // Private cache for GetManifest(), nil if not set yet. } @@ -156,7 +156,7 @@ func (s *Source) ensureCachedDataIsPresent() error { if err != nil { return err } - var parsedConfig image // Most fields ommitted, we only care about layer DiffIDs. + var parsedConfig manifest.Schema2Image // There's a lot of info there, but we only really care about layer DiffIDs. if err := json.Unmarshal(configBytes, &parsedConfig); err != nil { return errors.Wrapf(err, "Error decoding tar config %s", tarManifest[0].Config) } @@ -194,12 +194,12 @@ func (s *Source) LoadTarManifest() ([]ManifestItem, error) { return s.loadTarManifest() } -func (s *Source) prepareLayerData(tarManifest *ManifestItem, parsedConfig *image) (map[diffID]*layerInfo, error) { +func (s *Source) prepareLayerData(tarManifest *ManifestItem, parsedConfig *manifest.Schema2Image) (map[digest.Digest]*layerInfo, error) { // Collect layer data available in manifest and config. if len(tarManifest.Layers) != len(parsedConfig.RootFS.DiffIDs) { return nil, errors.Errorf("Inconsistent layer count: %d in manifest, %d in config", len(tarManifest.Layers), len(parsedConfig.RootFS.DiffIDs)) } - knownLayers := map[diffID]*layerInfo{} + knownLayers := map[digest.Digest]*layerInfo{} unknownLayerSizes := map[string]*layerInfo{} // Points into knownLayers, a "to do list" of items with unknown sizes. for i, diffID := range parsedConfig.RootFS.DiffIDs { if _, ok := knownLayers[diffID]; ok { @@ -249,28 +249,34 @@ func (s *Source) prepareLayerData(tarManifest *ManifestItem, parsedConfig *image // GetManifest returns the image's manifest along with its MIME type (which may be empty when it can't be determined but the manifest is available). // It may use a remote (= slow) service. -func (s *Source) GetManifest() ([]byte, string, error) { +// If instanceDigest is not nil, it contains a digest of the specific manifest instance to retrieve (when the primary manifest is a manifest list); +// this never happens if the primary manifest is not a manifest list (e.g. if the source never returns manifest lists). +func (s *Source) GetManifest(instanceDigest *digest.Digest) ([]byte, string, error) { + if instanceDigest != nil { + // How did we even get here? GetManifest(nil) has returned a manifest.DockerV2Schema2MediaType. + return nil, "", errors.Errorf(`Manifest lists are not supported by "docker-daemon:"`) + } if s.generatedManifest == nil { if err := s.ensureCachedDataIsPresent(); err != nil { return nil, "", err } - m := schema2Manifest{ + m := manifest.Schema2{ SchemaVersion: 2, MediaType: manifest.DockerV2Schema2MediaType, - Config: distributionDescriptor{ + ConfigDescriptor: manifest.Schema2Descriptor{ MediaType: manifest.DockerV2Schema2ConfigMediaType, Size: int64(len(s.configBytes)), Digest: s.configDigest, }, - Layers: []distributionDescriptor{}, + LayersDescriptors: []manifest.Schema2Descriptor{}, } for _, diffID := range s.orderedDiffIDList { li, ok := s.knownLayers[diffID] if !ok { return nil, "", errors.Errorf("Internal inconsistency: Information about layer %s missing", diffID) } - m.Layers = append(m.Layers, distributionDescriptor{ - Digest: digest.Digest(diffID), // diffID is a digest of the uncompressed tarball + m.LayersDescriptors = append(m.LayersDescriptors, manifest.Schema2Descriptor{ + Digest: diffID, // diffID is a digest of the uncompressed tarball MediaType: manifest.DockerV2Schema2LayerMediaType, Size: li.size, }) @@ -284,13 +290,6 @@ func (s *Source) GetManifest() ([]byte, string, error) { return s.generatedManifest, manifest.DockerV2Schema2MediaType, nil } -// GetTargetManifest returns an image's manifest given a digest. This is mainly used to retrieve a single image's manifest -// out of a manifest list. -func (s *Source) GetTargetManifest(digest digest.Digest) ([]byte, string, error) { - // How did we even get here? GetManifest() above has returned a manifest.DockerV2Schema2MediaType. - return nil, "", errors.Errorf(`Manifest lists are not supported by "docker-daemon:"`) -} - type readCloseWrapper struct { io.Reader closeFunc func() error @@ -313,7 +312,7 @@ func (s *Source) GetBlob(info types.BlobInfo) (io.ReadCloser, int64, error) { return ioutil.NopCloser(bytes.NewReader(s.configBytes)), int64(len(s.configBytes)), nil } - if li, ok := s.knownLayers[diffID(info.Digest)]; ok { // diffID is a digest of the uncompressed tarball, + if li, ok := s.knownLayers[info.Digest]; ok { // diffID is a digest of the uncompressed tarball, stream, err := s.openTarComponent(li.path) if err != nil { return nil, 0, err @@ -355,6 +354,13 @@ func (s *Source) GetBlob(info types.BlobInfo) (io.ReadCloser, int64, error) { } // GetSignatures returns the image's signatures. It may use a remote (= slow) service. -func (s *Source) GetSignatures(ctx context.Context) ([][]byte, error) { +// If instanceDigest is not nil, it contains a digest of the specific manifest instance to retrieve signatures for +// (when the primary manifest is a manifest list); this never happens if the primary manifest is not a manifest list +// (e.g. if the source never returns manifest lists). +func (s *Source) GetSignatures(ctx context.Context, instanceDigest *digest.Digest) ([][]byte, error) { + if instanceDigest != nil { + // How did we even get here? GetManifest(nil) has returned a manifest.DockerV2Schema2MediaType. + return nil, errors.Errorf(`Manifest lists are not supported by "docker-daemon:"`) + } return [][]byte{}, nil } diff --git a/vendor/github.com/containers/image/docker/tarfile/types.go b/vendor/github.com/containers/image/docker/tarfile/types.go index f16cc8c6..2aa56754 100644 --- a/vendor/github.com/containers/image/docker/tarfile/types.go +++ b/vendor/github.com/containers/image/docker/tarfile/types.go @@ -1,6 +1,9 @@ package tarfile -import "github.com/opencontainers/go-digest" +import ( + "github.com/containers/image/manifest" + "github.com/opencontainers/go-digest" +) // Various data structures. @@ -18,37 +21,8 @@ type ManifestItem struct { Config string RepoTags []string Layers []string - Parent imageID `json:",omitempty"` - LayerSources map[diffID]distributionDescriptor `json:",omitempty"` + Parent imageID `json:",omitempty"` + LayerSources map[digest.Digest]manifest.Schema2Descriptor `json:",omitempty"` } type imageID string -type diffID digest.Digest - -// Based on github.com/docker/distribution/blobs.go -type distributionDescriptor struct { - MediaType string `json:"mediaType,omitempty"` - Size int64 `json:"size,omitempty"` - Digest digest.Digest `json:"digest,omitempty"` - URLs []string `json:"urls,omitempty"` -} - -// Based on github.com/docker/distribution/manifest/schema2/manifest.go -// FIXME: We are repeating this all over the place; make a public copy? -type schema2Manifest struct { - SchemaVersion int `json:"schemaVersion"` - MediaType string `json:"mediaType,omitempty"` - Config distributionDescriptor `json:"config"` - Layers []distributionDescriptor `json:"layers"` -} - -// Based on github.com/docker/docker/image/image.go -// MOST CONTENT OMITTED AS UNNECESSARY -type image struct { - RootFS *rootFS `json:"rootfs,omitempty"` -} - -type rootFS struct { - Type string `json:"type"` - DiffIDs []diffID `json:"diff_ids,omitempty"` -} diff --git a/vendor/github.com/containers/image/image/docker_list.go b/vendor/github.com/containers/image/image/docker_list.go index c79adacc..412261dd 100644 --- a/vendor/github.com/containers/image/image/docker_list.go +++ b/vendor/github.com/containers/image/image/docker_list.go @@ -2,6 +2,7 @@ package image import ( "encoding/json" + "fmt" "runtime" "github.com/containers/image/manifest" @@ -21,7 +22,7 @@ type platformSpec struct { // A manifestDescriptor references a platform-specific manifest. type manifestDescriptor struct { - descriptor + manifest.Schema2Descriptor Platform platformSpec `json:"platform"` } @@ -31,22 +32,36 @@ type manifestList struct { Manifests []manifestDescriptor `json:"manifests"` } -func manifestSchema2FromManifestList(src types.ImageSource, manblob []byte) (genericManifest, error) { - list := manifestList{} - if err := json.Unmarshal(manblob, &list); err != nil { - return nil, err +// chooseDigestFromManifestList parses blob as a schema2 manifest list, +// and returns the digest of the image appropriate for the current environment. +func chooseDigestFromManifestList(ctx *types.SystemContext, blob []byte) (digest.Digest, error) { + wantedArch := runtime.GOARCH + if ctx != nil && ctx.ArchitectureChoice != "" { + wantedArch = ctx.ArchitectureChoice + } + wantedOS := runtime.GOOS + if ctx != nil && ctx.OSChoice != "" { + wantedOS = ctx.OSChoice + } + + list := manifestList{} + if err := json.Unmarshal(blob, &list); err != nil { + return "", err } - var targetManifestDigest digest.Digest for _, d := range list.Manifests { - if d.Platform.Architecture == runtime.GOARCH && d.Platform.OS == runtime.GOOS { - targetManifestDigest = d.Digest - break + if d.Platform.Architecture == wantedArch && d.Platform.OS == wantedOS { + return d.Digest, nil } } - if targetManifestDigest == "" { - return nil, errors.New("no supported platform found in manifest list") + return "", fmt.Errorf("no image found in manifest list for architecture %s, OS %s", wantedArch, wantedOS) +} + +func manifestSchema2FromManifestList(ctx *types.SystemContext, src types.ImageSource, manblob []byte) (genericManifest, error) { + targetManifestDigest, err := chooseDigestFromManifestList(ctx, manblob) + if err != nil { + return nil, err } - manblob, mt, err := src.GetTargetManifest(targetManifestDigest) + manblob, mt, err := src.GetManifest(&targetManifestDigest) if err != nil { return nil, err } @@ -59,5 +74,20 @@ func manifestSchema2FromManifestList(src types.ImageSource, manblob []byte) (gen return nil, errors.Errorf("Manifest image does not match selected manifest digest %s", targetManifestDigest) } - return manifestInstanceFromBlob(src, manblob, mt) + return manifestInstanceFromBlob(ctx, src, manblob, mt) +} + +// ChooseManifestInstanceFromManifestList returns a digest of a manifest appropriate +// for the current system from the manifest available from src. +func ChooseManifestInstanceFromManifestList(ctx *types.SystemContext, src types.UnparsedImage) (digest.Digest, error) { + // For now this only handles manifest.DockerV2ListMediaType; we can generalize it later, + // probably along with manifest list editing. + blob, mt, err := src.Manifest() + if err != nil { + return "", err + } + if mt != manifest.DockerV2ListMediaType { + return "", fmt.Errorf("Internal error: Trying to select an image from a non-manifest-list manifest type %s", mt) + } + return chooseDigestFromManifestList(ctx, blob) } diff --git a/vendor/github.com/containers/image/image/docker_schema1.go b/vendor/github.com/containers/image/image/docker_schema1.go index 4152b3cd..c6a6989d 100644 --- a/vendor/github.com/containers/image/image/docker_schema1.go +++ b/vendor/github.com/containers/image/image/docker_schema1.go @@ -2,9 +2,6 @@ package image import ( "encoding/json" - "regexp" - "strings" - "time" "github.com/containers/image/docker/reference" "github.com/containers/image/manifest" @@ -14,87 +11,25 @@ import ( "github.com/pkg/errors" ) -var ( - validHex = regexp.MustCompile(`^([a-f0-9]{64})$`) -) - -type fsLayersSchema1 struct { - BlobSum digest.Digest `json:"blobSum"` -} - -type historySchema1 struct { - V1Compatibility string `json:"v1Compatibility"` -} - -// historySchema1 is a string containing this. It is similar to v1Image but not the same, in particular note the ThrowAway field. -type v1Compatibility struct { - ID string `json:"id"` - Parent string `json:"parent,omitempty"` - Comment string `json:"comment,omitempty"` - Created time.Time `json:"created"` - ContainerConfig struct { - Cmd []string - } `json:"container_config,omitempty"` - Author string `json:"author,omitempty"` - ThrowAway bool `json:"throwaway,omitempty"` -} - type manifestSchema1 struct { - Name string `json:"name"` - Tag string `json:"tag"` - Architecture string `json:"architecture"` - FSLayers []fsLayersSchema1 `json:"fsLayers"` - History []historySchema1 `json:"history"` - SchemaVersion int `json:"schemaVersion"` + m *manifest.Schema1 } -func manifestSchema1FromManifest(manifest []byte) (genericManifest, error) { - mschema1 := &manifestSchema1{} - if err := json.Unmarshal(manifest, mschema1); err != nil { - return nil, err - } - if mschema1.SchemaVersion != 1 { - return nil, errors.Errorf("unsupported schema version %d", mschema1.SchemaVersion) - } - if len(mschema1.FSLayers) != len(mschema1.History) { - return nil, errors.New("length of history not equal to number of layers") - } - if len(mschema1.FSLayers) == 0 { - return nil, errors.New("no FSLayers in manifest") - } - - if err := fixManifestLayers(mschema1); err != nil { - return nil, err - } - return mschema1, nil -} - -// manifestSchema1FromComponents builds a new manifestSchema1 from the supplied data. -func manifestSchema1FromComponents(ref reference.Named, fsLayers []fsLayersSchema1, history []historySchema1, architecture string) genericManifest { - var name, tag string - if ref != nil { // Well, what to do if it _is_ nil? Most consumers actually don't use these fields nowadays, so we might as well try not supplying them. - name = reference.Path(ref) - if tagged, ok := ref.(reference.NamedTagged); ok { - tag = tagged.Tag() - } - } - return &manifestSchema1{ - Name: name, - Tag: tag, - Architecture: architecture, - FSLayers: fsLayers, - History: history, - SchemaVersion: 1, - } -} - -func (m *manifestSchema1) serialize() ([]byte, error) { - // docker/distribution requires a signature even if the incoming data uses the nominally unsigned DockerV2Schema1MediaType. - unsigned, err := json.Marshal(*m) +func manifestSchema1FromManifest(manifestBlob []byte) (genericManifest, error) { + m, err := manifest.Schema1FromManifest(manifestBlob) if err != nil { return nil, err } - return manifest.AddDummyV2S1Signature(unsigned) + return &manifestSchema1{m: m}, nil +} + +// manifestSchema1FromComponents builds a new manifestSchema1 from the supplied data. +func manifestSchema1FromComponents(ref reference.Named, fsLayers []manifest.Schema1FSLayers, history []manifest.Schema1History, architecture string) genericManifest { + return &manifestSchema1{m: manifest.Schema1FromComponents(ref, fsLayers, history, architecture)} +} + +func (m *manifestSchema1) serialize() ([]byte, error) { + return m.m.Serialize() } func (m *manifestSchema1) manifestMIMEType() string { @@ -104,7 +39,7 @@ func (m *manifestSchema1) manifestMIMEType() string { // ConfigInfo returns a complete BlobInfo for the separate config object, or a BlobInfo{Digest:""} if there isn't a separate object. // Note that the config object may not exist in the underlying storage in the return value of UpdatedImage! Use ConfigBlob() below. func (m *manifestSchema1) ConfigInfo() types.BlobInfo { - return types.BlobInfo{} + return m.m.ConfigInfo() } // ConfigBlob returns the blob described by ConfigInfo, iff ConfigInfo().Digest != ""; nil otherwise. @@ -128,11 +63,7 @@ func (m *manifestSchema1) OCIConfig() (*imgspecv1.Image, error) { // The Digest field is guaranteed to be provided; Size may be -1. // WARNING: The list may contain duplicates, and they are semantically relevant. func (m *manifestSchema1) LayerInfos() []types.BlobInfo { - layers := make([]types.BlobInfo, len(m.FSLayers)) - for i, layer := range m.FSLayers { // NOTE: This includes empty layers (where m.History.V1Compatibility->ThrowAway) - layers[(len(m.FSLayers)-1)-i] = types.BlobInfo{Digest: layer.BlobSum, Size: -1} - } - return layers + return m.m.LayerInfos() } // EmbeddedDockerReferenceConflicts whether a Docker reference embedded in the manifest, if any, conflicts with destination ref. @@ -153,22 +84,11 @@ func (m *manifestSchema1) EmbeddedDockerReferenceConflicts(ref reference.Named) } else { tag = "" } - return m.Name != name || m.Tag != tag + return m.m.Name != name || m.m.Tag != tag } func (m *manifestSchema1) imageInspectInfo() (*types.ImageInspectInfo, error) { - v1 := &v1Image{} - if err := json.Unmarshal([]byte(m.History[0].V1Compatibility), v1); err != nil { - return nil, err - } - return &types.ImageInspectInfo{ - Tag: m.Tag, - DockerVersion: v1.DockerVersion, - Created: v1.Created, - Labels: v1.Config.Labels, - Architecture: v1.Architecture, - Os: v1.OS, - }, nil + return m.m.Inspect(nil) } // UpdatedImageNeedsLayerDiffIDs returns true iff UpdatedImage(options) needs InformationOnly.LayerDiffIDs. @@ -181,25 +101,18 @@ func (m *manifestSchema1) UpdatedImageNeedsLayerDiffIDs(options types.ManifestUp // UpdatedImage returns a types.Image modified according to options. // This does not change the state of the original Image object. func (m *manifestSchema1) UpdatedImage(options types.ManifestUpdateOptions) (types.Image, error) { - copy := *m + copy := manifestSchema1{m: manifest.Schema1Clone(m.m)} if options.LayerInfos != nil { - // Our LayerInfos includes empty layers (where m.History.V1Compatibility->ThrowAway), so expect them to be included here as well. - if len(copy.FSLayers) != len(options.LayerInfos) { - return nil, errors.Errorf("Error preparing updated manifest: layer count changed from %d to %d", len(copy.FSLayers), len(options.LayerInfos)) - } - for i, info := range options.LayerInfos { - // (docker push) sets up m.History.V1Compatibility->{Id,Parent} based on values of info.Digest, - // but (docker pull) ignores them in favor of computing DiffIDs from uncompressed data, except verifying the child->parent links and uniqueness. - // So, we don't bother recomputing the IDs in m.History.V1Compatibility. - copy.FSLayers[(len(options.LayerInfos)-1)-i].BlobSum = info.Digest + if err := copy.m.UpdateLayerInfos(options.LayerInfos); err != nil { + return nil, err } } if options.EmbeddedDockerReference != nil { - copy.Name = reference.Path(options.EmbeddedDockerReference) + copy.m.Name = reference.Path(options.EmbeddedDockerReference) if tagged, isTagged := options.EmbeddedDockerReference.(reference.NamedTagged); isTagged { - copy.Tag = tagged.Tag() + copy.m.Tag = tagged.Tag() } else { - copy.Tag = "" + copy.m.Tag = "" } } @@ -209,7 +122,21 @@ func (m *manifestSchema1) UpdatedImage(options types.ManifestUpdateOptions) (typ // We have 2 MIME types for schema 1, which are basically equivalent (even the un-"Signed" MIME type will be rejected if there isn’t a signature; so, // handle conversions between them by doing nothing. case manifest.DockerV2Schema2MediaType: - return copy.convertToManifestSchema2(options.InformationOnly.LayerInfos, options.InformationOnly.LayerDiffIDs) + m2, err := copy.convertToManifestSchema2(options.InformationOnly.LayerInfos, options.InformationOnly.LayerDiffIDs) + if err != nil { + return nil, err + } + return memoryImageFromManifest(m2), nil + case imgspecv1.MediaTypeImageManifest: + // We can't directly convert to OCI, but we can transitively convert via a Docker V2.2 Distribution manifest + m2, err := copy.convertToManifestSchema2(options.InformationOnly.LayerInfos, options.InformationOnly.LayerDiffIDs) + if err != nil { + return nil, err + } + return m2.UpdatedImage(types.ManifestUpdateOptions{ + ManifestMIMEType: imgspecv1.MediaTypeImageManifest, + InformationOnly: options.InformationOnly, + }) default: return nil, errors.Errorf("Conversion of image manifest from %s to %s is not implemented", manifest.DockerV2Schema1SignedMediaType, options.ManifestMIMEType) } @@ -217,102 +144,32 @@ func (m *manifestSchema1) UpdatedImage(options types.ManifestUpdateOptions) (typ return memoryImageFromManifest(©), nil } -// fixManifestLayers, after validating the supplied manifest -// (to use correctly-formatted IDs, and to not have non-consecutive ID collisions in manifest.History), -// modifies manifest to only have one entry for each layer ID in manifest.History (deleting the older duplicates, -// both from manifest.History and manifest.FSLayers). -// Note that even after this succeeds, manifest.FSLayers may contain duplicate entries -// (for Dockerfile operations which change the configuration but not the filesystem). -func fixManifestLayers(manifest *manifestSchema1) error { - type imageV1 struct { - ID string - Parent string - } - // Per the specification, we can assume that len(manifest.FSLayers) == len(manifest.History) - imgs := make([]*imageV1, len(manifest.FSLayers)) - for i := range manifest.FSLayers { - img := &imageV1{} - - if err := json.Unmarshal([]byte(manifest.History[i].V1Compatibility), img); err != nil { - return err - } - - imgs[i] = img - if err := validateV1ID(img.ID); err != nil { - return err - } - } - if imgs[len(imgs)-1].Parent != "" { - return errors.New("Invalid parent ID in the base layer of the image") - } - // check general duplicates to error instead of a deadlock - idmap := make(map[string]struct{}) - var lastID string - for _, img := range imgs { - // skip IDs that appear after each other, we handle those later - if _, exists := idmap[img.ID]; img.ID != lastID && exists { - return errors.Errorf("ID %+v appears multiple times in manifest", img.ID) - } - lastID = img.ID - idmap[lastID] = struct{}{} - } - // backwards loop so that we keep the remaining indexes after removing items - for i := len(imgs) - 2; i >= 0; i-- { - if imgs[i].ID == imgs[i+1].ID { // repeated ID. remove and continue - manifest.FSLayers = append(manifest.FSLayers[:i], manifest.FSLayers[i+1:]...) - manifest.History = append(manifest.History[:i], manifest.History[i+1:]...) - } else if imgs[i].Parent != imgs[i+1].ID { - return errors.Errorf("Invalid parent ID. Expected %v, got %v", imgs[i+1].ID, imgs[i].Parent) - } - } - return nil -} - -func validateV1ID(id string) error { - if ok := validHex.MatchString(id); !ok { - return errors.Errorf("image ID %q is invalid", id) - } - return nil -} - // Based on github.com/docker/docker/distribution/pull_v2.go -func (m *manifestSchema1) convertToManifestSchema2(uploadedLayerInfos []types.BlobInfo, layerDiffIDs []digest.Digest) (types.Image, error) { - if len(m.History) == 0 { +func (m *manifestSchema1) convertToManifestSchema2(uploadedLayerInfos []types.BlobInfo, layerDiffIDs []digest.Digest) (genericManifest, error) { + if len(m.m.History) == 0 { // What would this even mean?! Anyhow, the rest of the code depends on fsLayers[0] and history[0] existing. return nil, errors.Errorf("Cannot convert an image with 0 history entries to %s", manifest.DockerV2Schema2MediaType) } - if len(m.History) != len(m.FSLayers) { - return nil, errors.Errorf("Inconsistent schema 1 manifest: %d history entries, %d fsLayers entries", len(m.History), len(m.FSLayers)) + if len(m.m.History) != len(m.m.FSLayers) { + return nil, errors.Errorf("Inconsistent schema 1 manifest: %d history entries, %d fsLayers entries", len(m.m.History), len(m.m.FSLayers)) } - if uploadedLayerInfos != nil && len(uploadedLayerInfos) != len(m.FSLayers) { - return nil, errors.Errorf("Internal error: uploaded %d blobs, but schema1 manifest has %d fsLayers", len(uploadedLayerInfos), len(m.FSLayers)) + if uploadedLayerInfos != nil && len(uploadedLayerInfos) != len(m.m.FSLayers) { + return nil, errors.Errorf("Internal error: uploaded %d blobs, but schema1 manifest has %d fsLayers", len(uploadedLayerInfos), len(m.m.FSLayers)) } - if layerDiffIDs != nil && len(layerDiffIDs) != len(m.FSLayers) { - return nil, errors.Errorf("Internal error: collected %d DiffID values, but schema1 manifest has %d fsLayers", len(layerDiffIDs), len(m.FSLayers)) + if layerDiffIDs != nil && len(layerDiffIDs) != len(m.m.FSLayers) { + return nil, errors.Errorf("Internal error: collected %d DiffID values, but schema1 manifest has %d fsLayers", len(layerDiffIDs), len(m.m.FSLayers)) } - rootFS := rootFS{ - Type: "layers", - DiffIDs: []digest.Digest{}, - BaseLayer: "", - } - var layers []descriptor - history := make([]imageHistory, len(m.History)) - for v1Index := len(m.History) - 1; v1Index >= 0; v1Index-- { - v2Index := (len(m.History) - 1) - v1Index + // Build a list of the diffIDs for the non-empty layers. + diffIDs := []digest.Digest{} + var layers []manifest.Schema2Descriptor + for v1Index := len(m.m.History) - 1; v1Index >= 0; v1Index-- { + v2Index := (len(m.m.History) - 1) - v1Index - var v1compat v1Compatibility - if err := json.Unmarshal([]byte(m.History[v1Index].V1Compatibility), &v1compat); err != nil { + var v1compat manifest.Schema1V1Compatibility + if err := json.Unmarshal([]byte(m.m.History[v1Index].V1Compatibility), &v1compat); err != nil { return nil, errors.Wrapf(err, "Error decoding history entry %d", v1Index) } - history[v2Index] = imageHistory{ - Created: v1compat.Created, - Author: v1compat.Author, - CreatedBy: strings.Join(v1compat.ContainerConfig.Cmd, " "), - Comment: v1compat.Comment, - EmptyLayer: v1compat.ThrowAway, - } - if !v1compat.ThrowAway { var size int64 if uploadedLayerInfos != nil { @@ -322,54 +179,23 @@ func (m *manifestSchema1) convertToManifestSchema2(uploadedLayerInfos []types.Bl if layerDiffIDs != nil { d = layerDiffIDs[v2Index] } - layers = append(layers, descriptor{ + layers = append(layers, manifest.Schema2Descriptor{ MediaType: "application/vnd.docker.image.rootfs.diff.tar.gzip", Size: size, - Digest: m.FSLayers[v1Index].BlobSum, + Digest: m.m.FSLayers[v1Index].BlobSum, }) - rootFS.DiffIDs = append(rootFS.DiffIDs, d) + diffIDs = append(diffIDs, d) } } - configJSON, err := configJSONFromV1Config([]byte(m.History[0].V1Compatibility), rootFS, history) + configJSON, err := m.m.ToSchema2(diffIDs) if err != nil { return nil, err } - configDescriptor := descriptor{ + configDescriptor := manifest.Schema2Descriptor{ MediaType: "application/vnd.docker.container.image.v1+json", Size: int64(len(configJSON)), Digest: digest.FromBytes(configJSON), } - m2 := manifestSchema2FromComponents(configDescriptor, nil, configJSON, layers) - return memoryImageFromManifest(m2), nil -} - -func configJSONFromV1Config(v1ConfigJSON []byte, rootFS rootFS, history []imageHistory) ([]byte, error) { - // github.com/docker/docker/image/v1/imagev1.go:MakeConfigFromV1Config unmarshals and re-marshals the input if docker_version is < 1.8.3 to remove blank fields; - // we don't do that here. FIXME? Should we? AFAICT it would only affect the digest value of the schema2 manifest, and we don't particularly need that to be - // a consistently reproducible value. - - // Preserve everything we don't specifically know about. - // (This must be a *json.RawMessage, even though *[]byte is fairly redundant, because only *RawMessage implements json.Marshaler.) - rawContents := map[string]*json.RawMessage{} - if err := json.Unmarshal(v1ConfigJSON, &rawContents); err != nil { // We have already unmarshaled it before, using a more detailed schema?! - return nil, err - } - - delete(rawContents, "id") - delete(rawContents, "parent") - delete(rawContents, "Size") - delete(rawContents, "parent_id") - delete(rawContents, "layer_id") - delete(rawContents, "throwaway") - - updates := map[string]interface{}{"rootfs": rootFS, "history": history} - for field, value := range updates { - encoded, err := json.Marshal(value) - if err != nil { - return nil, err - } - rawContents[field] = (*json.RawMessage)(&encoded) - } - return json.Marshal(rawContents) + return manifestSchema2FromComponents(configDescriptor, nil, configJSON, layers), nil } diff --git a/vendor/github.com/containers/image/image/docker_schema2.go b/vendor/github.com/containers/image/image/docker_schema2.go index 8cc3c495..b43bc17c 100644 --- a/vendor/github.com/containers/image/image/docker_schema2.go +++ b/vendor/github.com/containers/image/image/docker_schema2.go @@ -29,54 +29,44 @@ var gzippedEmptyLayer = []byte{ // gzippedEmptyLayerDigest is a digest of gzippedEmptyLayer const gzippedEmptyLayerDigest = digest.Digest("sha256:a3ed95caeb02ffe68cdd9fd84406680ae93d633cb16422d00e8a7c22955b46d4") -type descriptor struct { - MediaType string `json:"mediaType"` - Size int64 `json:"size"` - Digest digest.Digest `json:"digest"` - URLs []string `json:"urls,omitempty"` -} - type manifestSchema2 struct { - src types.ImageSource // May be nil if configBlob is not nil - configBlob []byte // If set, corresponds to contents of ConfigDescriptor. - SchemaVersion int `json:"schemaVersion"` - MediaType string `json:"mediaType"` - ConfigDescriptor descriptor `json:"config"` - LayersDescriptors []descriptor `json:"layers"` + src types.ImageSource // May be nil if configBlob is not nil + configBlob []byte // If set, corresponds to contents of ConfigDescriptor. + m *manifest.Schema2 } -func manifestSchema2FromManifest(src types.ImageSource, manifest []byte) (genericManifest, error) { - v2s2 := manifestSchema2{src: src} - if err := json.Unmarshal(manifest, &v2s2); err != nil { +func manifestSchema2FromManifest(src types.ImageSource, manifestBlob []byte) (genericManifest, error) { + m, err := manifest.Schema2FromManifest(manifestBlob) + if err != nil { return nil, err } - return &v2s2, nil + return &manifestSchema2{ + src: src, + m: m, + }, nil } // manifestSchema2FromComponents builds a new manifestSchema2 from the supplied data: -func manifestSchema2FromComponents(config descriptor, src types.ImageSource, configBlob []byte, layers []descriptor) genericManifest { +func manifestSchema2FromComponents(config manifest.Schema2Descriptor, src types.ImageSource, configBlob []byte, layers []manifest.Schema2Descriptor) genericManifest { return &manifestSchema2{ - src: src, - configBlob: configBlob, - SchemaVersion: 2, - MediaType: manifest.DockerV2Schema2MediaType, - ConfigDescriptor: config, - LayersDescriptors: layers, + src: src, + configBlob: configBlob, + m: manifest.Schema2FromComponents(config, layers), } } func (m *manifestSchema2) serialize() ([]byte, error) { - return json.Marshal(*m) + return m.m.Serialize() } func (m *manifestSchema2) manifestMIMEType() string { - return m.MediaType + return m.m.MediaType } // ConfigInfo returns a complete BlobInfo for the separate config object, or a BlobInfo{Digest:""} if there isn't a separate object. // Note that the config object may not exist in the underlying storage in the return value of UpdatedImage! Use ConfigBlob() below. func (m *manifestSchema2) ConfigInfo() types.BlobInfo { - return types.BlobInfo{Digest: m.ConfigDescriptor.Digest, Size: m.ConfigDescriptor.Size} + return m.m.ConfigInfo() } // OCIConfig returns the image configuration as per OCI v1 image-spec. Information about @@ -105,9 +95,9 @@ func (m *manifestSchema2) ConfigBlob() ([]byte, error) { return nil, errors.Errorf("Internal error: neither src nor configBlob set in manifestSchema2") } stream, _, err := m.src.GetBlob(types.BlobInfo{ - Digest: m.ConfigDescriptor.Digest, - Size: m.ConfigDescriptor.Size, - URLs: m.ConfigDescriptor.URLs, + Digest: m.m.ConfigDescriptor.Digest, + Size: m.m.ConfigDescriptor.Size, + URLs: m.m.ConfigDescriptor.URLs, }) if err != nil { return nil, err @@ -118,8 +108,8 @@ func (m *manifestSchema2) ConfigBlob() ([]byte, error) { return nil, err } computedDigest := digest.FromBytes(blob) - if computedDigest != m.ConfigDescriptor.Digest { - return nil, errors.Errorf("Download config.json digest %s does not match expected %s", computedDigest, m.ConfigDescriptor.Digest) + if computedDigest != m.m.ConfigDescriptor.Digest { + return nil, errors.Errorf("Download config.json digest %s does not match expected %s", computedDigest, m.m.ConfigDescriptor.Digest) } m.configBlob = blob } @@ -130,15 +120,7 @@ func (m *manifestSchema2) ConfigBlob() ([]byte, error) { // The Digest field is guaranteed to be provided; Size may be -1. // WARNING: The list may contain duplicates, and they are semantically relevant. func (m *manifestSchema2) LayerInfos() []types.BlobInfo { - blobs := []types.BlobInfo{} - for _, layer := range m.LayersDescriptors { - blobs = append(blobs, types.BlobInfo{ - Digest: layer.Digest, - Size: layer.Size, - URLs: layer.URLs, - }) - } - return blobs + return m.m.LayerInfos() } // EmbeddedDockerReferenceConflicts whether a Docker reference embedded in the manifest, if any, conflicts with destination ref. @@ -149,21 +131,18 @@ func (m *manifestSchema2) EmbeddedDockerReferenceConflicts(ref reference.Named) } func (m *manifestSchema2) imageInspectInfo() (*types.ImageInspectInfo, error) { - config, err := m.ConfigBlob() - if err != nil { - return nil, err + getter := func(info types.BlobInfo) ([]byte, error) { + if info.Digest != m.ConfigInfo().Digest { + // Shouldn't ever happen + return nil, errors.New("asked for a different config blob") + } + config, err := m.ConfigBlob() + if err != nil { + return nil, err + } + return config, nil } - v1 := &v1Image{} - if err := json.Unmarshal(config, v1); err != nil { - return nil, err - } - return &types.ImageInspectInfo{ - DockerVersion: v1.DockerVersion, - Created: v1.Created, - Labels: v1.Config.Labels, - Architecture: v1.Architecture, - Os: v1.OS, - }, nil + return m.m.Inspect(getter) } // UpdatedImageNeedsLayerDiffIDs returns true iff UpdatedImage(options) needs InformationOnly.LayerDiffIDs. @@ -176,17 +155,14 @@ func (m *manifestSchema2) UpdatedImageNeedsLayerDiffIDs(options types.ManifestUp // UpdatedImage returns a types.Image modified according to options. // This does not change the state of the original Image object. func (m *manifestSchema2) UpdatedImage(options types.ManifestUpdateOptions) (types.Image, error) { - copy := *m // NOTE: This is not a deep copy, it still shares slices etc. + copy := manifestSchema2{ // NOTE: This is not a deep copy, it still shares slices etc. + src: m.src, + configBlob: m.configBlob, + m: manifest.Schema2Clone(m.m), + } if options.LayerInfos != nil { - if len(copy.LayersDescriptors) != len(options.LayerInfos) { - return nil, errors.Errorf("Error preparing updated manifest: layer count changed from %d to %d", len(copy.LayersDescriptors), len(options.LayerInfos)) - } - copy.LayersDescriptors = make([]descriptor, len(options.LayerInfos)) - for i, info := range options.LayerInfos { - copy.LayersDescriptors[i].MediaType = m.LayersDescriptors[i].MediaType - copy.LayersDescriptors[i].Digest = info.Digest - copy.LayersDescriptors[i].Size = info.Size - copy.LayersDescriptors[i].URLs = info.URLs + if err := copy.m.UpdateLayerInfos(options.LayerInfos); err != nil { + return nil, err } } // Ignore options.EmbeddedDockerReference: it may be set when converting from schema1 to schema2, but we really don't care. @@ -204,6 +180,15 @@ func (m *manifestSchema2) UpdatedImage(options types.ManifestUpdateOptions) (typ return memoryImageFromManifest(©), nil } +func oci1DescriptorFromSchema2Descriptor(d manifest.Schema2Descriptor) imgspecv1.Descriptor { + return imgspecv1.Descriptor{ + MediaType: d.MediaType, + Size: d.Size, + Digest: d.Digest, + URLs: d.URLs, + } +} + func (m *manifestSchema2) convertToManifestOCI1() (types.Image, error) { configOCI, err := m.OCIConfig() if err != nil { @@ -214,18 +199,16 @@ func (m *manifestSchema2) convertToManifestOCI1() (types.Image, error) { return nil, err } - config := descriptorOCI1{ - descriptor: descriptor{ - MediaType: imgspecv1.MediaTypeImageConfig, - Size: int64(len(configOCIBytes)), - Digest: digest.FromBytes(configOCIBytes), - }, + config := imgspecv1.Descriptor{ + MediaType: imgspecv1.MediaTypeImageConfig, + Size: int64(len(configOCIBytes)), + Digest: digest.FromBytes(configOCIBytes), } - layers := make([]descriptorOCI1, len(m.LayersDescriptors)) + layers := make([]imgspecv1.Descriptor, len(m.m.LayersDescriptors)) for idx := range layers { - layers[idx] = descriptorOCI1{descriptor: m.LayersDescriptors[idx]} - if m.LayersDescriptors[idx].MediaType == manifest.DockerV2Schema2ForeignLayerMediaType { + layers[idx] = oci1DescriptorFromSchema2Descriptor(m.m.LayersDescriptors[idx]) + if m.m.LayersDescriptors[idx].MediaType == manifest.DockerV2Schema2ForeignLayerMediaType { layers[idx].MediaType = imgspecv1.MediaTypeImageLayerNonDistributable } else { // we assume layers are gzip'ed because docker v2s2 only deals with @@ -244,14 +227,14 @@ func (m *manifestSchema2) convertToManifestSchema1(dest types.ImageDestination) if err != nil { return nil, err } - imageConfig := &image{} + imageConfig := &manifest.Schema2Image{} if err := json.Unmarshal(configBytes, imageConfig); err != nil { return nil, err } // Build fsLayers and History, discarding all configs. We will patch the top-level config in later. - fsLayers := make([]fsLayersSchema1, len(imageConfig.History)) - history := make([]historySchema1, len(imageConfig.History)) + fsLayers := make([]manifest.Schema1FSLayers, len(imageConfig.History)) + history := make([]manifest.Schema1History, len(imageConfig.History)) nonemptyLayerIndex := 0 var parentV1ID string // Set in the loop v1ID := "" @@ -279,10 +262,10 @@ func (m *manifestSchema2) convertToManifestSchema1(dest types.ImageDestination) } blobDigest = gzippedEmptyLayerDigest } else { - if nonemptyLayerIndex >= len(m.LayersDescriptors) { - return nil, errors.Errorf("Invalid image configuration, needs more than the %d distributed layers", len(m.LayersDescriptors)) + if nonemptyLayerIndex >= len(m.m.LayersDescriptors) { + return nil, errors.Errorf("Invalid image configuration, needs more than the %d distributed layers", len(m.m.LayersDescriptors)) } - blobDigest = m.LayersDescriptors[nonemptyLayerIndex].Digest + blobDigest = m.m.LayersDescriptors[nonemptyLayerIndex].Digest nonemptyLayerIndex++ } @@ -293,7 +276,7 @@ func (m *manifestSchema2) convertToManifestSchema1(dest types.ImageDestination) } v1ID = v - fakeImage := v1Compatibility{ + fakeImage := manifest.Schema1V1Compatibility{ ID: v1ID, Parent: parentV1ID, Comment: historyEntry.Comment, @@ -307,8 +290,8 @@ func (m *manifestSchema2) convertToManifestSchema1(dest types.ImageDestination) return nil, errors.Errorf("Internal error: Error creating v1compatibility for %#v", fakeImage) } - fsLayers[v1Index] = fsLayersSchema1{BlobSum: blobDigest} - history[v1Index] = historySchema1{V1Compatibility: string(v1CompatibilityBytes)} + fsLayers[v1Index] = manifest.Schema1FSLayers{BlobSum: blobDigest} + history[v1Index] = manifest.Schema1History{V1Compatibility: string(v1CompatibilityBytes)} // Note that parentV1ID of the top layer is preserved when exiting this loop } diff --git a/vendor/github.com/containers/image/image/manifest.go b/vendor/github.com/containers/image/image/manifest.go index 75c9e711..cdd4233f 100644 --- a/vendor/github.com/containers/image/image/manifest.go +++ b/vendor/github.com/containers/image/image/manifest.go @@ -1,57 +1,14 @@ package image import ( - "time" + "fmt" "github.com/containers/image/docker/reference" "github.com/containers/image/manifest" - "github.com/containers/image/pkg/strslice" "github.com/containers/image/types" - "github.com/opencontainers/go-digest" imgspecv1 "github.com/opencontainers/image-spec/specs-go/v1" ) -type config struct { - Cmd strslice.StrSlice - Labels map[string]string -} - -type v1Image struct { - ID string `json:"id,omitempty"` - Parent string `json:"parent,omitempty"` - Comment string `json:"comment,omitempty"` - Created time.Time `json:"created"` - ContainerConfig *config `json:"container_config,omitempty"` - DockerVersion string `json:"docker_version,omitempty"` - Author string `json:"author,omitempty"` - // Config is the configuration of the container received from the client - Config *config `json:"config,omitempty"` - // Architecture is the hardware that the image is build and runs on - Architecture string `json:"architecture,omitempty"` - // OS is the operating system used to build and run the image - OS string `json:"os,omitempty"` -} - -type image struct { - v1Image - History []imageHistory `json:"history,omitempty"` - RootFS *rootFS `json:"rootfs,omitempty"` -} - -type imageHistory struct { - Created time.Time `json:"created"` - Author string `json:"author,omitempty"` - CreatedBy string `json:"created_by,omitempty"` - Comment string `json:"comment,omitempty"` - EmptyLayer bool `json:"empty_layer,omitempty"` -} - -type rootFS struct { - Type string `json:"type"` - DiffIDs []digest.Digest `json:"diff_ids,omitempty"` - BaseLayer string `json:"base_layer,omitempty"` -} - // genericManifest is an interface for parsing, modifying image manifests and related data. // Note that the public methods are intended to be a subset of types.Image // so that embedding a genericManifest into structs works. @@ -87,43 +44,24 @@ type genericManifest interface { UpdatedImage(options types.ManifestUpdateOptions) (types.Image, error) } -func manifestInstanceFromBlob(src types.ImageSource, manblob []byte, mt string) (genericManifest, error) { - switch mt { - // "application/json" is a valid v2s1 value per https://github.com/docker/distribution/blob/master/docs/spec/manifest-v2-1.md . - // This works for now, when nothing else seems to return "application/json"; if that were not true, the mapping/detection might - // need to happen within the ImageSource. - case manifest.DockerV2Schema1MediaType, manifest.DockerV2Schema1SignedMediaType, "application/json": +// manifestInstanceFromBlob returns a genericManifest implementation for (manblob, mt) in src. +// If manblob is a manifest list, it implicitly chooses an appropriate image from the list. +func manifestInstanceFromBlob(ctx *types.SystemContext, src types.ImageSource, manblob []byte, mt string) (genericManifest, error) { + switch manifest.NormalizedMIMEType(mt) { + case manifest.DockerV2Schema1MediaType, manifest.DockerV2Schema1SignedMediaType: return manifestSchema1FromManifest(manblob) case imgspecv1.MediaTypeImageManifest: return manifestOCI1FromManifest(src, manblob) case manifest.DockerV2Schema2MediaType: return manifestSchema2FromManifest(src, manblob) case manifest.DockerV2ListMediaType: - return manifestSchema2FromManifestList(src, manblob) - default: - // If it's not a recognized manifest media type, or we have failed determining the type, we'll try one last time - // to deserialize using v2s1 as per https://github.com/docker/distribution/blob/master/manifests.go#L108 - // and https://github.com/docker/distribution/blob/master/manifest/schema1/manifest.go#L50 - // - // Crane registries can also return "text/plain", or pretty much anything else depending on a file extension “recognized” in the tag. - // This makes no real sense, but it happens - // because requests for manifests are - // redirected to a content distribution - // network which is configured that way. See https://bugzilla.redhat.com/show_bug.cgi?id=1389442 - return manifestSchema1FromManifest(manblob) + return manifestSchema2FromManifestList(ctx, src, manblob) + default: // Note that this may not be reachable, manifest.NormalizedMIMEType has a default for unknown values. + return nil, fmt.Errorf("Unimplemented manifest MIME type %s", mt) } } // inspectManifest is an implementation of types.Image.Inspect func inspectManifest(m genericManifest) (*types.ImageInspectInfo, error) { - info, err := m.imageInspectInfo() - if err != nil { - return nil, err - } - layers := m.LayerInfos() - info.Layers = make([]string, len(layers)) - for i, layer := range layers { - info.Layers[i] = layer.Digest.String() - } - return info, nil + return m.imageInspectInfo() } diff --git a/vendor/github.com/containers/image/image/memory.go b/vendor/github.com/containers/image/image/memory.go index 62995f61..4639c49a 100644 --- a/vendor/github.com/containers/image/image/memory.go +++ b/vendor/github.com/containers/image/image/memory.go @@ -33,11 +33,6 @@ func (i *memoryImage) Reference() types.ImageReference { return nil } -// Close removes resources associated with an initialized UnparsedImage, if any. -func (i *memoryImage) Close() error { - return nil -} - // Size returns the size of the image as stored, if known, or -1 if not. func (i *memoryImage) Size() (int64, error) { return -1, nil @@ -67,7 +62,9 @@ func (i *memoryImage) Inspect() (*types.ImageInspectInfo, error) { return inspectManifest(i.genericManifest) } -// IsMultiImage returns true if the image's manifest is a list of images, false otherwise. -func (i *memoryImage) IsMultiImage() bool { - return false +// LayerInfosForCopy returns an updated set of layer blob information which may not match the manifest. +// The Digest field is guaranteed to be provided; Size may be -1. +// WARNING: The list may contain duplicates, and they are semantically relevant. +func (i *memoryImage) LayerInfosForCopy() []types.BlobInfo { + return nil } diff --git a/vendor/github.com/containers/image/image/oci.go b/vendor/github.com/containers/image/image/oci.go index 5f7c0728..3c03e49b 100644 --- a/vendor/github.com/containers/image/image/oci.go +++ b/vendor/github.com/containers/image/image/oci.go @@ -12,41 +12,34 @@ import ( "github.com/pkg/errors" ) -type descriptorOCI1 struct { - descriptor - Annotations map[string]string `json:"annotations,omitempty"` -} - type manifestOCI1 struct { - src types.ImageSource // May be nil if configBlob is not nil - configBlob []byte // If set, corresponds to contents of ConfigDescriptor. - SchemaVersion int `json:"schemaVersion"` - ConfigDescriptor descriptorOCI1 `json:"config"` - LayersDescriptors []descriptorOCI1 `json:"layers"` - Annotations map[string]string `json:"annotations,omitempty"` + src types.ImageSource // May be nil if configBlob is not nil + configBlob []byte // If set, corresponds to contents of m.Config. + m *manifest.OCI1 } -func manifestOCI1FromManifest(src types.ImageSource, manifest []byte) (genericManifest, error) { - oci := manifestOCI1{src: src} - if err := json.Unmarshal(manifest, &oci); err != nil { +func manifestOCI1FromManifest(src types.ImageSource, manifestBlob []byte) (genericManifest, error) { + m, err := manifest.OCI1FromManifest(manifestBlob) + if err != nil { return nil, err } - return &oci, nil + return &manifestOCI1{ + src: src, + m: m, + }, nil } // manifestOCI1FromComponents builds a new manifestOCI1 from the supplied data: -func manifestOCI1FromComponents(config descriptorOCI1, src types.ImageSource, configBlob []byte, layers []descriptorOCI1) genericManifest { +func manifestOCI1FromComponents(config imgspecv1.Descriptor, src types.ImageSource, configBlob []byte, layers []imgspecv1.Descriptor) genericManifest { return &manifestOCI1{ - src: src, - configBlob: configBlob, - SchemaVersion: 2, - ConfigDescriptor: config, - LayersDescriptors: layers, + src: src, + configBlob: configBlob, + m: manifest.OCI1FromComponents(config, layers), } } func (m *manifestOCI1) serialize() ([]byte, error) { - return json.Marshal(*m) + return m.m.Serialize() } func (m *manifestOCI1) manifestMIMEType() string { @@ -56,7 +49,7 @@ func (m *manifestOCI1) manifestMIMEType() string { // ConfigInfo returns a complete BlobInfo for the separate config object, or a BlobInfo{Digest:""} if there isn't a separate object. // Note that the config object may not exist in the underlying storage in the return value of UpdatedImage! Use ConfigBlob() below. func (m *manifestOCI1) ConfigInfo() types.BlobInfo { - return types.BlobInfo{Digest: m.ConfigDescriptor.Digest, Size: m.ConfigDescriptor.Size, Annotations: m.ConfigDescriptor.Annotations} + return m.m.ConfigInfo() } // ConfigBlob returns the blob described by ConfigInfo, iff ConfigInfo().Digest != ""; nil otherwise. @@ -67,9 +60,9 @@ func (m *manifestOCI1) ConfigBlob() ([]byte, error) { return nil, errors.Errorf("Internal error: neither src nor configBlob set in manifestOCI1") } stream, _, err := m.src.GetBlob(types.BlobInfo{ - Digest: m.ConfigDescriptor.Digest, - Size: m.ConfigDescriptor.Size, - URLs: m.ConfigDescriptor.URLs, + Digest: m.m.Config.Digest, + Size: m.m.Config.Size, + URLs: m.m.Config.URLs, }) if err != nil { return nil, err @@ -80,8 +73,8 @@ func (m *manifestOCI1) ConfigBlob() ([]byte, error) { return nil, err } computedDigest := digest.FromBytes(blob) - if computedDigest != m.ConfigDescriptor.Digest { - return nil, errors.Errorf("Download config.json digest %s does not match expected %s", computedDigest, m.ConfigDescriptor.Digest) + if computedDigest != m.m.Config.Digest { + return nil, errors.Errorf("Download config.json digest %s does not match expected %s", computedDigest, m.m.Config.Digest) } m.configBlob = blob } @@ -107,11 +100,7 @@ func (m *manifestOCI1) OCIConfig() (*imgspecv1.Image, error) { // The Digest field is guaranteed to be provided; Size may be -1. // WARNING: The list may contain duplicates, and they are semantically relevant. func (m *manifestOCI1) LayerInfos() []types.BlobInfo { - blobs := []types.BlobInfo{} - for _, layer := range m.LayersDescriptors { - blobs = append(blobs, types.BlobInfo{Digest: layer.Digest, Size: layer.Size, Annotations: layer.Annotations, URLs: layer.URLs}) - } - return blobs + return m.m.LayerInfos() } // EmbeddedDockerReferenceConflicts whether a Docker reference embedded in the manifest, if any, conflicts with destination ref. @@ -122,21 +111,18 @@ func (m *manifestOCI1) EmbeddedDockerReferenceConflicts(ref reference.Named) boo } func (m *manifestOCI1) imageInspectInfo() (*types.ImageInspectInfo, error) { - config, err := m.ConfigBlob() - if err != nil { - return nil, err + getter := func(info types.BlobInfo) ([]byte, error) { + if info.Digest != m.ConfigInfo().Digest { + // Shouldn't ever happen + return nil, errors.New("asked for a different config blob") + } + config, err := m.ConfigBlob() + if err != nil { + return nil, err + } + return config, nil } - v1 := &v1Image{} - if err := json.Unmarshal(config, v1); err != nil { - return nil, err - } - return &types.ImageInspectInfo{ - DockerVersion: v1.DockerVersion, - Created: v1.Created, - Labels: v1.Config.Labels, - Architecture: v1.Architecture, - Os: v1.OS, - }, nil + return m.m.Inspect(getter) } // UpdatedImageNeedsLayerDiffIDs returns true iff UpdatedImage(options) needs InformationOnly.LayerDiffIDs. @@ -149,18 +135,14 @@ func (m *manifestOCI1) UpdatedImageNeedsLayerDiffIDs(options types.ManifestUpdat // UpdatedImage returns a types.Image modified according to options. // This does not change the state of the original Image object. func (m *manifestOCI1) UpdatedImage(options types.ManifestUpdateOptions) (types.Image, error) { - copy := *m // NOTE: This is not a deep copy, it still shares slices etc. + copy := manifestOCI1{ // NOTE: This is not a deep copy, it still shares slices etc. + src: m.src, + configBlob: m.configBlob, + m: manifest.OCI1Clone(m.m), + } if options.LayerInfos != nil { - if len(copy.LayersDescriptors) != len(options.LayerInfos) { - return nil, errors.Errorf("Error preparing updated manifest: layer count changed from %d to %d", len(copy.LayersDescriptors), len(options.LayerInfos)) - } - copy.LayersDescriptors = make([]descriptorOCI1, len(options.LayerInfos)) - for i, info := range options.LayerInfos { - copy.LayersDescriptors[i].MediaType = m.LayersDescriptors[i].MediaType - copy.LayersDescriptors[i].Digest = info.Digest - copy.LayersDescriptors[i].Size = info.Size - copy.LayersDescriptors[i].Annotations = info.Annotations - copy.LayersDescriptors[i].URLs = info.URLs + if err := copy.m.UpdateLayerInfos(options.LayerInfos); err != nil { + return nil, err } } // Ignore options.EmbeddedDockerReference: it may be set when converting from schema1, but we really don't care. @@ -176,17 +158,26 @@ func (m *manifestOCI1) UpdatedImage(options types.ManifestUpdateOptions) (types. return memoryImageFromManifest(©), nil } +func schema2DescriptorFromOCI1Descriptor(d imgspecv1.Descriptor) manifest.Schema2Descriptor { + return manifest.Schema2Descriptor{ + MediaType: d.MediaType, + Size: d.Size, + Digest: d.Digest, + URLs: d.URLs, + } +} + func (m *manifestOCI1) convertToManifestSchema2() (types.Image, error) { // Create a copy of the descriptor. - config := m.ConfigDescriptor.descriptor + config := schema2DescriptorFromOCI1Descriptor(m.m.Config) // The only difference between OCI and DockerSchema2 is the mediatypes. The // media type of the manifest is handled by manifestSchema2FromComponents. config.MediaType = manifest.DockerV2Schema2ConfigMediaType - layers := make([]descriptor, len(m.LayersDescriptors)) + layers := make([]manifest.Schema2Descriptor, len(m.m.Layers)) for idx := range layers { - layers[idx] = m.LayersDescriptors[idx].descriptor + layers[idx] = schema2DescriptorFromOCI1Descriptor(m.m.Layers[idx]) layers[idx].MediaType = manifest.DockerV2Schema2LayerMediaType } diff --git a/vendor/github.com/containers/image/image/sourced.go b/vendor/github.com/containers/image/image/sourced.go index ef35b3c3..3477f341 100644 --- a/vendor/github.com/containers/image/image/sourced.go +++ b/vendor/github.com/containers/image/image/sourced.go @@ -4,12 +4,22 @@ package image import ( - "github.com/containers/image/manifest" "github.com/containers/image/types" ) -// FromSource returns a types.Image implementation for source. -// The caller must call .Close() on the returned Image. +// imageCloser implements types.ImageCloser, perhaps allowing simple users +// to use a single object without having keep a reference to a types.ImageSource +// only to call types.ImageSource.Close(). +type imageCloser struct { + types.Image + src types.ImageSource +} + +// FromSource returns a types.ImageCloser implementation for the default instance of source. +// If source is a manifest list, .Manifest() still returns the manifest list, +// but other methods transparently return data from an appropriate image instance. +// +// The caller must call .Close() on the returned ImageCloser. // // FromSource “takes ownership” of the input ImageSource and will call src.Close() // when the image is closed. (This does not prevent callers from using both the @@ -18,8 +28,19 @@ import ( // // NOTE: If any kind of signature verification should happen, build an UnparsedImage from the value returned by NewImageSource, // verify that UnparsedImage, and convert it into a real Image via image.FromUnparsedImage instead of calling this function. -func FromSource(src types.ImageSource) (types.Image, error) { - return FromUnparsedImage(UnparsedFromSource(src)) +func FromSource(ctx *types.SystemContext, src types.ImageSource) (types.ImageCloser, error) { + img, err := FromUnparsedImage(ctx, UnparsedInstance(src, nil)) + if err != nil { + return nil, err + } + return &imageCloser{ + Image: img, + src: src, + }, nil +} + +func (ic *imageCloser) Close() error { + return ic.src.Close() } // sourcedImage is a general set of utilities for working with container images, @@ -38,27 +59,22 @@ type sourcedImage struct { } // FromUnparsedImage returns a types.Image implementation for unparsed. -// The caller must call .Close() on the returned Image. +// If unparsed represents a manifest list, .Manifest() still returns the manifest list, +// but other methods transparently return data from an appropriate single image. // -// FromSource “takes ownership” of the input UnparsedImage and will call uparsed.Close() -// when the image is closed. (This does not prevent callers from using both the -// UnparsedImage and ImageSource objects simultaneously, but it means that they only need to -// keep a reference to the Image.) -func FromUnparsedImage(unparsed *UnparsedImage) (types.Image, error) { +// The Image must not be used after the underlying ImageSource is Close()d. +func FromUnparsedImage(ctx *types.SystemContext, unparsed *UnparsedImage) (types.Image, error) { // Note that the input parameter above is specifically *image.UnparsedImage, not types.UnparsedImage: // we want to be able to use unparsed.src. We could make that an explicit interface, but, well, // this is the only UnparsedImage implementation around, anyway. - // Also, we do not explicitly implement types.Image.Close; we let the implementation fall through to - // unparsed.Close. - // NOTE: It is essential for signature verification that all parsing done in this object happens on the same manifest which is returned by unparsed.Manifest(). manifestBlob, manifestMIMEType, err := unparsed.Manifest() if err != nil { return nil, err } - parsedManifest, err := manifestInstanceFromBlob(unparsed.src, manifestBlob, manifestMIMEType) + parsedManifest, err := manifestInstanceFromBlob(ctx, unparsed.src, manifestBlob, manifestMIMEType) if err != nil { return nil, err } @@ -85,6 +101,6 @@ func (i *sourcedImage) Inspect() (*types.ImageInspectInfo, error) { return inspectManifest(i.genericManifest) } -func (i *sourcedImage) IsMultiImage() bool { - return i.manifestMIMEType == manifest.DockerV2ListMediaType +func (i *sourcedImage) LayerInfosForCopy() []types.BlobInfo { + return i.UnparsedImage.LayerInfosForCopy() } diff --git a/vendor/github.com/containers/image/image/unparsed.go b/vendor/github.com/containers/image/image/unparsed.go index 483cfd04..aff06d8a 100644 --- a/vendor/github.com/containers/image/image/unparsed.go +++ b/vendor/github.com/containers/image/image/unparsed.go @@ -11,8 +11,10 @@ import ( ) // UnparsedImage implements types.UnparsedImage . +// An UnparsedImage is a pair of (ImageSource, instance digest); it can represent either a manifest list or a single image instance. type UnparsedImage struct { src types.ImageSource + instanceDigest *digest.Digest cachedManifest []byte // A private cache for Manifest(); nil if not yet known. // A private cache for Manifest(), may be the empty string if guessing failed. // Valid iff cachedManifest is not nil. @@ -20,49 +22,41 @@ type UnparsedImage struct { cachedSignatures [][]byte // A private cache for Signatures(); nil if not yet known. } -// UnparsedFromSource returns a types.UnparsedImage implementation for source. -// The caller must call .Close() on the returned UnparsedImage. +// UnparsedInstance returns a types.UnparsedImage implementation for (source, instanceDigest). +// If instanceDigest is not nil, it contains a digest of the specific manifest instance to retrieve (when the primary manifest is a manifest list). // -// UnparsedFromSource “takes ownership” of the input ImageSource and will call src.Close() -// when the image is closed. (This does not prevent callers from using both the -// UnparsedImage and ImageSource objects simultaneously, but it means that they only need to -// keep a reference to the UnparsedImage.) -func UnparsedFromSource(src types.ImageSource) *UnparsedImage { - return &UnparsedImage{src: src} +// The UnparsedImage must not be used after the underlying ImageSource is Close()d. +func UnparsedInstance(src types.ImageSource, instanceDigest *digest.Digest) *UnparsedImage { + return &UnparsedImage{ + src: src, + instanceDigest: instanceDigest, + } } // Reference returns the reference used to set up this source, _as specified by the user_ // (not as the image itself, or its underlying storage, claims). This can be used e.g. to determine which public keys are trusted for this image. func (i *UnparsedImage) Reference() types.ImageReference { + // Note that this does not depend on instanceDigest; e.g. all instances within a manifest list need to be signed with the manifest list identity. return i.src.Reference() } -// Close removes resources associated with an initialized UnparsedImage, if any. -func (i *UnparsedImage) Close() error { - return i.src.Close() -} - // Manifest is like ImageSource.GetManifest, but the result is cached; it is OK to call this however often you need. func (i *UnparsedImage) Manifest() ([]byte, string, error) { if i.cachedManifest == nil { - m, mt, err := i.src.GetManifest() + m, mt, err := i.src.GetManifest(i.instanceDigest) if err != nil { return nil, "", err } // ImageSource.GetManifest does not do digest verification, but we do; // this immediately protects also any user of types.Image. - ref := i.Reference().DockerReference() - if ref != nil { - if canonical, ok := ref.(reference.Canonical); ok { - digest := digest.Digest(canonical.Digest()) - matches, err := manifest.MatchesDigest(m, digest) - if err != nil { - return nil, "", errors.Wrap(err, "Error computing manifest digest") - } - if !matches { - return nil, "", errors.Errorf("Manifest does not match provided manifest digest %s", digest) - } + if digest, haveDigest := i.expectedManifestDigest(); haveDigest { + matches, err := manifest.MatchesDigest(m, digest) + if err != nil { + return nil, "", errors.Wrap(err, "Error computing manifest digest") + } + if !matches { + return nil, "", errors.Errorf("Manifest does not match provided manifest digest %s", digest) } } @@ -72,10 +66,26 @@ func (i *UnparsedImage) Manifest() ([]byte, string, error) { return i.cachedManifest, i.cachedManifestMIMEType, nil } +// expectedManifestDigest returns a the expected value of the manifest digest, and an indicator whether it is known. +// The bool return value seems redundant with digest != ""; it is used explicitly +// to refuse (unexpected) situations when the digest exists but is "". +func (i *UnparsedImage) expectedManifestDigest() (digest.Digest, bool) { + if i.instanceDigest != nil { + return *i.instanceDigest, true + } + ref := i.Reference().DockerReference() + if ref != nil { + if canonical, ok := ref.(reference.Canonical); ok { + return canonical.Digest(), true + } + } + return "", false +} + // Signatures is like ImageSource.GetSignatures, but the result is cached; it is OK to call this however often you need. func (i *UnparsedImage) Signatures(ctx context.Context) ([][]byte, error) { if i.cachedSignatures == nil { - sigs, err := i.src.GetSignatures(ctx) + sigs, err := i.src.GetSignatures(ctx, i.instanceDigest) if err != nil { return nil, err } @@ -83,3 +93,10 @@ func (i *UnparsedImage) Signatures(ctx context.Context) ([][]byte, error) { } return i.cachedSignatures, nil } + +// LayerInfosForCopy returns an updated set of layer blob information which may not match the manifest. +// The Digest field is guaranteed to be provided; Size may be -1. +// WARNING: The list may contain duplicates, and they are semantically relevant. +func (i *UnparsedImage) LayerInfosForCopy() []types.BlobInfo { + return i.src.LayerInfosForCopy() +} diff --git a/vendor/github.com/containers/image/internal/tmpdir/tmpdir.go b/vendor/github.com/containers/image/internal/tmpdir/tmpdir.go new file mode 100644 index 00000000..a28020ed --- /dev/null +++ b/vendor/github.com/containers/image/internal/tmpdir/tmpdir.go @@ -0,0 +1,19 @@ +package tmpdir + +import ( + "os" + "runtime" +) + +// TemporaryDirectoryForBigFiles returns a directory for temporary (big) files. +// On non Windows systems it avoids the use of os.TempDir(), because the default temporary directory usually falls under /tmp +// which on systemd based systems could be the unsuitable tmpfs filesystem. +func TemporaryDirectoryForBigFiles() string { + var temporaryDirectoryForBigFiles string + if runtime.GOOS == "windows" { + temporaryDirectoryForBigFiles = os.TempDir() + } else { + temporaryDirectoryForBigFiles = "/var/tmp" + } + return temporaryDirectoryForBigFiles +} diff --git a/vendor/github.com/containers/image/manifest/docker_schema1.go b/vendor/github.com/containers/image/manifest/docker_schema1.go new file mode 100644 index 00000000..b1c1cfe9 --- /dev/null +++ b/vendor/github.com/containers/image/manifest/docker_schema1.go @@ -0,0 +1,310 @@ +package manifest + +import ( + "encoding/json" + "regexp" + "strings" + "time" + + "github.com/containers/image/docker/reference" + "github.com/containers/image/types" + "github.com/docker/docker/api/types/versions" + "github.com/opencontainers/go-digest" + "github.com/pkg/errors" +) + +// Schema1FSLayers is an entry of the "fsLayers" array in docker/distribution schema 1. +type Schema1FSLayers struct { + BlobSum digest.Digest `json:"blobSum"` +} + +// Schema1History is an entry of the "history" array in docker/distribution schema 1. +type Schema1History struct { + V1Compatibility string `json:"v1Compatibility"` +} + +// Schema1 is a manifest in docker/distribution schema 1. +type Schema1 struct { + Name string `json:"name"` + Tag string `json:"tag"` + Architecture string `json:"architecture"` + FSLayers []Schema1FSLayers `json:"fsLayers"` + History []Schema1History `json:"history"` + SchemaVersion int `json:"schemaVersion"` +} + +// Schema1V1Compatibility is a v1Compatibility in docker/distribution schema 1. +type Schema1V1Compatibility struct { + ID string `json:"id"` + Parent string `json:"parent,omitempty"` + Comment string `json:"comment,omitempty"` + Created time.Time `json:"created"` + ContainerConfig struct { + Cmd []string + } `json:"container_config,omitempty"` + Author string `json:"author,omitempty"` + ThrowAway bool `json:"throwaway,omitempty"` +} + +// Schema1FromManifest creates a Schema1 manifest instance from a manifest blob. +// (NOTE: The instance is not necessary a literal representation of the original blob, +// layers with duplicate IDs are eliminated.) +func Schema1FromManifest(manifest []byte) (*Schema1, error) { + s1 := Schema1{} + if err := json.Unmarshal(manifest, &s1); err != nil { + return nil, err + } + if s1.SchemaVersion != 1 { + return nil, errors.Errorf("unsupported schema version %d", s1.SchemaVersion) + } + if len(s1.FSLayers) != len(s1.History) { + return nil, errors.New("length of history not equal to number of layers") + } + if len(s1.FSLayers) == 0 { + return nil, errors.New("no FSLayers in manifest") + } + if err := s1.fixManifestLayers(); err != nil { + return nil, err + } + return &s1, nil +} + +// Schema1FromComponents creates an Schema1 manifest instance from the supplied data. +func Schema1FromComponents(ref reference.Named, fsLayers []Schema1FSLayers, history []Schema1History, architecture string) *Schema1 { + var name, tag string + if ref != nil { // Well, what to do if it _is_ nil? Most consumers actually don't use these fields nowadays, so we might as well try not supplying them. + name = reference.Path(ref) + if tagged, ok := ref.(reference.NamedTagged); ok { + tag = tagged.Tag() + } + } + return &Schema1{ + Name: name, + Tag: tag, + Architecture: architecture, + FSLayers: fsLayers, + History: history, + SchemaVersion: 1, + } +} + +// Schema1Clone creates a copy of the supplied Schema1 manifest. +func Schema1Clone(src *Schema1) *Schema1 { + copy := *src + return © +} + +// ConfigInfo returns a complete BlobInfo for the separate config object, or a BlobInfo{Digest:""} if there isn't a separate object. +func (m *Schema1) ConfigInfo() types.BlobInfo { + return types.BlobInfo{} +} + +// LayerInfos returns a list of BlobInfos of layers referenced by this image, in order (the root layer first, and then successive layered layers). +// The Digest field is guaranteed to be provided; Size may be -1. +// WARNING: The list may contain duplicates, and they are semantically relevant. +func (m *Schema1) LayerInfos() []types.BlobInfo { + layers := make([]types.BlobInfo, len(m.FSLayers)) + for i, layer := range m.FSLayers { // NOTE: This includes empty layers (where m.History.V1Compatibility->ThrowAway) + layers[(len(m.FSLayers)-1)-i] = types.BlobInfo{Digest: layer.BlobSum, Size: -1} + } + return layers +} + +// UpdateLayerInfos replaces the original layers with the specified BlobInfos (size+digest+urls), in order (the root layer first, and then successive layered layers) +func (m *Schema1) UpdateLayerInfos(layerInfos []types.BlobInfo) error { + // Our LayerInfos includes empty layers (where m.History.V1Compatibility->ThrowAway), so expect them to be included here as well. + if len(m.FSLayers) != len(layerInfos) { + return errors.Errorf("Error preparing updated manifest: layer count changed from %d to %d", len(m.FSLayers), len(layerInfos)) + } + for i, info := range layerInfos { + // (docker push) sets up m.History.V1Compatibility->{Id,Parent} based on values of info.Digest, + // but (docker pull) ignores them in favor of computing DiffIDs from uncompressed data, except verifying the child->parent links and uniqueness. + // So, we don't bother recomputing the IDs in m.History.V1Compatibility. + m.FSLayers[(len(layerInfos)-1)-i].BlobSum = info.Digest + } + return nil +} + +// Serialize returns the manifest in a blob format. +// NOTE: Serialize() does not in general reproduce the original blob if this object was loaded from one, even if no modifications were made! +func (m *Schema1) Serialize() ([]byte, error) { + // docker/distribution requires a signature even if the incoming data uses the nominally unsigned DockerV2Schema1MediaType. + unsigned, err := json.Marshal(*m) + if err != nil { + return nil, err + } + return AddDummyV2S1Signature(unsigned) +} + +// fixManifestLayers, after validating the supplied manifest +// (to use correctly-formatted IDs, and to not have non-consecutive ID collisions in m.History), +// modifies manifest to only have one entry for each layer ID in m.History (deleting the older duplicates, +// both from m.History and m.FSLayers). +// Note that even after this succeeds, m.FSLayers may contain duplicate entries +// (for Dockerfile operations which change the configuration but not the filesystem). +func (m *Schema1) fixManifestLayers() error { + type imageV1 struct { + ID string + Parent string + } + // Per the specification, we can assume that len(m.FSLayers) == len(m.History) + imgs := make([]*imageV1, len(m.FSLayers)) + for i := range m.FSLayers { + img := &imageV1{} + + if err := json.Unmarshal([]byte(m.History[i].V1Compatibility), img); err != nil { + return err + } + + imgs[i] = img + if err := validateV1ID(img.ID); err != nil { + return err + } + } + if imgs[len(imgs)-1].Parent != "" { + return errors.New("Invalid parent ID in the base layer of the image") + } + // check general duplicates to error instead of a deadlock + idmap := make(map[string]struct{}) + var lastID string + for _, img := range imgs { + // skip IDs that appear after each other, we handle those later + if _, exists := idmap[img.ID]; img.ID != lastID && exists { + return errors.Errorf("ID %+v appears multiple times in manifest", img.ID) + } + lastID = img.ID + idmap[lastID] = struct{}{} + } + // backwards loop so that we keep the remaining indexes after removing items + for i := len(imgs) - 2; i >= 0; i-- { + if imgs[i].ID == imgs[i+1].ID { // repeated ID. remove and continue + m.FSLayers = append(m.FSLayers[:i], m.FSLayers[i+1:]...) + m.History = append(m.History[:i], m.History[i+1:]...) + } else if imgs[i].Parent != imgs[i+1].ID { + return errors.Errorf("Invalid parent ID. Expected %v, got %v", imgs[i+1].ID, imgs[i].Parent) + } + } + return nil +} + +var validHex = regexp.MustCompile(`^([a-f0-9]{64})$`) + +func validateV1ID(id string) error { + if ok := validHex.MatchString(id); !ok { + return errors.Errorf("image ID %q is invalid", id) + } + return nil +} + +// Inspect returns various information for (skopeo inspect) parsed from the manifest and configuration. +func (m *Schema1) Inspect(_ func(types.BlobInfo) ([]byte, error)) (*types.ImageInspectInfo, error) { + s1 := &Schema2V1Image{} + if err := json.Unmarshal([]byte(m.History[0].V1Compatibility), s1); err != nil { + return nil, err + } + return &types.ImageInspectInfo{ + Tag: m.Tag, + Created: s1.Created, + DockerVersion: s1.DockerVersion, + Labels: make(map[string]string), + Architecture: s1.Architecture, + Os: s1.OS, + Layers: LayerInfosToStrings(m.LayerInfos()), + }, nil +} + +// ToSchema2 builds a schema2-style configuration blob using the supplied diffIDs. +func (m *Schema1) ToSchema2(diffIDs []digest.Digest) ([]byte, error) { + // Convert the schema 1 compat info into a schema 2 config, constructing some of the fields + // that aren't directly comparable using info from the manifest. + if len(m.History) == 0 { + return nil, errors.New("image has no layers") + } + s2 := struct { + Schema2Image + ID string `json:"id,omitempty"` + Parent string `json:"parent,omitempty"` + ParentID string `json:"parent_id,omitempty"` + LayerID string `json:"layer_id,omitempty"` + ThrowAway bool `json:"throwaway,omitempty"` + Size int64 `json:",omitempty"` + }{} + config := []byte(m.History[0].V1Compatibility) + err := json.Unmarshal(config, &s2) + if err != nil { + return nil, errors.Wrapf(err, "error decoding configuration") + } + // Images created with versions prior to 1.8.3 require us to re-encode the encoded object, + // adding some fields that aren't "omitempty". + if s2.DockerVersion != "" && versions.LessThan(s2.DockerVersion, "1.8.3") { + config, err = json.Marshal(&s2) + if err != nil { + return nil, errors.Wrapf(err, "error re-encoding compat image config %#v", s2) + } + } + // Build the history. + convertedHistory := []Schema2History{} + for _, h := range m.History { + compat := Schema1V1Compatibility{} + if err := json.Unmarshal([]byte(h.V1Compatibility), &compat); err != nil { + return nil, errors.Wrapf(err, "error decoding history information") + } + hitem := Schema2History{ + Created: compat.Created, + CreatedBy: strings.Join(compat.ContainerConfig.Cmd, " "), + Author: compat.Author, + Comment: compat.Comment, + EmptyLayer: compat.ThrowAway, + } + convertedHistory = append([]Schema2History{hitem}, convertedHistory...) + } + // Build the rootfs information. We need the decompressed sums that we've been + // calculating to fill in the DiffIDs. It's expected (but not enforced by us) + // that the number of diffIDs corresponds to the number of non-EmptyLayer + // entries in the history. + rootFS := &Schema2RootFS{ + Type: "layers", + DiffIDs: diffIDs, + } + // And now for some raw manipulation. + raw := make(map[string]*json.RawMessage) + err = json.Unmarshal(config, &raw) + if err != nil { + return nil, errors.Wrapf(err, "error re-decoding compat image config %#v: %v", s2) + } + // Drop some fields. + delete(raw, "id") + delete(raw, "parent") + delete(raw, "parent_id") + delete(raw, "layer_id") + delete(raw, "throwaway") + delete(raw, "Size") + // Add the history and rootfs information. + rootfs, err := json.Marshal(rootFS) + if err != nil { + return nil, errors.Errorf("error encoding rootfs information %#v: %v", rootFS, err) + } + rawRootfs := json.RawMessage(rootfs) + raw["rootfs"] = &rawRootfs + history, err := json.Marshal(convertedHistory) + if err != nil { + return nil, errors.Errorf("error encoding history information %#v: %v", convertedHistory, err) + } + rawHistory := json.RawMessage(history) + raw["history"] = &rawHistory + // Encode the result. + config, err = json.Marshal(raw) + if err != nil { + return nil, errors.Errorf("error re-encoding compat image config %#v: %v", s2, err) + } + return config, nil +} + +// ImageID computes an ID which can uniquely identify this image by its contents. +func (m *Schema1) ImageID(diffIDs []digest.Digest) (string, error) { + image, err := m.ToSchema2(diffIDs) + if err != nil { + return "", err + } + return digest.FromBytes(image).Hex(), nil +} diff --git a/vendor/github.com/containers/image/manifest/docker_schema2.go b/vendor/github.com/containers/image/manifest/docker_schema2.go new file mode 100644 index 00000000..ef82ffc2 --- /dev/null +++ b/vendor/github.com/containers/image/manifest/docker_schema2.go @@ -0,0 +1,251 @@ +package manifest + +import ( + "encoding/json" + "time" + + "github.com/containers/image/pkg/strslice" + "github.com/containers/image/types" + "github.com/opencontainers/go-digest" + "github.com/pkg/errors" +) + +// Schema2Descriptor is a “descriptor” in docker/distribution schema 2. +type Schema2Descriptor struct { + MediaType string `json:"mediaType"` + Size int64 `json:"size"` + Digest digest.Digest `json:"digest"` + URLs []string `json:"urls,omitempty"` +} + +// Schema2 is a manifest in docker/distribution schema 2. +type Schema2 struct { + SchemaVersion int `json:"schemaVersion"` + MediaType string `json:"mediaType"` + ConfigDescriptor Schema2Descriptor `json:"config"` + LayersDescriptors []Schema2Descriptor `json:"layers"` +} + +// Schema2Port is a Port, a string containing port number and protocol in the +// format "80/tcp", from docker/go-connections/nat. +type Schema2Port string + +// Schema2PortSet is a PortSet, a collection of structs indexed by Port, from +// docker/go-connections/nat. +type Schema2PortSet map[Schema2Port]struct{} + +// Schema2HealthConfig is a HealthConfig, which holds configuration settings +// for the HEALTHCHECK feature, from docker/docker/api/types/container. +type Schema2HealthConfig struct { + // Test is the test to perform to check that the container is healthy. + // An empty slice means to inherit the default. + // The options are: + // {} : inherit healthcheck + // {"NONE"} : disable healthcheck + // {"CMD", args...} : exec arguments directly + // {"CMD-SHELL", command} : run command with system's default shell + Test []string `json:",omitempty"` + + // Zero means to inherit. Durations are expressed as integer nanoseconds. + Interval time.Duration `json:",omitempty"` // Interval is the time to wait between checks. + Timeout time.Duration `json:",omitempty"` // Timeout is the time to wait before considering the check to have hung. + + // Retries is the number of consecutive failures needed to consider a container as unhealthy. + // Zero means inherit. + Retries int `json:",omitempty"` +} + +// Schema2Config is a Config in docker/docker/api/types/container. +type Schema2Config struct { + Hostname string // Hostname + Domainname string // Domainname + User string // User that will run the command(s) inside the container, also support user:group + AttachStdin bool // Attach the standard input, makes possible user interaction + AttachStdout bool // Attach the standard output + AttachStderr bool // Attach the standard error + ExposedPorts Schema2PortSet `json:",omitempty"` // List of exposed ports + Tty bool // Attach standard streams to a tty, including stdin if it is not closed. + OpenStdin bool // Open stdin + StdinOnce bool // If true, close stdin after the 1 attached client disconnects. + Env []string // List of environment variable to set in the container + Cmd strslice.StrSlice // Command to run when starting the container + Healthcheck *Schema2HealthConfig `json:",omitempty"` // Healthcheck describes how to check the container is healthy + ArgsEscaped bool `json:",omitempty"` // True if command is already escaped (Windows specific) + Image string // Name of the image as it was passed by the operator (e.g. could be symbolic) + Volumes map[string]struct{} // List of volumes (mounts) used for the container + WorkingDir string // Current directory (PWD) in the command will be launched + Entrypoint strslice.StrSlice // Entrypoint to run when starting the container + NetworkDisabled bool `json:",omitempty"` // Is network disabled + MacAddress string `json:",omitempty"` // Mac Address of the container + OnBuild []string // ONBUILD metadata that were defined on the image Dockerfile + Labels map[string]string // List of labels set to this container + StopSignal string `json:",omitempty"` // Signal to stop a container + StopTimeout *int `json:",omitempty"` // Timeout (in seconds) to stop a container + Shell strslice.StrSlice `json:",omitempty"` // Shell for shell-form of RUN, CMD, ENTRYPOINT +} + +// Schema2V1Image is a V1Image in docker/docker/image. +type Schema2V1Image struct { + // ID is a unique 64 character identifier of the image + ID string `json:"id,omitempty"` + // Parent is the ID of the parent image + Parent string `json:"parent,omitempty"` + // Comment is the commit message that was set when committing the image + Comment string `json:"comment,omitempty"` + // Created is the timestamp at which the image was created + Created time.Time `json:"created"` + // Container is the id of the container used to commit + Container string `json:"container,omitempty"` + // ContainerConfig is the configuration of the container that is committed into the image + ContainerConfig Schema2Config `json:"container_config,omitempty"` + // DockerVersion specifies the version of Docker that was used to build the image + DockerVersion string `json:"docker_version,omitempty"` + // Author is the name of the author that was specified when committing the image + Author string `json:"author,omitempty"` + // Config is the configuration of the container received from the client + Config *Schema2Config `json:"config,omitempty"` + // Architecture is the hardware that the image is build and runs on + Architecture string `json:"architecture,omitempty"` + // OS is the operating system used to build and run the image + OS string `json:"os,omitempty"` + // Size is the total size of the image including all layers it is composed of + Size int64 `json:",omitempty"` +} + +// Schema2RootFS is a description of how to build up an image's root filesystem, from docker/docker/image. +type Schema2RootFS struct { + Type string `json:"type"` + DiffIDs []digest.Digest `json:"diff_ids,omitempty"` +} + +// Schema2History stores build commands that were used to create an image, from docker/docker/image. +type Schema2History struct { + // Created is the timestamp at which the image was created + Created time.Time `json:"created"` + // Author is the name of the author that was specified when committing the image + Author string `json:"author,omitempty"` + // CreatedBy keeps the Dockerfile command used while building the image + CreatedBy string `json:"created_by,omitempty"` + // Comment is the commit message that was set when committing the image + Comment string `json:"comment,omitempty"` + // EmptyLayer is set to true if this history item did not generate a + // layer. Otherwise, the history item is associated with the next + // layer in the RootFS section. + EmptyLayer bool `json:"empty_layer,omitempty"` +} + +// Schema2Image is an Image in docker/docker/image. +type Schema2Image struct { + Schema2V1Image + Parent digest.Digest `json:"parent,omitempty"` + RootFS *Schema2RootFS `json:"rootfs,omitempty"` + History []Schema2History `json:"history,omitempty"` + OSVersion string `json:"os.version,omitempty"` + OSFeatures []string `json:"os.features,omitempty"` + + // rawJSON caches the immutable JSON associated with this image. + rawJSON []byte + + // computedID is the ID computed from the hash of the image config. + // Not to be confused with the legacy V1 ID in V1Image. + computedID digest.Digest +} + +// Schema2FromManifest creates a Schema2 manifest instance from a manifest blob. +func Schema2FromManifest(manifest []byte) (*Schema2, error) { + s2 := Schema2{} + if err := json.Unmarshal(manifest, &s2); err != nil { + return nil, err + } + return &s2, nil +} + +// Schema2FromComponents creates an Schema2 manifest instance from the supplied data. +func Schema2FromComponents(config Schema2Descriptor, layers []Schema2Descriptor) *Schema2 { + return &Schema2{ + SchemaVersion: 2, + MediaType: DockerV2Schema2MediaType, + ConfigDescriptor: config, + LayersDescriptors: layers, + } +} + +// Schema2Clone creates a copy of the supplied Schema2 manifest. +func Schema2Clone(src *Schema2) *Schema2 { + copy := *src + return © +} + +// ConfigInfo returns a complete BlobInfo for the separate config object, or a BlobInfo{Digest:""} if there isn't a separate object. +func (m *Schema2) ConfigInfo() types.BlobInfo { + return types.BlobInfo{Digest: m.ConfigDescriptor.Digest, Size: m.ConfigDescriptor.Size} +} + +// LayerInfos returns a list of BlobInfos of layers referenced by this image, in order (the root layer first, and then successive layered layers). +// The Digest field is guaranteed to be provided; Size may be -1. +// WARNING: The list may contain duplicates, and they are semantically relevant. +func (m *Schema2) LayerInfos() []types.BlobInfo { + blobs := []types.BlobInfo{} + for _, layer := range m.LayersDescriptors { + blobs = append(blobs, types.BlobInfo{ + Digest: layer.Digest, + Size: layer.Size, + URLs: layer.URLs, + }) + } + return blobs +} + +// UpdateLayerInfos replaces the original layers with the specified BlobInfos (size+digest+urls), in order (the root layer first, and then successive layered layers) +func (m *Schema2) UpdateLayerInfos(layerInfos []types.BlobInfo) error { + if len(m.LayersDescriptors) != len(layerInfos) { + return errors.Errorf("Error preparing updated manifest: layer count changed from %d to %d", len(m.LayersDescriptors), len(layerInfos)) + } + original := m.LayersDescriptors + m.LayersDescriptors = make([]Schema2Descriptor, len(layerInfos)) + for i, info := range layerInfos { + m.LayersDescriptors[i].MediaType = original[i].MediaType + m.LayersDescriptors[i].Digest = info.Digest + m.LayersDescriptors[i].Size = info.Size + m.LayersDescriptors[i].URLs = info.URLs + } + return nil +} + +// Serialize returns the manifest in a blob format. +// NOTE: Serialize() does not in general reproduce the original blob if this object was loaded from one, even if no modifications were made! +func (m *Schema2) Serialize() ([]byte, error) { + return json.Marshal(*m) +} + +// Inspect returns various information for (skopeo inspect) parsed from the manifest and configuration. +func (m *Schema2) Inspect(configGetter func(types.BlobInfo) ([]byte, error)) (*types.ImageInspectInfo, error) { + config, err := configGetter(m.ConfigInfo()) + if err != nil { + return nil, err + } + s2 := &Schema2Image{} + if err := json.Unmarshal(config, s2); err != nil { + return nil, err + } + i := &types.ImageInspectInfo{ + Tag: "", + Created: s2.Created, + DockerVersion: s2.DockerVersion, + Architecture: s2.Architecture, + Os: s2.OS, + Layers: LayerInfosToStrings(m.LayerInfos()), + } + if s2.Config != nil { + i.Labels = s2.Config.Labels + } + return i, nil +} + +// ImageID computes an ID which can uniquely identify this image by its contents. +func (m *Schema2) ImageID([]digest.Digest) (string, error) { + if err := m.ConfigDescriptor.Digest.Validate(); err != nil { + return "", err + } + return m.ConfigDescriptor.Digest.Hex(), nil +} diff --git a/vendor/github.com/containers/image/manifest/manifest.go b/vendor/github.com/containers/image/manifest/manifest.go index e329ee57..2bc801d8 100644 --- a/vendor/github.com/containers/image/manifest/manifest.go +++ b/vendor/github.com/containers/image/manifest/manifest.go @@ -2,7 +2,9 @@ package manifest import ( "encoding/json" + "fmt" + "github.com/containers/image/types" "github.com/docker/libtrust" "github.com/opencontainers/go-digest" imgspecv1 "github.com/opencontainers/image-spec/specs-go/v1" @@ -35,7 +37,40 @@ var DefaultRequestedManifestMIMETypes = []string{ DockerV2Schema2MediaType, DockerV2Schema1SignedMediaType, DockerV2Schema1MediaType, - // DockerV2ListMediaType, // FIXME: Restore this ASAP + DockerV2ListMediaType, +} + +// Manifest is an interface for parsing, modifying image manifests in isolation. +// Callers can either use this abstract interface without understanding the details of the formats, +// or instantiate a specific implementation (e.g. manifest.OCI1) and access the public members +// directly. +// +// See types.Image for functionality not limited to manifests, including format conversions and config parsing. +// This interface is similar to, but not strictly equivalent to, the equivalent methods in types.Image. +type Manifest interface { + // ConfigInfo returns a complete BlobInfo for the separate config object, or a BlobInfo{Digest:""} if there isn't a separate object. + ConfigInfo() types.BlobInfo + // LayerInfos returns a list of BlobInfos of layers referenced by this image, in order (the root layer first, and then successive layered layers). + // The Digest field is guaranteed to be provided; Size may be -1. + // WARNING: The list may contain duplicates, and they are semantically relevant. + LayerInfos() []types.BlobInfo + // UpdateLayerInfos replaces the original layers with the specified BlobInfos (size+digest+urls), in order (the root layer first, and then successive layered layers) + UpdateLayerInfos(layerInfos []types.BlobInfo) error + + // ImageID computes an ID which can uniquely identify this image by its contents, irrespective + // of which (of possibly more than one simultaneously valid) reference was used to locate the + // image, and unchanged by whether or how the layers are compressed. The result takes the form + // of the hexadecimal portion of a digest.Digest. + ImageID(diffIDs []digest.Digest) (string, error) + + // Inspect returns various information for (skopeo inspect) parsed from the manifest, + // incorporating information from a configuration blob returned by configGetter, if + // the underlying image format is expected to include a configuration blob. + Inspect(configGetter func(types.BlobInfo) ([]byte, error)) (*types.ImageInspectInfo, error) + + // Serialize returns the manifest in a blob format. + // NOTE: Serialize() does not in general reproduce the original blob if this object was loaded from one, even if no modifications were made! + Serialize() ([]byte, error) } // GuessMIMEType guesses MIME type of a manifest and returns it _if it is recognized_, or "" if unknown or unrecognized. @@ -142,3 +177,62 @@ func AddDummyV2S1Signature(manifest []byte) ([]byte, error) { } return js.PrettySignature("signatures") } + +// MIMETypeIsMultiImage returns true if mimeType is a list of images +func MIMETypeIsMultiImage(mimeType string) bool { + return mimeType == DockerV2ListMediaType +} + +// NormalizedMIMEType returns the effective MIME type of a manifest MIME type returned by a server, +// centralizing various workarounds. +func NormalizedMIMEType(input string) string { + switch input { + // "application/json" is a valid v2s1 value per https://github.com/docker/distribution/blob/master/docs/spec/manifest-v2-1.md . + // This works for now, when nothing else seems to return "application/json"; if that were not true, the mapping/detection might + // need to happen within the ImageSource. + case "application/json": + return DockerV2Schema1SignedMediaType + case DockerV2Schema1MediaType, DockerV2Schema1SignedMediaType, + imgspecv1.MediaTypeImageManifest, + DockerV2Schema2MediaType, + DockerV2ListMediaType: + return input + default: + // If it's not a recognized manifest media type, or we have failed determining the type, we'll try one last time + // to deserialize using v2s1 as per https://github.com/docker/distribution/blob/master/manifests.go#L108 + // and https://github.com/docker/distribution/blob/master/manifest/schema1/manifest.go#L50 + // + // Crane registries can also return "text/plain", or pretty much anything else depending on a file extension “recognized” in the tag. + // This makes no real sense, but it happens + // because requests for manifests are + // redirected to a content distribution + // network which is configured that way. See https://bugzilla.redhat.com/show_bug.cgi?id=1389442 + return DockerV2Schema1SignedMediaType + } +} + +// FromBlob returns a Manifest instance for the specified manifest blob and the corresponding MIME type +func FromBlob(manblob []byte, mt string) (Manifest, error) { + switch NormalizedMIMEType(mt) { + case DockerV2Schema1MediaType, DockerV2Schema1SignedMediaType: + return Schema1FromManifest(manblob) + case imgspecv1.MediaTypeImageManifest: + return OCI1FromManifest(manblob) + case DockerV2Schema2MediaType: + return Schema2FromManifest(manblob) + case DockerV2ListMediaType: + return nil, fmt.Errorf("Treating manifest lists as individual manifests is not implemented") + default: // Note that this may not be reachable, NormalizedMIMEType has a default for unknown values. + return nil, fmt.Errorf("Unimplemented manifest MIME type %s", mt) + } +} + +// LayerInfosToStrings converts a list of layer infos, presumably obtained from a Manifest.LayerInfos() +// method call, into a format suitable for inclusion in a types.ImageInspectInfo structure. +func LayerInfosToStrings(infos []types.BlobInfo) []string { + layers := make([]string, len(infos)) + for i, info := range infos { + layers[i] = info.Digest.String() + } + return layers +} diff --git a/vendor/github.com/containers/image/manifest/oci.go b/vendor/github.com/containers/image/manifest/oci.go new file mode 100644 index 00000000..0ffb35b7 --- /dev/null +++ b/vendor/github.com/containers/image/manifest/oci.go @@ -0,0 +1,120 @@ +package manifest + +import ( + "encoding/json" + "time" + + "github.com/containers/image/types" + "github.com/opencontainers/go-digest" + "github.com/opencontainers/image-spec/specs-go" + imgspecv1 "github.com/opencontainers/image-spec/specs-go/v1" + "github.com/pkg/errors" +) + +// OCI1 is a manifest.Manifest implementation for OCI images. +// The underlying data from imgspecv1.Manifest is also available. +type OCI1 struct { + imgspecv1.Manifest +} + +// OCI1FromManifest creates an OCI1 manifest instance from a manifest blob. +func OCI1FromManifest(manifest []byte) (*OCI1, error) { + oci1 := OCI1{} + if err := json.Unmarshal(manifest, &oci1); err != nil { + return nil, err + } + return &oci1, nil +} + +// OCI1FromComponents creates an OCI1 manifest instance from the supplied data. +func OCI1FromComponents(config imgspecv1.Descriptor, layers []imgspecv1.Descriptor) *OCI1 { + return &OCI1{ + imgspecv1.Manifest{ + Versioned: specs.Versioned{SchemaVersion: 2}, + Config: config, + Layers: layers, + }, + } +} + +// OCI1Clone creates a copy of the supplied OCI1 manifest. +func OCI1Clone(src *OCI1) *OCI1 { + return &OCI1{ + Manifest: src.Manifest, + } +} + +// ConfigInfo returns a complete BlobInfo for the separate config object, or a BlobInfo{Digest:""} if there isn't a separate object. +func (m *OCI1) ConfigInfo() types.BlobInfo { + return types.BlobInfo{Digest: m.Config.Digest, Size: m.Config.Size, Annotations: m.Config.Annotations} +} + +// LayerInfos returns a list of BlobInfos of layers referenced by this image, in order (the root layer first, and then successive layered layers). +// The Digest field is guaranteed to be provided; Size may be -1. +// WARNING: The list may contain duplicates, and they are semantically relevant. +func (m *OCI1) LayerInfos() []types.BlobInfo { + blobs := []types.BlobInfo{} + for _, layer := range m.Layers { + blobs = append(blobs, types.BlobInfo{Digest: layer.Digest, Size: layer.Size, Annotations: layer.Annotations, URLs: layer.URLs, MediaType: layer.MediaType}) + } + return blobs +} + +// UpdateLayerInfos replaces the original layers with the specified BlobInfos (size+digest+urls), in order (the root layer first, and then successive layered layers) +func (m *OCI1) UpdateLayerInfos(layerInfos []types.BlobInfo) error { + if len(m.Layers) != len(layerInfos) { + return errors.Errorf("Error preparing updated manifest: layer count changed from %d to %d", len(m.Layers), len(layerInfos)) + } + original := m.Layers + m.Layers = make([]imgspecv1.Descriptor, len(layerInfos)) + for i, info := range layerInfos { + m.Layers[i].MediaType = original[i].MediaType + m.Layers[i].Digest = info.Digest + m.Layers[i].Size = info.Size + m.Layers[i].Annotations = info.Annotations + m.Layers[i].URLs = info.URLs + } + return nil +} + +// Serialize returns the manifest in a blob format. +// NOTE: Serialize() does not in general reproduce the original blob if this object was loaded from one, even if no modifications were made! +func (m *OCI1) Serialize() ([]byte, error) { + return json.Marshal(*m) +} + +// Inspect returns various information for (skopeo inspect) parsed from the manifest and configuration. +func (m *OCI1) Inspect(configGetter func(types.BlobInfo) ([]byte, error)) (*types.ImageInspectInfo, error) { + config, err := configGetter(m.ConfigInfo()) + if err != nil { + return nil, err + } + v1 := &imgspecv1.Image{} + if err := json.Unmarshal(config, v1); err != nil { + return nil, err + } + d1 := &Schema2V1Image{} + json.Unmarshal(config, d1) + created := time.Time{} + if v1.Created != nil { + created = *v1.Created + } + i := &types.ImageInspectInfo{ + Tag: "", + Created: created, + DockerVersion: d1.DockerVersion, + Labels: v1.Config.Labels, + Architecture: v1.Architecture, + Os: v1.OS, + Layers: LayerInfosToStrings(m.LayerInfos()), + } + return i, nil +} + +// ImageID computes an ID which can uniquely identify this image by its contents. +func (m *OCI1) ImageID([]digest.Digest) (string, error) { + if err := m.Config.Digest.Validate(); err != nil { + return "", err + } + return m.Config.Digest.Hex(), nil +} diff --git a/vendor/github.com/containers/image/oci/archive/oci_src.go b/vendor/github.com/containers/image/oci/archive/oci_src.go index 8644202f..aee5d8d5 100644 --- a/vendor/github.com/containers/image/oci/archive/oci_src.go +++ b/vendor/github.com/containers/image/oci/archive/oci_src.go @@ -68,14 +68,12 @@ func (s *ociArchiveImageSource) Close() error { return s.unpackedSrc.Close() } -// GetManifest returns the image's manifest along with its MIME type -// (which may be empty when it can't be determined but the manifest is available). -func (s *ociArchiveImageSource) GetManifest() ([]byte, string, error) { - return s.unpackedSrc.GetManifest() -} - -func (s *ociArchiveImageSource) GetTargetManifest(digest digest.Digest) ([]byte, string, error) { - return s.unpackedSrc.GetTargetManifest(digest) +// GetManifest returns the image's manifest along with its MIME type (which may be empty when it can't be determined but the manifest is available). +// It may use a remote (= slow) service. +// If instanceDigest is not nil, it contains a digest of the specific manifest instance to retrieve (when the primary manifest is a manifest list); +// this never happens if the primary manifest is not a manifest list (e.g. if the source never returns manifest lists). +func (s *ociArchiveImageSource) GetManifest(instanceDigest *digest.Digest) ([]byte, string, error) { + return s.unpackedSrc.GetManifest(instanceDigest) } // GetBlob returns a stream for the specified blob, and the blob's size. @@ -83,6 +81,15 @@ func (s *ociArchiveImageSource) GetBlob(info types.BlobInfo) (io.ReadCloser, int return s.unpackedSrc.GetBlob(info) } -func (s *ociArchiveImageSource) GetSignatures(c context.Context) ([][]byte, error) { - return s.unpackedSrc.GetSignatures(c) +// GetSignatures returns the image's signatures. It may use a remote (= slow) service. +// If instanceDigest is not nil, it contains a digest of the specific manifest instance to retrieve signatures for +// (when the primary manifest is a manifest list); this never happens if the primary manifest is not a manifest list +// (e.g. if the source never returns manifest lists). +func (s *ociArchiveImageSource) GetSignatures(ctx context.Context, instanceDigest *digest.Digest) ([][]byte, error) { + return s.unpackedSrc.GetSignatures(ctx, instanceDigest) +} + +// LayerInfosForCopy() returns updated layer info that should be used when reading, in preference to values in the manifest, if specified. +func (s *ociArchiveImageSource) LayerInfosForCopy() []types.BlobInfo { + return nil } diff --git a/vendor/github.com/containers/image/oci/archive/oci_transport.go b/vendor/github.com/containers/image/oci/archive/oci_transport.go index 31b19198..c4a4fa71 100644 --- a/vendor/github.com/containers/image/oci/archive/oci_transport.go +++ b/vendor/github.com/containers/image/oci/archive/oci_transport.go @@ -4,13 +4,13 @@ import ( "fmt" "io/ioutil" "os" - "path/filepath" - "regexp" "strings" "github.com/containers/image/directory/explicitfilepath" "github.com/containers/image/docker/reference" "github.com/containers/image/image" + "github.com/containers/image/internal/tmpdir" + "github.com/containers/image/oci/internal" ocilayout "github.com/containers/image/oci/layout" "github.com/containers/image/transports" "github.com/containers/image/types" @@ -48,51 +48,12 @@ func (t ociArchiveTransport) ParseReference(reference string) (types.ImageRefere // ValidatePolicyConfigurationScope checks that scope is a valid name for a signature.PolicyTransportScopes keys func (t ociArchiveTransport) ValidatePolicyConfigurationScope(scope string) error { - var file string - sep := strings.SplitN(scope, ":", 2) - file = sep[0] - - if len(sep) == 2 { - image := sep[1] - if !refRegexp.MatchString(image) { - return errors.Errorf("Invalid image %s", image) - } - } - - if !strings.HasPrefix(file, "/") { - return errors.Errorf("Invalid scope %s: must be an absolute path", scope) - } - // Refuse also "/", otherwise "/" and "" would have the same semantics, - // and "" could be unexpectedly shadowed by the "/" entry. - // (Note: we do allow "/:someimage", a bit ridiculous but why refuse it?) - if scope == "/" { - return errors.New(`Invalid scope "/": Use the generic default scope ""`) - } - cleaned := filepath.Clean(file) - if cleaned != file { - return errors.Errorf(`Invalid scope %s: Uses non-canonical path format, perhaps try with path %s`, scope, cleaned) - } - return nil + return internal.ValidateScope(scope) } -// annotation spex from https://github.com/opencontainers/image-spec/blob/master/annotations.md#pre-defined-annotation-keys -const ( - separator = `(?:[-._:@+]|--)` - alphanum = `(?:[A-Za-z0-9]+)` - component = `(?:` + alphanum + `(?:` + separator + alphanum + `)*)` -) - -var refRegexp = regexp.MustCompile(`^` + component + `(?:/` + component + `)*$`) - // ParseReference converts a string, which should not start with the ImageTransport.Name prefix, into an OCI ImageReference. func ParseReference(reference string) (types.ImageReference, error) { - var file, image string - sep := strings.SplitN(reference, ":", 2) - file = sep[0] - - if len(sep) == 2 { - image = sep[1] - } + file, image := internal.SplitPathAndImage(reference) return NewReference(file, image) } @@ -102,14 +63,15 @@ func NewReference(file, image string) (types.ImageReference, error) { if err != nil { return nil, err } - // This is necessary to prevent directory paths returned by PolicyConfigurationNamespaces - // from being ambiguous with values of PolicyConfigurationIdentity. - if strings.Contains(resolved, ":") { - return nil, errors.Errorf("Invalid OCI reference %s:%s: path %s contains a colon", file, image, resolved) + + if err := internal.ValidateOCIPath(file); err != nil { + return nil, err } - if len(image) > 0 && !refRegexp.MatchString(image) { - return nil, errors.Errorf("Invalid image %s", image) + + if err := internal.ValidateImageName(image); err != nil { + return nil, err } + return ociArchiveReference{file: file, resolvedFile: resolved, image: image}, nil } @@ -154,14 +116,17 @@ func (ref ociArchiveReference) PolicyConfigurationNamespaces() []string { return res } -// NewImage returns a types.Image for this reference, possibly specialized for this ImageTransport. -// The caller must call .Close() on the returned Image. -func (ref ociArchiveReference) NewImage(ctx *types.SystemContext) (types.Image, error) { +// NewImage returns a types.ImageCloser for this reference, possibly specialized for this ImageTransport. +// The caller must call .Close() on the returned ImageCloser. +// NOTE: If any kind of signature verification should happen, build an UnparsedImage from the value returned by NewImageSource, +// verify that UnparsedImage, and convert it into a real Image via image.FromUnparsedImage. +// WARNING: This may not do the right thing for a manifest list, see image.FromSource for details. +func (ref ociArchiveReference) NewImage(ctx *types.SystemContext) (types.ImageCloser, error) { src, err := newImageSource(ctx, ref) if err != nil { return nil, err } - return image.FromSource(src) + return image.FromSource(ctx, src) } // NewImageSource returns a types.ImageSource for this reference. @@ -194,7 +159,7 @@ func (t *tempDirOCIRef) deleteTempDir() error { // createOCIRef creates the oci reference of the image func createOCIRef(image string) (tempDirOCIRef, error) { - dir, err := ioutil.TempDir("/var/tmp", "oci") + dir, err := ioutil.TempDir(tmpdir.TemporaryDirectoryForBigFiles(), "oci") if err != nil { return tempDirOCIRef{}, errors.Wrapf(err, "error creating temp directory") } diff --git a/vendor/github.com/containers/image/oci/internal/oci_util.go b/vendor/github.com/containers/image/oci/internal/oci_util.go new file mode 100644 index 00000000..c2012e50 --- /dev/null +++ b/vendor/github.com/containers/image/oci/internal/oci_util.go @@ -0,0 +1,126 @@ +package internal + +import ( + "github.com/pkg/errors" + "path/filepath" + "regexp" + "runtime" + "strings" +) + +// annotation spex from https://github.com/opencontainers/image-spec/blob/master/annotations.md#pre-defined-annotation-keys +const ( + separator = `(?:[-._:@+]|--)` + alphanum = `(?:[A-Za-z0-9]+)` + component = `(?:` + alphanum + `(?:` + separator + alphanum + `)*)` +) + +var refRegexp = regexp.MustCompile(`^` + component + `(?:/` + component + `)*$`) +var windowsRefRegexp = regexp.MustCompile(`^([a-zA-Z]:\\.+?):(.*)$`) + +// ValidateImageName returns nil if the image name is empty or matches the open-containers image name specs. +// In any other case an error is returned. +func ValidateImageName(image string) error { + if len(image) == 0 { + return nil + } + + var err error + if !refRegexp.MatchString(image) { + err = errors.Errorf("Invalid image %s", image) + } + return err +} + +// SplitPathAndImage tries to split the provided OCI reference into the OCI path and image. +// Neither path nor image parts are validated at this stage. +func SplitPathAndImage(reference string) (string, string) { + if runtime.GOOS == "windows" { + return splitPathAndImageWindows(reference) + } + return splitPathAndImageNonWindows(reference) +} + +func splitPathAndImageWindows(reference string) (string, string) { + groups := windowsRefRegexp.FindStringSubmatch(reference) + // nil group means no match + if groups == nil { + return reference, "" + } + + // we expect three elements. First one full match, second the capture group for the path and + // the third the capture group for the image + if len(groups) != 3 { + return reference, "" + } + return groups[1], groups[2] +} + +func splitPathAndImageNonWindows(reference string) (string, string) { + sep := strings.SplitN(reference, ":", 2) + path := sep[0] + + var image string + if len(sep) == 2 { + image = sep[1] + } + return path, image +} + +// ValidateOCIPath takes the OCI path and validates it. +func ValidateOCIPath(path string) error { + if runtime.GOOS == "windows" { + // On Windows we must allow for a ':' as part of the path + if strings.Count(path, ":") > 1 { + return errors.Errorf("Invalid OCI reference: path %s contains more than one colon", path) + } + } else { + if strings.Contains(path, ":") { + return errors.Errorf("Invalid OCI reference: path %s contains a colon", path) + } + } + return nil +} + +// ValidateScope validates a policy configuration scope for an OCI transport. +func ValidateScope(scope string) error { + var err error + if runtime.GOOS == "windows" { + err = validateScopeWindows(scope) + } else { + err = validateScopeNonWindows(scope) + } + if err != nil { + return err + } + + cleaned := filepath.Clean(scope) + if cleaned != scope { + return errors.Errorf(`Invalid scope %s: Uses non-canonical path format, perhaps try with path %s`, scope, cleaned) + } + + return nil +} + +func validateScopeWindows(scope string) error { + matched, _ := regexp.Match(`^[a-zA-Z]:\\`, []byte(scope)) + if !matched { + return errors.Errorf("Invalid scope '%s'. Must be an absolute path", scope) + } + + return nil +} + +func validateScopeNonWindows(scope string) error { + if !strings.HasPrefix(scope, "/") { + return errors.Errorf("Invalid scope %s: must be an absolute path", scope) + } + + // Refuse also "/", otherwise "/" and "" would have the same semantics, + // and "" could be unexpectedly shadowed by the "/" entry. + if scope == "/" { + return errors.New(`Invalid scope "/": Use the generic default scope ""`) + } + + return nil +} diff --git a/vendor/github.com/containers/image/oci/layout/oci_dest.go b/vendor/github.com/containers/image/oci/layout/oci_dest.go index ce1e0c3e..e95f6516 100644 --- a/vendor/github.com/containers/image/oci/layout/oci_dest.go +++ b/vendor/github.com/containers/image/oci/layout/oci_dest.go @@ -18,21 +18,47 @@ import ( ) type ociImageDestination struct { - ref ociReference - index imgspecv1.Index + ref ociReference + index imgspecv1.Index + sharedBlobDir string } // newImageDestination returns an ImageDestination for writing to an existing directory. -func newImageDestination(ref ociReference) (types.ImageDestination, error) { +func newImageDestination(ctx *types.SystemContext, ref ociReference) (types.ImageDestination, error) { if ref.image == "" { return nil, errors.Errorf("cannot save image with empty image.ref.name") } - index := imgspecv1.Index{ - Versioned: imgspec.Versioned{ - SchemaVersion: 2, - }, + + var index *imgspecv1.Index + if indexExists(ref) { + var err error + index, err = ref.getIndex() + if err != nil { + return nil, err + } + } else { + index = &imgspecv1.Index{ + Versioned: imgspec.Versioned{ + SchemaVersion: 2, + }, + } } - return &ociImageDestination{ref: ref, index: index}, nil + + d := &ociImageDestination{ref: ref, index: *index} + if ctx != nil { + d.sharedBlobDir = ctx.OCISharedBlobDirPath + } + + if err := ensureDirectoryExists(d.ref.dir); err != nil { + return nil, err + } + // Per the OCI image specification, layouts MUST have a "blobs" subdirectory, + // but it MAY be empty (e.g. if we never end up calling PutBlob) + // https://github.com/opencontainers/image-spec/blame/7c889fafd04a893f5c5f50b7ab9963d5d64e5242/image-layout.md#L19 + if err := ensureDirectoryExists(filepath.Join(d.ref.dir, "blobs")); err != nil { + return nil, err + } + return d, nil } // Reference returns the reference used to set up this destination. Note that this should directly correspond to user's intent, @@ -81,16 +107,16 @@ func (d *ociImageDestination) MustMatchRuntimeOS() bool { // to any other readers for download using the supplied digest. // If stream.Read() at any time, ESPECIALLY at end of input, returns an error, PutBlob MUST 1) fail, and 2) delete any data stored so far. func (d *ociImageDestination) PutBlob(stream io.Reader, inputInfo types.BlobInfo) (types.BlobInfo, error) { - if err := ensureDirectoryExists(d.ref.dir); err != nil { - return types.BlobInfo{}, err - } blobFile, err := ioutil.TempFile(d.ref.dir, "oci-put-blob") if err != nil { return types.BlobInfo{}, err } succeeded := false + explicitClosed := false defer func() { - blobFile.Close() + if !explicitClosed { + blobFile.Close() + } if !succeeded { os.Remove(blobFile.Name()) } @@ -110,17 +136,28 @@ func (d *ociImageDestination) PutBlob(stream io.Reader, inputInfo types.BlobInfo if err := blobFile.Sync(); err != nil { return types.BlobInfo{}, err } - if err := blobFile.Chmod(0644); err != nil { - return types.BlobInfo{}, err + + // On POSIX systems, blobFile was created with mode 0600, so we need to make it readable. + // On Windows, the “permissions of newly created files” argument to syscall.Open is + // ignored and the file is already readable; besides, blobFile.Chmod, i.e. syscall.Fchmod, + // always fails on Windows. + if runtime.GOOS != "windows" { + if err := blobFile.Chmod(0644); err != nil { + return types.BlobInfo{}, err + } } - blobPath, err := d.ref.blobPath(computedDigest) + blobPath, err := d.ref.blobPath(computedDigest, d.sharedBlobDir) if err != nil { return types.BlobInfo{}, err } if err := ensureParentDirectoryExists(blobPath); err != nil { return types.BlobInfo{}, err } + + // need to explicitly close the file, since a rename won't otherwise not work on Windows + blobFile.Close() + explicitClosed = true if err := os.Rename(blobFile.Name(), blobPath); err != nil { return types.BlobInfo{}, err } @@ -136,7 +173,7 @@ func (d *ociImageDestination) HasBlob(info types.BlobInfo) (bool, int64, error) if info.Digest == "" { return false, -1, errors.Errorf(`"Can not check for a blob with unknown digest`) } - blobPath, err := d.ref.blobPath(info.Digest) + blobPath, err := d.ref.blobPath(info.Digest, d.sharedBlobDir) if err != nil { return false, -1, err } @@ -169,7 +206,7 @@ func (d *ociImageDestination) PutManifest(m []byte) error { desc.MediaType = imgspecv1.MediaTypeImageManifest desc.Size = int64(len(m)) - blobPath, err := d.ref.blobPath(digest) + blobPath, err := d.ref.blobPath(digest, d.sharedBlobDir) if err != nil { return err } @@ -191,23 +228,20 @@ func (d *ociImageDestination) PutManifest(m []byte) error { Architecture: runtime.GOARCH, OS: runtime.GOOS, } - d.index.Manifests = append(d.index.Manifests, desc) + d.addManifest(&desc) return nil } -func ensureDirectoryExists(path string) error { - if _, err := os.Stat(path); err != nil && os.IsNotExist(err) { - if err := os.MkdirAll(path, 0755); err != nil { - return err +func (d *ociImageDestination) addManifest(desc *imgspecv1.Descriptor) { + for i, manifest := range d.index.Manifests { + if manifest.Annotations["org.opencontainers.image.ref.name"] == desc.Annotations["org.opencontainers.image.ref.name"] { + // TODO Should there first be a cleanup based on the descriptor we are going to replace? + d.index.Manifests[i] = *desc + return } } - return nil -} - -// ensureParentDirectoryExists ensures the parent of the supplied path exists. -func ensureParentDirectoryExists(path string) error { - return ensureDirectoryExists(filepath.Dir(path)) + d.index.Manifests = append(d.index.Manifests, *desc) } func (d *ociImageDestination) PutSignatures(signatures [][]byte) error { @@ -231,3 +265,30 @@ func (d *ociImageDestination) Commit() error { } return ioutil.WriteFile(d.ref.indexPath(), indexJSON, 0644) } + +func ensureDirectoryExists(path string) error { + if _, err := os.Stat(path); err != nil && os.IsNotExist(err) { + if err := os.MkdirAll(path, 0755); err != nil { + return err + } + } + return nil +} + +// ensureParentDirectoryExists ensures the parent of the supplied path exists. +func ensureParentDirectoryExists(path string) error { + return ensureDirectoryExists(filepath.Dir(path)) +} + +// indexExists checks whether the index location specified in the OCI reference exists. +// The implementation is opinionated, since in case of unexpected errors false is returned +func indexExists(ref ociReference) bool { + _, err := os.Stat(ref.indexPath()) + if err == nil { + return true + } + if os.IsNotExist(err) { + return false + } + return true +} diff --git a/vendor/github.com/containers/image/oci/layout/oci_src.go b/vendor/github.com/containers/image/oci/layout/oci_src.go index be8a2aa7..1109f65c 100644 --- a/vendor/github.com/containers/image/oci/layout/oci_src.go +++ b/vendor/github.com/containers/image/oci/layout/oci_src.go @@ -17,9 +17,10 @@ import ( ) type ociImageSource struct { - ref ociReference - descriptor imgspecv1.Descriptor - client *http.Client + ref ociReference + descriptor imgspecv1.Descriptor + client *http.Client + sharedBlobDir string } // newImageSource returns an ImageSource for reading from an existing directory. @@ -40,7 +41,12 @@ func newImageSource(ctx *types.SystemContext, ref ociReference) (types.ImageSour if err != nil { return nil, err } - return &ociImageSource{ref: ref, descriptor: descriptor, client: client}, nil + d := &ociImageSource{ref: ref, descriptor: descriptor, client: client} + if ctx != nil { + // TODO(jonboulle): check dir existence? + d.sharedBlobDir = ctx.OCISharedBlobDirPath + } + return d, nil } // Reference returns the reference used to set up this source. @@ -55,8 +61,26 @@ func (s *ociImageSource) Close() error { // GetManifest returns the image's manifest along with its MIME type (which may be empty when it can't be determined but the manifest is available). // It may use a remote (= slow) service. -func (s *ociImageSource) GetManifest() ([]byte, string, error) { - manifestPath, err := s.ref.blobPath(digest.Digest(s.descriptor.Digest)) +// If instanceDigest is not nil, it contains a digest of the specific manifest instance to retrieve (when the primary manifest is a manifest list); +// this never happens if the primary manifest is not a manifest list (e.g. if the source never returns manifest lists). +func (s *ociImageSource) GetManifest(instanceDigest *digest.Digest) ([]byte, string, error) { + var dig digest.Digest + var mimeType string + if instanceDigest == nil { + dig = digest.Digest(s.descriptor.Digest) + mimeType = s.descriptor.MediaType + } else { + dig = *instanceDigest + // XXX: instanceDigest means that we don't immediately have the context of what + // mediaType the manifest has. In OCI this means that we don't know + // what reference it came from, so we just *assume* that its + // MediaTypeImageManifest. + // FIXME: We should actually be able to look up the manifest in the index, + // and see the MIME type there. + mimeType = imgspecv1.MediaTypeImageManifest + } + + manifestPath, err := s.ref.blobPath(dig, s.sharedBlobDir) if err != nil { return nil, "", err } @@ -65,25 +89,7 @@ func (s *ociImageSource) GetManifest() ([]byte, string, error) { return nil, "", err } - return m, s.descriptor.MediaType, nil -} - -func (s *ociImageSource) GetTargetManifest(digest digest.Digest) ([]byte, string, error) { - manifestPath, err := s.ref.blobPath(digest) - if err != nil { - return nil, "", err - } - - m, err := ioutil.ReadFile(manifestPath) - if err != nil { - return nil, "", err - } - - // XXX: GetTargetManifest means that we don't have the context of what - // mediaType the manifest has. In OCI this means that we don't know - // what reference it came from, so we just *assume* that its - // MediaTypeImageManifest. - return m, imgspecv1.MediaTypeImageManifest, nil + return m, mimeType, nil } // GetBlob returns a stream for the specified blob, and the blob's size. @@ -92,7 +98,7 @@ func (s *ociImageSource) GetBlob(info types.BlobInfo) (io.ReadCloser, int64, err return s.getExternalBlob(info.URLs) } - path, err := s.ref.blobPath(info.Digest) + path, err := s.ref.blobPath(info.Digest, s.sharedBlobDir) if err != nil { return nil, 0, err } @@ -108,7 +114,11 @@ func (s *ociImageSource) GetBlob(info types.BlobInfo) (io.ReadCloser, int64, err return r, fi.Size(), nil } -func (s *ociImageSource) GetSignatures(context.Context) ([][]byte, error) { +// GetSignatures returns the image's signatures. It may use a remote (= slow) service. +// If instanceDigest is not nil, it contains a digest of the specific manifest instance to retrieve signatures for +// (when the primary manifest is a manifest list); this never happens if the primary manifest is not a manifest list +// (e.g. if the source never returns manifest lists). +func (s *ociImageSource) GetSignatures(ctx context.Context, instanceDigest *digest.Digest) ([][]byte, error) { return [][]byte{}, nil } @@ -133,6 +143,11 @@ func (s *ociImageSource) getExternalBlob(urls []string) (io.ReadCloser, int64, e return nil, 0, errWrap } +// LayerInfosForCopy() returns updated layer info that should be used when reading, in preference to values in the manifest, if specified. +func (s *ociImageSource) LayerInfosForCopy() []types.BlobInfo { + return nil +} + func getBlobSize(resp *http.Response) int64 { size, err := strconv.ParseInt(resp.Header.Get("Content-Length"), 10, 64) if err != nil { diff --git a/vendor/github.com/containers/image/oci/layout/oci_transport.go b/vendor/github.com/containers/image/oci/layout/oci_transport.go index 312bc0e4..c181c4c7 100644 --- a/vendor/github.com/containers/image/oci/layout/oci_transport.go +++ b/vendor/github.com/containers/image/oci/layout/oci_transport.go @@ -5,12 +5,12 @@ import ( "fmt" "os" "path/filepath" - "regexp" "strings" "github.com/containers/image/directory/explicitfilepath" "github.com/containers/image/docker/reference" "github.com/containers/image/image" + "github.com/containers/image/oci/internal" "github.com/containers/image/transports" "github.com/containers/image/types" "github.com/opencontainers/go-digest" @@ -36,45 +36,12 @@ func (t ociTransport) ParseReference(reference string) (types.ImageReference, er return ParseReference(reference) } -// annotation spex from https://github.com/opencontainers/image-spec/blob/master/annotations.md#pre-defined-annotation-keys -const ( - separator = `(?:[-._:@+]|--)` - alphanum = `(?:[A-Za-z0-9]+)` - component = `(?:` + alphanum + `(?:` + separator + alphanum + `)*)` -) - -var refRegexp = regexp.MustCompile(`^` + component + `(?:/` + component + `)*$`) - // ValidatePolicyConfigurationScope checks that scope is a valid name for a signature.PolicyTransportScopes keys // (i.e. a valid PolicyConfigurationIdentity() or PolicyConfigurationNamespaces() return value). // It is acceptable to allow an invalid value which will never be matched, it can "only" cause user confusion. // scope passed to this function will not be "", that value is always allowed. func (t ociTransport) ValidatePolicyConfigurationScope(scope string) error { - var dir string - sep := strings.SplitN(scope, ":", 2) - dir = sep[0] - - if len(sep) == 2 { - image := sep[1] - if !refRegexp.MatchString(image) { - return errors.Errorf("Invalid image %s", image) - } - } - - if !strings.HasPrefix(dir, "/") { - return errors.Errorf("Invalid scope %s: must be an absolute path", scope) - } - // Refuse also "/", otherwise "/" and "" would have the same semantics, - // and "" could be unexpectedly shadowed by the "/" entry. - // (Note: we do allow "/:someimage", a bit ridiculous but why refuse it?) - if scope == "/" { - return errors.New(`Invalid scope "/": Use the generic default scope ""`) - } - cleaned := filepath.Clean(dir) - if cleaned != dir { - return errors.Errorf(`Invalid scope %s: Uses non-canonical path format, perhaps try with path %s`, scope, cleaned) - } - return nil + return internal.ValidateScope(scope) } // ociReference is an ImageReference for OCI directory paths. @@ -92,13 +59,7 @@ type ociReference struct { // ParseReference converts a string, which should not start with the ImageTransport.Name prefix, into an OCI ImageReference. func ParseReference(reference string) (types.ImageReference, error) { - var dir, image string - sep := strings.SplitN(reference, ":", 2) - dir = sep[0] - - if len(sep) == 2 { - image = sep[1] - } + dir, image := internal.SplitPathAndImage(reference) return NewReference(dir, image) } @@ -111,14 +72,15 @@ func NewReference(dir, image string) (types.ImageReference, error) { if err != nil { return nil, err } - // This is necessary to prevent directory paths returned by PolicyConfigurationNamespaces - // from being ambiguous with values of PolicyConfigurationIdentity. - if strings.Contains(resolved, ":") { - return nil, errors.Errorf("Invalid OCI reference %s:%s: path %s contains a colon", dir, image, resolved) + + if err := internal.ValidateOCIPath(dir); err != nil { + return nil, err } - if len(image) > 0 && !refRegexp.MatchString(image) { - return nil, errors.Errorf("Invalid image %s", image) + + if err = internal.ValidateImageName(image); err != nil { + return nil, err } + return ociReference{dir: dir, resolvedDir: resolved, image: image}, nil } @@ -177,28 +139,40 @@ func (ref ociReference) PolicyConfigurationNamespaces() []string { return res } -// NewImage returns a types.Image for this reference, possibly specialized for this ImageTransport. -// The caller must call .Close() on the returned Image. +// NewImage returns a types.ImageCloser for this reference, possibly specialized for this ImageTransport. +// The caller must call .Close() on the returned ImageCloser. // NOTE: If any kind of signature verification should happen, build an UnparsedImage from the value returned by NewImageSource, // verify that UnparsedImage, and convert it into a real Image via image.FromUnparsedImage. -func (ref ociReference) NewImage(ctx *types.SystemContext) (types.Image, error) { +// WARNING: This may not do the right thing for a manifest list, see image.FromSource for details. +func (ref ociReference) NewImage(ctx *types.SystemContext) (types.ImageCloser, error) { src, err := newImageSource(ctx, ref) if err != nil { return nil, err } - return image.FromSource(src) + return image.FromSource(ctx, src) +} + +// getIndex returns a pointer to the index references by this ociReference. If an error occurs opening an index nil is returned together +// with an error. +func (ref ociReference) getIndex() (*imgspecv1.Index, error) { + indexJSON, err := os.Open(ref.indexPath()) + if err != nil { + return nil, err + } + defer indexJSON.Close() + + index := &imgspecv1.Index{} + if err := json.NewDecoder(indexJSON).Decode(index); err != nil { + return nil, err + } + return index, nil } func (ref ociReference) getManifestDescriptor() (imgspecv1.Descriptor, error) { - indexJSON, err := os.Open(ref.indexPath()) + index, err := ref.getIndex() if err != nil { return imgspecv1.Descriptor{}, err } - defer indexJSON.Close() - index := imgspecv1.Index{} - if err := json.NewDecoder(indexJSON).Decode(&index); err != nil { - return imgspecv1.Descriptor{}, err - } var d *imgspecv1.Descriptor if ref.image == "" { @@ -250,7 +224,7 @@ func (ref ociReference) NewImageSource(ctx *types.SystemContext) (types.ImageSou // NewImageDestination returns a types.ImageDestination for this reference. // The caller must call .Close() on the returned ImageDestination. func (ref ociReference) NewImageDestination(ctx *types.SystemContext) (types.ImageDestination, error) { - return newImageDestination(ref) + return newImageDestination(ctx, ref) } // DeleteImage deletes the named image from the registry, if supported. @@ -269,9 +243,13 @@ func (ref ociReference) indexPath() string { } // blobPath returns a path for a blob within a directory using OCI image-layout conventions. -func (ref ociReference) blobPath(digest digest.Digest) (string, error) { +func (ref ociReference) blobPath(digest digest.Digest, sharedBlobDir string) (string, error) { if err := digest.Validate(); err != nil { return "", errors.Wrapf(err, "unexpected digest reference %s", digest) } - return filepath.Join(ref.dir, "blobs", digest.Algorithm().String(), digest.Hex()), nil + blobDir := filepath.Join(ref.dir, "blobs") + if sharedBlobDir != "" { + blobDir = sharedBlobDir + } + return filepath.Join(blobDir, digest.Algorithm().String(), digest.Hex()), nil } diff --git a/vendor/github.com/containers/image/openshift/openshift.go b/vendor/github.com/containers/image/openshift/openshift.go index 0117f2e0..54655914 100644 --- a/vendor/github.com/containers/image/openshift/openshift.go +++ b/vendor/github.com/containers/image/openshift/openshift.go @@ -200,20 +200,15 @@ func (s *openshiftImageSource) Close() error { return nil } -func (s *openshiftImageSource) GetTargetManifest(digest digest.Digest) ([]byte, string, error) { - if err := s.ensureImageIsResolved(context.TODO()); err != nil { - return nil, "", err - } - return s.docker.GetTargetManifest(digest) -} - // GetManifest returns the image's manifest along with its MIME type (which may be empty when it can't be determined but the manifest is available). // It may use a remote (= slow) service. -func (s *openshiftImageSource) GetManifest() ([]byte, string, error) { +// If instanceDigest is not nil, it contains a digest of the specific manifest instance to retrieve (when the primary manifest is a manifest list); +// this never happens if the primary manifest is not a manifest list (e.g. if the source never returns manifest lists). +func (s *openshiftImageSource) GetManifest(instanceDigest *digest.Digest) ([]byte, string, error) { if err := s.ensureImageIsResolved(context.TODO()); err != nil { return nil, "", err } - return s.docker.GetManifest() + return s.docker.GetManifest(instanceDigest) } // GetBlob returns a stream for the specified blob, and the blob’s size (or -1 if unknown). @@ -224,12 +219,21 @@ func (s *openshiftImageSource) GetBlob(info types.BlobInfo) (io.ReadCloser, int6 return s.docker.GetBlob(info) } -func (s *openshiftImageSource) GetSignatures(ctx context.Context) ([][]byte, error) { - if err := s.ensureImageIsResolved(ctx); err != nil { - return nil, err +// GetSignatures returns the image's signatures. It may use a remote (= slow) service. +// If instanceDigest is not nil, it contains a digest of the specific manifest instance to retrieve signatures for +// (when the primary manifest is a manifest list); this never happens if the primary manifest is not a manifest list +// (e.g. if the source never returns manifest lists). +func (s *openshiftImageSource) GetSignatures(ctx context.Context, instanceDigest *digest.Digest) ([][]byte, error) { + var imageName string + if instanceDigest == nil { + if err := s.ensureImageIsResolved(ctx); err != nil { + return nil, err + } + imageName = s.imageStreamImageName + } else { + imageName = instanceDigest.String() } - - image, err := s.client.getImage(ctx, s.imageStreamImageName) + image, err := s.client.getImage(ctx, imageName) if err != nil { return nil, err } @@ -242,6 +246,11 @@ func (s *openshiftImageSource) GetSignatures(ctx context.Context) ([][]byte, err return sigs, nil } +// LayerInfosForCopy() returns updated layer info that should be used when reading, in preference to values in the manifest, if specified. +func (s *openshiftImageSource) LayerInfosForCopy() []types.BlobInfo { + return nil +} + // ensureImageIsResolved sets up s.docker and s.imageStreamImageName func (s *openshiftImageSource) ensureImageIsResolved(ctx context.Context) error { if s.docker != nil { diff --git a/vendor/github.com/containers/image/openshift/openshift_transport.go b/vendor/github.com/containers/image/openshift/openshift_transport.go index 7db35d96..686d806f 100644 --- a/vendor/github.com/containers/image/openshift/openshift_transport.go +++ b/vendor/github.com/containers/image/openshift/openshift_transport.go @@ -125,16 +125,17 @@ func (ref openshiftReference) PolicyConfigurationNamespaces() []string { return policyconfiguration.DockerReferenceNamespaces(ref.dockerReference) } -// NewImage returns a types.Image for this reference, possibly specialized for this ImageTransport. -// The caller must call .Close() on the returned Image. +// NewImage returns a types.ImageCloser for this reference, possibly specialized for this ImageTransport. +// The caller must call .Close() on the returned ImageCloser. // NOTE: If any kind of signature verification should happen, build an UnparsedImage from the value returned by NewImageSource, // verify that UnparsedImage, and convert it into a real Image via image.FromUnparsedImage. -func (ref openshiftReference) NewImage(ctx *types.SystemContext) (types.Image, error) { +// WARNING: This may not do the right thing for a manifest list, see image.FromSource for details. +func (ref openshiftReference) NewImage(ctx *types.SystemContext) (types.ImageCloser, error) { src, err := newImageSource(ctx, ref) if err != nil { return nil, err } - return genericImage.FromSource(src) + return genericImage.FromSource(ctx, src) } // NewImageSource returns a types.ImageSource for this reference. diff --git a/vendor/github.com/containers/image/ostree/ostree_dest.go b/vendor/github.com/containers/image/ostree/ostree_dest.go index 26137431..704e1ece 100644 --- a/vendor/github.com/containers/image/ostree/ostree_dest.go +++ b/vendor/github.com/containers/image/ostree/ostree_dest.go @@ -4,6 +4,8 @@ package ostree import ( "bytes" + "compress/gzip" + "encoding/base64" "encoding/json" "fmt" "io" @@ -12,18 +14,27 @@ import ( "os/exec" "path/filepath" "strconv" - "strings" "time" "github.com/containers/image/manifest" "github.com/containers/image/types" "github.com/containers/storage/pkg/archive" "github.com/opencontainers/go-digest" - "github.com/pkg/errors" - "github.com/ostreedev/ostree-go/pkg/otbuiltin" + "github.com/pkg/errors" + "github.com/vbatts/tar-split/tar/asm" + "github.com/vbatts/tar-split/tar/storage" ) +// #cgo pkg-config: glib-2.0 gobject-2.0 ostree-1 +// #include +// #include +// #include +// #include +// #include +// #include +import "C" + type blobToImport struct { Size int64 Digest digest.Digest @@ -35,18 +46,24 @@ type descriptor struct { Digest digest.Digest `json:"digest"` } +type fsLayersSchema1 struct { + BlobSum digest.Digest `json:"blobSum"` +} + type manifestSchema struct { - ConfigDescriptor descriptor `json:"config"` - LayersDescriptors []descriptor `json:"layers"` + LayersDescriptors []descriptor `json:"layers"` + FSLayers []fsLayersSchema1 `json:"fsLayers"` } type ostreeImageDestination struct { - ref ostreeReference - manifest string - schema manifestSchema - tmpDirPath string - blobs map[string]*blobToImport - digest digest.Digest + ref ostreeReference + manifest string + schema manifestSchema + tmpDirPath string + blobs map[string]*blobToImport + digest digest.Digest + signaturesLen int + repo *C.struct_OstreeRepo } // newImageDestination returns an ImageDestination for writing to an existing ostree. @@ -55,7 +72,7 @@ func newImageDestination(ref ostreeReference, tmpDirPath string) (types.ImageDes if err := ensureDirectoryExists(tmpDirPath); err != nil { return nil, err } - return &ostreeImageDestination{ref, "", manifestSchema{}, tmpDirPath, map[string]*blobToImport{}, ""}, nil + return &ostreeImageDestination{ref, "", manifestSchema{}, tmpDirPath, map[string]*blobToImport{}, "", 0, nil}, nil } // Reference returns the reference used to set up this destination. Note that this should directly correspond to user's intent, @@ -66,6 +83,9 @@ func (d *ostreeImageDestination) Reference() types.ImageReference { // Close removes resources associated with an initialized ImageDestination, if any. func (d *ostreeImageDestination) Close() error { + if d.repo != nil { + C.g_object_unref(C.gpointer(d.repo)) + } return os.RemoveAll(d.tmpDirPath) } @@ -174,6 +194,35 @@ func (d *ostreeImageDestination) ostreeCommit(repo *otbuiltin.Repo, branch strin return err } +func generateTarSplitMetadata(output *bytes.Buffer, file string) error { + mfz := gzip.NewWriter(output) + defer mfz.Close() + metaPacker := storage.NewJSONPacker(mfz) + + stream, err := os.OpenFile(file, os.O_RDONLY, 0) + if err != nil { + return err + } + defer stream.Close() + + gzReader, err := gzip.NewReader(stream) + if err != nil { + return err + } + defer gzReader.Close() + + its, err := asm.NewInputTarStream(gzReader, metaPacker, nil) + if err != nil { + return err + } + + _, err = io.Copy(ioutil.Discard, its) + if err != nil { + return err + } + return nil +} + func (d *ostreeImageDestination) importBlob(repo *otbuiltin.Repo, blob *blobToImport) error { ostreeBranch := fmt.Sprintf("ociimage/%s", blob.Digest.Hex()) destinationPath := filepath.Join(d.tmpDirPath, blob.Digest.Hex(), "root") @@ -185,6 +234,11 @@ func (d *ostreeImageDestination) importBlob(repo *otbuiltin.Repo, blob *blobToIm os.RemoveAll(destinationPath) }() + var tarSplitOutput bytes.Buffer + if err := generateTarSplitMetadata(&tarSplitOutput, blob.BlobPath); err != nil { + return err + } + if os.Getuid() == 0 { if err := archive.UntarPath(blob.BlobPath, destinationPath); err != nil { return err @@ -202,28 +256,35 @@ func (d *ostreeImageDestination) importBlob(repo *otbuiltin.Repo, blob *blobToIm return err } } + return d.ostreeCommit(repo, ostreeBranch, destinationPath, []string{fmt.Sprintf("docker.size=%d", blob.Size), + fmt.Sprintf("tarsplit.output=%s", base64.StdEncoding.EncodeToString(tarSplitOutput.Bytes()))}) + +} + +func (d *ostreeImageDestination) importConfig(repo *otbuiltin.Repo, blob *blobToImport) error { + ostreeBranch := fmt.Sprintf("ociimage/%s", blob.Digest.Hex()) + destinationPath := filepath.Dir(blob.BlobPath) + return d.ostreeCommit(repo, ostreeBranch, destinationPath, []string{fmt.Sprintf("docker.size=%d", blob.Size)}) } -func (d *ostreeImageDestination) importConfig(blob *blobToImport) error { - ostreeBranch := fmt.Sprintf("ociimage/%s", blob.Digest.Hex()) - - return exec.Command("ostree", "commit", - "--repo", d.ref.repo, - fmt.Sprintf("--add-metadata-string=docker.size=%d", blob.Size), - "--branch", ostreeBranch, filepath.Dir(blob.BlobPath)).Run() -} - func (d *ostreeImageDestination) HasBlob(info types.BlobInfo) (bool, int64, error) { - branch := fmt.Sprintf("ociimage/%s", info.Digest.Hex()) - output, err := exec.Command("ostree", "show", "--repo", d.ref.repo, "--print-metadata-key=docker.size", branch).CombinedOutput() - if err != nil { - if bytes.Index(output, []byte("not found")) >= 0 || bytes.Index(output, []byte("No such")) >= 0 { - return false, -1, nil + + if d.repo == nil { + repo, err := openRepo(d.ref.repo) + if err != nil { + return false, 0, err } - return false, -1, err + d.repo = repo } - size, err := strconv.ParseInt(strings.Trim(string(output), "'\n"), 10, 64) + branch := fmt.Sprintf("ociimage/%s", info.Digest.Hex()) + + found, data, err := readMetadata(d.repo, branch, "docker.size") + if err != nil || !found { + return found, -1, err + } + + size, err := strconv.ParseInt(data, 10, 64) if err != nil { return false, -1, err } @@ -272,6 +333,7 @@ func (d *ostreeImageDestination) PutSignatures(signatures [][]byte) error { return err } } + d.signaturesLen = len(signatures) return nil } @@ -286,24 +348,37 @@ func (d *ostreeImageDestination) Commit() error { return err } - for _, layer := range d.schema.LayersDescriptors { - hash := layer.Digest.Hex() + checkLayer := func(hash string) error { blob := d.blobs[hash] // if the blob is not present in d.blobs then it is already stored in OSTree, // and we don't need to import it. if blob == nil { - continue + return nil } err := d.importBlob(repo, blob) if err != nil { return err } + + delete(d.blobs, hash) + return nil + } + for _, layer := range d.schema.LayersDescriptors { + hash := layer.Digest.Hex() + if err = checkLayer(hash); err != nil { + return err + } + } + for _, layer := range d.schema.FSLayers { + hash := layer.BlobSum.Hex() + if err = checkLayer(hash); err != nil { + return err + } } - hash := d.schema.ConfigDescriptor.Digest.Hex() - blob := d.blobs[hash] - if blob != nil { - err := d.importConfig(blob) + // Import the other blobs that are not layers + for _, blob := range d.blobs { + err := d.importConfig(repo, blob) if err != nil { return err } @@ -311,7 +386,9 @@ func (d *ostreeImageDestination) Commit() error { manifestPath := filepath.Join(d.tmpDirPath, "manifest") - metadata := []string{fmt.Sprintf("docker.manifest=%s", string(d.manifest)), fmt.Sprintf("docker.digest=%s", string(d.digest))} + metadata := []string{fmt.Sprintf("docker.manifest=%s", string(d.manifest)), + fmt.Sprintf("signatures=%d", d.signaturesLen), + fmt.Sprintf("docker.digest=%s", string(d.digest))} err = d.ostreeCommit(repo, fmt.Sprintf("ociimage/%s", d.ref.branchName), manifestPath, metadata) _, err = repo.CommitTransaction() diff --git a/vendor/github.com/containers/image/ostree/ostree_src.go b/vendor/github.com/containers/image/ostree/ostree_src.go new file mode 100644 index 00000000..c65a07b7 --- /dev/null +++ b/vendor/github.com/containers/image/ostree/ostree_src.go @@ -0,0 +1,354 @@ +// +build !containers_image_ostree_stub + +package ostree + +import ( + "bytes" + "compress/gzip" + "context" + "encoding/base64" + "fmt" + "io" + "io/ioutil" + "strconv" + "strings" + "unsafe" + + "github.com/containers/image/manifest" + "github.com/containers/image/types" + "github.com/containers/storage/pkg/ioutils" + "github.com/opencontainers/go-digest" + glib "github.com/ostreedev/ostree-go/pkg/glibobject" + "github.com/pkg/errors" + "github.com/vbatts/tar-split/tar/asm" + "github.com/vbatts/tar-split/tar/storage" +) + +// #cgo pkg-config: glib-2.0 gobject-2.0 ostree-1 +// #include +// #include +// #include +// #include +// #include +// #include +import "C" + +type ostreeImageSource struct { + ref ostreeReference + tmpDir string + repo *C.struct_OstreeRepo +} + +// newImageSource returns an ImageSource for reading from an existing directory. +func newImageSource(ctx *types.SystemContext, tmpDir string, ref ostreeReference) (types.ImageSource, error) { + return &ostreeImageSource{ref: ref, tmpDir: tmpDir}, nil +} + +// Reference returns the reference used to set up this source. +func (s *ostreeImageSource) Reference() types.ImageReference { + return s.ref +} + +// Close removes resources associated with an initialized ImageSource, if any. +func (s *ostreeImageSource) Close() error { + if s.repo != nil { + C.g_object_unref(C.gpointer(s.repo)) + } + return nil +} + +func (s *ostreeImageSource) getLayerSize(blob string) (int64, error) { + b := fmt.Sprintf("ociimage/%s", blob) + found, data, err := readMetadata(s.repo, b, "docker.size") + if err != nil || !found { + return 0, err + } + return strconv.ParseInt(data, 10, 64) +} + +func (s *ostreeImageSource) getLenSignatures() (int64, error) { + b := fmt.Sprintf("ociimage/%s", s.ref.branchName) + found, data, err := readMetadata(s.repo, b, "signatures") + if err != nil { + return -1, err + } + if !found { + // if 'signatures' is not present, just return 0 signatures. + return 0, nil + } + return strconv.ParseInt(data, 10, 64) +} + +func (s *ostreeImageSource) getTarSplitData(blob string) ([]byte, error) { + b := fmt.Sprintf("ociimage/%s", blob) + found, out, err := readMetadata(s.repo, b, "tarsplit.output") + if err != nil || !found { + return nil, err + } + return base64.StdEncoding.DecodeString(out) +} + +// GetManifest returns the image's manifest along with its MIME type (which may be empty when it can't be determined but the manifest is available). +// It may use a remote (= slow) service. +func (s *ostreeImageSource) GetManifest(instanceDigest *digest.Digest) ([]byte, string, error) { + if instanceDigest != nil { + return nil, "", errors.Errorf(`Manifest lists are not supported by "ostree:"`) + } + if s.repo == nil { + repo, err := openRepo(s.ref.repo) + if err != nil { + return nil, "", err + } + s.repo = repo + } + + b := fmt.Sprintf("ociimage/%s", s.ref.branchName) + found, out, err := readMetadata(s.repo, b, "docker.manifest") + if err != nil { + return nil, "", err + } + if !found { + return nil, "", errors.New("manifest not found") + } + m := []byte(out) + return m, manifest.GuessMIMEType(m), nil +} + +func (s *ostreeImageSource) GetTargetManifest(digest digest.Digest) ([]byte, string, error) { + return nil, "", errors.New("manifest lists are not supported by this transport") +} + +func openRepo(path string) (*C.struct_OstreeRepo, error) { + var cerr *C.GError + cpath := C.CString(path) + defer C.free(unsafe.Pointer(cpath)) + pathc := C.g_file_new_for_path(cpath) + defer C.g_object_unref(C.gpointer(pathc)) + repo := C.ostree_repo_new(pathc) + r := glib.GoBool(glib.GBoolean(C.ostree_repo_open(repo, nil, &cerr))) + if !r { + C.g_object_unref(C.gpointer(repo)) + return nil, glib.ConvertGError(glib.ToGError(unsafe.Pointer(cerr))) + } + return repo, nil +} + +type ostreePathFileGetter struct { + repo *C.struct_OstreeRepo + parentRoot *C.GFile +} + +type ostreeReader struct { + stream *C.GFileInputStream +} + +func (o ostreeReader) Close() error { + C.g_object_unref(C.gpointer(o.stream)) + return nil +} +func (o ostreeReader) Read(p []byte) (int, error) { + var cerr *C.GError + instanceCast := C.g_type_check_instance_cast((*C.GTypeInstance)(unsafe.Pointer(o.stream)), C.g_input_stream_get_type()) + stream := (*C.GInputStream)(unsafe.Pointer(instanceCast)) + + b := C.g_input_stream_read_bytes(stream, (C.gsize)(cap(p)), nil, &cerr) + if b == nil { + return 0, glib.ConvertGError(glib.ToGError(unsafe.Pointer(cerr))) + } + defer C.g_bytes_unref(b) + + count := int(C.g_bytes_get_size(b)) + if count == 0 { + return 0, io.EOF + } + data := (*[1 << 30]byte)(unsafe.Pointer(C.g_bytes_get_data(b, nil)))[:count:count] + copy(p, data) + return count, nil +} + +func readMetadata(repo *C.struct_OstreeRepo, commit, key string) (bool, string, error) { + var cerr *C.GError + var ref *C.char + defer C.free(unsafe.Pointer(ref)) + + cCommit := C.CString(commit) + defer C.free(unsafe.Pointer(cCommit)) + + if !glib.GoBool(glib.GBoolean(C.ostree_repo_resolve_rev(repo, cCommit, C.gboolean(1), &ref, &cerr))) { + return false, "", glib.ConvertGError(glib.ToGError(unsafe.Pointer(cerr))) + } + + if ref == nil { + return false, "", nil + } + + var variant *C.GVariant + if !glib.GoBool(glib.GBoolean(C.ostree_repo_load_variant(repo, C.OSTREE_OBJECT_TYPE_COMMIT, ref, &variant, &cerr))) { + return false, "", glib.ConvertGError(glib.ToGError(unsafe.Pointer(cerr))) + } + defer C.g_variant_unref(variant) + if variant != nil { + cKey := C.CString(key) + defer C.free(unsafe.Pointer(cKey)) + + metadata := C.g_variant_get_child_value(variant, 0) + defer C.g_variant_unref(metadata) + + data := C.g_variant_lookup_value(metadata, (*C.gchar)(cKey), nil) + if data != nil { + defer C.g_variant_unref(data) + ptr := (*C.char)(C.g_variant_get_string(data, nil)) + val := C.GoString(ptr) + return true, val, nil + } + } + return false, "", nil +} + +func newOSTreePathFileGetter(repo *C.struct_OstreeRepo, commit string) (*ostreePathFileGetter, error) { + var cerr *C.GError + var parentRoot *C.GFile + cCommit := C.CString(commit) + defer C.free(unsafe.Pointer(cCommit)) + if !glib.GoBool(glib.GBoolean(C.ostree_repo_read_commit(repo, cCommit, &parentRoot, nil, nil, &cerr))) { + return &ostreePathFileGetter{}, glib.ConvertGError(glib.ToGError(unsafe.Pointer(cerr))) + } + + C.g_object_ref(C.gpointer(repo)) + + return &ostreePathFileGetter{repo: repo, parentRoot: parentRoot}, nil +} + +func (o ostreePathFileGetter) Get(filename string) (io.ReadCloser, error) { + var file *C.GFile + if strings.HasPrefix(filename, "./") { + filename = filename[2:] + } + cfilename := C.CString(filename) + defer C.free(unsafe.Pointer(cfilename)) + + file = (*C.GFile)(C.g_file_resolve_relative_path(o.parentRoot, cfilename)) + + var cerr *C.GError + stream := C.g_file_read(file, nil, &cerr) + if stream == nil { + return nil, glib.ConvertGError(glib.ToGError(unsafe.Pointer(cerr))) + } + + return &ostreeReader{stream: stream}, nil +} + +func (o ostreePathFileGetter) Close() { + C.g_object_unref(C.gpointer(o.repo)) + C.g_object_unref(C.gpointer(o.parentRoot)) +} + +func (s *ostreeImageSource) readSingleFile(commit, path string) (io.ReadCloser, error) { + getter, err := newOSTreePathFileGetter(s.repo, commit) + if err != nil { + return nil, err + } + defer getter.Close() + + return getter.Get(path) +} + +// GetBlob returns a stream for the specified blob, and the blob's size. +func (s *ostreeImageSource) GetBlob(info types.BlobInfo) (io.ReadCloser, int64, error) { + blob := info.Digest.Hex() + branch := fmt.Sprintf("ociimage/%s", blob) + + if s.repo == nil { + repo, err := openRepo(s.ref.repo) + if err != nil { + return nil, 0, err + } + s.repo = repo + } + + layerSize, err := s.getLayerSize(blob) + if err != nil { + return nil, 0, err + } + + tarsplit, err := s.getTarSplitData(blob) + if err != nil { + return nil, 0, err + } + + // if tarsplit is nil we are looking at the manifest. Return directly the file in /content + if tarsplit == nil { + file, err := s.readSingleFile(branch, "/content") + if err != nil { + return nil, 0, err + } + return file, layerSize, nil + } + + mf := bytes.NewReader(tarsplit) + mfz, err := gzip.NewReader(mf) + if err != nil { + return nil, 0, err + } + defer mfz.Close() + metaUnpacker := storage.NewJSONUnpacker(mfz) + + getter, err := newOSTreePathFileGetter(s.repo, branch) + if err != nil { + return nil, 0, err + } + + ots := asm.NewOutputTarStream(getter, metaUnpacker) + + pipeReader, pipeWriter := io.Pipe() + go func() { + io.Copy(pipeWriter, ots) + pipeWriter.Close() + }() + + rc := ioutils.NewReadCloserWrapper(pipeReader, func() error { + getter.Close() + return ots.Close() + }) + return rc, layerSize, nil +} + +func (s *ostreeImageSource) GetSignatures(ctx context.Context, instanceDigest *digest.Digest) ([][]byte, error) { + if instanceDigest != nil { + return nil, errors.New("manifest lists are not supported by this transport") + } + lenSignatures, err := s.getLenSignatures() + if err != nil { + return nil, err + } + branch := fmt.Sprintf("ociimage/%s", s.ref.branchName) + + if s.repo == nil { + repo, err := openRepo(s.ref.repo) + if err != nil { + return nil, err + } + s.repo = repo + } + + signatures := [][]byte{} + for i := int64(1); i <= lenSignatures; i++ { + sigReader, err := s.readSingleFile(branch, fmt.Sprintf("/signature-%d", i)) + if err != nil { + return nil, err + } + defer sigReader.Close() + + sig, err := ioutil.ReadAll(sigReader) + if err != nil { + return nil, err + } + signatures = append(signatures, sig) + } + return signatures, nil +} + +// LayerInfosForCopy() returns updated layer info that should be used when reading, in preference to values in the manifest, if specified. +func (s *ostreeImageSource) LayerInfosForCopy() []types.BlobInfo { + return nil +} diff --git a/vendor/github.com/containers/image/ostree/ostree_transport.go b/vendor/github.com/containers/image/ostree/ostree_transport.go index 0de74a71..cc85a43f 100644 --- a/vendor/github.com/containers/image/ostree/ostree_transport.go +++ b/vendor/github.com/containers/image/ostree/ostree_transport.go @@ -10,12 +10,12 @@ import ( "regexp" "strings" - "github.com/pkg/errors" - "github.com/containers/image/directory/explicitfilepath" "github.com/containers/image/docker/reference" + "github.com/containers/image/image" "github.com/containers/image/transports" "github.com/containers/image/types" + "github.com/pkg/errors" ) const defaultOSTreeRepo = "/ostree/repo" @@ -66,6 +66,11 @@ type ostreeReference struct { repo string } +type ostreeImageCloser struct { + types.ImageCloser + size int64 +} + func (t ostreeTransport) ParseReference(ref string) (types.ImageReference, error) { var repo = "" var image = "" @@ -110,7 +115,7 @@ func NewReference(image string, repo string) (types.ImageReference, error) { // This is necessary to prevent directory paths returned by PolicyConfigurationNamespaces // from being ambiguous with values of PolicyConfigurationIdentity. if strings.Contains(resolved, ":") { - return nil, errors.Errorf("Invalid OSTreeCI reference %s@%s: path %s contains a colon", image, repo, resolved) + return nil, errors.Errorf("Invalid OSTree reference %s@%s: path %s contains a colon", image, repo, resolved) } return ostreeReference{ @@ -168,18 +173,38 @@ func (ref ostreeReference) PolicyConfigurationNamespaces() []string { return res } -// NewImage returns a types.Image for this reference, possibly specialized for this ImageTransport. -// The caller must call .Close() on the returned Image. +func (s *ostreeImageCloser) Size() (int64, error) { + return s.size, nil +} + +// NewImage returns a types.ImageCloser for this reference, possibly specialized for this ImageTransport. +// The caller must call .Close() on the returned ImageCloser. // NOTE: If any kind of signature verification should happen, build an UnparsedImage from the value returned by NewImageSource, // verify that UnparsedImage, and convert it into a real Image via image.FromUnparsedImage. -func (ref ostreeReference) NewImage(ctx *types.SystemContext) (types.Image, error) { - return nil, errors.New("Reading ostree: images is currently not supported") +func (ref ostreeReference) NewImage(ctx *types.SystemContext) (types.ImageCloser, error) { + var tmpDir string + if ctx == nil || ctx.OSTreeTmpDirPath == "" { + tmpDir = os.TempDir() + } else { + tmpDir = ctx.OSTreeTmpDirPath + } + src, err := newImageSource(ctx, tmpDir, ref) + if err != nil { + return nil, err + } + return image.FromSource(ctx, src) } // NewImageSource returns a types.ImageSource for this reference. // The caller must call .Close() on the returned ImageSource. func (ref ostreeReference) NewImageSource(ctx *types.SystemContext) (types.ImageSource, error) { - return nil, errors.New("Reading ostree: images is currently not supported") + var tmpDir string + if ctx == nil || ctx.OSTreeTmpDirPath == "" { + tmpDir = os.TempDir() + } else { + tmpDir = ctx.OSTreeTmpDirPath + } + return newImageSource(ctx, tmpDir, ref) } // NewImageDestination returns a types.ImageDestination for this reference. diff --git a/vendor/github.com/containers/image/signature/policy_config.go b/vendor/github.com/containers/image/signature/policy_config.go index bc6c5e9a..42cc12ab 100644 --- a/vendor/github.com/containers/image/signature/policy_config.go +++ b/vendor/github.com/containers/image/signature/policy_config.go @@ -70,7 +70,11 @@ func NewPolicyFromFile(fileName string) (*Policy, error) { if err != nil { return nil, err } - return NewPolicyFromBytes(contents) + policy, err := NewPolicyFromBytes(contents) + if err != nil { + return nil, errors.Wrapf(err, "invalid policy in %q", fileName) + } + return policy, nil } // NewPolicyFromBytes returns a policy parsed from the specified blob. diff --git a/vendor/github.com/containers/image/storage/storage_image.go b/vendor/github.com/containers/image/storage/storage_image.go index 08fa71b5..038195c1 100644 --- a/vendor/github.com/containers/image/storage/storage_image.go +++ b/vendor/github.com/containers/image/storage/storage_image.go @@ -1,14 +1,17 @@ +// +build !containers_image_storage_stub + package storage import ( "bytes" "context" "encoding/json" + "fmt" "io" "io/ioutil" - "time" - - "github.com/pkg/errors" + "os" + "path/filepath" + "sync/atomic" "github.com/containers/image/image" "github.com/containers/image/manifest" @@ -16,10 +19,14 @@ import ( "github.com/containers/storage" "github.com/containers/storage/pkg/archive" "github.com/containers/storage/pkg/ioutils" - ddigest "github.com/opencontainers/go-digest" + digest "github.com/opencontainers/go-digest" + imgspecv1 "github.com/opencontainers/image-spec/specs-go/v1" + "github.com/pkg/errors" "github.com/sirupsen/logrus" ) +const temporaryDirectoryForBigFiles = "/var/tmp" // Do not use the system default of os.TempDir(), usually /tmp, because with systemd it could be a tmpfs. + var ( // ErrBlobDigestMismatch is returned when PutBlob() is given a blob // with a digest-based name that doesn't match its contents. @@ -27,8 +34,8 @@ var ( // ErrBlobSizeMismatch is returned when PutBlob() is given a blob // with an expected size that doesn't match the reader. ErrBlobSizeMismatch = errors.New("blob size mismatch") - // ErrNoManifestLists is returned when GetTargetManifest() is - // called. + // ErrNoManifestLists is returned when GetManifest() is called. + // with a non-nil instanceDigest. ErrNoManifestLists = errors.New("manifest lists are not supported by this transport") // ErrNoSuchImage is returned when we attempt to access an image which // doesn't exist in the storage area. @@ -37,256 +44,318 @@ var ( type storageImageSource struct { imageRef storageReference - Tag string `json:"tag,omitempty"` - Created time.Time `json:"created-time,omitempty"` - ID string `json:"id"` - BlobList []types.BlobInfo `json:"blob-list,omitempty"` // Ordered list of every blob the image has been told to handle - Layers map[ddigest.Digest][]string `json:"layers,omitempty"` // Map from digests of blobs to lists of layer IDs - LayerPosition map[ddigest.Digest]int `json:"-"` // Where we are in reading a blob's layers - SignatureSizes []int `json:"signature-sizes"` // List of sizes of each signature slice + ID string + layerPosition map[digest.Digest]int // Where we are in reading a blob's layers + cachedManifest []byte // A cached copy of the manifest, if already known, or nil + SignatureSizes []int `json:"signature-sizes,omitempty"` // List of sizes of each signature slice } type storageImageDestination struct { - imageRef storageReference - Tag string `json:"tag,omitempty"` - Created time.Time `json:"created-time,omitempty"` - ID string `json:"id"` - BlobList []types.BlobInfo `json:"blob-list,omitempty"` // Ordered list of every blob the image has been told to handle - Layers map[ddigest.Digest][]string `json:"layers,omitempty"` // Map from digests of blobs to lists of layer IDs - BlobData map[ddigest.Digest][]byte `json:"-"` // Map from names of blobs that aren't layers to contents, temporary - Manifest []byte `json:"-"` // Manifest contents, temporary - Signatures []byte `json:"-"` // Signature contents, temporary - SignatureSizes []int `json:"signature-sizes"` // List of sizes of each signature slice + image types.ImageCloser + systemContext *types.SystemContext + imageRef storageReference // The reference we'll use to name the image + publicRef storageReference // The reference we return when asked about the name we'll give to the image + directory string // Temporary directory where we store blobs until Commit() time + nextTempFileID int32 // A counter that we use for computing filenames to assign to blobs + manifest []byte // Manifest contents, temporary + signatures []byte // Signature contents, temporary + blobDiffIDs map[digest.Digest]digest.Digest // Mapping from layer blobsums to their corresponding DiffIDs + fileSizes map[digest.Digest]int64 // Mapping from layer blobsums to their sizes + filenames map[digest.Digest]string // Mapping from layer blobsums to names of files we used to hold them + SignatureSizes []int `json:"signature-sizes,omitempty"` // List of sizes of each signature slice } -type storageLayerMetadata struct { - Digest string `json:"digest,omitempty"` - Size int64 `json:"size"` - CompressedSize int64 `json:"compressed-size,omitempty"` -} - -type storageImage struct { - types.Image +type storageImageCloser struct { + types.ImageCloser size int64 } -// newImageSource sets us up to read out an image, which needs to already exist. +// newImageSource sets up an image for reading. func newImageSource(imageRef storageReference) (*storageImageSource, error) { + // First, locate the image. img, err := imageRef.resolveImage() if err != nil { return nil, err } + + // Build the reader object. image := &storageImageSource{ imageRef: imageRef, - Created: time.Now(), ID: img.ID, - BlobList: []types.BlobInfo{}, - Layers: make(map[ddigest.Digest][]string), - LayerPosition: make(map[ddigest.Digest]int), + layerPosition: make(map[digest.Digest]int), SignatureSizes: []int{}, } - if err := json.Unmarshal([]byte(img.Metadata), image); err != nil { - return nil, errors.Wrap(err, "error decoding metadata for source image") - } - return image, nil -} - -// newImageDestination sets us up to write a new image. -func newImageDestination(imageRef storageReference) (*storageImageDestination, error) { - image := &storageImageDestination{ - imageRef: imageRef, - Tag: imageRef.reference, - Created: time.Now(), - ID: imageRef.id, - BlobList: []types.BlobInfo{}, - Layers: make(map[ddigest.Digest][]string), - BlobData: make(map[ddigest.Digest][]byte), - SignatureSizes: []int{}, + if img.Metadata != "" { + if err := json.Unmarshal([]byte(img.Metadata), image); err != nil { + return nil, errors.Wrap(err, "error decoding metadata for source image") + } } return image, nil } +// Reference returns the image reference that we used to find this image. func (s storageImageSource) Reference() types.ImageReference { return s.imageRef } -func (s storageImageDestination) Reference() types.ImageReference { - return s.imageRef -} - +// Close cleans up any resources we tied up while reading the image. func (s storageImageSource) Close() error { return nil } -func (s storageImageDestination) Close() error { - return nil +// GetBlob reads the data blob or filesystem layer which matches the digest and size, if given. +func (s *storageImageSource) GetBlob(info types.BlobInfo) (rc io.ReadCloser, n int64, err error) { + rc, n, _, err = s.getBlobAndLayerID(info) + return rc, n, err } +// getBlobAndLayer reads the data blob or filesystem layer which matches the digest and size, if given. +func (s *storageImageSource) getBlobAndLayerID(info types.BlobInfo) (rc io.ReadCloser, n int64, layerID string, err error) { + var layer storage.Layer + var diffOptions *storage.DiffOptions + // We need a valid digest value. + err = info.Digest.Validate() + if err != nil { + return nil, -1, "", err + } + // Check if the blob corresponds to a diff that was used to initialize any layers. Our + // callers should try to retrieve layers using their uncompressed digests, so no need to + // check if they're using one of the compressed digests, which we can't reproduce anyway. + layers, err := s.imageRef.transport.store.LayersByUncompressedDigest(info.Digest) + // If it's not a layer, then it must be a data item. + if len(layers) == 0 { + b, err := s.imageRef.transport.store.ImageBigData(s.ID, info.Digest.String()) + if err != nil { + return nil, -1, "", err + } + r := bytes.NewReader(b) + logrus.Debugf("exporting opaque data as blob %q", info.Digest.String()) + return ioutil.NopCloser(r), int64(r.Len()), "", nil + } + // Step through the list of matching layers. Tests may want to verify that if we have multiple layers + // which claim to have the same contents, that we actually do have multiple layers, otherwise we could + // just go ahead and use the first one every time. + i := s.layerPosition[info.Digest] + s.layerPosition[info.Digest] = i + 1 + if len(layers) > 0 { + layer = layers[i%len(layers)] + } + // Force the storage layer to not try to match any compression that was used when the layer was first + // handed to it. + noCompression := archive.Uncompressed + diffOptions = &storage.DiffOptions{ + Compression: &noCompression, + } + if layer.UncompressedSize < 0 { + n = -1 + } else { + n = layer.UncompressedSize + } + logrus.Debugf("exporting filesystem layer %q without compression for blob %q", layer.ID, info.Digest) + rc, err = s.imageRef.transport.store.Diff("", layer.ID, diffOptions) + if err != nil { + return nil, -1, "", err + } + return rc, n, layer.ID, err +} + +// GetManifest() reads the image's manifest. +func (s *storageImageSource) GetManifest(instanceDigest *digest.Digest) (manifestBlob []byte, MIMEType string, err error) { + if instanceDigest != nil { + return nil, "", ErrNoManifestLists + } + if len(s.cachedManifest) == 0 { + // We stored the manifest as an item named after storage.ImageDigestBigDataKey. + cachedBlob, err := s.imageRef.transport.store.ImageBigData(s.ID, storage.ImageDigestBigDataKey) + if err != nil { + return nil, "", err + } + s.cachedManifest = cachedBlob + } + return s.cachedManifest, manifest.GuessMIMEType(s.cachedManifest), err +} + +// LayerInfosForCopy() returns the list of layer blobs that make up the root filesystem of +// the image, after they've been decompressed. +func (s *storageImageSource) LayerInfosForCopy() []types.BlobInfo { + simg, err := s.imageRef.transport.store.Image(s.ID) + if err != nil { + logrus.Errorf("error reading image %q: %v", s.ID, err) + return nil + } + updatedBlobInfos := []types.BlobInfo{} + layerID := simg.TopLayer + _, manifestType, err := s.GetManifest(nil) + if err != nil { + logrus.Errorf("error reading image manifest for %q: %v", s.ID, err) + return nil + } + uncompressedLayerType := "" + switch manifestType { + case imgspecv1.MediaTypeImageManifest: + uncompressedLayerType = imgspecv1.MediaTypeImageLayer + case manifest.DockerV2Schema1MediaType, manifest.DockerV2Schema1SignedMediaType, manifest.DockerV2Schema2MediaType: + // This is actually a compressed type, but there's no uncompressed type defined + uncompressedLayerType = manifest.DockerV2Schema2LayerMediaType + } + for layerID != "" { + layer, err := s.imageRef.transport.store.Layer(layerID) + if err != nil { + logrus.Errorf("error reading layer %q in image %q: %v", layerID, s.ID, err) + return nil + } + if layer.UncompressedDigest == "" { + logrus.Errorf("uncompressed digest for layer %q is unknown", layerID) + return nil + } + if layer.UncompressedSize < 0 { + logrus.Errorf("uncompressed size for layer %q is unknown", layerID) + return nil + } + blobInfo := types.BlobInfo{ + Digest: layer.UncompressedDigest, + Size: layer.UncompressedSize, + MediaType: uncompressedLayerType, + } + updatedBlobInfos = append([]types.BlobInfo{blobInfo}, updatedBlobInfos...) + layerID = layer.Parent + } + return updatedBlobInfos +} + +// GetSignatures() parses the image's signatures blob into a slice of byte slices. +func (s *storageImageSource) GetSignatures(ctx context.Context, instanceDigest *digest.Digest) (signatures [][]byte, err error) { + if instanceDigest != nil { + return nil, ErrNoManifestLists + } + var offset int + sigslice := [][]byte{} + signature := []byte{} + if len(s.SignatureSizes) > 0 { + signatureBlob, err := s.imageRef.transport.store.ImageBigData(s.ID, "signatures") + if err != nil { + return nil, errors.Wrapf(err, "error looking up signatures data for image %q", s.ID) + } + signature = signatureBlob + } + for _, length := range s.SignatureSizes { + sigslice = append(sigslice, signature[offset:offset+length]) + offset += length + } + if offset != len(signature) { + return nil, errors.Errorf("signatures data contained %d extra bytes", len(signatures)-offset) + } + return sigslice, nil +} + +// newImageDestination sets us up to write a new image, caching blobs in a temporary directory until +// it's time to Commit() the image +func newImageDestination(ctx *types.SystemContext, imageRef storageReference) (*storageImageDestination, error) { + directory, err := ioutil.TempDir(temporaryDirectoryForBigFiles, "storage") + if err != nil { + return nil, errors.Wrapf(err, "error creating a temporary directory") + } + // Break reading of the reference we're writing, so that copy.Image() won't try to rewrite + // schema1 image manifests to remove embedded references, since that changes the manifest's + // digest, and that makes the image unusable if we subsequently try to access it using a + // reference that mentions the no-longer-correct digest. + publicRef := imageRef + publicRef.name = nil + image := &storageImageDestination{ + systemContext: ctx, + imageRef: imageRef, + publicRef: publicRef, + directory: directory, + blobDiffIDs: make(map[digest.Digest]digest.Digest), + fileSizes: make(map[digest.Digest]int64), + filenames: make(map[digest.Digest]string), + SignatureSizes: []int{}, + } + return image, nil +} + +// Reference returns a mostly-usable image reference that can't return a DockerReference, to +// avoid triggering logic in copy.Image() that rewrites schema 1 image manifests in order to +// remove image names that they contain which don't match the value we're using. +func (s storageImageDestination) Reference() types.ImageReference { + return s.publicRef +} + +// Close cleans up the temporary directory. +func (s *storageImageDestination) Close() error { + return os.RemoveAll(s.directory) +} + +// ShouldCompressLayers indicates whether or not a caller should compress not-already-compressed +// data when handing it to us. func (s storageImageDestination) ShouldCompressLayers() bool { - // We ultimately have to decompress layers to populate trees on disk, - // so callers shouldn't bother compressing them before handing them to - // us, if they're not already compressed. + // We ultimately have to decompress layers to populate trees on disk, so callers shouldn't + // bother compressing them before handing them to us, if they're not already compressed. return false } -// putBlob stores a layer or data blob, optionally enforcing that a digest in -// blobinfo matches the incoming data. -func (s *storageImageDestination) putBlob(stream io.Reader, blobinfo types.BlobInfo, enforceDigestAndSize bool) (types.BlobInfo, error) { - blobSize := blobinfo.Size - digest := blobinfo.Digest +// PutBlob stores a layer or data blob in our temporary directory, checking that any information +// in the blobinfo matches the incoming data. +func (s *storageImageDestination) PutBlob(stream io.Reader, blobinfo types.BlobInfo) (types.BlobInfo, error) { errorBlobInfo := types.BlobInfo{ Digest: "", Size: -1, } - // Try to read an initial snippet of the blob. - buf := [archive.HeaderSize]byte{} - n, err := io.ReadAtLeast(stream, buf[:], len(buf)) - if err != nil && err != io.EOF && err != io.ErrUnexpectedEOF { - return errorBlobInfo, err - } - // Set up to read the whole blob (the initial snippet, plus the rest) - // while digesting it with either the default, or the passed-in digest, - // if one was specified. - hasher := ddigest.Canonical.Digester() - if digest.Validate() == nil { - if a := digest.Algorithm(); a.Available() { + // Set up to digest the blob and count its size while saving it to a file. + hasher := digest.Canonical.Digester() + if blobinfo.Digest.Validate() == nil { + if a := blobinfo.Digest.Algorithm(); a.Available() { hasher = a.Digester() } } - hash := "" + diffID := digest.Canonical.Digester() + filename := filepath.Join(s.directory, fmt.Sprintf("%d", atomic.AddInt32(&s.nextTempFileID, 1))) + file, err := os.OpenFile(filename, os.O_CREATE|os.O_TRUNC|os.O_WRONLY|os.O_EXCL, 0600) + if err != nil { + return errorBlobInfo, errors.Wrapf(err, "error creating temporary file %q", filename) + } + defer file.Close() counter := ioutils.NewWriteCounter(hasher.Hash()) - defragmented := io.MultiReader(bytes.NewBuffer(buf[:n]), stream) - multi := io.TeeReader(defragmented, counter) - if (n > 0) && archive.IsArchive(buf[:n]) { - // It's a filesystem layer. If it's not the first one in the - // image, we assume that the most recently added layer is its - // parent. - parentLayer := "" - for _, blob := range s.BlobList { - if layerList, ok := s.Layers[blob.Digest]; ok { - parentLayer = layerList[len(layerList)-1] - } - } - // If we have an expected content digest, generate a layer ID - // based on the parent's ID and the expected content digest. - id := "" - if digest.Validate() == nil { - id = ddigest.Canonical.FromBytes([]byte(parentLayer + "+" + digest.String())).Hex() - } - // Attempt to create the identified layer and import its contents. - layer, uncompressedSize, err := s.imageRef.transport.store.PutLayer(id, parentLayer, nil, "", true, multi) - if err != nil && errors.Cause(err) != storage.ErrDuplicateID { - logrus.Debugf("error importing layer blob %q as %q: %v", blobinfo.Digest, id, err) - return errorBlobInfo, err - } - if errors.Cause(err) == storage.ErrDuplicateID { - // We specified an ID, and there's already a layer with - // the same ID. Drain the input so that we can look at - // its length and digest. - _, err := io.Copy(ioutil.Discard, multi) - if err != nil && err != io.EOF { - logrus.Debugf("error digesting layer blob %q: %v", blobinfo.Digest, id, err) - return errorBlobInfo, err - } - hash = hasher.Digest().String() - } else { - // Applied the layer with the specified ID. Note the - // size info and computed digest. - hash = hasher.Digest().String() - layerMeta := storageLayerMetadata{ - Digest: hash, - CompressedSize: counter.Count, - Size: uncompressedSize, - } - if metadata, err := json.Marshal(&layerMeta); len(metadata) != 0 && err == nil { - s.imageRef.transport.store.SetMetadata(layer.ID, string(metadata)) - } - // Hang on to the new layer's ID. - id = layer.ID - } - // Check if the size looks right. - if enforceDigestAndSize && blobinfo.Size >= 0 && blobinfo.Size != counter.Count { - logrus.Debugf("layer blob %q size is %d, not %d, rejecting", blobinfo.Digest, counter.Count, blobinfo.Size) - if layer != nil { - // Something's wrong; delete the newly-created layer. - s.imageRef.transport.store.DeleteLayer(layer.ID) - } - return errorBlobInfo, ErrBlobSizeMismatch - } - // If the content digest was specified, verify it. - if enforceDigestAndSize && digest.Validate() == nil && digest.String() != hash { - logrus.Debugf("layer blob %q digests to %q, rejecting", blobinfo.Digest, hash) - if layer != nil { - // Something's wrong; delete the newly-created layer. - s.imageRef.transport.store.DeleteLayer(layer.ID) - } - return errorBlobInfo, ErrBlobDigestMismatch - } - // If we didn't get a blob size, return the one we calculated. - if blobSize == -1 { - blobSize = counter.Count - } - // If we didn't get a digest, construct one. - if digest == "" { - digest = ddigest.Digest(hash) - } - // Record that this layer blob is a layer, and the layer ID it - // ended up having. This is a list, in case the same blob is - // being applied more than once. - s.Layers[digest] = append(s.Layers[digest], id) - s.BlobList = append(s.BlobList, types.BlobInfo{Digest: digest, Size: counter.Count}) - if layer != nil { - logrus.Debugf("blob %q imported as a filesystem layer %q", blobinfo.Digest, id) - } else { - logrus.Debugf("layer blob %q already present as layer %q", blobinfo.Digest, id) - } - } else { - // It's just data. Finish scanning it in, check that our - // computed digest matches the passed-in digest, and store it, - // but leave it out of the blob-to-layer-ID map so that we can - // tell that it's not a layer. - blob, err := ioutil.ReadAll(multi) - if err != nil && err != io.EOF { - return errorBlobInfo, err - } - hash = hasher.Digest().String() - if enforceDigestAndSize && blobinfo.Size >= 0 && int64(len(blob)) != blobinfo.Size { - logrus.Debugf("blob %q size is %d, not %d, rejecting", blobinfo.Digest, int64(len(blob)), blobinfo.Size) - return errorBlobInfo, ErrBlobSizeMismatch - } - // If we were given a digest, verify that the content matches - // it. - if enforceDigestAndSize && digest.Validate() == nil && digest.String() != hash { - logrus.Debugf("blob %q digests to %q, rejecting", blobinfo.Digest, hash) - return errorBlobInfo, ErrBlobDigestMismatch - } - // If we didn't get a blob size, return the one we calculated. - if blobSize == -1 { - blobSize = int64(len(blob)) - } - // If we didn't get a digest, construct one. - if digest == "" { - digest = ddigest.Digest(hash) - } - // Save the blob for when we Commit(). - s.BlobData[digest] = blob - s.BlobList = append(s.BlobList, types.BlobInfo{Digest: digest, Size: int64(len(blob))}) - logrus.Debugf("blob %q imported as opaque data %q", blobinfo.Digest, digest) + reader := io.TeeReader(io.TeeReader(stream, counter), file) + decompressed, err := archive.DecompressStream(reader) + if err != nil { + return errorBlobInfo, errors.Wrap(err, "error setting up to decompress blob") + } + // Copy the data to the file. + _, err = io.Copy(diffID.Hash(), decompressed) + decompressed.Close() + if err != nil { + return errorBlobInfo, errors.Wrapf(err, "error storing blob to file %q", filename) + } + // Ensure that any information that we were given about the blob is correct. + if blobinfo.Digest.Validate() == nil && blobinfo.Digest != hasher.Digest() { + return errorBlobInfo, ErrBlobDigestMismatch + } + if blobinfo.Size >= 0 && blobinfo.Size != counter.Count { + return errorBlobInfo, ErrBlobSizeMismatch + } + // Record information about the blob. + s.blobDiffIDs[hasher.Digest()] = diffID.Digest() + s.fileSizes[hasher.Digest()] = counter.Count + s.filenames[hasher.Digest()] = filename + blobDigest := blobinfo.Digest + if blobDigest.Validate() != nil { + blobDigest = hasher.Digest() + } + blobSize := blobinfo.Size + if blobSize < 0 { + blobSize = counter.Count } return types.BlobInfo{ - Digest: digest, - Size: blobSize, + Digest: blobDigest, + Size: blobSize, + MediaType: blobinfo.MediaType, }, nil } -// PutBlob is used to both store filesystem layers and binary data that is part -// of the image. Filesystem layers are assumed to be imported in order, as -// that is required by some of the underlying storage drivers. -func (s *storageImageDestination) PutBlob(stream io.Reader, blobinfo types.BlobInfo) (types.BlobInfo, error) { - return s.putBlob(stream, blobinfo, true) -} - -// HasBlob returns true iff the image destination already contains a blob with the matching digest which can be reapplied using ReapplyBlob. +// HasBlob returns true iff the image destination already contains a blob with the matching digest which can be +// reapplied using ReapplyBlob. +// // Unlike PutBlob, the digest can not be empty. If HasBlob returns true, the size of the blob must also be returned. // If the destination does not contain the blob, or it is unknown, HasBlob ordinarily returns (false, -1, nil); // it returns a non-nil error only on an unexpected failure. @@ -294,93 +363,289 @@ func (s *storageImageDestination) HasBlob(blobinfo types.BlobInfo) (bool, int64, if blobinfo.Digest == "" { return false, -1, errors.Errorf(`Can not check for a blob with unknown digest`) } - for _, blob := range s.BlobList { - if blob.Digest == blobinfo.Digest { - return true, blob.Size, nil - } + if err := blobinfo.Digest.Validate(); err != nil { + return false, -1, errors.Wrapf(err, `Can not check for a blob with invalid digest`) } + // Check if we've already cached it in a file. + if size, ok := s.fileSizes[blobinfo.Digest]; ok { + return true, size, nil + } + // Check if we have a wasn't-compressed layer in storage that's based on that blob. + layers, err := s.imageRef.transport.store.LayersByUncompressedDigest(blobinfo.Digest) + if err != nil && errors.Cause(err) != storage.ErrLayerUnknown { + return false, -1, errors.Wrapf(err, `Error looking for layers with digest %q`, blobinfo.Digest) + } + if len(layers) > 0 { + // Save this for completeness. + s.blobDiffIDs[blobinfo.Digest] = layers[0].UncompressedDigest + return true, layers[0].UncompressedSize, nil + } + // Check if we have a was-compressed layer in storage that's based on that blob. + layers, err = s.imageRef.transport.store.LayersByCompressedDigest(blobinfo.Digest) + if err != nil && errors.Cause(err) != storage.ErrLayerUnknown { + return false, -1, errors.Wrapf(err, `Error looking for compressed layers with digest %q`, blobinfo.Digest) + } + if len(layers) > 0 { + // Record the uncompressed value so that we can use it to calculate layer IDs. + s.blobDiffIDs[blobinfo.Digest] = layers[0].UncompressedDigest + return true, layers[0].CompressedSize, nil + } + // Nope, we don't have it. return false, -1, nil } +// ReapplyBlob is now a no-op, assuming HasBlob() says we already have it, since Commit() can just apply the +// same one when it walks the list in the manifest. func (s *storageImageDestination) ReapplyBlob(blobinfo types.BlobInfo) (types.BlobInfo, error) { - err := blobinfo.Digest.Validate() - if err != nil { - return types.BlobInfo{}, err + present, size, err := s.HasBlob(blobinfo) + if !present { + return types.BlobInfo{}, errors.Errorf("error reapplying blob %+v: blob was not previously applied", blobinfo) } - if layerList, ok := s.Layers[blobinfo.Digest]; !ok || len(layerList) < 1 { - b, err := s.imageRef.transport.store.ImageBigData(s.ID, blobinfo.Digest.String()) - if err != nil { - return types.BlobInfo{}, err + if err != nil { + return types.BlobInfo{}, errors.Wrapf(err, "error reapplying blob %+v", blobinfo) + } + blobinfo.Size = size + return blobinfo, nil +} + +// computeID computes a recommended image ID based on information we have so far. If +// the manifest is not of a type that we recognize, we return an empty value, indicating +// that since we don't have a recommendation, a random ID should be used if one needs +// to be allocated. +func (s *storageImageDestination) computeID(m manifest.Manifest) string { + // Build the diffID list. We need the decompressed sums that we've been calculating to + // fill in the DiffIDs. It's expected (but not enforced by us) that the number of + // diffIDs corresponds to the number of non-EmptyLayer entries in the history. + var diffIDs []digest.Digest + switch m.(type) { + case *manifest.Schema1: + // Build a list of the diffIDs we've generated for the non-throwaway FS layers, + // in reverse of the order in which they were originally listed. + s1, ok := m.(*manifest.Schema1) + if !ok { + // Shouldn't happen + logrus.Debugf("internal error reading schema 1 manifest") + return "" } - return types.BlobInfo{Digest: blobinfo.Digest, Size: int64(len(b))}, nil + for i, history := range s1.History { + compat := manifest.Schema1V1Compatibility{} + if err := json.Unmarshal([]byte(history.V1Compatibility), &compat); err != nil { + logrus.Debugf("internal error reading schema 1 history: %v", err) + return "" + } + if compat.ThrowAway { + continue + } + blobSum := s1.FSLayers[i].BlobSum + diffID, ok := s.blobDiffIDs[blobSum] + if !ok { + logrus.Infof("error looking up diffID for layer %q", blobSum.String()) + return "" + } + diffIDs = append([]digest.Digest{diffID}, diffIDs...) + } + case *manifest.Schema2, *manifest.OCI1: + // We know the ID calculation for these formats doesn't actually use the diffIDs, + // so we don't need to populate the diffID list. } - layerList := s.Layers[blobinfo.Digest] - rc, _, err := diffLayer(s.imageRef.transport.store, layerList[len(layerList)-1]) + id, err := m.ImageID(diffIDs) if err != nil { - return types.BlobInfo{}, err + return "" } - return s.putBlob(rc, blobinfo, false) + return id +} + +// getConfigBlob exists only to let us retrieve the configuration blob so that the manifest package can dig +// information out of it for Inspect(). +func (s *storageImageDestination) getConfigBlob(info types.BlobInfo) ([]byte, error) { + if info.Digest == "" { + return nil, errors.Errorf(`no digest supplied when reading blob`) + } + if err := info.Digest.Validate(); err != nil { + return nil, errors.Wrapf(err, `invalid digest supplied when reading blob`) + } + // Assume it's a file, since we're only calling this from a place that expects to read files. + if filename, ok := s.filenames[info.Digest]; ok { + contents, err2 := ioutil.ReadFile(filename) + if err2 != nil { + return nil, errors.Wrapf(err2, `error reading blob from file %q`, filename) + } + return contents, nil + } + // If it's not a file, it's a bug, because we're not expecting to be asked for a layer. + return nil, errors.New("blob not found") } func (s *storageImageDestination) Commit() error { - // Create the image record. - lastLayer := "" - for _, blob := range s.BlobList { - if layerList, ok := s.Layers[blob.Digest]; ok { - lastLayer = layerList[len(layerList)-1] - } + // Find the list of layer blobs. + if len(s.manifest) == 0 { + return errors.New("Internal error: storageImageDestination.Commit() called without PutManifest()") } - img, err := s.imageRef.transport.store.CreateImage(s.ID, nil, lastLayer, "", nil) + man, err := manifest.FromBlob(s.manifest, manifest.GuessMIMEType(s.manifest)) + if err != nil { + return errors.Wrapf(err, "error parsing manifest") + } + layerBlobs := man.LayerInfos() + // Extract or find the layers. + lastLayer := "" + addedLayers := []string{} + for _, blob := range layerBlobs { + var diff io.ReadCloser + // Check if there's already a layer with the ID that we'd give to the result of applying + // this layer blob to its parent, if it has one, or the blob's hex value otherwise. + diffID, haveDiffID := s.blobDiffIDs[blob.Digest] + if !haveDiffID { + // Check if it's elsewhere and the caller just forgot to pass it to us in a PutBlob(), + // or to even check if we had it. + logrus.Debugf("looking for diffID for blob %+v", blob.Digest) + has, _, err := s.HasBlob(blob) + if err != nil { + return errors.Wrapf(err, "error checking for a layer based on blob %q", blob.Digest.String()) + } + if !has { + return errors.Errorf("error determining uncompressed digest for blob %q", blob.Digest.String()) + } + diffID, haveDiffID = s.blobDiffIDs[blob.Digest] + if !haveDiffID { + return errors.Errorf("we have blob %q, but don't know its uncompressed digest", blob.Digest.String()) + } + } + id := diffID.Hex() + if lastLayer != "" { + id = digest.Canonical.FromBytes([]byte(lastLayer + "+" + diffID.Hex())).Hex() + } + if layer, err2 := s.imageRef.transport.store.Layer(id); layer != nil && err2 == nil { + // There's already a layer that should have the right contents, just reuse it. + lastLayer = layer.ID + continue + } + // Check if we cached a file with that blobsum. If we didn't already have a layer with + // the blob's contents, we should have gotten a copy. + if filename, ok := s.filenames[blob.Digest]; ok { + // Use the file's contents to initialize the layer. + file, err2 := os.Open(filename) + if err2 != nil { + return errors.Wrapf(err2, "error opening file %q", filename) + } + defer file.Close() + diff = file + } + if diff == nil { + // Try to find a layer with contents matching that blobsum. + layer := "" + layers, err2 := s.imageRef.transport.store.LayersByUncompressedDigest(blob.Digest) + if err2 == nil && len(layers) > 0 { + layer = layers[0].ID + } else { + layers, err2 = s.imageRef.transport.store.LayersByCompressedDigest(blob.Digest) + if err2 == nil && len(layers) > 0 { + layer = layers[0].ID + } + } + if layer == "" { + return errors.Wrapf(err2, "error locating layer for blob %q", blob.Digest) + } + // Use the layer's contents to initialize the new layer. + noCompression := archive.Uncompressed + diffOptions := &storage.DiffOptions{ + Compression: &noCompression, + } + diff, err2 = s.imageRef.transport.store.Diff("", layer, diffOptions) + if err2 != nil { + return errors.Wrapf(err2, "error reading layer %q for blob %q", layer, blob.Digest) + } + defer diff.Close() + } + if diff == nil { + // This shouldn't have happened. + return errors.Errorf("error applying blob %q: content not found", blob.Digest) + } + // Build the new layer using the diff, regardless of where it came from. + layer, _, err := s.imageRef.transport.store.PutLayer(id, lastLayer, nil, "", false, diff) + if err != nil { + return errors.Wrapf(err, "error adding layer with blob %q", blob.Digest) + } + lastLayer = layer.ID + addedLayers = append([]string{lastLayer}, addedLayers...) + } + // If one of those blobs was a configuration blob, then we can try to dig out the date when the image + // was originally created, in case we're just copying it. If not, no harm done. + options := &storage.ImageOptions{} + if inspect, err := man.Inspect(s.getConfigBlob); err == nil { + logrus.Debugf("setting image creation date to %s", inspect.Created) + options.CreationDate = inspect.Created + } + if manifestDigest, err := manifest.Digest(s.manifest); err == nil { + options.Digest = manifestDigest + } + // Create the image record, pointing to the most-recently added layer. + intendedID := s.imageRef.id + if intendedID == "" { + intendedID = s.computeID(man) + } + oldNames := []string{} + img, err := s.imageRef.transport.store.CreateImage(intendedID, nil, lastLayer, "", options) if err != nil { if errors.Cause(err) != storage.ErrDuplicateID { logrus.Debugf("error creating image: %q", err) - return errors.Wrapf(err, "error creating image %q", s.ID) + return errors.Wrapf(err, "error creating image %q", intendedID) } - img, err = s.imageRef.transport.store.Image(s.ID) + img, err = s.imageRef.transport.store.Image(intendedID) if err != nil { - return errors.Wrapf(err, "error reading image %q", s.ID) + return errors.Wrapf(err, "error reading image %q", intendedID) } if img.TopLayer != lastLayer { - logrus.Debugf("error creating image: image with ID %q exists, but uses different layers", s.ID) - return errors.Wrapf(storage.ErrDuplicateID, "image with ID %q already exists, but uses a different top layer", s.ID) + logrus.Debugf("error creating image: image with ID %q exists, but uses different layers", intendedID) + return errors.Wrapf(storage.ErrDuplicateID, "image with ID %q already exists, but uses a different top layer", intendedID) } logrus.Debugf("reusing image ID %q", img.ID) + oldNames = append(oldNames, img.Names...) } else { logrus.Debugf("created new image ID %q", img.ID) } - s.ID = img.ID - names := img.Names - if s.Tag != "" { - names = append(names, s.Tag) + // Add the non-layer blobs as data items. Since we only share layers, they should all be in files, so + // we just need to screen out the ones that are actually layers to get the list of non-layers. + dataBlobs := make(map[digest.Digest]struct{}) + for blob := range s.filenames { + dataBlobs[blob] = struct{}{} } - // We have names to set, so move those names to this image. - if len(names) > 0 { + for _, layerBlob := range layerBlobs { + delete(dataBlobs, layerBlob.Digest) + } + for blob := range dataBlobs { + v, err := ioutil.ReadFile(s.filenames[blob]) + if err != nil { + return errors.Wrapf(err, "error copying non-layer blob %q to image", blob) + } + if err := s.imageRef.transport.store.SetImageBigData(img.ID, blob.String(), v); err != nil { + if _, err2 := s.imageRef.transport.store.DeleteImage(img.ID, true); err2 != nil { + logrus.Debugf("error deleting incomplete image %q: %v", img.ID, err2) + } + logrus.Debugf("error saving big data %q for image %q: %v", blob.String(), img.ID, err) + return errors.Wrapf(err, "error saving big data %q for image %q", blob.String(), img.ID) + } + } + // Set the reference's name on the image. + if name := s.imageRef.DockerReference(); len(oldNames) > 0 || name != nil { + names := []string{} + if name != nil { + names = append(names, verboseName(name)) + } + if len(oldNames) > 0 { + names = append(names, oldNames...) + } if err := s.imageRef.transport.store.SetNames(img.ID, names); err != nil { if _, err2 := s.imageRef.transport.store.DeleteImage(img.ID, true); err2 != nil { logrus.Debugf("error deleting incomplete image %q: %v", img.ID, err2) } - logrus.Debugf("error setting names on image %q: %v", img.ID, err) - return err + logrus.Debugf("error setting names %v on image %q: %v", names, img.ID, err) + return errors.Wrapf(err, "error setting names %v on image %q", names, img.ID) } logrus.Debugf("set names of image %q to %v", img.ID, names) } - // Save the data blobs to disk, and drop their contents from memory. - keys := []ddigest.Digest{} - for k, v := range s.BlobData { - if err := s.imageRef.transport.store.SetImageBigData(img.ID, k.String(), v); err != nil { - if _, err2 := s.imageRef.transport.store.DeleteImage(img.ID, true); err2 != nil { - logrus.Debugf("error deleting incomplete image %q: %v", img.ID, err2) - } - logrus.Debugf("error saving big data %q for image %q: %v", k, img.ID, err) - return err - } - keys = append(keys, k) - } - for _, key := range keys { - delete(s.BlobData, key) - } - // Save the manifest, if we have one. - if err := s.imageRef.transport.store.SetImageBigData(s.ID, "manifest", s.Manifest); err != nil { + // Save the manifest. Use storage.ImageDigestBigDataKey as the item's + // name, so that its digest can be used to locate the image in the Store. + if err := s.imageRef.transport.store.SetImageBigData(img.ID, storage.ImageDigestBigDataKey, s.manifest); err != nil { if _, err2 := s.imageRef.transport.store.DeleteImage(img.ID, true); err2 != nil { logrus.Debugf("error deleting incomplete image %q: %v", img.ID, err2) } @@ -388,12 +653,14 @@ func (s *storageImageDestination) Commit() error { return err } // Save the signatures, if we have any. - if err := s.imageRef.transport.store.SetImageBigData(s.ID, "signatures", s.Signatures); err != nil { - if _, err2 := s.imageRef.transport.store.DeleteImage(img.ID, true); err2 != nil { - logrus.Debugf("error deleting incomplete image %q: %v", img.ID, err2) + if len(s.signatures) > 0 { + if err := s.imageRef.transport.store.SetImageBigData(img.ID, "signatures", s.signatures); err != nil { + if _, err2 := s.imageRef.transport.store.DeleteImage(img.ID, true); err2 != nil { + logrus.Debugf("error deleting incomplete image %q: %v", img.ID, err2) + } + logrus.Debugf("error saving signatures for image %q: %v", img.ID, err) + return err } - logrus.Debugf("error saving signatures for image %q: %v", img.ID, err) - return err } // Save our metadata. metadata, err := json.Marshal(s) @@ -405,7 +672,7 @@ func (s *storageImageDestination) Commit() error { return err } if len(metadata) != 0 { - if err = s.imageRef.transport.store.SetMetadata(s.ID, string(metadata)); err != nil { + if err = s.imageRef.transport.store.SetMetadata(img.ID, string(metadata)); err != nil { if _, err2 := s.imageRef.transport.store.DeleteImage(img.ID, true); err2 != nil { logrus.Debugf("error deleting incomplete image %q: %v", img.ID, err2) } @@ -418,7 +685,7 @@ func (s *storageImageDestination) Commit() error { } var manifestMIMETypes = []string{ - // TODO(runcom): we'll add OCI as part of another PR here + imgspecv1.MediaTypeImageManifest, manifest.DockerV2Schema2MediaType, manifest.DockerV2Schema1SignedMediaType, manifest.DockerV2Schema1MediaType, @@ -428,23 +695,20 @@ func (s *storageImageDestination) SupportedManifestMIMETypes() []string { return manifestMIMETypes } -// PutManifest writes manifest to the destination. -// FIXME? This should also receive a MIME type if known, to differentiate between schema versions. -// If the destination is in principle available, refuses this manifest type (e.g. it does not recognize the schema), -// but may accept a different manifest type, the returned error must be an ManifestTypeRejectedError. +// PutManifest writes the manifest to the destination. func (s *storageImageDestination) PutManifest(manifest []byte) error { - s.Manifest = make([]byte, len(manifest)) - copy(s.Manifest, manifest) + s.manifest = make([]byte, len(manifest)) + copy(s.manifest, manifest) return nil } -// SupportsSignatures returns an error if we can't expect GetSignatures() to -// return data that was previously supplied to PutSignatures(). +// SupportsSignatures returns an error if we can't expect GetSignatures() to return data that was +// previously supplied to PutSignatures(). func (s *storageImageDestination) SupportsSignatures() error { return nil } -// AcceptsForeignLayerURLs returns false iff foreign layers in manifest should be actually +// AcceptsForeignLayerURLs returns false iff foreign layers in the manifest should actually be // uploaded to the image destination, true otherwise. func (s *storageImageDestination) AcceptsForeignLayerURLs() bool { return false @@ -455,6 +719,7 @@ func (s *storageImageDestination) MustMatchRuntimeOS() bool { return true } +// PutSignatures records the image's signatures for committing as a single data blob. func (s *storageImageDestination) PutSignatures(signatures [][]byte) error { sizes := []int{} sigblob := []byte{} @@ -465,146 +730,73 @@ func (s *storageImageDestination) PutSignatures(signatures [][]byte) error { copy(newblob[len(sigblob):], sig) sigblob = newblob } - s.Signatures = sigblob + s.signatures = sigblob s.SignatureSizes = sizes return nil } -func (s *storageImageSource) GetBlob(info types.BlobInfo) (rc io.ReadCloser, n int64, err error) { - rc, n, _, err = s.getBlobAndLayerID(info) - return rc, n, err -} - -func (s *storageImageSource) getBlobAndLayerID(info types.BlobInfo) (rc io.ReadCloser, n int64, layerID string, err error) { - err = info.Digest.Validate() - if err != nil { - return nil, -1, "", err - } - if layerList, ok := s.Layers[info.Digest]; !ok || len(layerList) < 1 { - b, err := s.imageRef.transport.store.ImageBigData(s.ID, info.Digest.String()) - if err != nil { - return nil, -1, "", err - } - r := bytes.NewReader(b) - logrus.Debugf("exporting opaque data as blob %q", info.Digest.String()) - return ioutil.NopCloser(r), int64(r.Len()), "", nil - } - // If the blob was "put" more than once, we have multiple layer IDs - // which should all produce the same diff. For the sake of tests that - // want to make sure we created different layers each time the blob was - // "put", though, cycle through the layers. - layerList := s.Layers[info.Digest] - position, ok := s.LayerPosition[info.Digest] - if !ok { - position = 0 - } - s.LayerPosition[info.Digest] = (position + 1) % len(layerList) - logrus.Debugf("exporting filesystem layer %q for blob %q", layerList[position], info.Digest) - rc, n, err = diffLayer(s.imageRef.transport.store, layerList[position]) - return rc, n, layerList[position], err -} - -func diffLayer(store storage.Store, layerID string) (rc io.ReadCloser, n int64, err error) { - layer, err := store.Layer(layerID) - if err != nil { - return nil, -1, err - } - layerMeta := storageLayerMetadata{ - CompressedSize: -1, - } - if layer.Metadata != "" { - if err := json.Unmarshal([]byte(layer.Metadata), &layerMeta); err != nil { - return nil, -1, errors.Wrapf(err, "error decoding metadata for layer %q", layerID) - } - } - if layerMeta.CompressedSize <= 0 { - n = -1 - } else { - n = layerMeta.CompressedSize - } - diff, err := store.Diff("", layer.ID, nil) - if err != nil { - return nil, -1, err - } - return diff, n, nil -} - -func (s *storageImageSource) GetManifest() (manifestBlob []byte, MIMEType string, err error) { - manifestBlob, err = s.imageRef.transport.store.ImageBigData(s.ID, "manifest") - return manifestBlob, manifest.GuessMIMEType(manifestBlob), err -} - -func (s *storageImageSource) GetTargetManifest(digest ddigest.Digest) (manifestBlob []byte, MIMEType string, err error) { - return nil, "", ErrNoManifestLists -} - -func (s *storageImageSource) GetSignatures(ctx context.Context) (signatures [][]byte, err error) { - var offset int - signature, err := s.imageRef.transport.store.ImageBigData(s.ID, "signatures") - if err != nil { - return nil, err - } - sigslice := [][]byte{} - for _, length := range s.SignatureSizes { - sigslice = append(sigslice, signature[offset:offset+length]) - offset += length - } - if offset != len(signature) { - return nil, errors.Errorf("signatures data contained %d extra bytes", len(signatures)-offset) - } - return sigslice, nil -} - +// getSize() adds up the sizes of the image's data blobs (which includes the configuration blob), the +// signatures, and the uncompressed sizes of all of the image's layers. func (s *storageImageSource) getSize() (int64, error) { var sum int64 - names, err := s.imageRef.transport.store.ListImageBigData(s.imageRef.id) + // Size up the data blobs. + dataNames, err := s.imageRef.transport.store.ListImageBigData(s.ID) if err != nil { - return -1, errors.Wrapf(err, "error reading image %q", s.imageRef.id) + return -1, errors.Wrapf(err, "error reading image %q", s.ID) } - for _, name := range names { - bigSize, err := s.imageRef.transport.store.ImageBigDataSize(s.imageRef.id, name) + for _, dataName := range dataNames { + bigSize, err := s.imageRef.transport.store.ImageBigDataSize(s.ID, dataName) if err != nil { - return -1, errors.Wrapf(err, "error reading data blob size %q for %q", name, s.imageRef.id) + return -1, errors.Wrapf(err, "error reading data blob size %q for %q", dataName, s.ID) } sum += bigSize } + // Add the signature sizes. for _, sigSize := range s.SignatureSizes { sum += int64(sigSize) } - for _, layerList := range s.Layers { - for _, layerID := range layerList { - layer, err := s.imageRef.transport.store.Layer(layerID) - if err != nil { - return -1, err - } - layerMeta := storageLayerMetadata{ - Size: -1, - } - if layer.Metadata != "" { - if err := json.Unmarshal([]byte(layer.Metadata), &layerMeta); err != nil { - return -1, errors.Wrapf(err, "error decoding metadata for layer %q", layerID) - } - } - if layerMeta.Size < 0 { - return -1, errors.Errorf("size for layer %q is unknown, failing getSize()", layerID) - } - sum += layerMeta.Size + // Prepare to walk the layer list. + img, err := s.imageRef.transport.store.Image(s.ID) + if err != nil { + return -1, errors.Wrapf(err, "error reading image info %q", s.ID) + } + // Walk the layer list. + layerID := img.TopLayer + for layerID != "" { + layer, err := s.imageRef.transport.store.Layer(layerID) + if err != nil { + return -1, err } + if layer.UncompressedDigest == "" || layer.UncompressedSize < 0 { + return -1, errors.Errorf("size for layer %q is unknown, failing getSize()", layerID) + } + sum += layer.UncompressedSize + if layer.Parent == "" { + break + } + layerID = layer.Parent } return sum, nil } -func (s *storageImage) Size() (int64, error) { +// Size() adds up the sizes of the image's data blobs (which includes the configuration blob), the +// signatures, and the uncompressed sizes of all of the image's layers. +func (s *storageImageSource) Size() (int64, error) { + return s.getSize() +} + +// Size() returns the previously-computed size of the image, with no error. +func (s *storageImageCloser) Size() (int64, error) { return s.size, nil } // newImage creates an image that also knows its size -func newImage(s storageReference) (types.Image, error) { +func newImage(ctx *types.SystemContext, s storageReference) (types.ImageCloser, error) { src, err := newImageSource(s) if err != nil { return nil, err } - img, err := image.FromSource(src) + img, err := image.FromSource(ctx, src) if err != nil { return nil, err } @@ -612,5 +804,5 @@ func newImage(s storageReference) (types.Image, error) { if err != nil { return nil, err } - return &storageImage{Image: img, size: size}, nil + return &storageImageCloser{ImageCloser: img, size: size}, nil } diff --git a/vendor/github.com/containers/image/storage/storage_reference.go b/vendor/github.com/containers/image/storage/storage_reference.go index ded58705..96887142 100644 --- a/vendor/github.com/containers/image/storage/storage_reference.go +++ b/vendor/github.com/containers/image/storage/storage_reference.go @@ -1,3 +1,5 @@ +// +build !containers_image_storage_stub + package storage import ( @@ -6,6 +8,7 @@ import ( "github.com/containers/image/docker/reference" "github.com/containers/image/types" "github.com/containers/storage" + digest "github.com/opencontainers/go-digest" "github.com/pkg/errors" "github.com/sirupsen/logrus" ) @@ -18,9 +21,11 @@ type storageReference struct { reference string id string name reference.Named + tag string + digest digest.Digest } -func newReference(transport storageTransport, reference, id string, name reference.Named) *storageReference { +func newReference(transport storageTransport, reference, id string, name reference.Named, tag string, digest digest.Digest) *storageReference { // We take a copy of the transport, which contains a pointer to the // store that it used for resolving this reference, so that the // transport that we'll return from Transport() won't be affected by @@ -30,6 +35,8 @@ func newReference(transport storageTransport, reference, id string, name referen reference: reference, id: id, name: name, + tag: tag, + digest: digest, } } @@ -37,11 +44,32 @@ func newReference(transport storageTransport, reference, id string, name referen // one present with the same name or ID, and return the image. func (s *storageReference) resolveImage() (*storage.Image, error) { if s.id == "" { + // Look for an image that has the expanded reference name as an explicit Name value. image, err := s.transport.store.Image(s.reference) if image != nil && err == nil { s.id = image.ID } } + if s.id == "" && s.name != nil && s.digest != "" { + // Look for an image with the specified digest that has the same name, + // though possibly with a different tag or digest, as a Name value, so + // that the canonical reference can be implicitly resolved to the image. + images, err := s.transport.store.ImagesByDigest(s.digest) + if images != nil && err == nil { + repo := reference.FamiliarName(reference.TrimNamed(s.name)) + search: + for _, image := range images { + for _, name := range image.Names { + if named, err := reference.ParseNormalizedNamed(name); err == nil { + if reference.FamiliarName(reference.TrimNamed(named)) == repo { + s.id = image.ID + break search + } + } + } + } + } + } if s.id == "" { logrus.Errorf("reference %q does not resolve to an image ID", s.StringWithinTransport()) return nil, ErrNoSuchImage @@ -50,12 +78,15 @@ func (s *storageReference) resolveImage() (*storage.Image, error) { if err != nil { return nil, errors.Wrapf(err, "error reading image %q", s.id) } - if s.reference != "" { + if s.name != nil { + repo := reference.FamiliarName(reference.TrimNamed(s.name)) nameMatch := false for _, name := range img.Names { - if name == s.reference { - nameMatch = true - break + if named, err := reference.ParseNormalizedNamed(name); err == nil { + if reference.FamiliarName(reference.TrimNamed(named)) == repo { + nameMatch = true + break + } } } if !nameMatch { @@ -76,8 +107,21 @@ func (s storageReference) Transport() types.ImageTransport { } } -// Return a name with a tag, if we have a name to base them on. +// Return a name with a tag or digest, if we have either, else return it bare. func (s storageReference) DockerReference() reference.Named { + if s.name == nil { + return nil + } + if s.tag != "" { + if namedTagged, err := reference.WithTag(s.name, s.tag); err == nil { + return namedTagged + } + } + if s.digest != "" { + if canonical, err := reference.WithDigest(s.name, s.digest); err == nil { + return canonical + } + } return s.name } @@ -91,7 +135,7 @@ func (s storageReference) StringWithinTransport() string { optionsList = ":" + strings.Join(options, ",") } storeSpec := "[" + s.transport.store.GraphDriverName() + "@" + s.transport.store.GraphRoot() + "+" + s.transport.store.RunRoot() + optionsList + "]" - if s.name == nil { + if s.reference == "" { return storeSpec + "@" + s.id } if s.id == "" { @@ -120,11 +164,8 @@ func (s storageReference) PolicyConfigurationNamespaces() []string { driverlessStoreSpec := "[" + s.transport.store.GraphRoot() + "]" namespaces := []string{} if s.name != nil { - if s.id != "" { - // The reference without the ID is also a valid namespace. - namespaces = append(namespaces, storeSpec+s.reference) - } - components := strings.Split(s.name.Name(), "/") + name := reference.TrimNamed(s.name) + components := strings.Split(name.String(), "/") for len(components) > 0 { namespaces = append(namespaces, storeSpec+strings.Join(components, "/")) components = components[:len(components)-1] @@ -135,8 +176,13 @@ func (s storageReference) PolicyConfigurationNamespaces() []string { return namespaces } -func (s storageReference) NewImage(ctx *types.SystemContext) (types.Image, error) { - return newImage(s) +// NewImage returns a types.ImageCloser for this reference, possibly specialized for this ImageTransport. +// The caller must call .Close() on the returned ImageCloser. +// NOTE: If any kind of signature verification should happen, build an UnparsedImage from the value returned by NewImageSource, +// verify that UnparsedImage, and convert it into a real Image via image.FromUnparsedImage. +// WARNING: This may not do the right thing for a manifest list, see image.FromSource for details. +func (s storageReference) NewImage(ctx *types.SystemContext) (types.ImageCloser, error) { + return newImage(ctx, s) } func (s storageReference) DeleteImage(ctx *types.SystemContext) error { @@ -159,5 +205,5 @@ func (s storageReference) NewImageSource(ctx *types.SystemContext) (types.ImageS } func (s storageReference) NewImageDestination(ctx *types.SystemContext) (types.ImageDestination, error) { - return newImageDestination(s) + return newImageDestination(ctx, s) } diff --git a/vendor/github.com/containers/image/storage/storage_transport.go b/vendor/github.com/containers/image/storage/storage_transport.go index 1a0ebd04..f6ebcdc4 100644 --- a/vendor/github.com/containers/image/storage/storage_transport.go +++ b/vendor/github.com/containers/image/storage/storage_transport.go @@ -1,3 +1,5 @@ +// +build !containers_image_storage_stub + package storage import ( @@ -11,11 +13,14 @@ import ( "github.com/containers/image/types" "github.com/containers/storage" "github.com/containers/storage/pkg/idtools" - "github.com/opencontainers/go-digest" - ddigest "github.com/opencontainers/go-digest" + digest "github.com/opencontainers/go-digest" "github.com/sirupsen/logrus" ) +const ( + minimumTruncatedIDLength = 3 +) + func init() { transports.Register(Transport) } @@ -101,69 +106,133 @@ func (s *storageTransport) DefaultGIDMap() []idtools.IDMap { // relative to the given store, and returns it in a reference object. func (s storageTransport) ParseStoreReference(store storage.Store, ref string) (*storageReference, error) { var name reference.Named - var sum digest.Digest - var err error if ref == "" { - return nil, ErrInvalidReference + return nil, errors.Wrapf(ErrInvalidReference, "%q is an empty reference") } if ref[0] == '[' { // Ignore the store specifier. closeIndex := strings.IndexRune(ref, ']') if closeIndex < 1 { - return nil, ErrInvalidReference + return nil, errors.Wrapf(ErrInvalidReference, "store specifier in %q did not end", ref) } ref = ref[closeIndex+1:] } - refInfo := strings.SplitN(ref, "@", 2) - if len(refInfo) == 1 { - // A name. - name, err = reference.ParseNormalizedNamed(refInfo[0]) - if err != nil { - return nil, err + + // The last segment, if there's more than one, is either a digest from a reference, or an image ID. + split := strings.LastIndex(ref, "@") + idOrDigest := "" + if split != -1 { + // Peel off that last bit so that we can work on the rest. + idOrDigest = ref[split+1:] + if idOrDigest == "" { + return nil, errors.Wrapf(ErrInvalidReference, "%q does not look like a digest or image ID", idOrDigest) } - } else if len(refInfo) == 2 { - // An ID, possibly preceded by a name. - if refInfo[0] != "" { - name, err = reference.ParseNormalizedNamed(refInfo[0]) - if err != nil { - return nil, err - } - } - sum, err = digest.Parse(refInfo[1]) - if err != nil || sum.Validate() != nil { - sum, err = digest.Parse("sha256:" + refInfo[1]) - if err != nil || sum.Validate() != nil { - return nil, err - } - } - } else { // Coverage: len(refInfo) is always 1 or 2 - // Anything else: store specified in a form we don't - // recognize. - return nil, ErrInvalidReference + ref = ref[:split] } + + // The middle segment (now the last segment), if there is one, is a digest. + split = strings.LastIndex(ref, "@") + sum := digest.Digest("") + if split != -1 { + sum = digest.Digest(ref[split+1:]) + if sum == "" { + return nil, errors.Wrapf(ErrInvalidReference, "%q does not look like an image digest", sum) + } + ref = ref[:split] + } + + // If we have something that unambiguously should be a digest, validate it, and then the third part, + // if we have one, as an ID. + id := "" + if sum != "" { + if idSum, err := digest.Parse("sha256:" + idOrDigest); err != nil || idSum.Validate() != nil { + return nil, errors.Wrapf(ErrInvalidReference, "%q does not look like an image ID", idOrDigest) + } + if err := sum.Validate(); err != nil { + return nil, errors.Wrapf(ErrInvalidReference, "%q does not look like an image digest", sum) + } + id = idOrDigest + if img, err := store.Image(idOrDigest); err == nil && img != nil && len(idOrDigest) >= minimumTruncatedIDLength && strings.HasPrefix(img.ID, idOrDigest) { + // The ID is a truncated version of the ID of an image that's present in local storage, + // so we might as well use the expanded value. + id = img.ID + } + } else if idOrDigest != "" { + // There was no middle portion, so the final portion could be either a digest or an ID. + if idSum, err := digest.Parse("sha256:" + idOrDigest); err == nil && idSum.Validate() == nil { + // It's an ID. + id = idOrDigest + } else if idSum, err := digest.Parse(idOrDigest); err == nil && idSum.Validate() == nil { + // It's a digest. + sum = idSum + } else if img, err := store.Image(idOrDigest); err == nil && img != nil && len(idOrDigest) >= minimumTruncatedIDLength && strings.HasPrefix(img.ID, idOrDigest) { + // It's a truncated version of the ID of an image that's present in local storage, + // and we may need the expanded value. + id = img.ID + } else { + return nil, errors.Wrapf(ErrInvalidReference, "%q does not look like a digest or image ID", idOrDigest) + } + } + + // If we only had one portion, then _maybe_ it's a truncated image ID. Only check on that if it's + // at least of what we guess is a reasonable minimum length, because we don't want a really short value + // like "a" matching an image by ID prefix when the input was actually meant to specify an image name. + if len(ref) >= minimumTruncatedIDLength && sum == "" && id == "" { + if img, err := store.Image(ref); err == nil && img != nil && strings.HasPrefix(img.ID, ref) { + // It's a truncated version of the ID of an image that's present in local storage; + // we need to expand it. + id = img.ID + ref = "" + } + } + + // The initial portion is probably a name, possibly with a tag. + if ref != "" { + var err error + if name, err = reference.ParseNormalizedNamed(ref); err != nil { + return nil, errors.Wrapf(err, "error parsing named reference %q", ref) + } + } + if name == nil && sum == "" && id == "" { + return nil, errors.Errorf("error parsing reference") + } + + // Construct a copy of the store spec. optionsList := "" options := store.GraphOptions() if len(options) > 0 { optionsList = ":" + strings.Join(options, ",") } storeSpec := "[" + store.GraphDriverName() + "@" + store.GraphRoot() + "+" + store.RunRoot() + optionsList + "]" - id := "" - if sum.Validate() == nil { - id = sum.Hex() - } + + // Convert the name back into a reference string, if we got a name. refname := "" + tag := "" if name != nil { - name = reference.TagNameOnly(name) - refname = verboseName(name) + if sum.Validate() == nil { + canonical, err := reference.WithDigest(name, sum) + if err != nil { + return nil, errors.Wrapf(err, "error mixing name %q with digest %q", name, sum) + } + refname = verboseName(canonical) + } else { + name = reference.TagNameOnly(name) + tagged, ok := name.(reference.Tagged) + if !ok { + return nil, errors.Errorf("error parsing possibly-tagless name %q", ref) + } + refname = verboseName(name) + tag = tagged.Tag() + } } if refname == "" { - logrus.Debugf("parsed reference into %q", storeSpec+"@"+id) + logrus.Debugf("parsed reference to id into %q", storeSpec+"@"+id) } else if id == "" { - logrus.Debugf("parsed reference into %q", storeSpec+refname) + logrus.Debugf("parsed reference to refname into %q", storeSpec+refname) } else { - logrus.Debugf("parsed reference into %q", storeSpec+refname+"@"+id) + logrus.Debugf("parsed reference to refname@id into %q", storeSpec+refname+"@"+id) } - return newReference(storageTransport{store: store, defaultUIDMap: s.defaultUIDMap, defaultGIDMap: s.defaultGIDMap}, refname, id, name), nil + return newReference(storageTransport{store: store, defaultUIDMap: s.defaultUIDMap, defaultGIDMap: s.defaultGIDMap}, refname, id, name, tag, sum), nil } func (s *storageTransport) GetStore() (storage.Store, error) { @@ -182,11 +251,14 @@ func (s *storageTransport) GetStore() (storage.Store, error) { return s.store, nil } -// ParseReference takes a name and/or an ID ("_name_"/"@_id_"/"_name_@_id_"), +// ParseReference takes a name and a tag or digest and/or ID +// ("_name_"/"@_id_"/"_name_:_tag_"/"_name_:_tag_@_id_"/"_name_@_digest_"/"_name_@_digest_@_id_"), // possibly prefixed with a store specifier in the form "[_graphroot_]" or // "[_driver_@_graphroot_]" or "[_driver_@_graphroot_+_runroot_]" or // "[_driver_@_graphroot_:_options_]" or "[_driver_@_graphroot_+_runroot_:_options_]", // tries to figure out which it is, and returns it in a reference object. +// If _id_ is the ID of an image that's present in local storage, it can be truncated, and +// even be specified as if it were a _name_, value. func (s *storageTransport) ParseReference(reference string) (types.ImageReference, error) { var store storage.Store // Check if there's a store location prefix. If there is, then it @@ -265,17 +337,23 @@ func (s *storageTransport) ParseReference(reference string) (types.ImageReferenc func (s storageTransport) GetStoreImage(store storage.Store, ref types.ImageReference) (*storage.Image, error) { dref := ref.DockerReference() - if dref == nil { - if sref, ok := ref.(*storageReference); ok { - if sref.id != "" { - if img, err := store.Image(sref.id); err == nil { - return img, nil - } + if dref != nil { + if img, err := store.Image(verboseName(dref)); err == nil { + return img, nil + } + } + if sref, ok := ref.(*storageReference); ok { + if sref.id != "" { + if img, err := store.Image(sref.id); err == nil { + return img, nil } } - return nil, ErrInvalidReference + tmpRef := *sref + if img, err := tmpRef.resolveImage(); err == nil { + return img, nil + } } - return store.Image(verboseName(dref)) + return nil, storage.ErrImageUnknown } func (s *storageTransport) GetImage(ref types.ImageReference) (*storage.Image, error) { @@ -335,7 +413,7 @@ func (s storageTransport) ValidatePolicyConfigurationScope(scope string) error { if err != nil { return err } - _, err = ddigest.Parse("sha256:" + scopeInfo[1]) + _, err = digest.Parse("sha256:" + scopeInfo[1]) if err != nil { return err } @@ -345,11 +423,28 @@ func (s storageTransport) ValidatePolicyConfigurationScope(scope string) error { return nil } -func verboseName(name reference.Named) string { - name = reference.TagNameOnly(name) - tag := "" - if tagged, ok := name.(reference.NamedTagged); ok { - tag = ":" + tagged.Tag() +func verboseName(r reference.Reference) string { + if r == nil { + return "" } - return name.Name() + tag + named, isNamed := r.(reference.Named) + digested, isDigested := r.(reference.Digested) + tagged, isTagged := r.(reference.Tagged) + name := "" + tag := "" + sum := "" + if isNamed { + name = (reference.TrimNamed(named)).String() + } + if isTagged { + if tagged.Tag() != "" { + tag = ":" + tagged.Tag() + } + } + if isDigested { + if digested.Digest().Validate() == nil { + sum = "@" + digested.Digest().String() + } + } + return name + tag + sum } diff --git a/vendor/github.com/containers/image/tarball/doc.go b/vendor/github.com/containers/image/tarball/doc.go new file mode 100644 index 00000000..a6ced5a0 --- /dev/null +++ b/vendor/github.com/containers/image/tarball/doc.go @@ -0,0 +1,48 @@ +// Package tarball provides a way to generate images using one or more layer +// tarballs and an optional template configuration. +// +// An example: +// package main +// +// import ( +// "fmt" +// +// cp "github.com/containers/image/copy" +// "github.com/containers/image/tarball" +// "github.com/containers/image/transports/alltransports" +// +// imgspecv1 "github.com/containers/image/transports/alltransports" +// ) +// +// func imageFromTarball() { +// src, err := alltransports.ParseImageName("tarball:/var/cache/mock/fedora-26-x86_64/root_cache/cache.tar.gz") +// // - or - +// // src, err := tarball.Transport.ParseReference("/var/cache/mock/fedora-26-x86_64/root_cache/cache.tar.gz") +// if err != nil { +// panic(err) +// } +// updater, ok := src.(tarball.ConfigUpdater) +// if !ok { +// panic("unexpected: a tarball reference should implement tarball.ConfigUpdater") +// } +// config := imgspecv1.Image{ +// Config: imgspecv1.ImageConfig{ +// Cmd: []string{"/bin/bash"}, +// }, +// } +// annotations := make(map[string]string) +// annotations[imgspecv1.AnnotationDescription] = "test image built from a mock root cache" +// err = updater.ConfigUpdate(config, annotations) +// if err != nil { +// panic(err) +// } +// dest, err := alltransports.ParseImageName("docker-daemon:mock:latest") +// if err != nil { +// panic(err) +// } +// err = cp.Image(nil, dest, src, nil) +// if err != nil { +// panic(err) +// } +// } +package tarball diff --git a/vendor/github.com/containers/image/tarball/tarball_reference.go b/vendor/github.com/containers/image/tarball/tarball_reference.go new file mode 100644 index 00000000..4ccfb406 --- /dev/null +++ b/vendor/github.com/containers/image/tarball/tarball_reference.go @@ -0,0 +1,93 @@ +package tarball + +import ( + "fmt" + "os" + "strings" + + "github.com/containers/image/docker/reference" + "github.com/containers/image/image" + "github.com/containers/image/types" + + imgspecv1 "github.com/opencontainers/image-spec/specs-go/v1" +) + +// ConfigUpdater is an interface that ImageReferences for "tarball" images also +// implement. It can be used to set values for a configuration, and to set +// image annotations which will be present in the images returned by the +// reference's NewImage() or NewImageSource() methods. +type ConfigUpdater interface { + ConfigUpdate(config imgspecv1.Image, annotations map[string]string) error +} + +type tarballReference struct { + transport types.ImageTransport + config imgspecv1.Image + annotations map[string]string + filenames []string + stdin []byte +} + +// ConfigUpdate updates the image's default configuration and adds annotations +// which will be visible in source images created using this reference. +func (r *tarballReference) ConfigUpdate(config imgspecv1.Image, annotations map[string]string) error { + r.config = config + if r.annotations == nil { + r.annotations = make(map[string]string) + } + for k, v := range annotations { + r.annotations[k] = v + } + return nil +} + +func (r *tarballReference) Transport() types.ImageTransport { + return r.transport +} + +func (r *tarballReference) StringWithinTransport() string { + return strings.Join(r.filenames, ":") +} + +func (r *tarballReference) DockerReference() reference.Named { + return nil +} + +func (r *tarballReference) PolicyConfigurationIdentity() string { + return "" +} + +func (r *tarballReference) PolicyConfigurationNamespaces() []string { + return nil +} + +// NewImage returns a types.ImageCloser for this reference, possibly specialized for this ImageTransport. +// The caller must call .Close() on the returned ImageCloser. +// NOTE: If any kind of signature verification should happen, build an UnparsedImage from the value returned by NewImageSource, +// verify that UnparsedImage, and convert it into a real Image via image.FromUnparsedImage. +// WARNING: This may not do the right thing for a manifest list, see image.FromSource for details. +func (r *tarballReference) NewImage(ctx *types.SystemContext) (types.ImageCloser, error) { + src, err := r.NewImageSource(ctx) + if err != nil { + return nil, err + } + img, err := image.FromSource(ctx, src) + if err != nil { + src.Close() + return nil, err + } + return img, nil +} + +func (r *tarballReference) DeleteImage(ctx *types.SystemContext) error { + for _, filename := range r.filenames { + if err := os.Remove(filename); err != nil && !os.IsNotExist(err) { + return fmt.Errorf("error removing %q: %v", filename, err) + } + } + return nil +} + +func (r *tarballReference) NewImageDestination(ctx *types.SystemContext) (types.ImageDestination, error) { + return nil, fmt.Errorf("destination not implemented yet") +} diff --git a/vendor/github.com/containers/image/tarball/tarball_src.go b/vendor/github.com/containers/image/tarball/tarball_src.go new file mode 100644 index 00000000..8b5b496d --- /dev/null +++ b/vendor/github.com/containers/image/tarball/tarball_src.go @@ -0,0 +1,260 @@ +package tarball + +import ( + "bytes" + "compress/gzip" + "context" + "encoding/json" + "fmt" + "io" + "io/ioutil" + "os" + "runtime" + "strings" + "time" + + "github.com/containers/image/types" + + digest "github.com/opencontainers/go-digest" + imgspecs "github.com/opencontainers/image-spec/specs-go" + imgspecv1 "github.com/opencontainers/image-spec/specs-go/v1" +) + +type tarballImageSource struct { + reference tarballReference + filenames []string + diffIDs []digest.Digest + diffSizes []int64 + blobIDs []digest.Digest + blobSizes []int64 + blobTypes []string + config []byte + configID digest.Digest + configSize int64 + manifest []byte +} + +func (r *tarballReference) NewImageSource(ctx *types.SystemContext) (types.ImageSource, error) { + // Gather up the digests, sizes, and date information for all of the files. + filenames := []string{} + diffIDs := []digest.Digest{} + diffSizes := []int64{} + blobIDs := []digest.Digest{} + blobSizes := []int64{} + blobTimes := []time.Time{} + blobTypes := []string{} + for _, filename := range r.filenames { + var file *os.File + var err error + var blobSize int64 + var blobTime time.Time + var reader io.Reader + if filename == "-" { + blobSize = int64(len(r.stdin)) + blobTime = time.Now() + reader = bytes.NewReader(r.stdin) + } else { + file, err = os.Open(filename) + if err != nil { + return nil, fmt.Errorf("error opening %q for reading: %v", filename, err) + } + defer file.Close() + reader = file + fileinfo, err := file.Stat() + if err != nil { + return nil, fmt.Errorf("error reading size of %q: %v", filename, err) + } + blobSize = fileinfo.Size() + blobTime = fileinfo.ModTime() + } + + // Default to assuming the layer is compressed. + layerType := imgspecv1.MediaTypeImageLayerGzip + + // Set up to digest the file as it is. + blobIDdigester := digest.Canonical.Digester() + reader = io.TeeReader(reader, blobIDdigester.Hash()) + + // Set up to digest the file after we maybe decompress it. + diffIDdigester := digest.Canonical.Digester() + uncompressed, err := gzip.NewReader(reader) + if err == nil { + // It is compressed, so the diffID is the digest of the uncompressed version + reader = io.TeeReader(uncompressed, diffIDdigester.Hash()) + } else { + // It is not compressed, so the diffID and the blobID are going to be the same + diffIDdigester = blobIDdigester + layerType = imgspecv1.MediaTypeImageLayer + uncompressed = nil + } + n, err := io.Copy(ioutil.Discard, reader) + if err != nil { + return nil, fmt.Errorf("error reading %q: %v", filename, err) + } + if uncompressed != nil { + uncompressed.Close() + } + + // Grab our uncompressed and possibly-compressed digests and sizes. + filenames = append(filenames, filename) + diffIDs = append(diffIDs, diffIDdigester.Digest()) + diffSizes = append(diffSizes, n) + blobIDs = append(blobIDs, blobIDdigester.Digest()) + blobSizes = append(blobSizes, blobSize) + blobTimes = append(blobTimes, blobTime) + blobTypes = append(blobTypes, layerType) + } + + // Build the rootfs and history for the configuration blob. + rootfs := imgspecv1.RootFS{ + Type: "layers", + DiffIDs: diffIDs, + } + created := time.Time{} + history := []imgspecv1.History{} + // Pick up the layer comment from the configuration's history list, if one is set. + comment := "imported from tarball" + if len(r.config.History) > 0 && r.config.History[0].Comment != "" { + comment = r.config.History[0].Comment + } + for i := range diffIDs { + createdBy := fmt.Sprintf("/bin/sh -c #(nop) ADD file:%s in %c", diffIDs[i].Hex(), os.PathSeparator) + history = append(history, imgspecv1.History{ + Created: &blobTimes[i], + CreatedBy: createdBy, + Comment: comment, + }) + // Use the mtime of the most recently modified file as the image's creation time. + if created.Before(blobTimes[i]) { + created = blobTimes[i] + } + } + + // Pick up other defaults from the config in the reference. + config := r.config + if config.Created == nil { + config.Created = &created + } + if config.Architecture == "" { + config.Architecture = runtime.GOARCH + } + if config.OS == "" { + config.OS = runtime.GOOS + } + config.RootFS = rootfs + config.History = history + + // Encode and digest the image configuration blob. + configBytes, err := json.Marshal(&config) + if err != nil { + return nil, fmt.Errorf("error generating configuration blob for %q: %v", strings.Join(r.filenames, separator), err) + } + configID := digest.Canonical.FromBytes(configBytes) + configSize := int64(len(configBytes)) + + // Populate a manifest with the configuration blob and the file as the single layer. + layerDescriptors := []imgspecv1.Descriptor{} + for i := range blobIDs { + layerDescriptors = append(layerDescriptors, imgspecv1.Descriptor{ + Digest: blobIDs[i], + Size: blobSizes[i], + MediaType: blobTypes[i], + }) + } + annotations := make(map[string]string) + for k, v := range r.annotations { + annotations[k] = v + } + manifest := imgspecv1.Manifest{ + Versioned: imgspecs.Versioned{ + SchemaVersion: 2, + }, + Config: imgspecv1.Descriptor{ + Digest: configID, + Size: configSize, + MediaType: imgspecv1.MediaTypeImageConfig, + }, + Layers: layerDescriptors, + Annotations: annotations, + } + + // Encode the manifest. + manifestBytes, err := json.Marshal(&manifest) + if err != nil { + return nil, fmt.Errorf("error generating manifest for %q: %v", strings.Join(r.filenames, separator), err) + } + + // Return the image. + src := &tarballImageSource{ + reference: *r, + filenames: filenames, + diffIDs: diffIDs, + diffSizes: diffSizes, + blobIDs: blobIDs, + blobSizes: blobSizes, + blobTypes: blobTypes, + config: configBytes, + configID: configID, + configSize: configSize, + manifest: manifestBytes, + } + + return src, nil +} + +func (is *tarballImageSource) Close() error { + return nil +} + +func (is *tarballImageSource) GetBlob(blobinfo types.BlobInfo) (io.ReadCloser, int64, error) { + // We should only be asked about things in the manifest. Maybe the configuration blob. + if blobinfo.Digest == is.configID { + return ioutil.NopCloser(bytes.NewBuffer(is.config)), is.configSize, nil + } + // Maybe one of the layer blobs. + for i := range is.blobIDs { + if blobinfo.Digest == is.blobIDs[i] { + // We want to read that layer: open the file or memory block and hand it back. + if is.filenames[i] == "-" { + return ioutil.NopCloser(bytes.NewBuffer(is.reference.stdin)), int64(len(is.reference.stdin)), nil + } + reader, err := os.Open(is.filenames[i]) + if err != nil { + return nil, -1, fmt.Errorf("error opening %q: %v", is.filenames[i], err) + } + return reader, is.blobSizes[i], nil + } + } + return nil, -1, fmt.Errorf("no blob with digest %q found", blobinfo.Digest.String()) +} + +// GetManifest returns the image's manifest along with its MIME type (which may be empty when it can't be determined but the manifest is available). +// It may use a remote (= slow) service. +// If instanceDigest is not nil, it contains a digest of the specific manifest instance to retrieve (when the primary manifest is a manifest list); +// this never happens if the primary manifest is not a manifest list (e.g. if the source never returns manifest lists). +func (is *tarballImageSource) GetManifest(instanceDigest *digest.Digest) ([]byte, string, error) { + if instanceDigest != nil { + return nil, "", fmt.Errorf("manifest lists are not supported by the %q transport", transportName) + } + return is.manifest, imgspecv1.MediaTypeImageManifest, nil +} + +// GetSignatures returns the image's signatures. It may use a remote (= slow) service. +// If instanceDigest is not nil, it contains a digest of the specific manifest instance to retrieve signatures for +// (when the primary manifest is a manifest list); this never happens if the primary manifest is not a manifest list +// (e.g. if the source never returns manifest lists). +func (*tarballImageSource) GetSignatures(ctx context.Context, instanceDigest *digest.Digest) ([][]byte, error) { + if instanceDigest != nil { + return nil, fmt.Errorf("manifest lists are not supported by the %q transport", transportName) + } + return nil, nil +} + +func (is *tarballImageSource) Reference() types.ImageReference { + return &is.reference +} + +// LayerInfosForCopy() returns updated layer info that should be used when reading, in preference to values in the manifest, if specified. +func (*tarballImageSource) LayerInfosForCopy() []types.BlobInfo { + return nil +} diff --git a/vendor/github.com/containers/image/tarball/tarball_transport.go b/vendor/github.com/containers/image/tarball/tarball_transport.go new file mode 100644 index 00000000..72558b5e --- /dev/null +++ b/vendor/github.com/containers/image/tarball/tarball_transport.go @@ -0,0 +1,66 @@ +package tarball + +import ( + "errors" + "fmt" + "io/ioutil" + "os" + "strings" + + "github.com/containers/image/transports" + "github.com/containers/image/types" +) + +const ( + transportName = "tarball" + separator = ":" +) + +var ( + // Transport implements the types.ImageTransport interface for "tarball:" images, + // which are makeshift images constructed using one or more possibly-compressed tar + // archives. + Transport = &tarballTransport{} +) + +type tarballTransport struct { +} + +func (t *tarballTransport) Name() string { + return transportName +} + +func (t *tarballTransport) ParseReference(reference string) (types.ImageReference, error) { + var stdin []byte + var err error + filenames := strings.Split(reference, separator) + for _, filename := range filenames { + if filename == "-" { + stdin, err = ioutil.ReadAll(os.Stdin) + if err != nil { + return nil, fmt.Errorf("error buffering stdin: %v", err) + } + continue + } + f, err := os.Open(filename) + if err != nil { + return nil, fmt.Errorf("error opening %q: %v", filename, err) + } + f.Close() + } + ref := &tarballReference{ + transport: t, + filenames: filenames, + stdin: stdin, + } + return ref, nil +} + +func (t *tarballTransport) ValidatePolicyConfigurationScope(scope string) error { + // See the explanation in daemonReference.PolicyConfigurationIdentity. + return errors.New(`tarball: does not support any scopes except the default "" one`) +} + +func init() { + transports.Register(Transport) +} diff --git a/vendor/github.com/containers/image/transports/alltransports/alltransports.go b/vendor/github.com/containers/image/transports/alltransports/alltransports.go index 4279b9d2..b4552df6 100644 --- a/vendor/github.com/containers/image/transports/alltransports/alltransports.go +++ b/vendor/github.com/containers/image/transports/alltransports/alltransports.go @@ -13,8 +13,9 @@ import ( _ "github.com/containers/image/oci/archive" _ "github.com/containers/image/oci/layout" _ "github.com/containers/image/openshift" + _ "github.com/containers/image/tarball" // The ostree transport is registered by ostree*.go - _ "github.com/containers/image/storage" + // The storage transport is registered by storage*.go "github.com/containers/image/transports" "github.com/containers/image/types" "github.com/pkg/errors" diff --git a/vendor/github.com/containers/image/transports/alltransports/storage.go b/vendor/github.com/containers/image/transports/alltransports/storage.go new file mode 100644 index 00000000..a867c664 --- /dev/null +++ b/vendor/github.com/containers/image/transports/alltransports/storage.go @@ -0,0 +1,8 @@ +// +build !containers_image_storage_stub + +package alltransports + +import ( + // Register the storage transport + _ "github.com/containers/image/storage" +) diff --git a/vendor/github.com/containers/image/transports/alltransports/storage_stub.go b/vendor/github.com/containers/image/transports/alltransports/storage_stub.go new file mode 100644 index 00000000..4ac684e5 --- /dev/null +++ b/vendor/github.com/containers/image/transports/alltransports/storage_stub.go @@ -0,0 +1,9 @@ +// +build containers_image_storage_stub + +package alltransports + +import "github.com/containers/image/transports" + +func init() { + transports.Register(transports.NewStubTransport("containers-storage")) +} diff --git a/vendor/github.com/containers/image/types/types.go b/vendor/github.com/containers/image/types/types.go index 4ede907b..2e9c7105 100644 --- a/vendor/github.com/containers/image/types/types.go +++ b/vendor/github.com/containers/image/types/types.go @@ -73,11 +73,12 @@ type ImageReference interface { // and each following element to be a prefix of the element preceding it. PolicyConfigurationNamespaces() []string - // NewImage returns a types.Image for this reference, possibly specialized for this ImageTransport. - // The caller must call .Close() on the returned Image. + // NewImage returns a types.ImageCloser for this reference, possibly specialized for this ImageTransport. + // The caller must call .Close() on the returned ImageCloser. // NOTE: If any kind of signature verification should happen, build an UnparsedImage from the value returned by NewImageSource, // verify that UnparsedImage, and convert it into a real Image via image.FromUnparsedImage. - NewImage(ctx *SystemContext) (Image, error) + // WARNING: This may not do the right thing for a manifest list, see image.FromSource for details. + NewImage(ctx *SystemContext) (ImageCloser, error) // NewImageSource returns a types.ImageSource for this reference. // The caller must call .Close() on the returned ImageSource. NewImageSource(ctx *SystemContext) (ImageSource, error) @@ -96,9 +97,10 @@ type BlobInfo struct { Size int64 // -1 if unknown URLs []string Annotations map[string]string + MediaType string } -// ImageSource is a service, possibly remote (= slow), to download components of a single image. +// ImageSource is a service, possibly remote (= slow), to download components of a single image or a named image set (manifest list). // This is primarily useful for copying images around; for examining their properties, Image (below) // is usually more useful. // Each ImageSource should eventually be closed by calling Close(). @@ -113,15 +115,21 @@ type ImageSource interface { Close() error // GetManifest returns the image's manifest along with its MIME type (which may be empty when it can't be determined but the manifest is available). // It may use a remote (= slow) service. - GetManifest() ([]byte, string, error) - // GetTargetManifest returns an image's manifest given a digest. This is mainly used to retrieve a single image's manifest - // out of a manifest list. - GetTargetManifest(digest digest.Digest) ([]byte, string, error) + // If instanceDigest is not nil, it contains a digest of the specific manifest instance to retrieve (when the primary manifest is a manifest list); + // this never happens if the primary manifest is not a manifest list (e.g. if the source never returns manifest lists). + GetManifest(instanceDigest *digest.Digest) ([]byte, string, error) // GetBlob returns a stream for the specified blob, and the blob’s size (or -1 if unknown). - // The Digest field in BlobInfo is guaranteed to be provided; Size may be -1. + // The Digest field in BlobInfo is guaranteed to be provided, Size may be -1 and MediaType may be optionally provided. GetBlob(BlobInfo) (io.ReadCloser, int64, error) // GetSignatures returns the image's signatures. It may use a remote (= slow) service. - GetSignatures(context.Context) ([][]byte, error) + // If instanceDigest is not nil, it contains a digest of the specific manifest instance to retrieve signatures for + // (when the primary manifest is a manifest list); this never happens if the primary manifest is not a manifest list + // (e.g. if the source never returns manifest lists). + GetSignatures(ctx context.Context, instanceDigest *digest.Digest) ([][]byte, error) + // LayerInfosForCopy returns either nil (meaning the values in the manifest are fine), or updated values for the layer blobsums that are listed in the image's manifest. + // The Digest field is guaranteed to be provided; Size may be -1. + // WARNING: The list may contain duplicates, and they are semantically relevant. + LayerInfosForCopy() []BlobInfo } // ImageDestination is a service, possibly remote (= slow), to store components of a single image. @@ -153,9 +161,10 @@ type ImageDestination interface { AcceptsForeignLayerURLs() bool // MustMatchRuntimeOS returns true iff the destination can store only images targeted for the current runtime OS. False otherwise. MustMatchRuntimeOS() bool - // PutBlob writes contents of stream and returns data representing the result (with all data filled in). + // PutBlob writes contents of stream and returns data representing the result. // inputInfo.Digest can be optionally provided if known; it is not mandatory for the implementation to verify it. // inputInfo.Size is the expected length of stream, if known. + // inputInfo.MediaType describes the blob format, if known. // WARNING: The contents of stream are being verified on the fly. Until stream.Read() returns io.EOF, the contents of the data SHOULD NOT be available // to any other readers for download using the supplied digest. // If stream.Read() at any time, ESPECIALLY at end of input, returns an error, PutBlob MUST 1) fail, and 2) delete any data stored so far. @@ -194,28 +203,35 @@ func (e ManifestTypeRejectedError) Error() string { // Thus, an UnparsedImage can be created from an ImageSource simply by fetching blobs without interpreting them, // allowing cryptographic signature verification to happen first, before even fetching the manifest, or parsing anything else. // This also makes the UnparsedImage→Image conversion an explicitly visible step. -// Each UnparsedImage should eventually be closed by calling Close(). +// +// An UnparsedImage is a pair of (ImageSource, instance digest); it can represent either a manifest list or a single image instance. +// +// The UnparsedImage must not be used after the underlying ImageSource is Close()d. type UnparsedImage interface { // Reference returns the reference used to set up this source, _as specified by the user_ // (not as the image itself, or its underlying storage, claims). This can be used e.g. to determine which public keys are trusted for this image. Reference() ImageReference - // Close removes resources associated with an initialized UnparsedImage, if any. - Close() error // Manifest is like ImageSource.GetManifest, but the result is cached; it is OK to call this however often you need. Manifest() ([]byte, string, error) // Signatures is like ImageSource.GetSignatures, but the result is cached; it is OK to call this however often you need. Signatures(ctx context.Context) ([][]byte, error) + // LayerInfosForCopy returns either nil (meaning the values in the manifest are fine), or updated values for the layer blobsums that are listed in the image's manifest. + // The Digest field is guaranteed to be provided, Size may be -1 and MediaType may be optionally provided. + // WARNING: The list may contain duplicates, and they are semantically relevant. + LayerInfosForCopy() []BlobInfo } // Image is the primary API for inspecting properties of images. -// Each Image should eventually be closed by calling Close(). +// An Image is based on a pair of (ImageSource, instance digest); it can represent either a manifest list or a single image instance. +// +// The Image must not be used after the underlying ImageSource is Close()d. type Image interface { // Note that Reference may return nil in the return value of UpdatedImage! UnparsedImage // ConfigInfo returns a complete BlobInfo for the separate config object, or a BlobInfo{Digest:""} if there isn't a separate object. // Note that the config object may not exist in the underlying storage in the return value of UpdatedImage! Use ConfigBlob() below. ConfigInfo() BlobInfo - // ConfigBlob returns the blob described by ConfigInfo, iff ConfigInfo().Digest != ""; nil otherwise. + // ConfigBlob returns the blob described by ConfigInfo, if ConfigInfo().Digest != ""; nil otherwise. // The result is cached; it is OK to call this however often you need. ConfigBlob() ([]byte, error) // OCIConfig returns the image configuration as per OCI v1 image-spec. Information about @@ -223,7 +239,7 @@ type Image interface { // old image manifests work (docker v2s1 especially). OCIConfig() (*v1.Image, error) // LayerInfos returns a list of BlobInfos of layers referenced by this image, in order (the root layer first, and then successive layered layers). - // The Digest field is guaranteed to be provided; Size may be -1. + // The Digest field is guaranteed to be provided, Size may be -1 and MediaType may be optionally provided. // WARNING: The list may contain duplicates, and they are semantically relevant. LayerInfos() []BlobInfo // EmbeddedDockerReferenceConflicts whether a Docker reference embedded in the manifest, if any, conflicts with destination ref. @@ -240,16 +256,23 @@ type Image interface { // Everything in options.InformationOnly should be provided, other fields should be set only if a modification is desired. // This does not change the state of the original Image object. UpdatedImage(options ManifestUpdateOptions) (Image, error) - // IsMultiImage returns true if the image's manifest is a list of images, false otherwise. - IsMultiImage() bool // Size returns an approximation of the amount of disk space which is consumed by the image in its current // location. If the size is not known, -1 will be returned. Size() (int64, error) } +// ImageCloser is an Image with a Close() method which must be called by the user. +// This is returned by ImageReference.NewImage, which transparently instantiates a types.ImageSource, +// to ensure that the ImageSource is closed. +type ImageCloser interface { + Image + // Close removes resources associated with an initialized ImageCloser. + Close() error +} + // ManifestUpdateOptions is a way to pass named optional arguments to Image.UpdatedManifest type ManifestUpdateOptions struct { - LayerInfos []BlobInfo // Complete BlobInfos (size+digest+urls) which should replace the originals, in order (the root layer first, and then successive layered layers) + LayerInfos []BlobInfo // Complete BlobInfos (size+digest+urls+annotations) which should replace the originals, in order (the root layer first, and then successive layered layers). BlobInfos' MediaType fields are ignored. EmbeddedDockerReference reference.Named ManifestMIMEType string // The values below are NOT requests to modify the image; they provide optional context which may or may not be used. @@ -283,7 +306,7 @@ type DockerAuthConfig struct { Password string } -// SystemContext allows parametrizing access to implicitly-accessed resources, +// SystemContext allows parameterizing access to implicitly-accessed resources, // like configuration files in /etc and users' login state in their home directory. // Various components can share the same field only if their semantics is exactly // the same; if in doubt, add a new field. @@ -306,6 +329,10 @@ type SystemContext struct { SystemRegistriesConfPath string // If not "", overrides the default path for the authentication file AuthFilePath string + // If not "", overrides the use of platform.GOARCH when choosing an image or verifying architecture match. + ArchitectureChoice string + // If not "", overrides the use of platform.GOOS when choosing an image or verifying OS match. + OSChoice string // === OCI.Transport overrides === // If not "", a directory containing a CA certificate (ending with ".crt"), @@ -314,6 +341,8 @@ type SystemContext struct { OCICertPath string // Allow downloading OCI image layers over HTTP, or HTTPS with failed TLS verification. Note that this does not affect other TLS connections. OCIInsecureSkipTLSVerify bool + // If not "", use a shared directory for storing blobs rather than within OCI layouts + OCISharedBlobDirPath string // === docker.Transport overrides === // If not "", a directory containing a CA certificate (ending with ".crt"), @@ -322,8 +351,9 @@ type SystemContext struct { DockerCertPath string // If not "", overrides the system’s default path for a directory containing host[:port] subdirectories with the same structure as DockerCertPath above. // Ignored if DockerCertPath is non-empty. - DockerPerHostCertDirPath string - DockerInsecureSkipTLSVerify bool // Allow contacting docker registries over HTTP, or HTTPS with failed TLS verification. Note that this does not affect other TLS connections. + DockerPerHostCertDirPath string + // Allow contacting docker registries over HTTP, or HTTPS with failed TLS verification. Note that this does not affect other TLS connections. + DockerInsecureSkipTLSVerify bool // if nil, the library tries to parse ~/.docker/config.json to retrieve credentials DockerAuthConfig *DockerAuthConfig // if not "", an User-Agent header is added to each request when contacting a registry. @@ -334,6 +364,20 @@ type SystemContext struct { DockerDisableV1Ping bool // Directory to use for OSTree temporary files OSTreeTmpDirPath string + + // === docker/daemon.Transport overrides === + // A directory containing a CA certificate (ending with ".crt"), + // a client certificate (ending with ".cert") and a client certificate key + // (ending with ".key") used when talking to a Docker daemon. + DockerDaemonCertPath string + // The hostname or IP to the Docker daemon. If not set (aka ""), client.DefaultDockerHost is assumed. + DockerDaemonHost string + // Used to skip TLS verification, off by default. To take effect DockerDaemonCertPath needs to be specified as well. + DockerDaemonInsecureSkipTLSVerify bool + + // === dir.Transport overrides === + // DirForceCompress compresses the image layers if set to true + DirForceCompress bool } // ProgressProperties is used to pass information from the copy code to a monitor which diff --git a/vendor/github.com/containers/image/vendor.conf b/vendor/github.com/containers/image/vendor.conf index d5bae3b0..f3634b38 100644 --- a/vendor/github.com/containers/image/vendor.conf +++ b/vendor/github.com/containers/image/vendor.conf @@ -1,5 +1,5 @@ github.com/sirupsen/logrus v1.0.0 -github.com/containers/storage 47536c89fcc545a87745e1a1573addc439409165 +github.com/containers/storage master github.com/davecgh/go-spew 346938d642f2ec3594ed81d874461961cd0faa76 github.com/docker/docker-credential-helpers d68f9aeca33f5fd3f08eeae5e9d175edf4e731d1 github.com/docker/distribution 5f6282db7d65e6d72ad7c2cc66310724a57be716 @@ -22,7 +22,7 @@ github.com/pborman/uuid 1b00554d822231195d1babd97ff4a781231955c9 github.com/pkg/errors 248dadf4e9068a0b3e79f02ed0a610d935de5302 github.com/pmezard/go-difflib 792786c7400a136282c1664665ae0a8db921c6c2 github.com/stretchr/testify 4d4bfba8f1d1027c4fdbe371823030df51419987 -github.com/vbatts/tar-split bd4c5d64c3e9297f410025a3b1bd0c58f659e721 +github.com/vbatts/tar-split v0.10.2 golang.org/x/crypto 453249f01cfeb54c3d549ddb75ff152ca243f9d8 golang.org/x/net 6b27048ae5e6ad1ef927e72e437531493de612fe golang.org/x/sys 43e60d72a8e2bd92ee98319ba9a384a0e9837c08 @@ -36,4 +36,5 @@ github.com/tchap/go-patricia v2.2.6 github.com/opencontainers/selinux ba1aefe8057f1d0cfb8e88d0ec1dc85925ef987d github.com/BurntSushi/toml b26d9c308763d68093482582cea63d69be07a0f0 github.com/ostreedev/ostree-go aeb02c6b6aa2889db3ef62f7855650755befd460 -github.com/gogo/protobuf/proto fcdc5011193ff531a548e9b0301828d5a5b97fd8 +github.com/gogo/protobuf fcdc5011193ff531a548e9b0301828d5a5b97fd8 +github.com/pquerna/ffjson master diff --git a/vendor/github.com/containers/storage/drivers/overlay/overlay.go b/vendor/github.com/containers/storage/drivers/overlay/overlay.go index 249f98bc..4974a94e 100644 --- a/vendor/github.com/containers/storage/drivers/overlay/overlay.go +++ b/vendor/github.com/containers/storage/drivers/overlay/overlay.go @@ -3,7 +3,6 @@ package overlay import ( - "bufio" "fmt" "io" "io/ioutil" @@ -26,7 +25,6 @@ import ( "github.com/containers/storage/pkg/locker" "github.com/containers/storage/pkg/mount" "github.com/containers/storage/pkg/parsers" - "github.com/containers/storage/pkg/parsers/kernel" "github.com/containers/storage/pkg/system" units "github.com/docker/go-units" "github.com/opencontainers/selinux/go-selinux/label" @@ -124,22 +122,6 @@ func Init(home string, options []string, uidMaps, gidMaps []idtools.IDMap) (grap return nil, err } - if err := supportsOverlay(); err != nil { - return nil, errors.Wrap(graphdriver.ErrNotSupported, "kernel does not support overlay fs") - } - - // require kernel 4.0.0 to ensure multiple lower dirs are supported - v, err := kernel.GetKernelVersion() - if err != nil { - return nil, err - } - if kernel.CompareKernelVersion(*v, kernel.VersionInfo{Kernel: 4, Major: 0, Minor: 0}) < 0 { - if !opts.overrideKernelCheck { - return nil, errors.Wrap(graphdriver.ErrNotSupported, "kernel too old to provide multiple lowers feature for overlay") - } - logrus.Warn("Using pre-4.0.0 kernel for overlay, mount failures may require kernel update") - } - fsMagic, err := graphdriver.GetFSMagic(home) if err != nil { return nil, err @@ -153,22 +135,18 @@ func Init(home string, options []string, uidMaps, gidMaps []idtools.IDMap) (grap case graphdriver.FsMagicAufs, graphdriver.FsMagicZfs, graphdriver.FsMagicOverlay, graphdriver.FsMagicEcryptfs: logrus.Errorf("'overlay' is not supported over %s", backingFs) return nil, errors.Wrapf(graphdriver.ErrIncompatibleFS, "'overlay' is not supported over %s", backingFs) - case graphdriver.FsMagicBtrfs: - // Support for OverlayFS on BTRFS was added in kernel 4.7 - // See https://btrfs.wiki.kernel.org/index.php/Changelog - if kernel.CompareKernelVersion(*v, kernel.VersionInfo{Kernel: 4, Major: 7, Minor: 0}) < 0 { - if !opts.overrideKernelCheck { - logrus.Errorf("'overlay' requires kernel 4.7 to use on %s", backingFs) - return nil, errors.Wrapf(graphdriver.ErrIncompatibleFS, "'overlay' requires kernel 4.7 to use on %s", backingFs) - } - logrus.Warn("Using pre-4.7.0 kernel for overlay on btrfs, may require kernel update") - } } rootUID, rootGID, err := idtools.GetRootUIDGID(uidMaps, gidMaps) if err != nil { return nil, err } + + supportsDType, err := supportsOverlay(home, fsMagic, rootUID, rootGID) + if err != nil { + return nil, errors.Wrap(graphdriver.ErrNotSupported, "kernel does not support overlay fs") + } + // Create the driver home dir if err := idtools.MkdirAllAs(path.Join(home, linkDir), 0700, rootUID, rootGID); err != nil && !os.IsExist(err) { return nil, err @@ -178,16 +156,6 @@ func Init(home string, options []string, uidMaps, gidMaps []idtools.IDMap) (grap return nil, err } - supportsDType, err := fsutils.SupportsDType(home) - if err != nil { - return nil, err - } - if !supportsDType { - logrus.Warn(overlayutils.ErrDTypeNotSupported("overlay", backingFs)) - // TODO: Will make fatal when CRI-O Has AMI built on RHEL7.4 - // return nil, overlayutils.ErrDTypeNotSupported("overlay", backingFs) - } - d := &Driver{ name: "overlay", home: home, @@ -210,10 +178,10 @@ func Init(home string, options []string, uidMaps, gidMaps []idtools.IDMap) (grap } } else if opts.quota.Size > 0 { // if xfs is not the backing fs then error out if the storage-opt overlay.size is used. - return nil, fmt.Errorf("Storage Option overlay.size only supported for backingFS XFS. Found %v", backingFs) + return nil, fmt.Errorf("Storage option overlay.size only supported for backingFS XFS. Found %v", backingFs) } - logrus.Debugf("backingFs=%s, projectQuotaSupported=%v", backingFs, projectQuotaSupported) + logrus.Debugf("backingFs=%s, projectQuotaSupported=%v", backingFs, projectQuotaSupported) return d, nil } @@ -227,20 +195,20 @@ func parseOptions(options []string) (*overlayOptions, error) { } key = strings.ToLower(key) switch key { - case "overlay.override_kernel_check", "overlay2.override_kernel_check": + case ".override_kernel_check", "overlay.override_kernel_check", "overlay2.override_kernel_check": logrus.Debugf("overlay: override_kernelcheck=%s", val) o.overrideKernelCheck, err = strconv.ParseBool(val) if err != nil { return nil, err } - case "overlay.size", "overlay2.size": + case ".size", "overlay.size", "overlay2.size": logrus.Debugf("overlay: size=%s", val) size, err := units.RAMInBytes(val) if err != nil { return nil, err } o.quota.Size = uint64(size) - case "overlay.imagestore", "overlay2.imagestore": + case ".imagestore", "overlay.imagestore", "overlay2.imagestore": logrus.Debugf("overlay: imagestore=%s", val) // Additional read only image stores to use for lower paths for _, store := range strings.Split(val, ",") { @@ -264,25 +232,59 @@ func parseOptions(options []string) (*overlayOptions, error) { return o, nil } -func supportsOverlay() error { - // We can try to modprobe overlay first before looking at - // proc/filesystems for when overlay is supported +func supportsOverlay(home string, homeMagic graphdriver.FsMagic, rootUID, rootGID int) (supportsDType bool, err error) { + // We can try to modprobe overlay first exec.Command("modprobe", "overlay").Run() - f, err := os.Open("/proc/filesystems") - if err != nil { - return err - } - defer f.Close() - - s := bufio.NewScanner(f) - for s.Scan() { - if s.Text() == "nodev\toverlay" { - return nil + layerDir, err := ioutil.TempDir(home, "compat") + if err == nil { + // Check if reading the directory's contents populates the d_type field, which is required + // for proper operation of the overlay filesystem. + supportsDType, err = fsutils.SupportsDType(layerDir) + if err != nil { + return false, err } + if !supportsDType { + logrus.Warn(overlayutils.ErrDTypeNotSupported("overlay", backingFs)) + // TODO: Will make fatal when CRI-O Has AMI built on RHEL7.4 + // return nil, overlayutils.ErrDTypeNotSupported("overlay", backingFs) + } + + // Try a test mount in the specific location we're looking at using. + mergedDir := filepath.Join(layerDir, "merged") + lower1Dir := filepath.Join(layerDir, "lower1") + lower2Dir := filepath.Join(layerDir, "lower2") + defer func() { + // Permitted to fail, since the various subdirectories + // can be empty or not even there, and the home might + // legitimately be not empty + _ = unix.Unmount(mergedDir, unix.MNT_DETACH) + _ = os.RemoveAll(layerDir) + _ = os.Remove(home) + }() + _ = idtools.MkdirAs(mergedDir, 0700, rootUID, rootGID) + _ = idtools.MkdirAs(lower1Dir, 0700, rootUID, rootGID) + _ = idtools.MkdirAs(lower2Dir, 0700, rootUID, rootGID) + flags := fmt.Sprintf("lowerdir=%s:%s", lower1Dir, lower2Dir) + if len(flags) < unix.Getpagesize() { + if mountFrom(filepath.Dir(home), "overlay", mergedDir, "overlay", 0, flags) == nil { + logrus.Debugf("overlay test mount with multiple lowers succeeded") + return supportsDType, nil + } + } + flags = fmt.Sprintf("lowerdir=%s", lower1Dir) + if len(flags) < unix.Getpagesize() { + if mountFrom(filepath.Dir(home), "overlay", mergedDir, "overlay", 0, flags) == nil { + logrus.Errorf("overlay test mount with multiple lowers failed, but succeeded with a single lower") + return supportsDType, errors.Wrap(graphdriver.ErrNotSupported, "kernel too old to provide multiple lowers feature for overlay") + } + } + logrus.Errorf("'overlay' is not supported over %s at %q", backingFs, home) + return supportsDType, errors.Wrapf(graphdriver.ErrIncompatibleFS, "'overlay' is not supported over %s at %q", backingFs, home) } + logrus.Error("'overlay' not found as a supported filesystem on this host. Please ensure kernel is new enough and has overlay support loaded.") - return errors.Wrap(graphdriver.ErrNotSupported, "'overlay' not found as a supported filesystem on this host. Please ensure kernel is new enough and has overlay support loaded.") + return supportsDType, errors.Wrap(graphdriver.ErrNotSupported, "'overlay' not found as a supported filesystem on this host. Please ensure kernel is new enough and has overlay support loaded.") } func useNaiveDiff(home string) bool { @@ -650,10 +652,21 @@ func (d *Driver) Get(id, mountLabel string) (_ string, retErr error) { func (d *Driver) Put(id string) error { d.locker.Lock(id) defer d.locker.Unlock(id) + dir := d.dir(id) + if _, err := os.Stat(dir); err != nil { + return err + } mountpoint := path.Join(d.dir(id), "merged") if count := d.ctr.Decrement(mountpoint); count > 0 { return nil } + if _, err := ioutil.ReadFile(path.Join(dir, lowerFile)); err != nil { + // If no lower, we used the diff directory, so no work to do + if os.IsNotExist(err) { + return nil + } + return err + } if err := unix.Unmount(mountpoint, unix.MNT_DETACH); err != nil { logrus.Debugf("Failed to unmount %s overlay: %s - %v", id, mountpoint, err) } diff --git a/vendor/github.com/containers/storage/drivers/vfs/driver.go b/vendor/github.com/containers/storage/drivers/vfs/driver.go index cf8eca91..ae62207d 100644 --- a/vendor/github.com/containers/storage/drivers/vfs/driver.go +++ b/vendor/github.com/containers/storage/drivers/vfs/driver.go @@ -36,6 +36,11 @@ func Init(home string, options []string, uidMaps, gidMaps []idtools.IDMap) (grap for _, option := range options { if strings.HasPrefix(option, "vfs.imagestore=") { d.homes = append(d.homes, strings.Split(option[15:], ",")...) + continue + } + if strings.HasPrefix(option, ".imagestore=") { + d.homes = append(d.homes, strings.Split(option[12:], ",")...) + continue } } return graphdriver.NewNaiveDiffDriver(d, uidMaps, gidMaps), nil diff --git a/vendor/github.com/containers/storage/images.go b/vendor/github.com/containers/storage/images.go index ed22e131..962e1bb7 100644 --- a/vendor/github.com/containers/storage/images.go +++ b/vendor/github.com/containers/storage/images.go @@ -14,12 +14,22 @@ import ( "github.com/pkg/errors" ) +const ( + // ImageDigestBigDataKey is the name of the big data item whose + // contents we consider useful for computing a "digest" of the + // image, by which we can locate the image later. + ImageDigestBigDataKey = "manifest" +) + // An Image is a reference to a layer and an associated metadata string. type Image struct { // ID is either one which was specified at create-time, or a random // value which was generated by the library. ID string `json:"id"` + // Digest is a digest value that we can use to locate the image. + Digest digest.Digest `json:"digest,omitempty"` + // Names is an optional set of user-defined convenience values. The // image can be referred to by its ID or any of its names. Names are // unique among images. @@ -28,7 +38,7 @@ type Image struct { // TopLayer is the ID of the topmost layer of the image itself, if the // image contains one or more layers. Multiple images can refer to the // same top layer. - TopLayer string `json:"layer"` + TopLayer string `json:"layer,omitempty"` // Metadata is data we keep for the convenience of the caller. It is not // expected to be large, since it is kept in memory. @@ -74,6 +84,10 @@ type ROImageStore interface { // Images returns a slice enumerating the known images. Images() ([]Image, error) + + // Images returns a slice enumerating the images which have a big data + // item with the name ImageDigestBigDataKey and the specified digest. + ByDigest(d digest.Digest) ([]*Image, error) } // ImageStore provides bookkeeping for information about Images. @@ -87,7 +101,7 @@ type ImageStore interface { // Create creates an image that has a specified ID (or a random one) and // optional names, using the specified layer as its topmost (hopefully // read-only) layer. That layer can be referenced by multiple images. - Create(id string, names []string, layer, metadata string, created time.Time) (*Image, error) + Create(id string, names []string, layer, metadata string, created time.Time, searchableDigest digest.Digest) (*Image, error) // SetNames replaces the list of names associated with an image with the // supplied values. @@ -107,6 +121,7 @@ type imageStore struct { idindex *truncindex.TruncIndex byid map[string]*Image byname map[string]*Image + bydigest map[digest.Digest][]*Image } func (r *imageStore) Images() ([]Image, error) { @@ -140,6 +155,7 @@ func (r *imageStore) Load() error { idlist := []string{} ids := make(map[string]*Image) names := make(map[string]*Image) + digests := make(map[digest.Digest][]*Image) if err = json.Unmarshal(data, &images); len(data) == 0 || err == nil { idlist = make([]string, 0, len(images)) for n, image := range images { @@ -152,6 +168,16 @@ func (r *imageStore) Load() error { } names[name] = images[n] } + // Implicit digest + if digest, ok := image.BigDataDigests[ImageDigestBigDataKey]; ok { + digests[digest] = append(digests[digest], images[n]) + } + // Explicit digest + if image.Digest == "" { + image.Digest = image.BigDataDigests[ImageDigestBigDataKey] + } else if image.Digest != image.BigDataDigests[ImageDigestBigDataKey] { + digests[image.Digest] = append(digests[image.Digest], images[n]) + } } } if shouldSave && !r.IsReadWrite() { @@ -161,6 +187,7 @@ func (r *imageStore) Load() error { r.idindex = truncindex.NewTruncIndex(idlist) r.byid = ids r.byname = names + r.bydigest = digests if shouldSave { return r.Save() } @@ -199,6 +226,7 @@ func newImageStore(dir string) (ImageStore, error) { images: []*Image{}, byid: make(map[string]*Image), byname: make(map[string]*Image), + bydigest: make(map[digest.Digest][]*Image), } if err := istore.Load(); err != nil { return nil, err @@ -219,6 +247,7 @@ func newROImageStore(dir string) (ROImageStore, error) { images: []*Image{}, byid: make(map[string]*Image), byname: make(map[string]*Image), + bydigest: make(map[digest.Digest][]*Image), } if err := istore.Load(); err != nil { return nil, err @@ -265,7 +294,7 @@ func (r *imageStore) SetFlag(id string, flag string, value interface{}) error { return r.Save() } -func (r *imageStore) Create(id string, names []string, layer, metadata string, created time.Time) (image *Image, err error) { +func (r *imageStore) Create(id string, names []string, layer, metadata string, created time.Time, searchableDigest digest.Digest) (image *Image, err error) { if !r.IsReadWrite() { return nil, errors.Wrapf(ErrStoreIsReadOnly, "not allowed to create new images at %q", r.imagespath()) } @@ -292,6 +321,7 @@ func (r *imageStore) Create(id string, names []string, layer, metadata string, c if err == nil { image = &Image{ ID: id, + Digest: searchableDigest, Names: names, TopLayer: layer, Metadata: metadata, @@ -304,6 +334,10 @@ func (r *imageStore) Create(id string, names []string, layer, metadata string, c r.images = append(r.images, image) r.idindex.Add(id) r.byid[id] = image + if searchableDigest != "" { + list := r.bydigest[searchableDigest] + r.bydigest[searchableDigest] = append(list, image) + } for _, name := range names { r.byname[name] = image } @@ -383,6 +417,28 @@ func (r *imageStore) Delete(id string) error { r.images = append(r.images[:toDeleteIndex], r.images[toDeleteIndex+1:]...) } } + if digest, ok := image.BigDataDigests[ImageDigestBigDataKey]; ok { + // remove the image from the digest-based index + if list, ok := r.bydigest[digest]; ok { + prunedList := imageSliceWithoutValue(list, image) + if len(prunedList) == 0 { + delete(r.bydigest, digest) + } else { + r.bydigest[digest] = prunedList + } + } + } + if image.Digest != "" { + // remove the image's hard-coded digest from the digest-based index + if list, ok := r.bydigest[image.Digest]; ok { + prunedList := imageSliceWithoutValue(list, image) + if len(prunedList) == 0 { + delete(r.bydigest, image.Digest) + } else { + r.bydigest[image.Digest] = prunedList + } + } + } if err := r.Save(); err != nil { return err } @@ -411,6 +467,13 @@ func (r *imageStore) Exists(id string) bool { return ok } +func (r *imageStore) ByDigest(d digest.Digest) ([]*Image, error) { + if images, ok := r.bydigest[d]; ok { + return images, nil + } + return nil, ErrImageUnknown +} + func (r *imageStore) BigData(id, key string) ([]byte, error) { if key == "" { return nil, errors.Wrapf(ErrInvalidBigDataName, "can't retrieve image big data value for empty name") @@ -486,6 +549,17 @@ func (r *imageStore) BigDataNames(id string) ([]string, error) { return image.BigDataNames, nil } +func imageSliceWithoutValue(slice []*Image, value *Image) []*Image { + modified := make([]*Image, 0, len(slice)) + for _, v := range slice { + if v == value { + continue + } + modified = append(modified, v) + } + return modified +} + func (r *imageStore) SetBigData(id, key string, data []byte) error { if key == "" { return errors.Wrapf(ErrInvalidBigDataName, "can't set empty name for image big data item") @@ -528,6 +602,29 @@ func (r *imageStore) SetBigData(id, key string, data []byte) error { image.BigDataNames = append(image.BigDataNames, key) save = true } + if key == ImageDigestBigDataKey { + if oldDigest != "" && oldDigest != newDigest && oldDigest != image.Digest { + // remove the image from the list of images in the digest-based + // index which corresponds to the old digest for this item, unless + // it's also the hard-coded digest + if list, ok := r.bydigest[oldDigest]; ok { + prunedList := imageSliceWithoutValue(list, image) + if len(prunedList) == 0 { + delete(r.bydigest, oldDigest) + } else { + r.bydigest[oldDigest] = prunedList + } + } + } + // add the image to the list of images in the digest-based index which + // corresponds to the new digest for this item, unless it's already there + list := r.bydigest[newDigest] + if len(list) == len(imageSliceWithoutValue(list, image)) { + // the list isn't shortened by trying to prune this image from it, + // so it's not in there yet + r.bydigest[newDigest] = append(list, image) + } + } if save { err = r.Save() } diff --git a/vendor/github.com/containers/storage/images_ffjson.go b/vendor/github.com/containers/storage/images_ffjson.go index fd3ef11a..f6a8b065 100644 --- a/vendor/github.com/containers/storage/images_ffjson.go +++ b/vendor/github.com/containers/storage/images_ffjson.go @@ -38,6 +38,11 @@ func (j *Image) MarshalJSONBuf(buf fflib.EncodingBuffer) error { buf.WriteString(`{ "id":`) fflib.WriteJsonString(buf, string(j.ID)) buf.WriteByte(',') + if len(j.Digest) != 0 { + buf.WriteString(`"digest":`) + fflib.WriteJsonString(buf, string(j.Digest)) + buf.WriteByte(',') + } if len(j.Names) != 0 { buf.WriteString(`"names":`) if j.Names != nil { @@ -54,9 +59,11 @@ func (j *Image) MarshalJSONBuf(buf fflib.EncodingBuffer) error { } buf.WriteByte(',') } - buf.WriteString(`"layer":`) - fflib.WriteJsonString(buf, string(j.TopLayer)) - buf.WriteByte(',') + if len(j.TopLayer) != 0 { + buf.WriteString(`"layer":`) + fflib.WriteJsonString(buf, string(j.TopLayer)) + buf.WriteByte(',') + } if len(j.Metadata) != 0 { buf.WriteString(`"metadata":`) fflib.WriteJsonString(buf, string(j.Metadata)) @@ -144,6 +151,8 @@ const ( ffjtImageID + ffjtImageDigest + ffjtImageNames ffjtImageTopLayer @@ -163,6 +172,8 @@ const ( var ffjKeyImageID = []byte("id") +var ffjKeyImageDigest = []byte("digest") + var ffjKeyImageNames = []byte("names") var ffjKeyImageTopLayer = []byte("layer") @@ -266,6 +277,14 @@ mainparse: goto mainparse } + case 'd': + + if bytes.Equal(ffjKeyImageDigest, kn) { + currentKey = ffjtImageDigest + state = fflib.FFParse_want_colon + goto mainparse + } + case 'f': if bytes.Equal(ffjKeyImageFlags, kn) { @@ -356,6 +375,12 @@ mainparse: goto mainparse } + if fflib.EqualFoldRight(ffjKeyImageDigest, kn) { + currentKey = ffjtImageDigest + state = fflib.FFParse_want_colon + goto mainparse + } + if fflib.SimpleLetterEqualFold(ffjKeyImageID, kn) { currentKey = ffjtImageID state = fflib.FFParse_want_colon @@ -382,6 +407,9 @@ mainparse: case ffjtImageID: goto handle_ID + case ffjtImageDigest: + goto handle_Digest + case ffjtImageNames: goto handle_Names @@ -446,6 +474,32 @@ handle_ID: state = fflib.FFParse_after_value goto mainparse +handle_Digest: + + /* handler: j.Digest type=digest.Digest kind=string quoted=false*/ + + { + + { + if tok != fflib.FFTok_string && tok != fflib.FFTok_null { + return fs.WrapErr(fmt.Errorf("cannot unmarshal %s into Go value for Digest", tok)) + } + } + + if tok == fflib.FFTok_null { + + } else { + + outBuf := fs.Output.Bytes() + + j.Digest = digest.Digest(string(outBuf)) + + } + } + + state = fflib.FFParse_after_value + goto mainparse + handle_Names: /* handler: j.Names type=[]string kind=slice quoted=false*/ diff --git a/vendor/github.com/containers/storage/store.go b/vendor/github.com/containers/storage/store.go index c84bfaf9..de605432 100644 --- a/vendor/github.com/containers/storage/store.go +++ b/vendor/github.com/containers/storage/store.go @@ -370,6 +370,10 @@ type Store interface { // and may have different metadata, big data items, and flags. ImagesByTopLayer(id string) ([]*Image, error) + // ImagesByDigest returns a list of images which contain a big data item + // named ImageDigestBigDataKey whose contents have the specified digest. + ImagesByDigest(d digest.Digest) ([]*Image, error) + // Container returns a specific container. Container(id string) (*Container, error) @@ -430,6 +434,8 @@ type ImageOptions struct { // CreationDate, if not zero, will override the default behavior of marking the image as having been // created when CreateImage() was called, recording CreationDate instead. CreationDate time.Time + // Digest is a hard-coded digest value that we can use to look up the image. It is optional. + Digest digest.Digest } // ContainerOptions is used for passing options to a Store's CreateContainer() method. @@ -487,11 +493,6 @@ func GetStore(options StoreOptions) (Store, error) { if err := os.MkdirAll(options.RunRoot, 0700); err != nil && !os.IsExist(err) { return nil, err } - for _, subdir := range []string{} { - if err := os.MkdirAll(filepath.Join(options.RunRoot, subdir), 0700); err != nil && !os.IsExist(err) { - return nil, err - } - } if err := os.MkdirAll(options.GraphRoot, 0700); err != nil && !os.IsExist(err) { return nil, err } @@ -834,11 +835,11 @@ func (s *store) CreateImage(id string, names []string, layer, metadata string, o } creationDate := time.Now().UTC() - if options != nil { + if options != nil && !options.CreationDate.IsZero() { creationDate = options.CreationDate } - return ristore.Create(id, names, layer, metadata, creationDate) + return ristore.Create(id, names, layer, metadata, creationDate, options.Digest) } func (s *store) CreateContainer(id string, names []string, image, layer, metadata string, options *ContainerOptions) (*Container, error) { @@ -1888,10 +1889,16 @@ func (s *store) layersByMappedDigest(m func(ROLayerStore, digest.Digest) ([]Laye } storeLayers, err := m(store, d) if err != nil { - return nil, err + if errors.Cause(err) != ErrLayerUnknown { + return nil, err + } + continue } layers = append(layers, storeLayers...) } + if len(layers) == 0 { + return nil, ErrLayerUnknown + } return layers, nil } @@ -2080,6 +2087,33 @@ func (s *store) ImagesByTopLayer(id string) ([]*Image, error) { return images, nil } +func (s *store) ImagesByDigest(d digest.Digest) ([]*Image, error) { + images := []*Image{} + + istore, err := s.ImageStore() + if err != nil { + return nil, err + } + + istores, err := s.ROImageStores() + if err != nil { + return nil, err + } + for _, store := range append([]ROImageStore{istore}, istores...) { + store.Lock() + defer store.Unlock() + if modified, err := store.Modified(); modified || err != nil { + store.Load() + } + imageList, err := store.ByDigest(d) + if err != nil && err != ErrImageUnknown { + return nil, err + } + images = append(images, imageList...) + } + return images, nil +} + func (s *store) Container(id string) (*Container, error) { rcstore, err := s.ContainerStore() if err != nil { diff --git a/vendor/github.com/containers/storage/vendor.conf b/vendor/github.com/containers/storage/vendor.conf index 9958101e..a30f8feb 100644 --- a/vendor/github.com/containers/storage/vendor.conf +++ b/vendor/github.com/containers/storage/vendor.conf @@ -15,7 +15,7 @@ github.com/pmezard/go-difflib v1.0.0 github.com/sirupsen/logrus v1.0.0 github.com/stretchr/testify 4d4bfba8f1d1027c4fdbe371823030df51419987 github.com/tchap/go-patricia v2.2.6 -github.com/vbatts/tar-split bd4c5d64c3e9297f410025a3b1bd0c58f659e721 +github.com/vbatts/tar-split v0.10.2 golang.org/x/net 7dcfb8076726a3fdd9353b6b8a1f1b6be6811bd6 golang.org/x/sys 07c182904dbd53199946ba614a412c61d3c548f5 github.com/pquerna/ffjson d49c2bc1aa135aad0c6f4fc2056623ec78f5d5ac diff --git a/vendor/github.com/opencontainers/runtime-tools/README.md b/vendor/github.com/opencontainers/runtime-tools/README.md index bbcafc26..5f16436b 100644 --- a/vendor/github.com/opencontainers/runtime-tools/README.md +++ b/vendor/github.com/opencontainers/runtime-tools/README.md @@ -8,7 +8,7 @@ To build from source code, runtime-tools requires Go 1.7.x or above. [`oci-runtime-tool generate`][generate.1] generates [configuration JSON][config.json] for an [OCI bundle][bundle]. [OCI-compatible runtimes][runtime-spec] like [runC][] expect to read the configuration from `config.json`. -```sh +```console $ oci-runtime-tool generate --output config.json $ cat config.json { @@ -22,7 +22,7 @@ $ cat config.json [`oci-runtime-tool validate`][validate.1] validates an OCI bundle. The error message will be printed if the OCI bundle failed the validation procedure. -```sh +```console $ oci-runtime-tool generate $ oci-runtime-tool validate INFO[0000] Bundle validation succeeded. @@ -30,55 +30,153 @@ INFO[0000] Bundle validation succeeded. ## Testing OCI runtimes -```sh -$ sudo make RUNTIME=runc localvalidation -RUNTIME=runc go test -tags "" -v github.com/opencontainers/runtime-tools/validation -=== RUN TestValidateBasic +The runtime validation suite uses [node-tap][], which is packaged for some distributions (for example, it is in [Debian's `node-tap` package][debian-node-tap]). +If your distribution does not package node-tap, you can install [npm][] (for example, from [Gentoo's `nodejs` package][gentoo-nodejs]) and use it: + +```console +$ npm install tap +``` + +```console +$ make runtimetest validation-executables +RUNTIME=runc tap validation/linux_rootfs_propagation_shared.t validation/create.t validation/default.t validation/linux_readonly_paths.t validation/linux_masked_paths.t validation/mounts.t validation/process.t validation/root_readonly_false.t validation/linux_sysctl.t validation/linux_devices.t validation/linux_gid_mappings.t validation/process_oom_score_adj.t validation/process_capabilities.t validation/process_rlimits.t validation/root_readonly_true.t validation/linux_rootfs_propagation_unbindable.t validation/hostname.t validation/linux_uid_mappings.t +validation/linux_rootfs_propagation_shared.t ........ 18/19 + not ok rootfs propagation + +validation/create.t ................................... 4/4 +validation/default.t ................................ 19/19 +validation/linux_readonly_paths.t ................... 19/19 +validation/linux_masked_paths.t ..................... 18/19 + not ok masked paths + +validation/mounts.t ................................... 0/1 + Skipped: 1 + TODO: mounts generation options have not been implemented + +validation/process.t ................................ 19/19 +validation/root_readonly_false.t .................... 19/19 +validation/linux_sysctl.t ........................... 19/19 +validation/linux_devices.t .......................... 19/19 +validation/linux_gid_mappings.t ..................... 18/19 + not ok gid mappings + +validation/process_oom_score_adj.t .................. 19/19 +validation/process_capabilities.t ................... 19/19 +validation/process_rlimits.t ........................ 19/19 +validation/root_readonly_true.t ...................failed to create the container +rootfsPropagation=unbindable is not supported +exit status 1 +validation/root_readonly_true.t ..................... 19/19 +validation/linux_rootfs_propagation_unbindable.t ...... 0/1 + not ok validation/linux_rootfs_propagation_unbindable.t + timeout: 30000 + file: validation/linux_rootfs_propagation_unbindable.t + command: validation/linux_rootfs_propagation_unbindable.t + args: [] + stdio: + - 0 + - pipe + - 2 + cwd: /…/go/src/github.com/opencontainers/runtime-tools + exitCode: 1 + +validation/hostname.t ...................failed to create the container +User namespace mappings specified, but USER namespace isn't enabled in the config +exit status 1 +validation/hostname.t ............................... 19/19 +validation/linux_uid_mappings.t ....................... 0/1 + not ok validation/linux_uid_mappings.t + timeout: 30000 + file: validation/linux_uid_mappings.t + command: validation/linux_uid_mappings.t + args: [] + stdio: + - 0 + - pipe + - 2 + cwd: /…/go/src/github.com/opencontainers/runtime-tools + exitCode: 1 + +total ............................................. 267/273 + + + 267 passing (31s) + 1 pending + 5 failing + +make: *** [Makefile:43: localvalidation] Error 1 +``` + +You can also run an individual test executable directly: + +```console +$ RUNTIME=runc validation/default.t TAP version 13 ok 1 - root filesystem ok 2 - hostname -ok 3 - mounts -ok 4 - capabilities -ok 5 - default symlinks -ok 6 - default devices -ok 7 - linux devices -ok 8 - linux process -ok 9 - masked paths -ok 10 - oom score adj -ok 11 - read only paths -ok 12 - rlimits -ok 13 - sysctls -ok 14 - uid mappings -ok 15 - gid mappings -1..15 ---- PASS: TestValidateBasic (0.08s) -=== RUN TestValidateSysctls -TAP version 13 -ok 1 - root filesystem -ok 2 - hostname -ok 3 - mounts -ok 4 - capabilities -ok 5 - default symlinks -ok 6 - default devices -ok 7 - linux devices -ok 8 - linux process -ok 9 - masked paths -ok 10 - oom score adj -ok 11 - read only paths -ok 12 - rlimits -ok 13 - sysctls -ok 14 - uid mappings -ok 15 - gid mappings -1..15 ---- PASS: TestValidateSysctls (0.20s) -PASS -ok github.com/opencontainers/runtime-tools/validation 0.281s +ok 3 - process +ok 4 - mounts +ok 5 - user +ok 6 - rlimits +ok 7 - capabilities +ok 8 - default symlinks +ok 9 - default file system +ok 10 - default devices +ok 11 - linux devices +ok 12 - linux process +ok 13 - masked paths +ok 14 - oom score adj +ok 15 - read only paths +ok 16 - rootfs propagation +ok 17 - sysctls +ok 18 - uid mappings +ok 19 - gid mappings +1..19 +``` + +If you cannot install node-tap, you can probably run the test suite with another [TAP consumer][tap-consumers]. +For example, with [`prove`][prove]: + +```console +$ sudo make TAP='prove -Q -j9' RUNTIME=runc localvalidation +RUNTIME=runc prove -Q -j9 validation/linux_rootfs_propagation_shared.t validation/create.t validation/default.t validation/linux_readonly_paths.t validation/linux_masked_paths.t validation/mounts.t validation/process.t validation/root_readonly_false.t validation/linux_sysctl.t validation/linux_devices.t validation/linux_gid_mappings.t validation/process_oom_score_adj.t validation/process_capabilities.t validation/process_rlimits.t validation/root_readonly_true.t validation/linux_rootfs_propagation_unbindable.t validation/hostname.t validation/linux_uid_mappings.t +failed to create the container +rootfsPropagation=unbindable is not supported +exit status 1 +failed to create the container +User namespace mappings specified, but USER namespace isn't enabled in the config +exit status 1 + +Test Summary Report +------------------- +validation/linux_rootfs_propagation_shared.t (Wstat: 0 Tests: 19 Failed: 1) + Failed test: 16 +validation/linux_masked_paths.t (Wstat: 0 Tests: 19 Failed: 1) + Failed test: 13 +validation/linux_rootfs_propagation_unbindable.t (Wstat: 256 Tests: 0 Failed: 0) + Non-zero exit status: 1 + Parse errors: No plan found in TAP output +validation/linux_uid_mappings.t (Wstat: 256 Tests: 0 Failed: 0) + Non-zero exit status: 1 + Parse errors: No plan found in TAP output +validation/linux_gid_mappings.t (Wstat: 0 Tests: 19 Failed: 1) + Failed test: 19 +Files=18, Tests=271, 6 wallclock secs ( 0.06 usr 0.01 sys + 0.59 cusr 0.24 csys = 0.90 CPU) +Result: FAIL +make: *** [Makefile:43: localvalidation] Error 1 ``` [bundle]: https://github.com/opencontainers/runtime-spec/blob/master/bundle.md [config.json]: https://github.com/opencontainers/runtime-spec/blob/master/config.md +[debian-node-tap]: https://packages.debian.org/stretch/node-tap +[debian-nodejs]: https://packages.debian.org/stretch/nodejs +[gentoo-nodejs]: https://packages.gentoo.org/packages/net-libs/nodejs +[node-tap]: http://www.node-tap.org/ +[npm]: https://www.npmjs.com/ +[prove]: http://search.cpan.org/~leont/Test-Harness-3.39/bin/prove [runC]: https://github.com/opencontainers/runc [runtime-spec]: https://github.com/opencontainers/runtime-spec +[tap-consumers]: https://testanything.org/consumers.html [generate.1]: man/oci-runtime-tool-generate.1.md [validate.1]: man/oci-runtime-tool-validate.1.md diff --git a/vendor/github.com/opencontainers/runtime-tools/filepath/abs.go b/vendor/github.com/opencontainers/runtime-tools/filepath/abs.go new file mode 100644 index 00000000..e4ab7453 --- /dev/null +++ b/vendor/github.com/opencontainers/runtime-tools/filepath/abs.go @@ -0,0 +1,48 @@ +package filepath + +import ( + "regexp" + "strings" +) + +var windowsAbs = regexp.MustCompile(`^[a-zA-Z]:\\.*$`) + +// Abs is a version of path/filepath's Abs with an explicit operating +// system and current working directory. +func Abs(os, path, cwd string) (_ string, err error) { + if IsAbs(os, path) { + return Clean(os, path), nil + } + return Clean(os, Join(os, cwd, path)), nil +} + +// IsAbs is a version of path/filepath's IsAbs with an explicit +// operating system. +func IsAbs(os, path string) bool { + if os == "windows" { + // FIXME: copy hideous logic from Go's + // src/path/filepath/path_windows.go into somewhere where we can + // put 3-clause BSD licensed code. + return windowsAbs.MatchString(path) + } + sep := Separator(os) + + // POSIX has [1]: + // + // > If a pathname begins with two successive characters, + // > the first component following the leading characters + // > may be interpreted in an implementation-defined manner, + // > although more than two leading characters shall be + // > treated as a single character. + // + // And Boost treats // as non-absolute [2], but Linux [3,4], Python + // [5] and Go [6] all treat // as absolute. + // + // [1]: http://pubs.opengroup.org/onlinepubs/9699919799/basedefs/V1_chap04.html#tag_04_13 + // [2]: https://github.com/boostorg/filesystem/blob/boost-1.64.0/test/path_test.cpp#L861 + // [3]: http://man7.org/linux/man-pages/man7/path_resolution.7.html + // [4]: https://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git/tree/Documentation/filesystems/path-lookup.md?h=v4.12#n41 + // [5]: https://github.com/python/cpython/blob/v3.6.1/Lib/posixpath.py#L64-L66 + // [6]: https://go.googlesource.com/go/+/go1.8.3/src/path/path.go#199 + return strings.HasPrefix(path, string(sep)) +} diff --git a/vendor/github.com/opencontainers/runtime-tools/filepath/ancestor.go b/vendor/github.com/opencontainers/runtime-tools/filepath/ancestor.go new file mode 100644 index 00000000..896cd820 --- /dev/null +++ b/vendor/github.com/opencontainers/runtime-tools/filepath/ancestor.go @@ -0,0 +1,32 @@ +package filepath + +import ( + "fmt" + "strings" +) + +// IsAncestor returns true when pathB is an strict ancestor of pathA, +// and false where the paths are equal or pathB is outside of pathA. +// Paths that are not absolute will be made absolute with Abs. +func IsAncestor(os, pathA, pathB, cwd string) (_ bool, err error) { + if pathA == pathB { + return false, nil + } + + pathA, err = Abs(os, pathA, cwd) + if err != nil { + return false, err + } + pathB, err = Abs(os, pathB, cwd) + if err != nil { + return false, err + } + sep := Separator(os) + if !strings.HasSuffix(pathA, string(sep)) { + pathA = fmt.Sprintf("%s%c", pathA, sep) + } + if pathA == pathB { + return false, nil + } + return strings.HasPrefix(pathB, pathA), nil +} diff --git a/vendor/github.com/opencontainers/runtime-tools/filepath/clean.go b/vendor/github.com/opencontainers/runtime-tools/filepath/clean.go new file mode 100644 index 00000000..d5dd65ae --- /dev/null +++ b/vendor/github.com/opencontainers/runtime-tools/filepath/clean.go @@ -0,0 +1,74 @@ +package filepath + +import ( + "fmt" + "strings" +) + +// Clean is an explicit-OS version of path/filepath's Clean. +func Clean(os, path string) string { + abs := IsAbs(os, path) + sep := Separator(os) + elements := strings.Split(path, string(sep)) + + // Replace multiple Separator elements with a single one. + for i := 0; i < len(elements); i++ { + if len(elements[i]) == 0 { + elements = append(elements[:i], elements[i+1:]...) + i-- + } + } + + // Eliminate each . path name element (the current directory). + for i := 0; i < len(elements); i++ { + if elements[i] == "." && len(elements) > 1 { + elements = append(elements[:i], elements[i+1:]...) + i-- + } + } + + // Eliminate each inner .. path name element (the parent directory) + // along with the non-.. element that precedes it. + for i := 1; i < len(elements); i++ { + if i == 1 && abs && sep == '\\' { + continue + } + if i > 0 && elements[i] == ".." { + elements = append(elements[:i-1], elements[i+1:]...) + i -= 2 + } + } + + // Eliminate .. elements that begin a rooted path: + // that is, replace "/.." by "/" at the beginning of a path, + // assuming Separator is '/'. + offset := 0 + if sep == '\\' { + offset = 1 + } + if abs { + for len(elements) > offset && elements[offset] == ".." { + elements = append(elements[:offset], elements[offset+1:]...) + } + } + + cleaned := strings.Join(elements, string(sep)) + if abs { + if sep == '/' { + cleaned = fmt.Sprintf("%c%s", sep, cleaned) + } else if len(elements) == 1 { + cleaned = fmt.Sprintf("%s%c", cleaned, sep) + } + } + + // If the result of this process is an empty string, Clean returns + // the string ".". + if len(cleaned) == 0 { + cleaned = "." + } + + if cleaned == path { + return path + } + return Clean(os, cleaned) +} diff --git a/vendor/github.com/opencontainers/runtime-tools/filepath/doc.go b/vendor/github.com/opencontainers/runtime-tools/filepath/doc.go new file mode 100644 index 00000000..7ee085bf --- /dev/null +++ b/vendor/github.com/opencontainers/runtime-tools/filepath/doc.go @@ -0,0 +1,6 @@ +// Package filepath implements Go's filepath package with explicit +// operating systems (and for some functions and explicit working +// directory). This allows tools built for one OS to operate on paths +// targeting another OS. For example, a Linux build can determine +// whether a path is absolute on Linux or on Windows. +package filepath diff --git a/vendor/github.com/opencontainers/runtime-tools/filepath/join.go b/vendor/github.com/opencontainers/runtime-tools/filepath/join.go new file mode 100644 index 00000000..b865d237 --- /dev/null +++ b/vendor/github.com/opencontainers/runtime-tools/filepath/join.go @@ -0,0 +1,9 @@ +package filepath + +import "strings" + +// Join is an explicit-OS version of path/filepath's Join. +func Join(os string, elem ...string) string { + sep := Separator(os) + return Clean(os, strings.Join(elem, string(sep))) +} diff --git a/vendor/github.com/opencontainers/runtime-tools/filepath/separator.go b/vendor/github.com/opencontainers/runtime-tools/filepath/separator.go new file mode 100644 index 00000000..2c5e8905 --- /dev/null +++ b/vendor/github.com/opencontainers/runtime-tools/filepath/separator.go @@ -0,0 +1,9 @@ +package filepath + +// Separator is an explicit-OS version of path/filepath's Separator. +func Separator(os string) rune { + if os == "windows" { + return '\\' + } + return '/' +} diff --git a/vendor/github.com/opencontainers/runtime-tools/generate/generate.go b/vendor/github.com/opencontainers/runtime-tools/generate/generate.go index fce88f5e..d2951b52 100644 --- a/vendor/github.com/opencontainers/runtime-tools/generate/generate.go +++ b/vendor/github.com/opencontainers/runtime-tools/generate/generate.go @@ -17,6 +17,12 @@ import ( var ( // Namespaces include the names of supported namespaces. Namespaces = []string{"network", "pid", "mount", "ipc", "uts", "user", "cgroup"} + + // we don't care about order...and this is way faster... + removeFunc = func(s []string, i int) []string { + s[i] = s[len(s)-1] + return s[:len(s)-1] + } ) // Generator represents a generator for a container spec. @@ -35,7 +41,7 @@ func New() Generator { spec := rspec.Spec{ Version: rspec.Version, Root: &rspec.Root{ - Path: "", + Path: "rootfs", Readonly: false, }, Process: &rspec.Process{ @@ -227,6 +233,7 @@ func NewFromFile(path string) (Generator, error) { if os.IsNotExist(err) { return Generator{}, fmt.Errorf("template configuration at %s not found", path) } + return Generator{}, err } defer cf.Close() @@ -392,7 +399,7 @@ func (g *Generator) SetProcessArgs(args []string) { // ClearProcessEnv clears g.spec.Process.Env. func (g *Generator) ClearProcessEnv() { - if g.spec == nil { + if g.spec == nil || g.spec.Process == nil { return } g.spec.Process.Env = []string{} @@ -433,22 +440,21 @@ func (g *Generator) AddProcessRlimits(rType string, rHard uint64, rSoft uint64) } // RemoveProcessRlimits removes a rlimit from g.spec.Process.Rlimits. -func (g *Generator) RemoveProcessRlimits(rType string) error { - if g.spec == nil { - return nil +func (g *Generator) RemoveProcessRlimits(rType string) { + if g.spec == nil || g.spec.Process == nil { + return } for i, rlimit := range g.spec.Process.Rlimits { if rlimit.Type == rType { g.spec.Process.Rlimits = append(g.spec.Process.Rlimits[:i], g.spec.Process.Rlimits[i+1:]...) - return nil + return } } - return nil } // ClearProcessRlimits clear g.spec.Process.Rlimits. func (g *Generator) ClearProcessRlimits() { - if g.spec == nil { + if g.spec == nil || g.spec.Process == nil { return } g.spec.Process.Rlimits = []rspec.POSIXRlimit{} @@ -456,7 +462,7 @@ func (g *Generator) ClearProcessRlimits() { // ClearProcessAdditionalGids clear g.spec.Process.AdditionalGids. func (g *Generator) ClearProcessAdditionalGids() { - if g.spec == nil { + if g.spec == nil || g.spec.Process == nil { return } g.spec.Process.User.AdditionalGids = []uint32{} @@ -485,6 +491,12 @@ func (g *Generator) SetLinuxCgroupsPath(path string) { g.spec.Linux.CgroupsPath = path } +// SetLinuxIntelRdtL3CacheSchema sets g.spec.Linux.IntelRdt.L3CacheSchema +func (g *Generator) SetLinuxIntelRdtL3CacheSchema(schema string) { + g.initSpecLinuxIntelRdt() + g.spec.Linux.IntelRdt.L3CacheSchema = schema +} + // SetLinuxMountLabel sets g.spec.Linux.MountLabel. func (g *Generator) SetLinuxMountLabel(label string) { g.initSpecLinux() @@ -497,6 +509,162 @@ func (g *Generator) SetProcessOOMScoreAdj(adj int) { g.spec.Process.OOMScoreAdj = &adj } +// SetLinuxResourcesBlockIOLeafWeight sets g.spec.Linux.Resources.BlockIO.LeafWeight. +func (g *Generator) SetLinuxResourcesBlockIOLeafWeight(weight uint16) { + g.initSpecLinuxResourcesBlockIO() + g.spec.Linux.Resources.BlockIO.LeafWeight = &weight +} + +// AddLinuxResourcesBlockIOLeafWeightDevice adds or sets g.spec.Linux.Resources.BlockIO.WeightDevice.LeafWeight. +func (g *Generator) AddLinuxResourcesBlockIOLeafWeightDevice(major int64, minor int64, weight uint16) { + g.initSpecLinuxResourcesBlockIO() + for i, weightDevice := range g.spec.Linux.Resources.BlockIO.WeightDevice { + if weightDevice.Major == major && weightDevice.Minor == minor { + g.spec.Linux.Resources.BlockIO.WeightDevice[i].LeafWeight = &weight + return + } + } + weightDevice := new(rspec.LinuxWeightDevice) + weightDevice.Major = major + weightDevice.Minor = minor + weightDevice.LeafWeight = &weight + g.spec.Linux.Resources.BlockIO.WeightDevice = append(g.spec.Linux.Resources.BlockIO.WeightDevice, *weightDevice) +} + +// DropLinuxResourcesBlockIOLeafWeightDevice drops a item form g.spec.Linux.Resources.BlockIO.WeightDevice.LeafWeight +func (g *Generator) DropLinuxResourcesBlockIOLeafWeightDevice(major int64, minor int64) { + if g.spec == nil || g.spec.Linux == nil || g.spec.Linux.Resources == nil || g.spec.Linux.Resources.BlockIO == nil { + return + } + + for i, weightDevice := range g.spec.Linux.Resources.BlockIO.WeightDevice { + if weightDevice.Major == major && weightDevice.Minor == minor { + if weightDevice.Weight != nil { + newWeightDevice := new(rspec.LinuxWeightDevice) + newWeightDevice.Major = major + newWeightDevice.Minor = minor + newWeightDevice.Weight = weightDevice.Weight + g.spec.Linux.Resources.BlockIO.WeightDevice[i] = *newWeightDevice + } else { + g.spec.Linux.Resources.BlockIO.WeightDevice = append(g.spec.Linux.Resources.BlockIO.WeightDevice[:i], g.spec.Linux.Resources.BlockIO.WeightDevice[i+1:]...) + } + return + } + } +} + +// SetLinuxResourcesBlockIOWeight sets g.spec.Linux.Resources.BlockIO.Weight. +func (g *Generator) SetLinuxResourcesBlockIOWeight(weight uint16) { + g.initSpecLinuxResourcesBlockIO() + g.spec.Linux.Resources.BlockIO.Weight = &weight +} + +// AddLinuxResourcesBlockIOWeightDevice adds or sets g.spec.Linux.Resources.BlockIO.WeightDevice.Weight. +func (g *Generator) AddLinuxResourcesBlockIOWeightDevice(major int64, minor int64, weight uint16) { + g.initSpecLinuxResourcesBlockIO() + for i, weightDevice := range g.spec.Linux.Resources.BlockIO.WeightDevice { + if weightDevice.Major == major && weightDevice.Minor == minor { + g.spec.Linux.Resources.BlockIO.WeightDevice[i].Weight = &weight + return + } + } + weightDevice := new(rspec.LinuxWeightDevice) + weightDevice.Major = major + weightDevice.Minor = minor + weightDevice.Weight = &weight + g.spec.Linux.Resources.BlockIO.WeightDevice = append(g.spec.Linux.Resources.BlockIO.WeightDevice, *weightDevice) +} + +// DropLinuxResourcesBlockIOWeightDevice drops a item form g.spec.Linux.Resources.BlockIO.WeightDevice.Weight +func (g *Generator) DropLinuxResourcesBlockIOWeightDevice(major int64, minor int64) { + if g.spec == nil || g.spec.Linux == nil || g.spec.Linux.Resources == nil || g.spec.Linux.Resources.BlockIO == nil { + return + } + + for i, weightDevice := range g.spec.Linux.Resources.BlockIO.WeightDevice { + if weightDevice.Major == major && weightDevice.Minor == minor { + if weightDevice.LeafWeight != nil { + newWeightDevice := new(rspec.LinuxWeightDevice) + newWeightDevice.Major = major + newWeightDevice.Minor = minor + newWeightDevice.LeafWeight = weightDevice.LeafWeight + g.spec.Linux.Resources.BlockIO.WeightDevice[i] = *newWeightDevice + } else { + g.spec.Linux.Resources.BlockIO.WeightDevice = append(g.spec.Linux.Resources.BlockIO.WeightDevice[:i], g.spec.Linux.Resources.BlockIO.WeightDevice[i+1:]...) + } + return + } + } +} + +// AddLinuxResourcesBlockIOThrottleReadBpsDevice adds or sets g.spec.Linux.Resources.BlockIO.ThrottleReadBpsDevice. +func (g *Generator) AddLinuxResourcesBlockIOThrottleReadBpsDevice(major int64, minor int64, rate uint64) { + g.initSpecLinuxResourcesBlockIO() + throttleDevices := addOrReplaceBlockIOThrottleDevice(g.spec.Linux.Resources.BlockIO.ThrottleReadBpsDevice, major, minor, rate) + g.spec.Linux.Resources.BlockIO.ThrottleReadBpsDevice = throttleDevices +} + +// DropLinuxResourcesBlockIOThrottleReadBpsDevice drops a item from g.spec.Linux.Resources.BlockIO.ThrottleReadBpsDevice. +func (g *Generator) DropLinuxResourcesBlockIOThrottleReadBpsDevice(major int64, minor int64) { + if g.spec == nil || g.spec.Linux == nil || g.spec.Linux.Resources == nil || g.spec.Linux.Resources.BlockIO == nil { + return + } + + throttleDevices := dropBlockIOThrottleDevice(g.spec.Linux.Resources.BlockIO.ThrottleReadBpsDevice, major, minor) + g.spec.Linux.Resources.BlockIO.ThrottleReadBpsDevice = throttleDevices +} + +// AddLinuxResourcesBlockIOThrottleReadIOPSDevice adds or sets g.spec.Linux.Resources.BlockIO.ThrottleReadIOPSDevice. +func (g *Generator) AddLinuxResourcesBlockIOThrottleReadIOPSDevice(major int64, minor int64, rate uint64) { + g.initSpecLinuxResourcesBlockIO() + throttleDevices := addOrReplaceBlockIOThrottleDevice(g.spec.Linux.Resources.BlockIO.ThrottleReadIOPSDevice, major, minor, rate) + g.spec.Linux.Resources.BlockIO.ThrottleReadIOPSDevice = throttleDevices +} + +// DropLinuxResourcesBlockIOThrottleReadIOPSDevice drops a item from g.spec.Linux.Resources.BlockIO.ThrottleReadIOPSDevice. +func (g *Generator) DropLinuxResourcesBlockIOThrottleReadIOPSDevice(major int64, minor int64) { + if g.spec == nil || g.spec.Linux == nil || g.spec.Linux.Resources == nil || g.spec.Linux.Resources.BlockIO == nil { + return + } + + throttleDevices := dropBlockIOThrottleDevice(g.spec.Linux.Resources.BlockIO.ThrottleReadIOPSDevice, major, minor) + g.spec.Linux.Resources.BlockIO.ThrottleReadIOPSDevice = throttleDevices +} + +// AddLinuxResourcesBlockIOThrottleWriteBpsDevice adds or sets g.spec.Linux.Resources.BlockIO.ThrottleWriteBpsDevice. +func (g *Generator) AddLinuxResourcesBlockIOThrottleWriteBpsDevice(major int64, minor int64, rate uint64) { + g.initSpecLinuxResourcesBlockIO() + throttleDevices := addOrReplaceBlockIOThrottleDevice(g.spec.Linux.Resources.BlockIO.ThrottleWriteBpsDevice, major, minor, rate) + g.spec.Linux.Resources.BlockIO.ThrottleWriteBpsDevice = throttleDevices +} + +// DropLinuxResourcesBlockIOThrottleWriteBpsDevice drops a item from g.spec.Linux.Resources.BlockIO.ThrottleWriteBpsDevice. +func (g *Generator) DropLinuxResourcesBlockIOThrottleWriteBpsDevice(major int64, minor int64) { + if g.spec == nil || g.spec.Linux == nil || g.spec.Linux.Resources == nil || g.spec.Linux.Resources.BlockIO == nil { + return + } + + throttleDevices := dropBlockIOThrottleDevice(g.spec.Linux.Resources.BlockIO.ThrottleWriteBpsDevice, major, minor) + g.spec.Linux.Resources.BlockIO.ThrottleWriteBpsDevice = throttleDevices +} + +// AddLinuxResourcesBlockIOThrottleWriteIOPSDevice adds or sets g.spec.Linux.Resources.BlockIO.ThrottleWriteIOPSDevice. +func (g *Generator) AddLinuxResourcesBlockIOThrottleWriteIOPSDevice(major int64, minor int64, rate uint64) { + g.initSpecLinuxResourcesBlockIO() + throttleDevices := addOrReplaceBlockIOThrottleDevice(g.spec.Linux.Resources.BlockIO.ThrottleWriteIOPSDevice, major, minor, rate) + g.spec.Linux.Resources.BlockIO.ThrottleWriteIOPSDevice = throttleDevices +} + +// DropLinuxResourcesBlockIOThrottleWriteIOPSDevice drops a item from g.spec.Linux.Resources.BlockIO.ThrottleWriteIOPSDevice. +func (g *Generator) DropLinuxResourcesBlockIOThrottleWriteIOPSDevice(major int64, minor int64) { + if g.spec == nil || g.spec.Linux == nil || g.spec.Linux.Resources == nil || g.spec.Linux.Resources.BlockIO == nil { + return + } + + throttleDevices := dropBlockIOThrottleDevice(g.spec.Linux.Resources.BlockIO.ThrottleWriteIOPSDevice, major, minor) + g.spec.Linux.Resources.BlockIO.ThrottleWriteIOPSDevice = throttleDevices +} + // SetLinuxResourcesCPUShares sets g.spec.Linux.Resources.CPU.Shares. func (g *Generator) SetLinuxResourcesCPUShares(shares uint64) { g.initSpecLinuxResourcesCPU() @@ -557,16 +725,17 @@ func (g *Generator) AddLinuxResourcesHugepageLimit(pageSize string, limit uint64 } // DropLinuxResourcesHugepageLimit drops a hugepage limit from g.spec.Linux.Resources.HugepageLimits. -func (g *Generator) DropLinuxResourcesHugepageLimit(pageSize string) error { - g.initSpecLinuxResources() +func (g *Generator) DropLinuxResourcesHugepageLimit(pageSize string) { + if g.spec == nil || g.spec.Linux == nil || g.spec.Linux.Resources == nil { + return + } + for i, pageLimit := range g.spec.Linux.Resources.HugepageLimits { if pageLimit.Pagesize == pageSize { g.spec.Linux.Resources.HugepageLimits = append(g.spec.Linux.Resources.HugepageLimits[:i], g.spec.Linux.Resources.HugepageLimits[i+1:]...) - return nil + return } } - - return nil } // SetLinuxResourcesMemoryLimit sets g.spec.Linux.Resources.Memory.Limit. @@ -634,7 +803,10 @@ func (g *Generator) AddLinuxResourcesNetworkPriorities(name string, prio uint32) // DropLinuxResourcesNetworkPriorities drops one item from g.spec.Linux.Resources.Network.Priorities. func (g *Generator) DropLinuxResourcesNetworkPriorities(name string) { - g.initSpecLinuxResourcesNetwork() + if g.spec == nil || g.spec.Linux == nil || g.spec.Linux.Resources == nil || g.spec.Linux.Resources.Network == nil { + return + } + for i, netPriority := range g.spec.Linux.Resources.Network.Priorities { if netPriority.Name == name { g.spec.Linux.Resources.Network.Priorities = append(g.spec.Linux.Resources.Network.Priorities[:i], g.spec.Linux.Resources.Network.Priorities[i+1:]...) @@ -721,8 +893,10 @@ func (g *Generator) SetLinuxRootPropagation(rp string) error { case "rslave": case "shared": case "rshared": + case "unbindable": + case "runbindable": default: - return fmt.Errorf("rootfs-propagation must be empty or one of private|rprivate|slave|rslave|shared|rshared") + return fmt.Errorf("rootfs-propagation %q must be empty or one of (r)private|(r)slave|(r)shared|(r)unbindable", rp) } g.initSpecLinux() g.spec.Linux.RootfsPropagation = rp @@ -731,217 +905,99 @@ func (g *Generator) SetLinuxRootPropagation(rp string) error { // ClearPreStartHooks clear g.spec.Hooks.Prestart. func (g *Generator) ClearPreStartHooks() { - if g.spec == nil { - return - } - if g.spec.Hooks == nil { + if g.spec == nil || g.spec.Hooks == nil { return } g.spec.Hooks.Prestart = []rspec.Hook{} } // AddPreStartHook add a prestart hook into g.spec.Hooks.Prestart. -func (g *Generator) AddPreStartHook(path string, args []string) { - g.initSpecHooks() - hook := rspec.Hook{Path: path, Args: args} - for i, hook := range g.spec.Hooks.Prestart { - if hook.Path == path { - g.spec.Hooks.Prestart[i] = hook - return - } - } - g.spec.Hooks.Prestart = append(g.spec.Hooks.Prestart, hook) -} - -// AddPreStartHookEnv adds envs of a prestart hook into g.spec.Hooks.Prestart. -func (g *Generator) AddPreStartHookEnv(path string, envs []string) { +func (g *Generator) AddPreStartHook(preStartHook rspec.Hook) error { g.initSpecHooks() for i, hook := range g.spec.Hooks.Prestart { - if hook.Path == path { - g.spec.Hooks.Prestart[i].Env = envs - return + if hook.Path == preStartHook.Path { + g.spec.Hooks.Prestart[i] = preStartHook + return nil } } - hook := rspec.Hook{Path: path, Env: envs} - g.spec.Hooks.Prestart = append(g.spec.Hooks.Prestart, hook) -} - -// AddPreStartHookTimeout adds timeout of a prestart hook into g.spec.Hooks.Prestart. -func (g *Generator) AddPreStartHookTimeout(path string, timeout int) { - g.initSpecHooks() - for i, hook := range g.spec.Hooks.Prestart { - if hook.Path == path { - g.spec.Hooks.Prestart[i].Timeout = &timeout - return - } - } - hook := rspec.Hook{Path: path, Timeout: &timeout} - g.spec.Hooks.Prestart = append(g.spec.Hooks.Prestart, hook) + g.spec.Hooks.Prestart = append(g.spec.Hooks.Prestart, preStartHook) + return nil } // ClearPostStopHooks clear g.spec.Hooks.Poststop. func (g *Generator) ClearPostStopHooks() { - if g.spec == nil { - return - } - if g.spec.Hooks == nil { + if g.spec == nil || g.spec.Hooks == nil { return } g.spec.Hooks.Poststop = []rspec.Hook{} } // AddPostStopHook adds a poststop hook into g.spec.Hooks.Poststop. -func (g *Generator) AddPostStopHook(path string, args []string) { - g.initSpecHooks() - hook := rspec.Hook{Path: path, Args: args} - for i, hook := range g.spec.Hooks.Poststop { - if hook.Path == path { - g.spec.Hooks.Poststop[i] = hook - return - } - } - g.spec.Hooks.Poststop = append(g.spec.Hooks.Poststop, hook) -} - -// AddPostStopHookEnv adds envs of a poststop hook into g.spec.Hooks.Poststop. -func (g *Generator) AddPostStopHookEnv(path string, envs []string) { +func (g *Generator) AddPostStopHook(postStopHook rspec.Hook) error { g.initSpecHooks() for i, hook := range g.spec.Hooks.Poststop { - if hook.Path == path { - g.spec.Hooks.Poststop[i].Env = envs - return + if hook.Path == postStopHook.Path { + g.spec.Hooks.Poststop[i] = postStopHook + return nil } } - hook := rspec.Hook{Path: path, Env: envs} - g.spec.Hooks.Poststop = append(g.spec.Hooks.Poststop, hook) -} - -// AddPostStopHookTimeout adds timeout of a poststop hook into g.spec.Hooks.Poststop. -func (g *Generator) AddPostStopHookTimeout(path string, timeout int) { - g.initSpecHooks() - for i, hook := range g.spec.Hooks.Poststop { - if hook.Path == path { - g.spec.Hooks.Poststop[i].Timeout = &timeout - return - } - } - hook := rspec.Hook{Path: path, Timeout: &timeout} - g.spec.Hooks.Poststop = append(g.spec.Hooks.Poststop, hook) + g.spec.Hooks.Poststop = append(g.spec.Hooks.Poststop, postStopHook) + return nil } // ClearPostStartHooks clear g.spec.Hooks.Poststart. func (g *Generator) ClearPostStartHooks() { - if g.spec == nil { - return - } - if g.spec.Hooks == nil { + if g.spec == nil || g.spec.Hooks == nil { return } g.spec.Hooks.Poststart = []rspec.Hook{} } // AddPostStartHook adds a poststart hook into g.spec.Hooks.Poststart. -func (g *Generator) AddPostStartHook(path string, args []string) { - g.initSpecHooks() - hook := rspec.Hook{Path: path, Args: args} - for i, hook := range g.spec.Hooks.Poststart { - if hook.Path == path { - g.spec.Hooks.Poststart[i] = hook - return - } - } - g.spec.Hooks.Poststart = append(g.spec.Hooks.Poststart, hook) -} - -// AddPostStartHookEnv adds envs of a poststart hook into g.spec.Hooks.Poststart. -func (g *Generator) AddPostStartHookEnv(path string, envs []string) { +func (g *Generator) AddPostStartHook(postStartHook rspec.Hook) error { g.initSpecHooks() for i, hook := range g.spec.Hooks.Poststart { - if hook.Path == path { - g.spec.Hooks.Poststart[i].Env = envs - return + if hook.Path == postStartHook.Path { + g.spec.Hooks.Poststart[i] = postStartHook + return nil } } - hook := rspec.Hook{Path: path, Env: envs} - g.spec.Hooks.Poststart = append(g.spec.Hooks.Poststart, hook) -} - -// AddPostStartHookTimeout adds timeout of a poststart hook into g.spec.Hooks.Poststart. -func (g *Generator) AddPostStartHookTimeout(path string, timeout int) { - g.initSpecHooks() - for i, hook := range g.spec.Hooks.Poststart { - if hook.Path == path { - g.spec.Hooks.Poststart[i].Timeout = &timeout - return - } - } - hook := rspec.Hook{Path: path, Timeout: &timeout} - g.spec.Hooks.Poststart = append(g.spec.Hooks.Poststart, hook) -} - -// AddTmpfsMount adds a tmpfs mount into g.spec.Mounts. -func (g *Generator) AddTmpfsMount(dest string, options []string) { - mnt := rspec.Mount{ - Destination: dest, - Type: "tmpfs", - Source: "tmpfs", - Options: options, - } - - g.initSpec() - g.spec.Mounts = append(g.spec.Mounts, mnt) -} - -// AddCgroupsMount adds a cgroup mount into g.spec.Mounts. -func (g *Generator) AddCgroupsMount(mountCgroupOption string) error { - switch mountCgroupOption { - case "ro": - case "rw": - case "no": - return nil - default: - return fmt.Errorf("--mount-cgroups should be one of (ro,rw,no)") - } - - mnt := rspec.Mount{ - Destination: "/sys/fs/cgroup", - Type: "cgroup", - Source: "cgroup", - Options: []string{"nosuid", "noexec", "nodev", "relatime", mountCgroupOption}, - } - g.initSpec() - g.spec.Mounts = append(g.spec.Mounts, mnt) - + g.spec.Hooks.Poststart = append(g.spec.Hooks.Poststart, postStartHook) return nil } -// AddBindMount adds a bind mount into g.spec.Mounts. -func (g *Generator) AddBindMount(source, dest string, options []string) { - if len(options) == 0 { - options = []string{"rw"} - } +// AddMount adds a mount into g.spec.Mounts. +func (g *Generator) AddMount(mnt rspec.Mount) { + g.initSpec() - // We have to make sure that there is a bind option set, otherwise it won't - // be an actual bindmount. - foundBindOption := false - for _, opt := range options { - if opt == "bind" || opt == "rbind" { - foundBindOption = true - break + g.spec.Mounts = append(g.spec.Mounts, mnt) +} + +// RemoveMount removes a mount point on the dest directory +func (g *Generator) RemoveMount(dest string) { + g.initSpec() + + for index, mount := range g.spec.Mounts { + if mount.Destination == dest { + g.spec.Mounts = append(g.spec.Mounts[:index], g.spec.Mounts[index+1:]...) + return } } - if !foundBindOption { - options = append(options, "bind") - } +} - mnt := rspec.Mount{ - Destination: dest, - Type: "bind", - Source: source, - Options: options, - } +// Mounts returns the list of mounts +func (g *Generator) Mounts() []rspec.Mount { g.initSpec() - g.spec.Mounts = append(g.spec.Mounts, mnt) + + return g.spec.Mounts +} + +// ClearMounts clear g.spec.Mounts +func (g *Generator) ClearMounts() { + if g.spec == nil { + return + } + g.spec.Mounts = []rspec.Mount{} } // SetupPrivileged sets up the privilege-related fields inside g.spec. @@ -970,7 +1026,7 @@ func (g *Generator) SetupPrivileged(privileged bool) { // ClearProcessCapabilities clear g.spec.Process.Capabilities. func (g *Generator) ClearProcessCapabilities() { - if g.spec == nil { + if g.spec == nil || g.spec.Process == nil || g.spec.Process.Capabilities == nil { return } g.spec.Process.Capabilities.Bounding = []string{} @@ -980,8 +1036,32 @@ func (g *Generator) ClearProcessCapabilities() { g.spec.Process.Capabilities.Ambient = []string{} } -// AddProcessCapability adds a process capability into g.spec.Process.Capabilities. -func (g *Generator) AddProcessCapability(c string) error { +// AddProcessCapabilityAmbient adds a process capability into g.spec.Process.Capabilities.Ambient. +func (g *Generator) AddProcessCapabilityAmbient(c string) error { + cp := strings.ToUpper(c) + if err := validate.CapValid(cp, g.HostSpecific); err != nil { + return err + } + + g.initSpecProcessCapabilities() + + var foundAmbient bool + for _, cap := range g.spec.Process.Capabilities.Ambient { + if strings.ToUpper(cap) == cp { + foundAmbient = true + break + } + } + + if !foundAmbient { + g.spec.Process.Capabilities.Ambient = append(g.spec.Process.Capabilities.Ambient, cp) + } + + return nil +} + +// AddProcessCapabilityBounding adds a process capability into g.spec.Process.Capabilities.Bounding. +func (g *Generator) AddProcessCapabilityBounding(c string) error { cp := strings.ToUpper(c) if err := validate.CapValid(cp, g.HostSpecific); err != nil { return err @@ -1000,6 +1080,18 @@ func (g *Generator) AddProcessCapability(c string) error { g.spec.Process.Capabilities.Bounding = append(g.spec.Process.Capabilities.Bounding, cp) } + return nil +} + +// AddProcessCapabilityEffective adds a process capability into g.spec.Process.Capabilities.Effective. +func (g *Generator) AddProcessCapabilityEffective(c string) error { + cp := strings.ToUpper(c) + if err := validate.CapValid(cp, g.HostSpecific); err != nil { + return err + } + + g.initSpecProcessCapabilities() + var foundEffective bool for _, cap := range g.spec.Process.Capabilities.Effective { if strings.ToUpper(cap) == cp { @@ -1011,6 +1103,18 @@ func (g *Generator) AddProcessCapability(c string) error { g.spec.Process.Capabilities.Effective = append(g.spec.Process.Capabilities.Effective, cp) } + return nil +} + +// AddProcessCapabilityInheritable adds a process capability into g.spec.Process.Capabilities.Inheritable. +func (g *Generator) AddProcessCapabilityInheritable(c string) error { + cp := strings.ToUpper(c) + if err := validate.CapValid(cp, g.HostSpecific); err != nil { + return err + } + + g.initSpecProcessCapabilities() + var foundInheritable bool for _, cap := range g.spec.Process.Capabilities.Inheritable { if strings.ToUpper(cap) == cp { @@ -1022,6 +1126,18 @@ func (g *Generator) AddProcessCapability(c string) error { g.spec.Process.Capabilities.Inheritable = append(g.spec.Process.Capabilities.Inheritable, cp) } + return nil +} + +// AddProcessCapabilityPermitted adds a process capability into g.spec.Process.Capabilities.Permitted. +func (g *Generator) AddProcessCapabilityPermitted(c string) error { + cp := strings.ToUpper(c) + if err := validate.CapValid(cp, g.HostSpecific); err != nil { + return err + } + + g.initSpecProcessCapabilities() + var foundPermitted bool for _, cap := range g.spec.Process.Capabilities.Permitted { if strings.ToUpper(cap) == cp { @@ -1033,66 +1149,87 @@ func (g *Generator) AddProcessCapability(c string) error { g.spec.Process.Capabilities.Permitted = append(g.spec.Process.Capabilities.Permitted, cp) } - var foundAmbient bool - for _, cap := range g.spec.Process.Capabilities.Ambient { - if strings.ToUpper(cap) == cp { - foundAmbient = true - break - } - } - if !foundAmbient { - g.spec.Process.Capabilities.Ambient = append(g.spec.Process.Capabilities.Ambient, cp) - } - return nil } -// DropProcessCapability drops a process capability from g.spec.Process.Capabilities. -func (g *Generator) DropProcessCapability(c string) error { +// DropProcessCapabilityAmbient drops a process capability from g.spec.Process.Capabilities.Ambient. +func (g *Generator) DropProcessCapabilityAmbient(c string) error { + if g.spec == nil || g.spec.Process == nil || g.spec.Process.Capabilities == nil { + return nil + } + cp := strings.ToUpper(c) - if err := validate.CapValid(cp, g.HostSpecific); err != nil { - return err - } - - g.initSpecProcessCapabilities() - - // we don't care about order...and this is way faster... - removeFunc := func(s []string, i int) []string { - s[i] = s[len(s)-1] - return s[:len(s)-1] - } - - for i, cap := range g.spec.Process.Capabilities.Bounding { - if strings.ToUpper(cap) == cp { - g.spec.Process.Capabilities.Bounding = removeFunc(g.spec.Process.Capabilities.Bounding, i) - } - } - - for i, cap := range g.spec.Process.Capabilities.Effective { - if strings.ToUpper(cap) == cp { - g.spec.Process.Capabilities.Effective = removeFunc(g.spec.Process.Capabilities.Effective, i) - } - } - - for i, cap := range g.spec.Process.Capabilities.Inheritable { - if strings.ToUpper(cap) == cp { - g.spec.Process.Capabilities.Inheritable = removeFunc(g.spec.Process.Capabilities.Inheritable, i) - } - } - - for i, cap := range g.spec.Process.Capabilities.Permitted { - if strings.ToUpper(cap) == cp { - g.spec.Process.Capabilities.Permitted = removeFunc(g.spec.Process.Capabilities.Permitted, i) - } - } - for i, cap := range g.spec.Process.Capabilities.Ambient { if strings.ToUpper(cap) == cp { g.spec.Process.Capabilities.Ambient = removeFunc(g.spec.Process.Capabilities.Ambient, i) } } - return nil + return validate.CapValid(cp, false) +} + +// DropProcessCapabilityBounding drops a process capability from g.spec.Process.Capabilities.Bounding. +func (g *Generator) DropProcessCapabilityBounding(c string) error { + if g.spec == nil || g.spec.Process == nil || g.spec.Process.Capabilities == nil { + return nil + } + + cp := strings.ToUpper(c) + for i, cap := range g.spec.Process.Capabilities.Bounding { + if strings.ToUpper(cap) == cp { + g.spec.Process.Capabilities.Bounding = removeFunc(g.spec.Process.Capabilities.Bounding, i) + } + } + + return validate.CapValid(cp, false) +} + +// DropProcessCapabilityEffective drops a process capability from g.spec.Process.Capabilities.Effective. +func (g *Generator) DropProcessCapabilityEffective(c string) error { + if g.spec == nil || g.spec.Process == nil || g.spec.Process.Capabilities == nil { + return nil + } + + cp := strings.ToUpper(c) + for i, cap := range g.spec.Process.Capabilities.Effective { + if strings.ToUpper(cap) == cp { + g.spec.Process.Capabilities.Effective = removeFunc(g.spec.Process.Capabilities.Effective, i) + } + } + + return validate.CapValid(cp, false) +} + +// DropProcessCapabilityInheritable drops a process capability from g.spec.Process.Capabilities.Inheritable. +func (g *Generator) DropProcessCapabilityInheritable(c string) error { + if g.spec == nil || g.spec.Process == nil || g.spec.Process.Capabilities == nil { + return nil + } + + cp := strings.ToUpper(c) + for i, cap := range g.spec.Process.Capabilities.Inheritable { + if strings.ToUpper(cap) == cp { + g.spec.Process.Capabilities.Inheritable = removeFunc(g.spec.Process.Capabilities.Inheritable, i) + } + } + + return validate.CapValid(cp, false) +} + +// DropProcessCapabilityPermitted drops a process capability from g.spec.Process.Capabilities.Permitted. +func (g *Generator) DropProcessCapabilityPermitted(c string) error { + if g.spec == nil || g.spec.Process == nil || g.spec.Process.Capabilities == nil { + return nil + } + + cp := strings.ToUpper(c) + for i, cap := range g.spec.Process.Capabilities.Permitted { + if strings.ToUpper(cap) == cp { + g.spec.Process.Capabilities.Ambient = removeFunc(g.spec.Process.Capabilities.Ambient, i) + } + } + + return validate.CapValid(cp, false) } func mapStrToNamespace(ns string, path string) (rspec.LinuxNamespace, error) { @@ -1180,18 +1317,17 @@ func (g *Generator) AddDevice(device rspec.LinuxDevice) { } // RemoveDevice remove a device from g.spec.Linux.Devices -func (g *Generator) RemoveDevice(path string) error { +func (g *Generator) RemoveDevice(path string) { if g.spec == nil || g.spec.Linux == nil || g.spec.Linux.Devices == nil { - return nil + return } for i, device := range g.spec.Linux.Devices { if device.Path == path { g.spec.Linux.Devices = append(g.spec.Linux.Devices[:i], g.spec.Linux.Devices[i+1:]...) - return nil + return } } - return nil } // ClearLinuxDevices clears g.spec.Linux.Devices @@ -1203,6 +1339,39 @@ func (g *Generator) ClearLinuxDevices() { g.spec.Linux.Devices = []rspec.LinuxDevice{} } +// AddLinuxResourcesDevice - add a device into g.spec.Linux.Resources.Devices +func (g *Generator) AddLinuxResourcesDevice(allow bool, devType string, major, minor *int64, access string) { + g.initSpecLinuxResources() + + device := rspec.LinuxDeviceCgroup{ + Allow: allow, + Type: devType, + Access: access, + Major: major, + Minor: minor, + } + g.spec.Linux.Resources.Devices = append(g.spec.Linux.Resources.Devices, device) +} + +// RemoveLinuxResourcesDevice - remove a device from g.spec.Linux.Resources.Devices +func (g *Generator) RemoveLinuxResourcesDevice(allow bool, devType string, major, minor *int64, access string) { + if g.spec == nil || g.spec.Linux == nil || g.spec.Linux.Resources == nil { + return + } + for i, device := range g.spec.Linux.Resources.Devices { + if device.Allow == allow && + (devType == device.Type || (devType != "" && device.Type != "" && devType == device.Type)) && + (access == device.Access || (access != "" && device.Access != "" && access == device.Access)) && + (major == device.Major || (major != nil && device.Major != nil && *major == *device.Major)) && + (minor == device.Minor || (minor != nil && device.Minor != nil && *minor == *device.Minor)) { + + g.spec.Linux.Resources.Devices = append(g.spec.Linux.Resources.Devices[:i], g.spec.Linux.Resources.Devices[i+1:]...) + return + } + } + return +} + // strPtr returns the pointer pointing to the string s. func strPtr(s string) *string { return &s } @@ -1254,3 +1423,122 @@ func (g *Generator) AddLinuxReadonlyPaths(path string) { g.initSpecLinux() g.spec.Linux.ReadonlyPaths = append(g.spec.Linux.ReadonlyPaths, path) } + +func addOrReplaceBlockIOThrottleDevice(tmpList []rspec.LinuxThrottleDevice, major int64, minor int64, rate uint64) []rspec.LinuxThrottleDevice { + throttleDevices := tmpList + for i, throttleDevice := range throttleDevices { + if throttleDevice.Major == major && throttleDevice.Minor == minor { + throttleDevices[i].Rate = rate + return throttleDevices + } + } + throttleDevice := new(rspec.LinuxThrottleDevice) + throttleDevice.Major = major + throttleDevice.Minor = minor + throttleDevice.Rate = rate + throttleDevices = append(throttleDevices, *throttleDevice) + + return throttleDevices +} + +func dropBlockIOThrottleDevice(tmpList []rspec.LinuxThrottleDevice, major int64, minor int64) []rspec.LinuxThrottleDevice { + throttleDevices := tmpList + for i, throttleDevice := range throttleDevices { + if throttleDevice.Major == major && throttleDevice.Minor == minor { + throttleDevices = append(throttleDevices[:i], throttleDevices[i+1:]...) + return throttleDevices + } + } + + return throttleDevices +} + +// AddSolarisAnet adds network into g.spec.Solaris.Anet +func (g *Generator) AddSolarisAnet(anet rspec.SolarisAnet) { + g.initSpecSolaris() + g.spec.Solaris.Anet = append(g.spec.Solaris.Anet, anet) +} + +// SetSolarisCappedCPUNcpus sets g.spec.Solaris.CappedCPU.Ncpus +func (g *Generator) SetSolarisCappedCPUNcpus(ncpus string) { + g.initSpecSolarisCappedCPU() + g.spec.Solaris.CappedCPU.Ncpus = ncpus +} + +// SetSolarisCappedMemoryPhysical sets g.spec.Solaris.CappedMemory.Physical +func (g *Generator) SetSolarisCappedMemoryPhysical(physical string) { + g.initSpecSolarisCappedMemory() + g.spec.Solaris.CappedMemory.Physical = physical +} + +// SetSolarisCappedMemorySwap sets g.spec.Solaris.CappedMemory.Swap +func (g *Generator) SetSolarisCappedMemorySwap(swap string) { + g.initSpecSolarisCappedMemory() + g.spec.Solaris.CappedMemory.Swap = swap +} + +// SetSolarisLimitPriv sets g.spec.Solaris.LimitPriv +func (g *Generator) SetSolarisLimitPriv(limitPriv string) { + g.initSpecSolaris() + g.spec.Solaris.LimitPriv = limitPriv +} + +// SetSolarisMaxShmMemory sets g.spec.Solaris.MaxShmMemory +func (g *Generator) SetSolarisMaxShmMemory(memory string) { + g.initSpecSolaris() + g.spec.Solaris.MaxShmMemory = memory +} + +// SetSolarisMilestone sets g.spec.Solaris.Milestone +func (g *Generator) SetSolarisMilestone(milestone string) { + g.initSpecSolaris() + g.spec.Solaris.Milestone = milestone +} + +// SetWindowsHypervUntilityVMPath sets g.spec.Windows.HyperV.UtilityVMPath. +func (g *Generator) SetWindowsHypervUntilityVMPath(path string) { + g.initSpecWindowsHyperV() + g.spec.Windows.HyperV.UtilityVMPath = path +} + +// SetWinodwsIgnoreFlushesDuringBoot sets g.spec.Winodws.IgnoreFlushesDuringBoot. +func (g *Generator) SetWinodwsIgnoreFlushesDuringBoot(ignore bool) { + g.initSpecWindows() + g.spec.Windows.IgnoreFlushesDuringBoot = ignore +} + +// AddWindowsLayerFolders adds layer folders into g.spec.Windows.LayerFolders. +func (g *Generator) AddWindowsLayerFolders(folder string) { + g.initSpecWindows() + g.spec.Windows.LayerFolders = append(g.spec.Windows.LayerFolders, folder) +} + +// SetWindowsNetwork sets g.spec.Windows.Network. +func (g *Generator) SetWindowsNetwork(network rspec.WindowsNetwork) { + g.initSpecWindows() + g.spec.Windows.Network = &network +} + +// SetWindowsResourcesCPU sets g.spec.Windows.Resources.CPU. +func (g *Generator) SetWindowsResourcesCPU(cpu rspec.WindowsCPUResources) { + g.initSpecWindowsResources() + g.spec.Windows.Resources.CPU = &cpu +} + +// SetWindowsResourcesMemoryLimit sets g.spec.Windows.Resources.Memory.Limit. +func (g *Generator) SetWindowsResourcesMemoryLimit(limit uint64) { + g.initSpecWindowsResourcesMemory() + g.spec.Windows.Resources.Memory.Limit = &limit +} + +// SetWindowsResourcesStorage sets g.spec.Windows.Resources.Storage. +func (g *Generator) SetWindowsResourcesStorage(storage rspec.WindowsStorageResources) { + g.initSpecWindowsResources() + g.spec.Windows.Resources.Storage = &storage +} + +// SetWinodwsServicing sets g.spec.Winodws.Servicing. +func (g *Generator) SetWinodwsServicing(servicing bool) { + g.initSpecWindows() + g.spec.Windows.Servicing = servicing +} diff --git a/vendor/github.com/opencontainers/runtime-tools/generate/spec.go b/vendor/github.com/opencontainers/runtime-tools/generate/spec.go index 519d2544..d7a6da81 100644 --- a/vendor/github.com/opencontainers/runtime-tools/generate/spec.go +++ b/vendor/github.com/opencontainers/runtime-tools/generate/spec.go @@ -59,6 +59,13 @@ func (g *Generator) initSpecLinux() { } } +func (g *Generator) initSpecLinuxIntelRdt() { + g.initSpecLinux() + if g.spec.Linux.IntelRdt == nil { + g.spec.Linux.IntelRdt = &rspec.LinuxIntelRdt{} + } +} + func (g *Generator) initSpecLinuxSysctl() { g.initSpecLinux() if g.spec.Linux.Sysctl == nil { @@ -80,6 +87,13 @@ func (g *Generator) initSpecLinuxResources() { } } +func (g *Generator) initSpecLinuxResourcesBlockIO() { + g.initSpecLinuxResources() + if g.spec.Linux.Resources.BlockIO == nil { + g.spec.Linux.Resources.BlockIO = &rspec.LinuxBlockIO{} + } +} + func (g *Generator) initSpecLinuxResourcesCPU() { g.initSpecLinuxResources() if g.spec.Linux.Resources.CPU == nil { @@ -107,3 +121,52 @@ func (g *Generator) initSpecLinuxResourcesPids() { g.spec.Linux.Resources.Pids = &rspec.LinuxPids{} } } + +func (g *Generator) initSpecSolaris() { + g.initSpec() + if g.spec.Solaris == nil { + g.spec.Solaris = &rspec.Solaris{} + } +} + +func (g *Generator) initSpecSolarisCappedCPU() { + g.initSpecSolaris() + if g.spec.Solaris.CappedCPU == nil { + g.spec.Solaris.CappedCPU = &rspec.SolarisCappedCPU{} + } +} + +func (g *Generator) initSpecSolarisCappedMemory() { + g.initSpecSolaris() + if g.spec.Solaris.CappedMemory == nil { + g.spec.Solaris.CappedMemory = &rspec.SolarisCappedMemory{} + } +} + +func (g *Generator) initSpecWindows() { + g.initSpec() + if g.spec.Windows == nil { + g.spec.Windows = &rspec.Windows{} + } +} + +func (g *Generator) initSpecWindowsHyperV() { + g.initSpecWindows() + if g.spec.Windows.HyperV == nil { + g.spec.Windows.HyperV = &rspec.WindowsHyperV{} + } +} + +func (g *Generator) initSpecWindowsResources() { + g.initSpecWindows() + if g.spec.Windows.Resources == nil { + g.spec.Windows.Resources = &rspec.WindowsResources{} + } +} + +func (g *Generator) initSpecWindowsResourcesMemory() { + g.initSpecWindowsResources() + if g.spec.Windows.Resources.Memory == nil { + g.spec.Windows.Resources.Memory = &rspec.WindowsMemoryResources{} + } +} diff --git a/vendor/github.com/opencontainers/runtime-tools/specerror/bundle.go b/vendor/github.com/opencontainers/runtime-tools/specerror/bundle.go new file mode 100644 index 00000000..dbe32af3 --- /dev/null +++ b/vendor/github.com/opencontainers/runtime-tools/specerror/bundle.go @@ -0,0 +1,29 @@ +package specerror + +import ( + "fmt" + + rfc2119 "github.com/opencontainers/runtime-tools/error" +) + +// define error codes +const ( + // ConfigInRootBundleDir represents "This REQUIRED file MUST reside in the root of the bundle directory" + ConfigInRootBundleDir Code = 0xa001 + iota + // ConfigConstName represents "This REQUIRED file MUST be named `config.json`." + ConfigConstName + // ArtifactsInSingleDir represents "When supplied, while these artifacts MUST all be present in a single directory on the local filesystem, that directory itself is not part of the bundle." + ArtifactsInSingleDir +) + +var ( + containerFormatRef = func(version string) (reference string, err error) { + return fmt.Sprintf(referenceTemplate, version, "bundle.md#container-format"), nil + } +) + +func init() { + register(ConfigInRootBundleDir, rfc2119.Must, containerFormatRef) + register(ConfigConstName, rfc2119.Must, containerFormatRef) + register(ArtifactsInSingleDir, rfc2119.Must, containerFormatRef) +} diff --git a/vendor/github.com/opencontainers/runtime-tools/specerror/config-linux.go b/vendor/github.com/opencontainers/runtime-tools/specerror/config-linux.go new file mode 100644 index 00000000..aa2a284c --- /dev/null +++ b/vendor/github.com/opencontainers/runtime-tools/specerror/config-linux.go @@ -0,0 +1,134 @@ +package specerror + +import ( + "fmt" + + rfc2119 "github.com/opencontainers/runtime-tools/error" +) + +// define error codes +const ( + // DefaultFilesystems represents "The following filesystems SHOULD be made available in each container's filesystem:" + DefaultFilesystems Code = 0xc001 + iota + // NSPathAbs represents "This value MUST be an absolute path in the runtime mount namespace." + NSPathAbs + // NSProcInPath represents "The runtime MUST place the container process in the namespace associated with that `path`." + NSProcInPath + // NSPathMatchTypeError represents "The runtime MUST generate an error if `path` is not associated with a namespace of type `type`." + NSPathMatchTypeError + // NSNewNSWithoutPath represents "If `path` is not specified, the runtime MUST create a new container namespace of type `type`." + NSNewNSWithoutPath + // NSInheritWithoutType represents "If a namespace type is not specified in the `namespaces` array, the container MUST inherit the runtime namespace of that type." + NSInheritWithoutType + // NSErrorOnDup represents "If a `namespaces` field contains duplicated namespaces with same `type`, the runtime MUST generate an error." + NSErrorOnDup + // UserNSMapOwnershipRO represents "The runtime SHOULD NOT modify the ownership of referenced filesystems to realize the mapping." + UserNSMapOwnershipRO + // DevicesAvailable represents "devices (array of objects, OPTIONAL) lists devices that MUST be available in the container." + DevicesAvailable + // DevicesFileNotMatch represents "If a file already exists at `path` that does not match the requested device, the runtime MUST generate an error." + DevicesFileNotMatch + // DevicesMajMinRequired represents "`major, minor` (int64, REQUIRED unless `type` is `p`) - major, minor numbers for the device." + DevicesMajMinRequired + // DevicesErrorOnDup represents "The same `type`, `major` and `minor` SHOULD NOT be used for multiple devices." + DevicesErrorOnDup + // DefaultDevices represents "In addition to any devices configured with this setting, the runtime MUST also supply default devices." + DefaultDevices + // CgroupsPathAbsOrRel represents "The value of `cgroupsPath` MUST be either an absolute path or a relative path." + CgroupsPathAbsOrRel + // CgroupsAbsPathRelToMount represents "In the case of an absolute path (starting with `/`), the runtime MUST take the path to be relative to the cgroups mount point." + CgroupsAbsPathRelToMount + // CgroupsPathAttach represents "If the value is specified, the runtime MUST consistently attach to the same place in the cgroups hierarchy given the same value of `cgroupsPath`." + CgroupsPathAttach + // CgroupsPathError represents "Runtimes MAY consider certain `cgroupsPath` values to be invalid, and MUST generate an error if this is the case." + CgroupsPathError + // DevicesApplyInOrder represents "The runtime MUST apply entries in the listed order." + DevicesApplyInOrder + // BlkIOWeightOrLeafWeightExist represents "You MUST specify at least one of `weight` or `leafWeight` in a given entry, and MAY specify both." + BlkIOWeightOrLeafWeightExist + // IntelRdtPIDWrite represents "If `intelRdt` is set, the runtime MUST write the container process ID to the `/tasks` file in a mounted `resctrl` pseudo-filesystem, using the container ID from `start` and creating the `container-id` directory if necessary." + IntelRdtPIDWrite + // IntelRdtNoMountedResctrlError represents "If no mounted `resctrl` pseudo-filesystem is available in the runtime mount namespace, the runtime MUST generate an error." + IntelRdtNoMountedResctrlError + // NotManipResctrlWithoutIntelRdt represents "If `intelRdt` is not set, the runtime MUST NOT manipulate any `resctrl` pseudo-filesystems." + NotManipResctrlWithoutIntelRdt + // IntelRdtL3CacheSchemaWrite represents "If `l3CacheSchema` is set, runtimes MUST write the value to the `schemata` file in the `` directory discussed in `intelRdt`." + IntelRdtL3CacheSchemaWrite + // IntelRdtL3CacheSchemaNotWrite represents "If `l3CacheSchema` is not set, runtimes MUST NOT write to `schemata` files in any `resctrl` pseudo-filesystems." + IntelRdtL3CacheSchemaNotWrite + // SeccSyscallsNamesRequired represents "`names` MUST contain at least one entry." + SeccSyscallsNamesRequired + // MaskedPathsAbs represents "maskedPaths (array of strings, OPTIONAL) will mask over the provided paths inside the container so that they cannot be read. The values MUST be absolute paths in the container namespace." + MaskedPathsAbs + // ReadonlyPathsAbs represents "readonlyPaths (array of strings, OPTIONAL) will set the provided paths as readonly inside the container. The values MUST be absolute paths in the container namespace." + ReadonlyPathsAbs +) + +var ( + defaultFilesystemsRef = func(version string) (reference string, err error) { + return fmt.Sprintf(referenceTemplate, version, "config-linux.md#default-filesystems"), nil + } + namespacesRef = func(version string) (reference string, err error) { + return fmt.Sprintf(referenceTemplate, version, "config-linux.md#namespaces"), nil + } + userNamespaceMappingsRef = func(version string) (reference string, err error) { + return fmt.Sprintf(referenceTemplate, version, "config-linux.md#user-namespace-mappings"), nil + } + devicesRef = func(version string) (reference string, err error) { + return fmt.Sprintf(referenceTemplate, version, "config-linux.md#devices"), nil + } + defaultDevicesRef = func(version string) (reference string, err error) { + return fmt.Sprintf(referenceTemplate, version, "config-linux.md#default-devices"), nil + } + cgroupsPathRef = func(version string) (reference string, err error) { + return fmt.Sprintf(referenceTemplate, version, "config-linux.md#cgroups-path"), nil + } + deviceWhitelistRef = func(version string) (reference string, err error) { + return fmt.Sprintf(referenceTemplate, version, "config-linux.md#device-whitelist"), nil + } + blockIoRef = func(version string) (reference string, err error) { + return fmt.Sprintf(referenceTemplate, version, "config-linux.md#block-io"), nil + } + intelrdtRef = func(version string) (reference string, err error) { + return fmt.Sprintf(referenceTemplate, version, "config-linux.md#intelrdt"), nil + } + seccompRef = func(version string) (reference string, err error) { + return fmt.Sprintf(referenceTemplate, version, "config-linux.md#seccomp"), nil + } + maskedPathsRef = func(version string) (reference string, err error) { + return fmt.Sprintf(referenceTemplate, version, "config-linux.md#masked-paths"), nil + } + readonlyPathsRef = func(version string) (reference string, err error) { + return fmt.Sprintf(referenceTemplate, version, "config-linux.md#readonly-paths"), nil + } +) + +func init() { + register(DefaultFilesystems, rfc2119.Should, defaultFilesystemsRef) + register(NSPathAbs, rfc2119.Must, namespacesRef) + register(NSProcInPath, rfc2119.Must, namespacesRef) + register(NSPathMatchTypeError, rfc2119.Must, namespacesRef) + register(NSNewNSWithoutPath, rfc2119.Must, namespacesRef) + register(NSInheritWithoutType, rfc2119.Must, namespacesRef) + register(NSErrorOnDup, rfc2119.Must, namespacesRef) + register(UserNSMapOwnershipRO, rfc2119.Should, userNamespaceMappingsRef) + register(DevicesAvailable, rfc2119.Must, devicesRef) + register(DevicesFileNotMatch, rfc2119.Must, devicesRef) + register(DevicesMajMinRequired, rfc2119.Required, devicesRef) + register(DevicesErrorOnDup, rfc2119.Should, devicesRef) + register(DefaultDevices, rfc2119.Must, defaultDevicesRef) + register(CgroupsPathAbsOrRel, rfc2119.Must, cgroupsPathRef) + register(CgroupsAbsPathRelToMount, rfc2119.Must, cgroupsPathRef) + register(CgroupsPathAttach, rfc2119.Must, cgroupsPathRef) + register(CgroupsPathError, rfc2119.Must, cgroupsPathRef) + register(DevicesApplyInOrder, rfc2119.Must, deviceWhitelistRef) + register(BlkIOWeightOrLeafWeightExist, rfc2119.Must, blockIoRef) + register(IntelRdtPIDWrite, rfc2119.Must, intelrdtRef) + register(IntelRdtNoMountedResctrlError, rfc2119.Must, intelrdtRef) + register(NotManipResctrlWithoutIntelRdt, rfc2119.Must, intelrdtRef) + register(IntelRdtL3CacheSchemaWrite, rfc2119.Must, intelrdtRef) + register(IntelRdtL3CacheSchemaNotWrite, rfc2119.Must, intelrdtRef) + register(SeccSyscallsNamesRequired, rfc2119.Must, seccompRef) + register(MaskedPathsAbs, rfc2119.Must, maskedPathsRef) + register(ReadonlyPathsAbs, rfc2119.Must, readonlyPathsRef) +} diff --git a/vendor/github.com/opencontainers/runtime-tools/specerror/config-windows.go b/vendor/github.com/opencontainers/runtime-tools/specerror/config-windows.go new file mode 100644 index 00000000..c8134867 --- /dev/null +++ b/vendor/github.com/opencontainers/runtime-tools/specerror/config-windows.go @@ -0,0 +1,32 @@ +package specerror + +import ( + "fmt" + + rfc2119 "github.com/opencontainers/runtime-tools/error" +) + +// define error codes +const ( + // WindowsLayerFoldersRequired represents "`layerFolders` MUST contain at least one entry." + WindowsLayerFoldersRequired Code = 0xd001 + iota + // WindowsHyperVPresent represents "If present, the container MUST be run with Hyper-V isolation." + WindowsHyperVPresent + // WindowsHyperVOmit represents "If omitted, the container MUST be run as a Windows Server container." + WindowsHyperVOmit +) + +var ( + layerfoldersRef = func(version string) (reference string, err error) { + return fmt.Sprintf(referenceTemplate, version, "config-windows.md#layerfolders"), nil + } + hypervRef = func(version string) (reference string, err error) { + return fmt.Sprintf(referenceTemplate, version, "config-windows.md#hyperv"), nil + } +) + +func init() { + register(WindowsLayerFoldersRequired, rfc2119.Must, layerfoldersRef) + register(WindowsHyperVPresent, rfc2119.Must, hypervRef) + register(WindowsHyperVOmit, rfc2119.Must, hypervRef) +} diff --git a/vendor/github.com/opencontainers/runtime-tools/specerror/config.go b/vendor/github.com/opencontainers/runtime-tools/specerror/config.go new file mode 100644 index 00000000..8a73f47f --- /dev/null +++ b/vendor/github.com/opencontainers/runtime-tools/specerror/config.go @@ -0,0 +1,188 @@ +package specerror + +import ( + "fmt" + + rfc2119 "github.com/opencontainers/runtime-tools/error" +) + +// define error codes +const ( + // SpecVersionInSemVer represents "`ociVersion` (string, REQUIRED) MUST be in SemVer v2.0.0 format and specifies the version of the Open Container Initiative Runtime Specification with which the bundle complies." + SpecVersionInSemVer Code = 0xb001 + iota + // RootOnWindowsRequired represents "On Windows, for Windows Server Containers, this field is REQUIRED." + RootOnWindowsRequired + // RootOnHyperVNotSet represents "For Hyper-V Containers, this field MUST NOT be set." + RootOnHyperVNotSet + // RootOnNonHyperVRequired represents "On all other platforms, this field is REQUIRED." + RootOnNonHyperVRequired + // RootPathOnWindowsGUID represents "On Windows, `path` MUST be a volume GUID path." + RootPathOnWindowsGUID + // RootPathOnPosixConvention represents "The value SHOULD be the conventional `rootfs`." + RootPathOnPosixConvention + // RootPathExist represents "A directory MUST exist at the path declared by the field." + RootPathExist + // RootReadonlyImplement represents "`readonly` (bool, OPTIONAL) If true then the root filesystem MUST be read-only inside the container, defaults to false." + RootReadonlyImplement + // RootReadonlyOnWindowsFalse represents "* On Windows, this field MUST be omitted or false." + RootReadonlyOnWindowsFalse + // MountsInOrder represents "The runtime MUST mount entries in the listed order." + MountsInOrder + // MountsDestAbs represents "Destination of mount point: path inside container. This value MUST be an absolute path." + MountsDestAbs + // MountsDestOnWindowsNotNested represents "Windows: one mount destination MUST NOT be nested within another mount (e.g., c:\\foo and c:\\foo\\bar)." + MountsDestOnWindowsNotNested + // MountsOptionsOnWindowsROSupport represents "Windows: runtimes MUST support `ro`, mounting the filesystem read-only when `ro` is given." + MountsOptionsOnWindowsROSupport + // ProcRequiredAtStart represents "This property is REQUIRED when `start` is called." + ProcRequiredAtStart + // ProcConsoleSizeIgnore represents "Runtimes MUST ignore `consoleSize` if `terminal` is `false` or unset." + ProcConsoleSizeIgnore + // ProcCwdAbs represents "cwd (string, REQUIRED) is the working directory that will be set for the executable. This value MUST be an absolute path." + ProcCwdAbs + // ProcArgsOneEntryRequired represents "This specification extends the IEEE standard in that at least one entry is REQUIRED, and that entry is used with the same semantics as `execvp`'s *file*." + ProcArgsOneEntryRequired + // PosixProcRlimitsTypeGenError represents "The runtime MUST generate an error for any values which cannot be mapped to a relevant kernel interface." + PosixProcRlimitsTypeGenError + // PosixProcRlimitsTypeGet represents "For each entry in `rlimits`, a `getrlimit(3)` on `type` MUST succeed." + PosixProcRlimitsTypeGet + // PosixProcRlimitsTypeValueError represents "valid values are defined in the ... man page" + PosixProcRlimitsTypeValueError + // PosixProcRlimitsSoftMatchCur represents "`rlim.rlim_cur` MUST match the configured value." + PosixProcRlimitsSoftMatchCur + // PosixProcRlimitsHardMatchMax represents "`rlim.rlim_max` MUST match the configured value." + PosixProcRlimitsHardMatchMax + // PosixProcRlimitsErrorOnDup represents "If `rlimits` contains duplicated entries with same `type`, the runtime MUST generate an error." + PosixProcRlimitsErrorOnDup + // LinuxProcCapError represents "Any value which cannot be mapped to a relevant kernel interface MUST cause an error." + LinuxProcCapError + // LinuxProcOomScoreAdjSet represents "If `oomScoreAdj` is set, the runtime MUST set `oom_score_adj` to the given value." + LinuxProcOomScoreAdjSet + // LinuxProcOomScoreAdjNotSet represents "If `oomScoreAdj` is not set, the runtime MUST NOT change the value of `oom_score_adj`." + LinuxProcOomScoreAdjNotSet + // PlatformSpecConfOnWindowsSet represents "This MUST be set if the target platform of this spec is `windows`." + PlatformSpecConfOnWindowsSet + // PosixHooksPathAbs represents "This specification extends the IEEE standard in that `path` MUST be absolute." + PosixHooksPathAbs + // PosixHooksTimeoutPositive represents "If set, `timeout` MUST be greater than zero." + PosixHooksTimeoutPositive + // PosixHooksCalledInOrder represents "Hooks MUST be called in the listed order." + PosixHooksCalledInOrder + // PosixHooksStateToStdin represents "The state of the container MUST be passed to hooks over stdin so that they may do work appropriate to the current state of the container." + PosixHooksStateToStdin + // PrestartTiming represents "The pre-start hooks MUST be called after the `start` operation is called but before the user-specified program command is executed." + PrestartTiming + // PoststartTiming represents "The post-start hooks MUST be called after the user-specified process is executed but before the `start` operation returns." + PoststartTiming + // PoststopTiming represents "The post-stop hooks MUST be called after the container is deleted but before the `delete` operation returns." + PoststopTiming + // AnnotationsKeyValueMap represents "Annotations MUST be a key-value map." + AnnotationsKeyValueMap + // AnnotationsKeyString represents "Keys MUST be strings." + AnnotationsKeyString + // AnnotationsKeyRequired represents "Keys MUST NOT be an empty string." + AnnotationsKeyRequired + // AnnotationsKeyReversedDomain represents "Keys SHOULD be named using a reverse domain notation - e.g. `com.example.myKey`." + AnnotationsKeyReversedDomain + // AnnotationsKeyReservedNS represents "Keys using the `org.opencontainers` namespace are reserved and MUST NOT be used by subsequent specifications." + AnnotationsKeyReservedNS + // AnnotationsKeyIgnoreUnknown represents "Implementations that are reading/processing this configuration file MUST NOT generate an error if they encounter an unknown annotation key." + AnnotationsKeyIgnoreUnknown + // AnnotationsValueString represents "Values MUST be strings." + AnnotationsValueString + // ExtensibilityIgnoreUnknownProp represents "Runtimes that are reading or processing this configuration file MUST NOT generate an error if they encounter an unknown property." + ExtensibilityIgnoreUnknownProp + // ValidValues represents "Runtimes that are reading or processing this configuration file MUST generate an error when invalid or unsupported values are encountered." + ValidValues +) + +var ( + specificationVersionRef = func(version string) (reference string, err error) { + return fmt.Sprintf(referenceTemplate, version, "config.md#specification-version"), nil + } + rootRef = func(version string) (reference string, err error) { + return fmt.Sprintf(referenceTemplate, version, "config.md#root"), nil + } + mountsRef = func(version string) (reference string, err error) { + return fmt.Sprintf(referenceTemplate, version, "config.md#mounts"), nil + } + processRef = func(version string) (reference string, err error) { + return fmt.Sprintf(referenceTemplate, version, "config.md#process"), nil + } + posixProcessRef = func(version string) (reference string, err error) { + return fmt.Sprintf(referenceTemplate, version, "config.md#posix-process"), nil + } + linuxProcessRef = func(version string) (reference string, err error) { + return fmt.Sprintf(referenceTemplate, version, "config.md#linux-process"), nil + } + platformSpecificConfigurationRef = func(version string) (reference string, err error) { + return fmt.Sprintf(referenceTemplate, version, "config.md#platform-specific-configuration"), nil + } + posixPlatformHooksRef = func(version string) (reference string, err error) { + return fmt.Sprintf(referenceTemplate, version, "config.md#posix-platform-hooks"), nil + } + prestartRef = func(version string) (reference string, err error) { + return fmt.Sprintf(referenceTemplate, version, "config.md#prestart"), nil + } + poststartRef = func(version string) (reference string, err error) { + return fmt.Sprintf(referenceTemplate, version, "config.md#poststart"), nil + } + poststopRef = func(version string) (reference string, err error) { + return fmt.Sprintf(referenceTemplate, version, "config.md#poststop"), nil + } + annotationsRef = func(version string) (reference string, err error) { + return fmt.Sprintf(referenceTemplate, version, "config.md#annotations"), nil + } + extensibilityRef = func(version string) (reference string, err error) { + return fmt.Sprintf(referenceTemplate, version, "config.md#extensibility"), nil + } + validValuesRef = func(version string) (reference string, err error) { + return fmt.Sprintf(referenceTemplate, version, "config.md#valid-values"), nil + } +) + +func init() { + register(SpecVersionInSemVer, rfc2119.Must, specificationVersionRef) + register(RootOnWindowsRequired, rfc2119.Required, rootRef) + register(RootOnHyperVNotSet, rfc2119.Must, rootRef) + register(RootOnNonHyperVRequired, rfc2119.Required, rootRef) + register(RootPathOnWindowsGUID, rfc2119.Must, rootRef) + register(RootPathOnPosixConvention, rfc2119.Should, rootRef) + register(RootPathExist, rfc2119.Must, rootRef) + register(RootReadonlyImplement, rfc2119.Must, rootRef) + register(RootReadonlyOnWindowsFalse, rfc2119.Must, rootRef) + register(MountsInOrder, rfc2119.Must, mountsRef) + register(MountsDestAbs, rfc2119.Must, mountsRef) + register(MountsDestOnWindowsNotNested, rfc2119.Must, mountsRef) + register(MountsOptionsOnWindowsROSupport, rfc2119.Must, mountsRef) + register(ProcRequiredAtStart, rfc2119.Required, processRef) + register(ProcConsoleSizeIgnore, rfc2119.Must, processRef) + register(ProcCwdAbs, rfc2119.Must, processRef) + register(ProcArgsOneEntryRequired, rfc2119.Required, processRef) + register(PosixProcRlimitsTypeGenError, rfc2119.Must, posixProcessRef) + register(PosixProcRlimitsTypeGet, rfc2119.Must, posixProcessRef) + register(PosixProcRlimitsTypeValueError, rfc2119.Should, posixProcessRef) + register(PosixProcRlimitsSoftMatchCur, rfc2119.Must, posixProcessRef) + register(PosixProcRlimitsHardMatchMax, rfc2119.Must, posixProcessRef) + register(PosixProcRlimitsErrorOnDup, rfc2119.Must, posixProcessRef) + register(LinuxProcCapError, rfc2119.Must, linuxProcessRef) + register(LinuxProcOomScoreAdjSet, rfc2119.Must, linuxProcessRef) + register(LinuxProcOomScoreAdjNotSet, rfc2119.Must, linuxProcessRef) + register(PlatformSpecConfOnWindowsSet, rfc2119.Must, platformSpecificConfigurationRef) + register(PosixHooksPathAbs, rfc2119.Must, posixPlatformHooksRef) + register(PosixHooksTimeoutPositive, rfc2119.Must, posixPlatformHooksRef) + register(PosixHooksCalledInOrder, rfc2119.Must, posixPlatformHooksRef) + register(PosixHooksStateToStdin, rfc2119.Must, posixPlatformHooksRef) + register(PrestartTiming, rfc2119.Must, prestartRef) + register(PoststartTiming, rfc2119.Must, poststartRef) + register(PoststopTiming, rfc2119.Must, poststopRef) + register(AnnotationsKeyValueMap, rfc2119.Must, annotationsRef) + register(AnnotationsKeyString, rfc2119.Must, annotationsRef) + register(AnnotationsKeyRequired, rfc2119.Must, annotationsRef) + register(AnnotationsKeyReversedDomain, rfc2119.Should, annotationsRef) + register(AnnotationsKeyReservedNS, rfc2119.Must, annotationsRef) + register(AnnotationsKeyIgnoreUnknown, rfc2119.Must, annotationsRef) + register(AnnotationsValueString, rfc2119.Must, annotationsRef) + register(ExtensibilityIgnoreUnknownProp, rfc2119.Must, extensibilityRef) + register(ValidValues, rfc2119.Must, validValuesRef) +} diff --git a/vendor/github.com/opencontainers/runtime-tools/specerror/error.go b/vendor/github.com/opencontainers/runtime-tools/specerror/error.go index c75bb6b1..0300f7df 100644 --- a/vendor/github.com/opencontainers/runtime-tools/specerror/error.go +++ b/vendor/github.com/opencontainers/runtime-tools/specerror/error.go @@ -13,46 +13,13 @@ const referenceTemplate = "https://github.com/opencontainers/runtime-spec/blob/v // Code represents the spec violation, enumerating both // configuration violations and runtime violations. -type Code int +type Code int64 const ( // NonError represents that an input is not an error - NonError Code = iota + NonError Code = 0x1a001 + iota // NonRFCError represents that an error is not a rfc2119 error NonRFCError - - // ConfigFileExistence represents the error code of 'config.json' existence test - ConfigFileExistence - // ArtifactsInSingleDir represents the error code of artifacts place test - ArtifactsInSingleDir - - // SpecVersion represents the error code of specfication version test - SpecVersion - - // RootOnNonHyperV represents the error code of root setting test on non hyper-v containers - RootOnNonHyperV - // RootOnHyperV represents the error code of root setting test on hyper-v containers - RootOnHyperV - // PathFormatOnWindows represents the error code of the path format test on Window - PathFormatOnWindows - // PathName represents the error code of the path name test - PathName - // PathExistence represents the error code of the path existence test - PathExistence - // ReadonlyFilesystem represents the error code of readonly test - ReadonlyFilesystem - // ReadonlyOnWindows represents the error code of readonly setting test on Windows - ReadonlyOnWindows - - // DefaultFilesystems represents the error code of default filesystems test - DefaultFilesystems - - // CreateWithID represents the error code of 'create' lifecyle test with 'id' provided - CreateWithID - // CreateWithUniqueID represents the error code of 'create' lifecyle test with unique 'id' provided - CreateWithUniqueID - // CreateNewContainer represents the error code 'create' lifecyle test that creates new container - CreateNewContainer ) type errorTemplate struct { @@ -69,52 +36,24 @@ type Error struct { Code Code } -var ( - containerFormatRef = func(version string) (reference string, err error) { - return fmt.Sprintf(referenceTemplate, version, "bundle.md#container-format"), nil - } - specVersionRef = func(version string) (reference string, err error) { - return fmt.Sprintf(referenceTemplate, version, "config.md#specification-version"), nil - } - rootRef = func(version string) (reference string, err error) { - return fmt.Sprintf(referenceTemplate, version, "config.md#root"), nil - } - defaultFSRef = func(version string) (reference string, err error) { - return fmt.Sprintf(referenceTemplate, version, "config-linux.md#default-filesystems"), nil - } - runtimeCreateRef = func(version string) (reference string, err error) { - return fmt.Sprintf(referenceTemplate, version, "runtime.md#create"), nil - } -) +// LevelErrors represents Errors filtered into fatal and warnings. +type LevelErrors struct { + // Warnings holds Errors that were below a compliance-level threshold. + Warnings []*Error -var ociErrors = map[Code]errorTemplate{ - // Bundle.md - // Container Format - ConfigFileExistence: {Level: rfc2119.Must, Reference: containerFormatRef}, - ArtifactsInSingleDir: {Level: rfc2119.Must, Reference: containerFormatRef}, + // Error holds errors that were at or above a compliance-level + // threshold, as well as errors that are not Errors. + Error *multierror.Error +} - // Config.md - // Specification Version - SpecVersion: {Level: rfc2119.Must, Reference: specVersionRef}, - // Root - RootOnNonHyperV: {Level: rfc2119.Required, Reference: rootRef}, - RootOnHyperV: {Level: rfc2119.Must, Reference: rootRef}, - // TODO: add tests for 'PathFormatOnWindows' - PathFormatOnWindows: {Level: rfc2119.Must, Reference: rootRef}, - PathName: {Level: rfc2119.Should, Reference: rootRef}, - PathExistence: {Level: rfc2119.Must, Reference: rootRef}, - ReadonlyFilesystem: {Level: rfc2119.Must, Reference: rootRef}, - ReadonlyOnWindows: {Level: rfc2119.Must, Reference: rootRef}, +var ociErrors = map[Code]errorTemplate{} - // Config-Linux.md - // Default Filesystems - DefaultFilesystems: {Level: rfc2119.Should, Reference: defaultFSRef}, +func register(code Code, level rfc2119.Level, ref func(versiong string) (string, error)) { + if _, ok := ociErrors[code]; ok { + panic(fmt.Sprintf("should not regist a same code twice: %v", code)) + } - // Runtime.md - // Create - CreateWithID: {Level: rfc2119.Must, Reference: runtimeCreateRef}, - CreateWithUniqueID: {Level: rfc2119.Must, Reference: runtimeCreateRef}, - CreateNewContainer: {Level: rfc2119.Must, Reference: runtimeCreateRef}, + ociErrors[code] = errorTemplate{Level: level, Reference: ref} } // Error returns the error message with specification reference. @@ -168,3 +107,23 @@ func FindError(err error, code Code) Code { } return NonRFCError } + +// SplitLevel removes RFC 2119 errors with a level less than 'level' +// from the source error. If the source error is not a multierror, it +// is returned unchanged. +func SplitLevel(errIn error, level rfc2119.Level) (levelErrors LevelErrors, errOut error) { + merr, ok := errIn.(*multierror.Error) + if !ok { + return levelErrors, errIn + } + for _, err := range merr.Errors { + e, ok := err.(*Error) + if ok && e.Err.Level < level { + fmt.Println(e) + levelErrors.Warnings = append(levelErrors.Warnings, e) + continue + } + levelErrors.Error = multierror.Append(levelErrors.Error, err) + } + return levelErrors, nil +} diff --git a/vendor/github.com/opencontainers/runtime-tools/specerror/runtime-linux.go b/vendor/github.com/opencontainers/runtime-tools/specerror/runtime-linux.go new file mode 100644 index 00000000..8eb259ba --- /dev/null +++ b/vendor/github.com/opencontainers/runtime-tools/specerror/runtime-linux.go @@ -0,0 +1,23 @@ +package specerror + +import ( + "fmt" + + rfc2119 "github.com/opencontainers/runtime-tools/error" +) + +// define error codes +const ( + // DefaultRuntimeLinuxSymlinks represents "While creating the container (step 2 in the lifecycle), runtimes MUST create default symlinks if the source file exists after processing `mounts`." + DefaultRuntimeLinuxSymlinks Code = 0xf001 + iota +) + +var ( + devSymbolicLinksRef = func(version string) (reference string, err error) { + return fmt.Sprintf(referenceTemplate, version, "runtime-linux.md#dev-symbolic-links"), nil + } +) + +func init() { + register(DefaultRuntimeLinuxSymlinks, rfc2119.Must, devSymbolicLinksRef) +} diff --git a/vendor/github.com/opencontainers/runtime-tools/specerror/runtime.go b/vendor/github.com/opencontainers/runtime-tools/specerror/runtime.go new file mode 100644 index 00000000..0144f669 --- /dev/null +++ b/vendor/github.com/opencontainers/runtime-tools/specerror/runtime.go @@ -0,0 +1,179 @@ +package specerror + +import ( + "fmt" + + rfc2119 "github.com/opencontainers/runtime-tools/error" +) + +// define error codes +const ( + // EntityOperSameContainer represents "The entity using a runtime to create a container MUST be able to use the operations defined in this specification against that same container." + EntityOperSameContainer Code = 0xe001 + iota + // StateIDUniq represents "`id` (string, REQUIRED) is the container's ID. This MUST be unique across all containers on this host." + StateIDUniq + // StateNewStatus represents "Additional values MAY be defined by the runtime, however, they MUST be used to represent new runtime states not defined above." + StateNewStatus + // DefaultStateJSONPattern represents "When serialized in JSON, the format MUST adhere to the default pattern." + DefaultStateJSONPattern + // EnvCreateImplement represents "The container's runtime environment MUST be created according to the configuration in `config.json`." + EnvCreateImplement + // EnvCreateError represents "If the runtime is unable to create the environment specified in the `config.json`, it MUST generate an error." + EnvCreateError + // ProcNotRunAtResRequest represents "While the resources requested in the `config.json` MUST be created, the user-specified program (from `process`) MUST NOT be run at this time." + ProcNotRunAtResRequest + // ConfigUpdatesWithoutAffect represents "Any updates to `config.json` after this step MUST NOT affect the container." + ConfigUpdatesWithoutAffect + // PrestartHooksInvoke represents "The prestart hooks MUST be invoked by the runtime." + PrestartHooksInvoke + // PrestartHookFailGenError represents "If any prestart hook fails, the runtime MUST generate an error, stop the container, and continue the lifecycle at step 9." + PrestartHookFailGenError + // ProcImplement represents "The runtime MUST run the user-specified program, as specified by `process`." + ProcImplement + // PoststartHooksInvoke represents "The poststart hooks MUST be invoked by the runtime." + PoststartHooksInvoke + // PoststartHookFailGenWarn represents "If any poststart hook fails, the runtime MUST log a warning, but the remaining hooks and lifecycle continue as if the hook had succeeded." + PoststartHookFailGenWarn + // UndoCreateSteps represents "The container MUST be destroyed by undoing the steps performed during create phase (step 2)." + UndoCreateSteps + // PoststopHooksInvoke represents "The poststop hooks MUST be invoked by the runtime." + PoststopHooksInvoke + // PoststopHookFailGenWarn represents "If any poststop hook fails, the runtime MUST log a warning, but the remaining hooks and lifecycle continue as if the hook had succeeded." + PoststopHookFailGenWarn + // ErrorsLeaveStateUnchange represents "Unless otherwise stated, generating an error MUST leave the state of the environment as if the operation were never attempted - modulo any possible trivial ancillary changes such as logging." + ErrorsLeaveStateUnchange + // WarnsLeaveFlowUnchange represents "Unless otherwise stated, logging a warning does not change the flow of the operation; it MUST continue as if the warning had not been logged." + WarnsLeaveFlowUnchange + // DefaultOperations represents "Unless otherwise stated, runtimes MUST support the default operations." + DefaultOperations + // QueryWithoutIDGenError represents "This operation MUST generate an error if it is not provided the ID of a container." + QueryWithoutIDGenError + // QueryNonExistGenError represents "Attempting to query a container that does not exist MUST generate an error." + QueryNonExistGenError + // QueryStateImplement represents "This operation MUST return the state of a container as specified in the State section." + QueryStateImplement + // CreateWithBundlePathAndID represents "This operation MUST generate an error if it is not provided a path to the bundle and the container ID to associate with the container." + CreateWithBundlePathAndID + // CreateWithUniqueID represents "If the ID provided is not unique across all containers within the scope of the runtime, or is not valid in any other way, the implementation MUST generate an error and a new container MUST NOT be created." + CreateWithUniqueID + // CreateNewContainer represents "This operation MUST create a new container." + CreateNewContainer + // PropsApplyExceptProcOnCreate represents "All of the properties configured in `config.json` except for `process` MUST be applied." + PropsApplyExceptProcOnCreate + // ProcArgsApplyUntilStart represents `process.args` MUST NOT be applied until triggered by the `start` operation." + ProcArgsApplyUntilStart + // PropApplyFailGenError represents "If the runtime cannot apply a property as specified in the configuration, it MUST generate an error." + PropApplyFailGenError + // PropApplyFailNotCreate represents "If the runtime cannot apply a property as specified in the configuration, a new container MUST NOT be created." + PropApplyFailNotCreate + // StartWithoutIDGenError represents "`start` operation MUST generate an error if it is not provided the container ID." + StartWithoutIDGenError + // StartNonCreateHaveNoEffect represents "Attempting to `start` a container that is not `created` MUST have no effect on the container." + StartNonCreateHaveNoEffect + // StartNonCreateGenError represents "Attempting to `start` a container that is not `created` MUST generate an error." + StartNonCreateGenError + // StartProcImplement represents "`start` operation MUST run the user-specified program as specified by `process`." + StartProcImplement + // StartWithProcUnsetGenError represents "`start` operation MUST generate an error if `process` was not set." + StartWithProcUnsetGenError + // KillWithoutIDGenError represents "`kill` operation MUST generate an error if it is not provided the container ID." + KillWithoutIDGenError + // KillNonCreateRunHaveNoEffect represents "Attempting to send a signal to a container that is neither `created` nor `running` MUST have no effect on the container." + KillNonCreateRunHaveNoEffect + // KillNonCreateRunGenError represents "Attempting to send a signal to a container that is neither `created` nor `running` MUST generate an error." + KillNonCreateRunGenError + // KillSignalImplement represents "`kill` operation MUST send the specified signal to the container process." + KillSignalImplement + // DeleteWithoutIDGenError represents "`delete` operation MUST generate an error if it is not provided the container ID." + DeleteWithoutIDGenError + // DeleteNonStopHaveNoEffect represents "Attempting to `delete` a container that is not `stopped` MUST have no effect on the container." + DeleteNonStopHaveNoEffect + // DeleteNonStopGenError represents "Attempting to `delete` a container that is not `stopped` MUST generate an error." + DeleteNonStopGenError + // DeleteResImplement represents "Deleting a container MUST delete the resources that were created during the `create` step." + DeleteResImplement + // DeleteOnlyCreatedRes represents "Note that resources associated with the container, but not created by this container, MUST NOT be deleted." + DeleteOnlyCreatedRes +) + +var ( + scopeOfAContainerRef = func(version string) (reference string, err error) { + return fmt.Sprintf(referenceTemplate, version, "runtime.md#scope-of-a-container"), nil + } + stateRef = func(version string) (reference string, err error) { + return fmt.Sprintf(referenceTemplate, version, "runtime.md#state"), nil + } + lifecycleRef = func(version string) (reference string, err error) { + return fmt.Sprintf(referenceTemplate, version, "runtime.md#lifecycle"), nil + } + errorsRef = func(version string) (reference string, err error) { + return fmt.Sprintf(referenceTemplate, version, "runtime.md#errors"), nil + } + warningsRef = func(version string) (reference string, err error) { + return fmt.Sprintf(referenceTemplate, version, "runtime.md#warnings"), nil + } + operationsRef = func(version string) (reference string, err error) { + return fmt.Sprintf(referenceTemplate, version, "runtime.md#operations"), nil + } + queryStateRef = func(version string) (reference string, err error) { + return fmt.Sprintf(referenceTemplate, version, "runtime.md#query-state"), nil + } + createRef = func(version string) (reference string, err error) { + return fmt.Sprintf(referenceTemplate, version, "runtime.md#create"), nil + } + startRef = func(version string) (reference string, err error) { + return fmt.Sprintf(referenceTemplate, version, "runtime.md#start"), nil + } + killRef = func(version string) (reference string, err error) { + return fmt.Sprintf(referenceTemplate, version, "runtime.md#kill"), nil + } + deleteRef = func(version string) (reference string, err error) { + return fmt.Sprintf(referenceTemplate, version, "runtime.md#delete"), nil + } +) + +func init() { + register(EntityOperSameContainer, rfc2119.Must, scopeOfAContainerRef) + register(StateIDUniq, rfc2119.Must, stateRef) + register(StateNewStatus, rfc2119.Must, stateRef) + register(DefaultStateJSONPattern, rfc2119.Must, stateRef) + register(EnvCreateImplement, rfc2119.Must, lifecycleRef) + register(EnvCreateError, rfc2119.Must, lifecycleRef) + register(ProcNotRunAtResRequest, rfc2119.Must, lifecycleRef) + register(ConfigUpdatesWithoutAffect, rfc2119.Must, lifecycleRef) + register(PrestartHooksInvoke, rfc2119.Must, lifecycleRef) + register(PrestartHookFailGenError, rfc2119.Must, lifecycleRef) + register(ProcImplement, rfc2119.Must, lifecycleRef) + register(PoststartHooksInvoke, rfc2119.Must, lifecycleRef) + register(PoststartHookFailGenWarn, rfc2119.Must, lifecycleRef) + register(UndoCreateSteps, rfc2119.Must, lifecycleRef) + register(PoststopHooksInvoke, rfc2119.Must, lifecycleRef) + register(PoststopHookFailGenWarn, rfc2119.Must, lifecycleRef) + register(ErrorsLeaveStateUnchange, rfc2119.Must, errorsRef) + register(WarnsLeaveFlowUnchange, rfc2119.Must, warningsRef) + register(DefaultOperations, rfc2119.Must, operationsRef) + register(QueryWithoutIDGenError, rfc2119.Must, queryStateRef) + register(QueryNonExistGenError, rfc2119.Must, queryStateRef) + register(QueryStateImplement, rfc2119.Must, queryStateRef) + register(CreateWithBundlePathAndID, rfc2119.Must, createRef) + register(CreateWithUniqueID, rfc2119.Must, createRef) + register(CreateNewContainer, rfc2119.Must, createRef) + register(PropsApplyExceptProcOnCreate, rfc2119.Must, createRef) + register(ProcArgsApplyUntilStart, rfc2119.Must, createRef) + register(PropApplyFailGenError, rfc2119.Must, createRef) + register(PropApplyFailNotCreate, rfc2119.Must, createRef) + register(StartWithoutIDGenError, rfc2119.Must, startRef) + register(StartNonCreateHaveNoEffect, rfc2119.Must, startRef) + register(StartNonCreateGenError, rfc2119.Must, startRef) + register(StartProcImplement, rfc2119.Must, startRef) + register(StartWithProcUnsetGenError, rfc2119.Must, startRef) + register(KillWithoutIDGenError, rfc2119.Must, killRef) + register(KillNonCreateRunHaveNoEffect, rfc2119.Must, killRef) + register(KillNonCreateRunGenError, rfc2119.Must, killRef) + register(KillSignalImplement, rfc2119.Must, killRef) + register(DeleteWithoutIDGenError, rfc2119.Must, deleteRef) + register(DeleteNonStopHaveNoEffect, rfc2119.Must, deleteRef) + register(DeleteNonStopGenError, rfc2119.Must, deleteRef) + register(DeleteResImplement, rfc2119.Must, deleteRef) + register(DeleteOnlyCreatedRes, rfc2119.Must, deleteRef) +} diff --git a/vendor/github.com/opencontainers/runtime-tools/validate/validate.go b/vendor/github.com/opencontainers/runtime-tools/validate/validate.go index bbdb29c6..1030099d 100644 --- a/vendor/github.com/opencontainers/runtime-tools/validate/validate.go +++ b/vendor/github.com/opencontainers/runtime-tools/validate/validate.go @@ -20,33 +20,41 @@ import ( "github.com/blang/semver" "github.com/hashicorp/go-multierror" rspec "github.com/opencontainers/runtime-spec/specs-go" + osFilepath "github.com/opencontainers/runtime-tools/filepath" "github.com/sirupsen/logrus" "github.com/syndtr/gocapability/capability" "github.com/opencontainers/runtime-tools/specerror" + "github.com/xeipuuv/gojsonschema" ) const specConfig = "config.json" var ( - defaultRlimits = []string{ + // http://pubs.opengroup.org/onlinepubs/9699919799/functions/getrlimit.html + posixRlimits = []string{ "RLIMIT_AS", "RLIMIT_CORE", "RLIMIT_CPU", "RLIMIT_DATA", "RLIMIT_FSIZE", - "RLIMIT_LOCKS", + "RLIMIT_NOFILE", + "RLIMIT_STACK", + } + + // https://git.kernel.org/pub/scm/docs/man-pages/man-pages.git/tree/man2/getrlimit.2?h=man-pages-4.13 + linuxRlimits = append(posixRlimits, []string{ "RLIMIT_MEMLOCK", "RLIMIT_MSGQUEUE", "RLIMIT_NICE", - "RLIMIT_NOFILE", "RLIMIT_NPROC", "RLIMIT_RSS", "RLIMIT_RTPRIO", "RLIMIT_RTTIME", "RLIMIT_SIGPENDING", - "RLIMIT_STACK", - } + }...) + + configSchemaTemplate = "https://raw.githubusercontent.com/opencontainers/runtime-spec/v%s/schema/config-schema.json" ) // Validator represents a validator for runtime bundle @@ -58,23 +66,20 @@ type Validator struct { } // NewValidator creates a Validator -func NewValidator(spec *rspec.Spec, bundlePath string, hostSpecific bool, platform string) Validator { +func NewValidator(spec *rspec.Spec, bundlePath string, hostSpecific bool, platform string) (Validator, error) { if hostSpecific && platform != runtime.GOOS { - platform = runtime.GOOS + return Validator{}, fmt.Errorf("When hostSpecific is set, platform must be same as the host platform") } return Validator{ spec: spec, bundlePath: bundlePath, HostSpecific: hostSpecific, platform: platform, - } + }, nil } // NewValidatorFromPath creates a Validator with specified bundle path func NewValidatorFromPath(bundlePath string, hostSpecific bool, platform string) (Validator, error) { - if hostSpecific && platform != runtime.GOOS { - platform = runtime.GOOS - } if bundlePath == "" { return Validator{}, fmt.Errorf("bundle path shouldn't be empty") } @@ -86,7 +91,7 @@ func NewValidatorFromPath(bundlePath string, hostSpecific bool, platform string) configPath := filepath.Join(bundlePath, specConfig) content, err := ioutil.ReadFile(configPath) if err != nil { - return Validator{}, specerror.NewError(specerror.ConfigFileExistence, err, rspec.Version) + return Validator{}, specerror.NewError(specerror.ConfigInRootBundleDir, err, rspec.Version) } if !utf8.Valid(content) { return Validator{}, fmt.Errorf("%q is not encoded in UTF-8", configPath) @@ -96,21 +101,68 @@ func NewValidatorFromPath(bundlePath string, hostSpecific bool, platform string) return Validator{}, err } - return NewValidator(&spec, bundlePath, hostSpecific, platform), nil + return NewValidator(&spec, bundlePath, hostSpecific, platform) } // CheckAll checks all parts of runtime bundle -func (v *Validator) CheckAll() (errs error) { +func (v *Validator) CheckAll() error { + var errs *multierror.Error + errs = multierror.Append(errs, v.CheckJSONSchema()) errs = multierror.Append(errs, v.CheckPlatform()) errs = multierror.Append(errs, v.CheckRoot()) errs = multierror.Append(errs, v.CheckMandatoryFields()) errs = multierror.Append(errs, v.CheckSemVer()) errs = multierror.Append(errs, v.CheckMounts()) errs = multierror.Append(errs, v.CheckProcess()) - errs = multierror.Append(errs, v.CheckHooks()) errs = multierror.Append(errs, v.CheckLinux()) + if v.platform == "linux" || v.platform == "solaris" { + errs = multierror.Append(errs, v.CheckHooks()) + } - return + return errs.ErrorOrNil() +} + +// JSONSchemaURL returns the URL for the JSON Schema specifying the +// configuration format. It consumes configSchemaTemplate, but we +// provide it as a function to isolate consumers from inconsistent +// naming as runtime-spec evolves. +func JSONSchemaURL(version string) (url string, err error) { + ver, err := semver.Parse(version) + if err != nil { + return "", specerror.NewError(specerror.SpecVersionInSemVer, err, rspec.Version) + } + configRenamedToConfigSchemaVersion, err := semver.Parse("1.0.0-rc2") // config.json became config-schema.json in 1.0.0-rc2 + if ver.Compare(configRenamedToConfigSchemaVersion) == -1 { + return "", fmt.Errorf("unsupported configuration version (older than %s)", configRenamedToConfigSchemaVersion) + } + return fmt.Sprintf(configSchemaTemplate, version), nil +} + +// CheckJSONSchema validates the configuration against the +// runtime-spec JSON Schema, using the version of the schema that +// matches the configuration's declared version. +func (v *Validator) CheckJSONSchema() (errs error) { + url, err := JSONSchemaURL(v.spec.Version) + if err != nil { + errs = multierror.Append(errs, err) + return errs + } + + schemaLoader := gojsonschema.NewReferenceLoader(url) + documentLoader := gojsonschema.NewGoLoader(v.spec) + result, err := gojsonschema.Validate(schemaLoader, documentLoader) + if err != nil { + errs = multierror.Append(errs, err) + return errs + } + + if !result.Valid() { + for _, resultError := range result.Errors() { + errs = multierror.Append(errs, errors.New(resultError.String())) + } + } + + return errs } // CheckRoot checks status of v.spec.Root @@ -120,13 +172,30 @@ func (v *Validator) CheckRoot() (errs error) { if v.platform == "windows" && v.spec.Windows != nil && v.spec.Windows.HyperV != nil { if v.spec.Root != nil { errs = multierror.Append(errs, - specerror.NewError(specerror.RootOnHyperV, fmt.Errorf("for Hyper-V containers, Root must not be set"), rspec.Version)) + specerror.NewError(specerror.RootOnHyperVNotSet, fmt.Errorf("for Hyper-V containers, Root must not be set"), rspec.Version)) return } return } else if v.spec.Root == nil { errs = multierror.Append(errs, - specerror.NewError(specerror.RootOnNonHyperV, fmt.Errorf("for non-Hyper-V containers, Root must be set"), rspec.Version)) + specerror.NewError(specerror.RootOnNonHyperVRequired, fmt.Errorf("for non-Hyper-V containers, Root must be set"), rspec.Version)) + return + } + + if v.platform == "windows" { + matched, err := regexp.MatchString(`\\\\[?]\\Volume[{][a-fA-F0-9]{8}-[a-fA-F0-9]{4}-[a-fA-F0-9]{4}-[a-fA-F0-9]{4}-[a-fA-F0-9]{12}[}]\\`, v.spec.Root.Path) + if err != nil { + errs = multierror.Append(errs, err) + } else if !matched { + errs = multierror.Append(errs, + specerror.NewError(specerror.RootPathOnWindowsGUID, fmt.Errorf("root.path is %q, but it MUST be a volume GUID path when target platform is windows", v.spec.Root.Path), rspec.Version)) + } + + if v.spec.Root.Readonly { + errs = multierror.Append(errs, + specerror.NewError(specerror.RootReadonlyOnWindowsFalse, fmt.Errorf("root.readonly field MUST be omitted or false when target platform is windows"), rspec.Version)) + } + return } @@ -138,7 +207,7 @@ func (v *Validator) CheckRoot() (errs error) { if filepath.Base(v.spec.Root.Path) != "rootfs" { errs = multierror.Append(errs, - specerror.NewError(specerror.PathName, fmt.Errorf("path name should be the conventional 'rootfs'"), rspec.Version)) + specerror.NewError(specerror.RootPathOnPosixConvention, fmt.Errorf("path name should be the conventional 'rootfs'"), rspec.Version)) } var rootfsPath string @@ -158,10 +227,10 @@ func (v *Validator) CheckRoot() (errs error) { if fi, err := os.Stat(rootfsPath); err != nil { errs = multierror.Append(errs, - specerror.NewError(specerror.PathExistence, fmt.Errorf("cannot find the root path %q", rootfsPath), rspec.Version)) + specerror.NewError(specerror.RootPathExist, fmt.Errorf("cannot find the root path %q", rootfsPath), rspec.Version)) } else if !fi.IsDir() { errs = multierror.Append(errs, - specerror.NewError(specerror.PathExistence, fmt.Errorf("root.path %q is not a directory", rootfsPath), rspec.Version)) + specerror.NewError(specerror.RootPathExist, fmt.Errorf("root.path %q is not a directory", rootfsPath), rspec.Version)) } rootParent := filepath.Dir(absRootPath) @@ -170,13 +239,6 @@ func (v *Validator) CheckRoot() (errs error) { specerror.NewError(specerror.ArtifactsInSingleDir, fmt.Errorf("root.path is %q, but it MUST be a child of %q", v.spec.Root.Path, absBundlePath), rspec.Version)) } - if v.platform == "windows" { - if v.spec.Root.Readonly { - errs = multierror.Append(errs, - specerror.NewError(specerror.ReadonlyOnWindows, fmt.Errorf("root.readonly field MUST be omitted or false when target platform is windows"), rspec.Version)) - } - } - return } @@ -188,7 +250,7 @@ func (v *Validator) CheckSemVer() (errs error) { _, err := semver.Parse(version) if err != nil { errs = multierror.Append(errs, - specerror.NewError(specerror.SpecVersion, fmt.Errorf("%q is not valid SemVer: %s", version, err.Error()), rspec.Version)) + specerror.NewError(specerror.SpecVersionInSemVer, fmt.Errorf("%q is not valid SemVer: %s", version, err.Error()), rspec.Version)) } if version != rspec.Version { errs = multierror.Append(errs, fmt.Errorf("validate currently only handles version %s, but the supplied configuration targets %s", rspec.Version, version)) @@ -201,19 +263,29 @@ func (v *Validator) CheckSemVer() (errs error) { func (v *Validator) CheckHooks() (errs error) { logrus.Debugf("check hooks") + if v.platform != "linux" && v.platform != "solaris" { + errs = multierror.Append(errs, fmt.Errorf("For %q platform, the configuration structure does not support hooks", v.platform)) + return + } + if v.spec.Hooks != nil { - errs = multierror.Append(errs, checkEventHooks("pre-start", v.spec.Hooks.Prestart, v.HostSpecific)) - errs = multierror.Append(errs, checkEventHooks("post-start", v.spec.Hooks.Poststart, v.HostSpecific)) - errs = multierror.Append(errs, checkEventHooks("post-stop", v.spec.Hooks.Poststop, v.HostSpecific)) + errs = multierror.Append(errs, v.checkEventHooks("prestart", v.spec.Hooks.Prestart, v.HostSpecific)) + errs = multierror.Append(errs, v.checkEventHooks("poststart", v.spec.Hooks.Poststart, v.HostSpecific)) + errs = multierror.Append(errs, v.checkEventHooks("poststop", v.spec.Hooks.Poststop, v.HostSpecific)) } return } -func checkEventHooks(hookType string, hooks []rspec.Hook, hostSpecific bool) (errs error) { - for _, hook := range hooks { - if !filepath.IsAbs(hook.Path) { - errs = multierror.Append(errs, fmt.Errorf("the %s hook %v: is not absolute path", hookType, hook.Path)) +func (v *Validator) checkEventHooks(hookType string, hooks []rspec.Hook, hostSpecific bool) (errs error) { + for i, hook := range hooks { + if !osFilepath.IsAbs(v.platform, hook.Path) { + errs = multierror.Append(errs, + specerror.NewError( + specerror.PosixHooksPathAbs, + fmt.Errorf("hooks.%s[%d].path %v: is not absolute path", + hookType, i, hook.Path), + rspec.Version)) } if hostSpecific { @@ -245,8 +317,12 @@ func (v *Validator) CheckProcess() (errs error) { } process := v.spec.Process - if !filepath.IsAbs(process.Cwd) { - errs = multierror.Append(errs, fmt.Errorf("cwd %q is not an absolute path", process.Cwd)) + if !osFilepath.IsAbs(v.platform, process.Cwd) { + errs = multierror.Append(errs, + specerror.NewError( + specerror.ProcCwdAbs, + fmt.Errorf("cwd %q is not an absolute path", process.Cwd), + rspec.Version)) } for _, env := range process.Env { @@ -256,9 +332,13 @@ func (v *Validator) CheckProcess() (errs error) { } if len(process.Args) == 0 { - errs = multierror.Append(errs, fmt.Errorf("args must not be empty")) + errs = multierror.Append(errs, + specerror.NewError( + specerror.ProcArgsOneEntryRequired, + fmt.Errorf("args must not be empty"), + rspec.Version)) } else { - if filepath.IsAbs(process.Args[0]) { + if filepath.IsAbs(process.Args[0]) && v.spec.Root != nil { var rootfsPath string if filepath.IsAbs(v.spec.Root.Path) { rootfsPath = v.spec.Root.Path @@ -280,12 +360,15 @@ func (v *Validator) CheckProcess() (errs error) { } } - if v.spec.Process.Capabilities != nil { - errs = multierror.Append(errs, v.CheckCapabilities()) + if v.platform == "linux" || v.platform == "solaris" { + errs = multierror.Append(errs, v.CheckRlimits()) } - errs = multierror.Append(errs, v.CheckRlimits()) if v.platform == "linux" { + if v.spec.Process.Capabilities != nil { + errs = multierror.Append(errs, v.CheckCapabilities()) + } + if len(process.ApparmorProfile) > 0 { profilePath := filepath.Join(v.bundlePath, v.spec.Root.Path, "/etc/apparmor.d", process.ApparmorProfile) _, err := os.Stat(profilePath) @@ -300,60 +383,61 @@ func (v *Validator) CheckProcess() (errs error) { // CheckCapabilities checks v.spec.Process.Capabilities func (v *Validator) CheckCapabilities() (errs error) { + if v.platform != "linux" { + errs = multierror.Append(errs, fmt.Errorf("For %q platform, the configuration structure does not support process.capabilities", v.platform)) + return + } + process := v.spec.Process - if v.platform == "linux" { - var effective, permitted, inheritable, ambient bool - caps := make(map[string][]string) + var effective, permitted, inheritable, ambient bool + caps := make(map[string][]string) - for _, cap := range process.Capabilities.Bounding { - caps[cap] = append(caps[cap], "bounding") - } - for _, cap := range process.Capabilities.Effective { - caps[cap] = append(caps[cap], "effective") - } - for _, cap := range process.Capabilities.Inheritable { - caps[cap] = append(caps[cap], "inheritable") - } - for _, cap := range process.Capabilities.Permitted { - caps[cap] = append(caps[cap], "permitted") - } - for _, cap := range process.Capabilities.Ambient { - caps[cap] = append(caps[cap], "ambient") + for _, cap := range process.Capabilities.Bounding { + caps[cap] = append(caps[cap], "bounding") + } + for _, cap := range process.Capabilities.Effective { + caps[cap] = append(caps[cap], "effective") + } + for _, cap := range process.Capabilities.Inheritable { + caps[cap] = append(caps[cap], "inheritable") + } + for _, cap := range process.Capabilities.Permitted { + caps[cap] = append(caps[cap], "permitted") + } + for _, cap := range process.Capabilities.Ambient { + caps[cap] = append(caps[cap], "ambient") + } + + for capability, owns := range caps { + if err := CapValid(capability, v.HostSpecific); err != nil { + errs = multierror.Append(errs, fmt.Errorf("capability %q is not valid, man capabilities(7)", capability)) } - for capability, owns := range caps { - if err := CapValid(capability, v.HostSpecific); err != nil { - errs = multierror.Append(errs, fmt.Errorf("capability %q is not valid, man capabilities(7)", capability)) + effective, permitted, ambient, inheritable = false, false, false, false + for _, set := range owns { + if set == "effective" { + effective = true + continue } - - effective, permitted, ambient, inheritable = false, false, false, false - for _, set := range owns { - if set == "effective" { - effective = true - continue - } - if set == "inheritable" { - inheritable = true - continue - } - if set == "permitted" { - permitted = true - continue - } - if set == "ambient" { - ambient = true - continue - } + if set == "inheritable" { + inheritable = true + continue } - if effective && !permitted { - errs = multierror.Append(errs, fmt.Errorf("effective capability %q is not allowed, as it's not permitted", capability)) + if set == "permitted" { + permitted = true + continue } - if ambient && !(effective && inheritable) { - errs = multierror.Append(errs, fmt.Errorf("ambient capability %q is not allowed, as it's not permitted and inheribate", capability)) + if set == "ambient" { + ambient = true + continue } } - } else { - logrus.Warnf("process.capabilities validation not yet implemented for OS %q", v.platform) + if effective && !permitted { + errs = multierror.Append(errs, fmt.Errorf("effective capability %q is not allowed, as it's not permitted", capability)) + } + if ambient && !(permitted && inheritable) { + errs = multierror.Append(errs, fmt.Errorf("ambient capability %q is not allowed, as it's not permitted and inheribate", capability)) + } } return @@ -361,11 +445,21 @@ func (v *Validator) CheckCapabilities() (errs error) { // CheckRlimits checks v.spec.Process.Rlimits func (v *Validator) CheckRlimits() (errs error) { + if v.platform != "linux" && v.platform != "solaris" { + errs = multierror.Append(errs, fmt.Errorf("For %q platform, the configuration structure does not support process.rlimits", v.platform)) + return + } + process := v.spec.Process for index, rlimit := range process.Rlimits { for i := index + 1; i < len(process.Rlimits); i++ { if process.Rlimits[index].Type == process.Rlimits[i].Type { - errs = multierror.Append(errs, fmt.Errorf("rlimit can not contain the same type %q", process.Rlimits[index].Type)) + errs = multierror.Append(errs, + specerror.NewError( + specerror.PosixProcRlimitsErrorOnDup, + fmt.Errorf("rlimit can not contain the same type %q", + process.Rlimits[index].Type), + rspec.Version)) } } errs = multierror.Append(errs, v.rlimitValid(rlimit)) @@ -429,31 +523,33 @@ func (v *Validator) CheckMounts() (errs error) { if supportedTypes != nil && !supportedTypes[mountA.Type] { errs = multierror.Append(errs, fmt.Errorf("unsupported mount type %q", mountA.Type)) } - if v.platform == "windows" { - if err := pathValid(v.platform, mountA.Destination); err != nil { - errs = multierror.Append(errs, err) - } - if err := pathValid(v.platform, mountA.Source); err != nil { - errs = multierror.Append(errs, err) - } - } else { - if err := pathValid(v.platform, mountA.Destination); err != nil { - errs = multierror.Append(errs, err) - } + if !osFilepath.IsAbs(v.platform, mountA.Destination) { + errs = multierror.Append(errs, + specerror.NewError( + specerror.MountsDestAbs, + fmt.Errorf("mounts[%d].destination %q is not absolute", + i, + mountA.Destination), + rspec.Version)) } for j, mountB := range v.spec.Mounts { if i == j { continue } // whether B.Desination is nested within A.Destination - nested, err := nestedValid(v.platform, mountA.Destination, mountB.Destination) + nested, err := osFilepath.IsAncestor(v.platform, mountA.Destination, mountB.Destination, ".") if err != nil { errs = multierror.Append(errs, err) continue } if nested { if v.platform == "windows" && i < j { - errs = multierror.Append(errs, fmt.Errorf("on Windows, %v nested within %v is forbidden", mountB.Destination, mountA.Destination)) + errs = multierror.Append(errs, + specerror.NewError( + specerror.MountsDestOnWindowsNotNested, + fmt.Errorf("on Windows, %v nested within %v is forbidden", + mountB.Destination, mountA.Destination), + rspec.Version)) } if i > j { logrus.Warnf("%v will be covered by %v", mountB.Destination, mountA.Destination) @@ -476,7 +572,11 @@ func (v *Validator) CheckPlatform() (errs error) { if v.platform == "windows" { if v.spec.Windows == nil { - errs = multierror.Append(errs, errors.New("'windows' MUST be set when platform is `windows`")) + errs = multierror.Append(errs, + specerror.NewError( + specerror.PlatformSpecConfOnWindowsSet, + fmt.Errorf("'windows' MUST be set when platform is `windows`"), + rspec.Version)) } } @@ -506,14 +606,14 @@ func (v *Validator) CheckLinux() (errs error) { for index := 0; index < len(v.spec.Linux.Namespaces); index++ { ns := v.spec.Linux.Namespaces[index] - if !namespaceValid(ns) { - errs = multierror.Append(errs, fmt.Errorf("namespace %v is invalid", ns)) + if ns.Path != "" && !osFilepath.IsAbs(v.platform, ns.Path) { + errs = multierror.Append(errs, specerror.NewError(specerror.NSPathAbs, fmt.Errorf("namespace.path %q is not an absolute path", ns.Path), rspec.Version)) } tmpItem := nsTypeList[ns.Type] tmpItem.num = tmpItem.num + 1 if tmpItem.num > 1 { - errs = multierror.Append(errs, fmt.Errorf("duplicated namespace %q", ns.Type)) + errs = multierror.Append(errs, specerror.NewError(specerror.NSErrorOnDup, fmt.Errorf("duplicated namespace %q", ns.Type), rspec.Version)) } if len(ns.Path) == 0 { @@ -524,10 +624,6 @@ func (v *Validator) CheckLinux() (errs error) { if (len(v.spec.Linux.UIDMappings) > 0 || len(v.spec.Linux.GIDMappings) > 0) && !nsTypeList[rspec.UserNamespace].newExist { errs = multierror.Append(errs, errors.New("the UID/GID mappings requires a new User namespace to be specified as well")) - } else if len(v.spec.Linux.UIDMappings) > 5 { - errs = multierror.Append(errs, errors.New("only 5 UID mappings are allowed (linux kernel restriction)")) - } else if len(v.spec.Linux.GIDMappings) > 5 { - errs = multierror.Append(errs, errors.New("only 5 GID mappings are allowed (linux kernel restriction)")) } for k := range v.spec.Linux.Sysctl { @@ -572,7 +668,8 @@ func (v *Validator) CheckLinux() (errs error) { } else { fStat, ok := fi.Sys().(*syscall.Stat_t) if !ok { - errs = multierror.Append(errs, fmt.Errorf("cannot determine state for device %s", device.Path)) + errs = multierror.Append(errs, specerror.NewError(specerror.DevicesAvailable, + fmt.Errorf("cannot determine state for device %s", device.Path), rspec.Version)) continue } var devType string @@ -587,7 +684,8 @@ func (v *Validator) CheckLinux() (errs error) { devType = "unmatched" } if devType != device.Type || (devType == "c" && device.Type == "u") { - errs = multierror.Append(errs, fmt.Errorf("unmatched %s already exists in filesystem", device.Path)) + errs = multierror.Append(errs, specerror.NewError(specerror.DevicesFileNotMatch, + fmt.Errorf("unmatched %s already exists in filesystem", device.Path), rspec.Version)) continue } if devType != "p" { @@ -595,7 +693,8 @@ func (v *Validator) CheckLinux() (errs error) { major := (dev >> 8) & 0xfff minor := (dev & 0xff) | ((dev >> 12) & 0xfff00) if int64(major) != device.Major || int64(minor) != device.Minor { - errs = multierror.Append(errs, fmt.Errorf("unmatched %s already exists in filesystem", device.Path)) + errs = multierror.Append(errs, specerror.NewError(specerror.DevicesFileNotMatch, + fmt.Errorf("unmatched %s already exists in filesystem", device.Path), rspec.Version)) continue } } @@ -603,19 +702,22 @@ func (v *Validator) CheckLinux() (errs error) { expectedPerm := *device.FileMode & os.ModePerm actualPerm := fi.Mode() & os.ModePerm if expectedPerm != actualPerm { - errs = multierror.Append(errs, fmt.Errorf("unmatched %s already exists in filesystem", device.Path)) + errs = multierror.Append(errs, specerror.NewError(specerror.DevicesFileNotMatch, + fmt.Errorf("unmatched %s already exists in filesystem", device.Path), rspec.Version)) continue } } if device.UID != nil { if *device.UID != fStat.Uid { - errs = multierror.Append(errs, fmt.Errorf("unmatched %s already exists in filesystem", device.Path)) + errs = multierror.Append(errs, specerror.NewError(specerror.DevicesFileNotMatch, + fmt.Errorf("unmatched %s already exists in filesystem", device.Path), rspec.Version)) continue } } if device.GID != nil { if *device.GID != fStat.Gid { - errs = multierror.Append(errs, fmt.Errorf("unmatched %s already exists in filesystem", device.Path)) + errs = multierror.Append(errs, specerror.NewError(specerror.DevicesFileNotMatch, + fmt.Errorf("unmatched %s already exists in filesystem", device.Path), rspec.Version)) continue } } @@ -641,33 +743,23 @@ func (v *Validator) CheckLinux() (errs error) { errs = multierror.Append(errs, v.CheckLinuxResources()) } - if v.spec.Linux.Seccomp != nil { - errs = multierror.Append(errs, v.CheckSeccomp()) - } - - switch v.spec.Linux.RootfsPropagation { - case "": - case "private": - case "rprivate": - case "slave": - case "rslave": - case "shared": - case "rshared": - case "unbindable": - case "runbindable": - default: - errs = multierror.Append(errs, errors.New("rootfsPropagation must be empty or one of \"private|rprivate|slave|rslave|shared|rshared|unbindable|runbindable\"")) - } - for _, maskedPath := range v.spec.Linux.MaskedPaths { if !strings.HasPrefix(maskedPath, "/") { - errs = multierror.Append(errs, fmt.Errorf("maskedPath %v is not an absolute path", maskedPath)) + errs = multierror.Append(errs, + specerror.NewError( + specerror.MaskedPathsAbs, + fmt.Errorf("maskedPath %v is not an absolute path", maskedPath), + rspec.Version)) } } for _, readonlyPath := range v.spec.Linux.ReadonlyPaths { if !strings.HasPrefix(readonlyPath, "/") { - errs = multierror.Append(errs, fmt.Errorf("readonlyPath %v is not an absolute path", readonlyPath)) + errs = multierror.Append(errs, + specerror.NewError( + specerror.ReadonlyPathsAbs, + fmt.Errorf("readonlyPath %v is not an absolute path", readonlyPath), + rspec.Version)) } } @@ -709,7 +801,7 @@ func (v *Validator) CheckLinuxResources() (errs error) { } for index := 0; index < len(r.Devices); index++ { switch r.Devices[index].Type { - case "a", "b", "c": + case "a", "b", "c", "": default: errs = multierror.Append(errs, fmt.Errorf("type of devices %s is invalid", r.Devices[index].Type)) } @@ -728,47 +820,6 @@ func (v *Validator) CheckLinuxResources() (errs error) { return } -// CheckSeccomp checkc v.spec.Linux.Seccomp -func (v *Validator) CheckSeccomp() (errs error) { - logrus.Debugf("check linux seccomp") - - s := v.spec.Linux.Seccomp - if !seccompActionValid(s.DefaultAction) { - errs = multierror.Append(errs, fmt.Errorf("seccomp defaultAction %q is invalid", s.DefaultAction)) - } - for index := 0; index < len(s.Syscalls); index++ { - if !syscallValid(s.Syscalls[index]) { - errs = multierror.Append(errs, fmt.Errorf("syscall %v is invalid", s.Syscalls[index])) - } - } - for index := 0; index < len(s.Architectures); index++ { - switch s.Architectures[index] { - case rspec.ArchX86: - case rspec.ArchX86_64: - case rspec.ArchX32: - case rspec.ArchARM: - case rspec.ArchAARCH64: - case rspec.ArchMIPS: - case rspec.ArchMIPS64: - case rspec.ArchMIPS64N32: - case rspec.ArchMIPSEL: - case rspec.ArchMIPSEL64: - case rspec.ArchMIPSEL64N32: - case rspec.ArchPPC: - case rspec.ArchPPC64: - case rspec.ArchPPC64LE: - case rspec.ArchS390: - case rspec.ArchS390X: - case rspec.ArchPARISC: - case rspec.ArchPARISC64: - default: - errs = multierror.Append(errs, fmt.Errorf("seccomp architecture %q is invalid", s.Architectures[index])) - } - } - - return -} - // CapValid checks whether a capability is valid func CapValid(c string, hostSpecific bool) error { isValid := false @@ -825,12 +876,19 @@ func (v *Validator) rlimitValid(rlimit rspec.POSIXRlimit) (errs error) { } if v.platform == "linux" { - for _, val := range defaultRlimits { + for _, val := range linuxRlimits { if val == rlimit.Type { return } } - errs = multierror.Append(errs, fmt.Errorf("rlimit type %q is invalid", rlimit.Type)) + errs = multierror.Append(errs, specerror.NewError(specerror.PosixProcRlimitsTypeValueError, fmt.Errorf("rlimit type %q may not be valid", rlimit.Type), v.spec.Version)) + } else if v.platform == "solaris" { + for _, val := range posixRlimits { + if val == rlimit.Type { + return + } + } + errs = multierror.Append(errs, specerror.NewError(specerror.PosixProcRlimitsTypeValueError, fmt.Errorf("rlimit type %q may not be valid", rlimit.Type), v.spec.Version)) } else { logrus.Warnf("process.rlimits validation not yet implemented for platform %q", v.platform) } @@ -838,85 +896,6 @@ func (v *Validator) rlimitValid(rlimit rspec.POSIXRlimit) (errs error) { return } -func namespaceValid(ns rspec.LinuxNamespace) bool { - switch ns.Type { - case rspec.PIDNamespace: - case rspec.NetworkNamespace: - case rspec.MountNamespace: - case rspec.IPCNamespace: - case rspec.UTSNamespace: - case rspec.UserNamespace: - case rspec.CgroupNamespace: - default: - return false - } - - if ns.Path != "" && !filepath.IsAbs(ns.Path) { - return false - } - - return true -} - -func pathValid(os, path string) error { - if os == "windows" { - matched, err := regexp.MatchString("^[a-zA-Z]:(\\\\[^\\\\/<>|:*?\"]+)+$", path) - if err != nil { - return err - } - if !matched { - return fmt.Errorf("invalid windows path %v", path) - } - return nil - } - if !filepath.IsAbs(path) { - return fmt.Errorf("%v is not an absolute path", path) - } - return nil -} - -// Check whether pathB is nested whithin pathA -func nestedValid(os, pathA, pathB string) (bool, error) { - if pathA == pathB { - return false, nil - } - if pathA == "/" && pathB != "" { - return true, nil - } - - var sep string - if os == "windows" { - sep = "\\" - } else { - sep = "/" - } - - splitedPathA := strings.Split(filepath.Clean(pathA), sep) - splitedPathB := strings.Split(filepath.Clean(pathB), sep) - lenA := len(splitedPathA) - lenB := len(splitedPathB) - - if lenA > lenB { - if (lenA - lenB) == 1 { - // if pathA is longer but not end with separator - if splitedPathA[lenA-1] != "" { - return false, nil - } - splitedPathA = splitedPathA[:lenA-1] - } else { - return false, nil - } - } - - for i, partA := range splitedPathA { - if partA != splitedPathB[i] { - return false, nil - } - } - - return true, nil -} - func deviceValid(d rspec.LinuxDevice) bool { switch d.Type { case "b", "c", "u": @@ -924,7 +903,7 @@ func deviceValid(d rspec.LinuxDevice) bool { return false } case "p": - if d.Major > 0 || d.Minor > 0 { + if d.Major != 0 || d.Minor != 0 { return false } default: @@ -933,41 +912,6 @@ func deviceValid(d rspec.LinuxDevice) bool { return true } -func seccompActionValid(secc rspec.LinuxSeccompAction) bool { - switch secc { - case "": - case rspec.ActKill: - case rspec.ActTrap: - case rspec.ActErrno: - case rspec.ActTrace: - case rspec.ActAllow: - default: - return false - } - return true -} - -func syscallValid(s rspec.LinuxSyscall) bool { - if !seccompActionValid(s.Action) { - return false - } - for index := 0; index < len(s.Args); index++ { - arg := s.Args[index] - switch arg.Op { - case rspec.OpNotEqual: - case rspec.OpLessThan: - case rspec.OpLessEqual: - case rspec.OpEqualTo: - case rspec.OpGreaterEqual: - case rspec.OpGreaterThan: - case rspec.OpMaskedEqual: - default: - return false - } - } - return true -} - func isStruct(t reflect.Type) bool { return t.Kind() == reflect.Struct } diff --git a/vendor/github.com/xeipuuv/gojsonpointer/LICENSE-APACHE-2.0.txt b/vendor/github.com/xeipuuv/gojsonpointer/LICENSE-APACHE-2.0.txt new file mode 100644 index 00000000..55ede8a4 --- /dev/null +++ b/vendor/github.com/xeipuuv/gojsonpointer/LICENSE-APACHE-2.0.txt @@ -0,0 +1,202 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright 2015 xeipuuv + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/vendor/github.com/xeipuuv/gojsonpointer/README.md b/vendor/github.com/xeipuuv/gojsonpointer/README.md new file mode 100644 index 00000000..dbe4d508 --- /dev/null +++ b/vendor/github.com/xeipuuv/gojsonpointer/README.md @@ -0,0 +1,8 @@ +# gojsonpointer +An implementation of JSON Pointer - Go language + +## References +http://tools.ietf.org/html/draft-ietf-appsawg-json-pointer-07 + +### Note +The 4.Evaluation part of the previous reference, starting with 'If the currently referenced value is a JSON array, the reference token MUST contain either...' is not implemented. diff --git a/vendor/github.com/xeipuuv/gojsonpointer/pointer.go b/vendor/github.com/xeipuuv/gojsonpointer/pointer.go new file mode 100644 index 00000000..06f1918e --- /dev/null +++ b/vendor/github.com/xeipuuv/gojsonpointer/pointer.go @@ -0,0 +1,190 @@ +// Copyright 2015 xeipuuv ( https://github.com/xeipuuv ) +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// author xeipuuv +// author-github https://github.com/xeipuuv +// author-mail xeipuuv@gmail.com +// +// repository-name gojsonpointer +// repository-desc An implementation of JSON Pointer - Go language +// +// description Main and unique file. +// +// created 25-02-2013 + +package gojsonpointer + +import ( + "errors" + "fmt" + "reflect" + "strconv" + "strings" +) + +const ( + const_empty_pointer = `` + const_pointer_separator = `/` + + const_invalid_start = `JSON pointer must be empty or start with a "` + const_pointer_separator + `"` +) + +type implStruct struct { + mode string // "SET" or "GET" + + inDocument interface{} + + setInValue interface{} + + getOutNode interface{} + getOutKind reflect.Kind + outError error +} + +type JsonPointer struct { + referenceTokens []string +} + +// NewJsonPointer parses the given string JSON pointer and returns an object +func NewJsonPointer(jsonPointerString string) (p JsonPointer, err error) { + + // Pointer to the root of the document + if len(jsonPointerString) == 0 { + // Keep referenceTokens nil + return + } + if jsonPointerString[0] != '/' { + return p, errors.New(const_invalid_start) + } + + p.referenceTokens = strings.Split(jsonPointerString[1:], const_pointer_separator) + return +} + +// Uses the pointer to retrieve a value from a JSON document +func (p *JsonPointer) Get(document interface{}) (interface{}, reflect.Kind, error) { + + is := &implStruct{mode: "GET", inDocument: document} + p.implementation(is) + return is.getOutNode, is.getOutKind, is.outError + +} + +// Uses the pointer to update a value from a JSON document +func (p *JsonPointer) Set(document interface{}, value interface{}) (interface{}, error) { + + is := &implStruct{mode: "SET", inDocument: document, setInValue: value} + p.implementation(is) + return document, is.outError + +} + +// Both Get and Set functions use the same implementation to avoid code duplication +func (p *JsonPointer) implementation(i *implStruct) { + + kind := reflect.Invalid + + // Full document when empty + if len(p.referenceTokens) == 0 { + i.getOutNode = i.inDocument + i.outError = nil + i.getOutKind = kind + i.outError = nil + return + } + + node := i.inDocument + + for ti, token := range p.referenceTokens { + + isLastToken := ti == len(p.referenceTokens)-1 + + switch v := node.(type) { + + case map[string]interface{}: + decodedToken := decodeReferenceToken(token) + if _, ok := v[decodedToken]; ok { + node = v[decodedToken] + if isLastToken && i.mode == "SET" { + v[decodedToken] = i.setInValue + } + } else { + i.outError = fmt.Errorf("Object has no key '%s'", decodedToken) + i.getOutKind = reflect.Map + i.getOutNode = nil + return + } + + case []interface{}: + tokenIndex, err := strconv.Atoi(token) + if err != nil { + i.outError = fmt.Errorf("Invalid array index '%s'", token) + i.getOutKind = reflect.Slice + i.getOutNode = nil + return + } + if tokenIndex < 0 || tokenIndex >= len(v) { + i.outError = fmt.Errorf("Out of bound array[0,%d] index '%d'", len(v), tokenIndex) + i.getOutKind = reflect.Slice + i.getOutNode = nil + return + } + + node = v[tokenIndex] + if isLastToken && i.mode == "SET" { + v[tokenIndex] = i.setInValue + } + + default: + i.outError = fmt.Errorf("Invalid token reference '%s'", token) + i.getOutKind = reflect.ValueOf(node).Kind() + i.getOutNode = nil + return + } + + } + + i.getOutNode = node + i.getOutKind = reflect.ValueOf(node).Kind() + i.outError = nil +} + +// Pointer to string representation function +func (p *JsonPointer) String() string { + + if len(p.referenceTokens) == 0 { + return const_empty_pointer + } + + pointerString := const_pointer_separator + strings.Join(p.referenceTokens, const_pointer_separator) + + return pointerString +} + +// Specific JSON pointer encoding here +// ~0 => ~ +// ~1 => / +// ... and vice versa + +func decodeReferenceToken(token string) string { + step1 := strings.Replace(token, `~1`, `/`, -1) + step2 := strings.Replace(step1, `~0`, `~`, -1) + return step2 +} + +func encodeReferenceToken(token string) string { + step1 := strings.Replace(token, `~`, `~0`, -1) + step2 := strings.Replace(step1, `/`, `~1`, -1) + return step2 +} diff --git a/vendor/github.com/xeipuuv/gojsonreference/LICENSE-APACHE-2.0.txt b/vendor/github.com/xeipuuv/gojsonreference/LICENSE-APACHE-2.0.txt new file mode 100644 index 00000000..55ede8a4 --- /dev/null +++ b/vendor/github.com/xeipuuv/gojsonreference/LICENSE-APACHE-2.0.txt @@ -0,0 +1,202 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright 2015 xeipuuv + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/vendor/github.com/xeipuuv/gojsonreference/README.md b/vendor/github.com/xeipuuv/gojsonreference/README.md new file mode 100644 index 00000000..9ab6e1eb --- /dev/null +++ b/vendor/github.com/xeipuuv/gojsonreference/README.md @@ -0,0 +1,10 @@ +# gojsonreference +An implementation of JSON Reference - Go language + +## Dependencies +https://github.com/xeipuuv/gojsonpointer + +## References +http://tools.ietf.org/html/draft-ietf-appsawg-json-pointer-07 + +http://tools.ietf.org/html/draft-pbryan-zyp-json-ref-03 diff --git a/vendor/github.com/xeipuuv/gojsonreference/reference.go b/vendor/github.com/xeipuuv/gojsonreference/reference.go new file mode 100644 index 00000000..d4d2eca0 --- /dev/null +++ b/vendor/github.com/xeipuuv/gojsonreference/reference.go @@ -0,0 +1,141 @@ +// Copyright 2015 xeipuuv ( https://github.com/xeipuuv ) +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// author xeipuuv +// author-github https://github.com/xeipuuv +// author-mail xeipuuv@gmail.com +// +// repository-name gojsonreference +// repository-desc An implementation of JSON Reference - Go language +// +// description Main and unique file. +// +// created 26-02-2013 + +package gojsonreference + +import ( + "errors" + "github.com/xeipuuv/gojsonpointer" + "net/url" + "path/filepath" + "runtime" + "strings" +) + +const ( + const_fragment_char = `#` +) + +func NewJsonReference(jsonReferenceString string) (JsonReference, error) { + + var r JsonReference + err := r.parse(jsonReferenceString) + return r, err + +} + +type JsonReference struct { + referenceUrl *url.URL + referencePointer gojsonpointer.JsonPointer + + HasFullUrl bool + HasUrlPathOnly bool + HasFragmentOnly bool + HasFileScheme bool + HasFullFilePath bool +} + +func (r *JsonReference) GetUrl() *url.URL { + return r.referenceUrl +} + +func (r *JsonReference) GetPointer() *gojsonpointer.JsonPointer { + return &r.referencePointer +} + +func (r *JsonReference) String() string { + + if r.referenceUrl != nil { + return r.referenceUrl.String() + } + + if r.HasFragmentOnly { + return const_fragment_char + r.referencePointer.String() + } + + return r.referencePointer.String() +} + +func (r *JsonReference) IsCanonical() bool { + return (r.HasFileScheme && r.HasFullFilePath) || (!r.HasFileScheme && r.HasFullUrl) +} + +// "Constructor", parses the given string JSON reference +func (r *JsonReference) parse(jsonReferenceString string) (err error) { + + r.referenceUrl, err = url.Parse(jsonReferenceString) + if err != nil { + return + } + refUrl := r.referenceUrl + + if refUrl.Scheme != "" && refUrl.Host != "" { + r.HasFullUrl = true + } else { + if refUrl.Path != "" { + r.HasUrlPathOnly = true + } else if refUrl.RawQuery == "" && refUrl.Fragment != "" { + r.HasFragmentOnly = true + } + } + + r.HasFileScheme = refUrl.Scheme == "file" + if runtime.GOOS == "windows" { + // on Windows, a file URL may have an extra leading slash, and if it + // doesn't then its first component will be treated as the host by the + // Go runtime + if refUrl.Host == "" && strings.HasPrefix(refUrl.Path, "/") { + r.HasFullFilePath = filepath.IsAbs(refUrl.Path[1:]) + } else { + r.HasFullFilePath = filepath.IsAbs(refUrl.Host + refUrl.Path) + } + } else { + r.HasFullFilePath = filepath.IsAbs(refUrl.Path) + } + + // invalid json-pointer error means url has no json-pointer fragment. simply ignore error + r.referencePointer, _ = gojsonpointer.NewJsonPointer(refUrl.Fragment) + + return +} + +// Creates a new reference from a parent and a child +// If the child cannot inherit from the parent, an error is returned +func (r *JsonReference) Inherits(child JsonReference) (*JsonReference, error) { + childUrl := child.GetUrl() + parentUrl := r.GetUrl() + if childUrl == nil { + return nil, errors.New("childUrl is nil!") + } + if parentUrl == nil { + return nil, errors.New("parentUrl is nil!") + } + + ref, err := NewJsonReference(parentUrl.ResolveReference(childUrl).String()) + if err != nil { + return nil, err + } + return &ref, err +} diff --git a/vendor/github.com/xeipuuv/gojsonschema/LICENSE-APACHE-2.0.txt b/vendor/github.com/xeipuuv/gojsonschema/LICENSE-APACHE-2.0.txt new file mode 100644 index 00000000..55ede8a4 --- /dev/null +++ b/vendor/github.com/xeipuuv/gojsonschema/LICENSE-APACHE-2.0.txt @@ -0,0 +1,202 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright 2015 xeipuuv + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/vendor/github.com/xeipuuv/gojsonschema/README.md b/vendor/github.com/xeipuuv/gojsonschema/README.md new file mode 100644 index 00000000..fe43659d --- /dev/null +++ b/vendor/github.com/xeipuuv/gojsonschema/README.md @@ -0,0 +1,295 @@ +[![GoDoc](https://godoc.org/github.com/xeipuuv/gojsonschema?status.svg)](https://godoc.org/github.com/xeipuuv/gojsonschema) +[![Build Status](https://travis-ci.org/xeipuuv/gojsonschema.svg)](https://travis-ci.org/xeipuuv/gojsonschema) + +# gojsonschema + +## Description + +An implementation of JSON Schema, based on IETF's draft v4 - Go language + +References : + +* http://json-schema.org +* http://json-schema.org/latest/json-schema-core.html +* http://json-schema.org/latest/json-schema-validation.html + +## Installation + +``` +go get github.com/xeipuuv/gojsonschema +``` + +Dependencies : +* [github.com/xeipuuv/gojsonpointer](https://github.com/xeipuuv/gojsonpointer) +* [github.com/xeipuuv/gojsonreference](https://github.com/xeipuuv/gojsonreference) +* [github.com/stretchr/testify/assert](https://github.com/stretchr/testify#assert-package) + +## Usage + +### Example + +```go + +package main + +import ( + "fmt" + "github.com/xeipuuv/gojsonschema" +) + +func main() { + + schemaLoader := gojsonschema.NewReferenceLoader("file:///home/me/schema.json") + documentLoader := gojsonschema.NewReferenceLoader("file:///home/me/document.json") + + result, err := gojsonschema.Validate(schemaLoader, documentLoader) + if err != nil { + panic(err.Error()) + } + + if result.Valid() { + fmt.Printf("The document is valid\n") + } else { + fmt.Printf("The document is not valid. see errors :\n") + for _, desc := range result.Errors() { + fmt.Printf("- %s\n", desc) + } + } + +} + + +``` + +#### Loaders + +There are various ways to load your JSON data. +In order to load your schemas and documents, +first declare an appropriate loader : + +* Web / HTTP, using a reference : + +```go +loader := gojsonschema.NewReferenceLoader("http://www.some_host.com/schema.json") +``` + +* Local file, using a reference : + +```go +loader := gojsonschema.NewReferenceLoader("file:///home/me/schema.json") +``` + +References use the URI scheme, the prefix (file://) and a full path to the file are required. + +* JSON strings : + +```go +loader := gojsonschema.NewStringLoader(`{"type": "string"}`) +``` + +* Custom Go types : + +```go +m := map[string]interface{}{"type": "string"} +loader := gojsonschema.NewGoLoader(m) +``` + +And + +```go +type Root struct { + Users []User `json:"users"` +} + +type User struct { + Name string `json:"name"` +} + +... + +data := Root{} +data.Users = append(data.Users, User{"John"}) +data.Users = append(data.Users, User{"Sophia"}) +data.Users = append(data.Users, User{"Bill"}) + +loader := gojsonschema.NewGoLoader(data) +``` + +#### Validation + +Once the loaders are set, validation is easy : + +```go +result, err := gojsonschema.Validate(schemaLoader, documentLoader) +``` + +Alternatively, you might want to load a schema only once and process to multiple validations : + +```go +schema, err := gojsonschema.NewSchema(schemaLoader) +... +result1, err := schema.Validate(documentLoader1) +... +result2, err := schema.Validate(documentLoader2) +... +// etc ... +``` + +To check the result : + +```go + if result.Valid() { + fmt.Printf("The document is valid\n") + } else { + fmt.Printf("The document is not valid. see errors :\n") + for _, err := range result.Errors() { + // Err implements the ResultError interface + fmt.Printf("- %s\n", err) + } + } +``` + +## Working with Errors + +The library handles string error codes which you can customize by creating your own gojsonschema.locale and setting it +```go +gojsonschema.Locale = YourCustomLocale{} +``` + +However, each error contains additional contextual information. + +**err.Type()**: *string* Returns the "type" of error that occurred. Note you can also type check. See below + +Note: An error of RequiredType has an err.Type() return value of "required" + + "required": RequiredError + "invalid_type": InvalidTypeError + "number_any_of": NumberAnyOfError + "number_one_of": NumberOneOfError + "number_all_of": NumberAllOfError + "number_not": NumberNotError + "missing_dependency": MissingDependencyError + "internal": InternalError + "enum": EnumError + "array_no_additional_items": ArrayNoAdditionalItemsError + "array_min_items": ArrayMinItemsError + "array_max_items": ArrayMaxItemsError + "unique": ItemsMustBeUniqueError + "array_min_properties": ArrayMinPropertiesError + "array_max_properties": ArrayMaxPropertiesError + "additional_property_not_allowed": AdditionalPropertyNotAllowedError + "invalid_property_pattern": InvalidPropertyPatternError + "string_gte": StringLengthGTEError + "string_lte": StringLengthLTEError + "pattern": DoesNotMatchPatternError + "multiple_of": MultipleOfError + "number_gte": NumberGTEError + "number_gt": NumberGTError + "number_lte": NumberLTEError + "number_lt": NumberLTError + +**err.Value()**: *interface{}* Returns the value given + +**err.Context()**: *gojsonschema.jsonContext* Returns the context. This has a String() method that will print something like this: (root).firstName + +**err.Field()**: *string* Returns the fieldname in the format firstName, or for embedded properties, person.firstName. This returns the same as the String() method on *err.Context()* but removes the (root). prefix. + +**err.Description()**: *string* The error description. This is based on the locale you are using. See the beginning of this section for overwriting the locale with a custom implementation. + +**err.Details()**: *gojsonschema.ErrorDetails* Returns a map[string]interface{} of additional error details specific to the error. For example, GTE errors will have a "min" value, LTE will have a "max" value. See errors.go for a full description of all the error details. Every error always contains a "field" key that holds the value of *err.Field()* + +Note in most cases, the err.Details() will be used to generate replacement strings in your locales, and not used directly. These strings follow the text/template format i.e. +``` +{{.field}} must be greater than or equal to {{.min}} +``` + +The library allows you to specify custom template functions, should you require more complex error message handling. +```go +gojsonschema.ErrorTemplateFuncs = map[string]interface{}{ + "allcaps": func(s string) string { + return strings.ToUpper(s) + }, +} +``` + +Given the above definition, you can use the custom function `"allcaps"` in your localization templates: +``` +{{allcaps .field}} must be greater than or equal to {{.min}} +``` + +The above error message would then be rendered with the `field` value in capital letters. For example: +``` +"PASSWORD must be greater than or equal to 8" +``` + +Learn more about what types of template functions you can use in `ErrorTemplateFuncs` by referring to Go's [text/template FuncMap](https://golang.org/pkg/text/template/#FuncMap) type. + +## Formats +JSON Schema allows for optional "format" property to validate instances against well-known formats. gojsonschema ships with all of the formats defined in the spec that you can use like this: +````json +{"type": "string", "format": "email"} +```` +Available formats: date-time, hostname, email, ipv4, ipv6, uri, uri-reference. + +For repetitive or more complex formats, you can create custom format checkers and add them to gojsonschema like this: + +```go +// Define the format checker +type RoleFormatChecker struct {} + +// Ensure it meets the gojsonschema.FormatChecker interface +func (f RoleFormatChecker) IsFormat(input interface{}) bool { + + asString, ok := input.(string) + if ok == false { + return false + } + + return strings.HasPrefix("ROLE_", asString) +} + +// Add it to the library +gojsonschema.FormatCheckers.Add("role", RoleFormatChecker{}) +```` + +Now to use in your json schema: +````json +{"type": "string", "format": "role"} +```` + +Another example would be to check if the provided integer matches an id on database: + +JSON schema: +```json +{"type": "integer", "format": "ValidUserId"} +``` + +```go +// Define the format checker +type ValidUserIdFormatChecker struct {} + +// Ensure it meets the gojsonschema.FormatChecker interface +func (f ValidUserIdFormatChecker) IsFormat(input interface{}) bool { + + asFloat64, ok := input.(float64) // Numbers are always float64 here + if ok == false { + return false + } + + // XXX + // do the magic on the database looking for the int(asFloat64) + + return true +} + +// Add it to the library +gojsonschema.FormatCheckers.Add("ValidUserId", ValidUserIdFormatChecker{}) +```` + + + +## Uses + +gojsonschema uses the following test suite : + +https://github.com/json-schema/JSON-Schema-Test-Suite diff --git a/vendor/github.com/xeipuuv/gojsonschema/errors.go b/vendor/github.com/xeipuuv/gojsonschema/errors.go new file mode 100644 index 00000000..d39f0195 --- /dev/null +++ b/vendor/github.com/xeipuuv/gojsonschema/errors.go @@ -0,0 +1,283 @@ +package gojsonschema + +import ( + "bytes" + "sync" + "text/template" +) + +var errorTemplates errorTemplate = errorTemplate{template.New("errors-new"), sync.RWMutex{}} + +// template.Template is not thread-safe for writing, so some locking is done +// sync.RWMutex is used for efficiently locking when new templates are created +type errorTemplate struct { + *template.Template + sync.RWMutex +} + +type ( + // RequiredError. ErrorDetails: property string + RequiredError struct { + ResultErrorFields + } + + // InvalidTypeError. ErrorDetails: expected, given + InvalidTypeError struct { + ResultErrorFields + } + + // NumberAnyOfError. ErrorDetails: - + NumberAnyOfError struct { + ResultErrorFields + } + + // NumberOneOfError. ErrorDetails: - + NumberOneOfError struct { + ResultErrorFields + } + + // NumberAllOfError. ErrorDetails: - + NumberAllOfError struct { + ResultErrorFields + } + + // NumberNotError. ErrorDetails: - + NumberNotError struct { + ResultErrorFields + } + + // MissingDependencyError. ErrorDetails: dependency + MissingDependencyError struct { + ResultErrorFields + } + + // InternalError. ErrorDetails: error + InternalError struct { + ResultErrorFields + } + + // EnumError. ErrorDetails: allowed + EnumError struct { + ResultErrorFields + } + + // ArrayNoAdditionalItemsError. ErrorDetails: - + ArrayNoAdditionalItemsError struct { + ResultErrorFields + } + + // ArrayMinItemsError. ErrorDetails: min + ArrayMinItemsError struct { + ResultErrorFields + } + + // ArrayMaxItemsError. ErrorDetails: max + ArrayMaxItemsError struct { + ResultErrorFields + } + + // ItemsMustBeUniqueError. ErrorDetails: type + ItemsMustBeUniqueError struct { + ResultErrorFields + } + + // ArrayMinPropertiesError. ErrorDetails: min + ArrayMinPropertiesError struct { + ResultErrorFields + } + + // ArrayMaxPropertiesError. ErrorDetails: max + ArrayMaxPropertiesError struct { + ResultErrorFields + } + + // AdditionalPropertyNotAllowedError. ErrorDetails: property + AdditionalPropertyNotAllowedError struct { + ResultErrorFields + } + + // InvalidPropertyPatternError. ErrorDetails: property, pattern + InvalidPropertyPatternError struct { + ResultErrorFields + } + + // StringLengthGTEError. ErrorDetails: min + StringLengthGTEError struct { + ResultErrorFields + } + + // StringLengthLTEError. ErrorDetails: max + StringLengthLTEError struct { + ResultErrorFields + } + + // DoesNotMatchPatternError. ErrorDetails: pattern + DoesNotMatchPatternError struct { + ResultErrorFields + } + + // DoesNotMatchFormatError. ErrorDetails: format + DoesNotMatchFormatError struct { + ResultErrorFields + } + + // MultipleOfError. ErrorDetails: multiple + MultipleOfError struct { + ResultErrorFields + } + + // NumberGTEError. ErrorDetails: min + NumberGTEError struct { + ResultErrorFields + } + + // NumberGTError. ErrorDetails: min + NumberGTError struct { + ResultErrorFields + } + + // NumberLTEError. ErrorDetails: max + NumberLTEError struct { + ResultErrorFields + } + + // NumberLTError. ErrorDetails: max + NumberLTError struct { + ResultErrorFields + } +) + +// newError takes a ResultError type and sets the type, context, description, details, value, and field +func newError(err ResultError, context *jsonContext, value interface{}, locale locale, details ErrorDetails) { + var t string + var d string + switch err.(type) { + case *RequiredError: + t = "required" + d = locale.Required() + case *InvalidTypeError: + t = "invalid_type" + d = locale.InvalidType() + case *NumberAnyOfError: + t = "number_any_of" + d = locale.NumberAnyOf() + case *NumberOneOfError: + t = "number_one_of" + d = locale.NumberOneOf() + case *NumberAllOfError: + t = "number_all_of" + d = locale.NumberAllOf() + case *NumberNotError: + t = "number_not" + d = locale.NumberNot() + case *MissingDependencyError: + t = "missing_dependency" + d = locale.MissingDependency() + case *InternalError: + t = "internal" + d = locale.Internal() + case *EnumError: + t = "enum" + d = locale.Enum() + case *ArrayNoAdditionalItemsError: + t = "array_no_additional_items" + d = locale.ArrayNoAdditionalItems() + case *ArrayMinItemsError: + t = "array_min_items" + d = locale.ArrayMinItems() + case *ArrayMaxItemsError: + t = "array_max_items" + d = locale.ArrayMaxItems() + case *ItemsMustBeUniqueError: + t = "unique" + d = locale.Unique() + case *ArrayMinPropertiesError: + t = "array_min_properties" + d = locale.ArrayMinProperties() + case *ArrayMaxPropertiesError: + t = "array_max_properties" + d = locale.ArrayMaxProperties() + case *AdditionalPropertyNotAllowedError: + t = "additional_property_not_allowed" + d = locale.AdditionalPropertyNotAllowed() + case *InvalidPropertyPatternError: + t = "invalid_property_pattern" + d = locale.InvalidPropertyPattern() + case *StringLengthGTEError: + t = "string_gte" + d = locale.StringGTE() + case *StringLengthLTEError: + t = "string_lte" + d = locale.StringLTE() + case *DoesNotMatchPatternError: + t = "pattern" + d = locale.DoesNotMatchPattern() + case *DoesNotMatchFormatError: + t = "format" + d = locale.DoesNotMatchFormat() + case *MultipleOfError: + t = "multiple_of" + d = locale.MultipleOf() + case *NumberGTEError: + t = "number_gte" + d = locale.NumberGTE() + case *NumberGTError: + t = "number_gt" + d = locale.NumberGT() + case *NumberLTEError: + t = "number_lte" + d = locale.NumberLTE() + case *NumberLTError: + t = "number_lt" + d = locale.NumberLT() + } + + err.SetType(t) + err.SetContext(context) + err.SetValue(value) + err.SetDetails(details) + details["field"] = err.Field() + + if _, exists := details["context"]; !exists && context != nil { + details["context"] = context.String() + } + + err.SetDescription(formatErrorDescription(d, details)) +} + +// formatErrorDescription takes a string in the default text/template +// format and converts it to a string with replacements. The fields come +// from the ErrorDetails struct and vary for each type of error. +func formatErrorDescription(s string, details ErrorDetails) string { + + var tpl *template.Template + var descrAsBuffer bytes.Buffer + var err error + + errorTemplates.RLock() + tpl = errorTemplates.Lookup(s) + errorTemplates.RUnlock() + + if tpl == nil { + errorTemplates.Lock() + tpl = errorTemplates.New(s) + + if ErrorTemplateFuncs != nil { + tpl.Funcs(ErrorTemplateFuncs) + } + + tpl, err = tpl.Parse(s) + errorTemplates.Unlock() + + if err != nil { + return err.Error() + } + } + + err = tpl.Execute(&descrAsBuffer, details) + if err != nil { + return err.Error() + } + + return descrAsBuffer.String() +} diff --git a/vendor/github.com/xeipuuv/gojsonschema/format_checkers.go b/vendor/github.com/xeipuuv/gojsonschema/format_checkers.go new file mode 100644 index 00000000..c6a07923 --- /dev/null +++ b/vendor/github.com/xeipuuv/gojsonschema/format_checkers.go @@ -0,0 +1,250 @@ +package gojsonschema + +import ( + "net" + "net/url" + "regexp" + "strings" + "time" +) + +type ( + // FormatChecker is the interface all formatters added to FormatCheckerChain must implement + FormatChecker interface { + IsFormat(input interface{}) bool + } + + // FormatCheckerChain holds the formatters + FormatCheckerChain struct { + formatters map[string]FormatChecker + } + + // EmailFormatter verifies email address formats + EmailFormatChecker struct{} + + // IPV4FormatChecker verifies IP addresses in the ipv4 format + IPV4FormatChecker struct{} + + // IPV6FormatChecker verifies IP addresses in the ipv6 format + IPV6FormatChecker struct{} + + // DateTimeFormatChecker verifies date/time formats per RFC3339 5.6 + // + // Valid formats: + // Partial Time: HH:MM:SS + // Full Date: YYYY-MM-DD + // Full Time: HH:MM:SSZ-07:00 + // Date Time: YYYY-MM-DDTHH:MM:SSZ-0700 + // + // Where + // YYYY = 4DIGIT year + // MM = 2DIGIT month ; 01-12 + // DD = 2DIGIT day-month ; 01-28, 01-29, 01-30, 01-31 based on month/year + // HH = 2DIGIT hour ; 00-23 + // MM = 2DIGIT ; 00-59 + // SS = 2DIGIT ; 00-58, 00-60 based on leap second rules + // T = Literal + // Z = Literal + // + // Note: Nanoseconds are also suported in all formats + // + // http://tools.ietf.org/html/rfc3339#section-5.6 + DateTimeFormatChecker struct{} + + // URIFormatChecker validates a URI with a valid Scheme per RFC3986 + URIFormatChecker struct{} + + // URIReferenceFormatChecker validates a URI or relative-reference per RFC3986 + URIReferenceFormatChecker struct{} + + // HostnameFormatChecker validates a hostname is in the correct format + HostnameFormatChecker struct{} + + // UUIDFormatChecker validates a UUID is in the correct format + UUIDFormatChecker struct{} + + // RegexFormatChecker validates a regex is in the correct format + RegexFormatChecker struct{} +) + +var ( + // Formatters holds the valid formatters, and is a public variable + // so library users can add custom formatters + FormatCheckers = FormatCheckerChain{ + formatters: map[string]FormatChecker{ + "date-time": DateTimeFormatChecker{}, + "hostname": HostnameFormatChecker{}, + "email": EmailFormatChecker{}, + "ipv4": IPV4FormatChecker{}, + "ipv6": IPV6FormatChecker{}, + "uri": URIFormatChecker{}, + "uri-reference": URIReferenceFormatChecker{}, + "uuid": UUIDFormatChecker{}, + "regex": RegexFormatChecker{}, + }, + } + + // Regex credit: https://github.com/asaskevich/govalidator + rxEmail = regexp.MustCompile("^(((([a-zA-Z]|\\d|[!#\\$%&'\\*\\+\\-\\/=\\?\\^_`{\\|}~]|[\\x{00A0}-\\x{D7FF}\\x{F900}-\\x{FDCF}\\x{FDF0}-\\x{FFEF}])+(\\.([a-zA-Z]|\\d|[!#\\$%&'\\*\\+\\-\\/=\\?\\^_`{\\|}~]|[\\x{00A0}-\\x{D7FF}\\x{F900}-\\x{FDCF}\\x{FDF0}-\\x{FFEF}])+)*)|((\\x22)((((\\x20|\\x09)*(\\x0d\\x0a))?(\\x20|\\x09)+)?(([\\x01-\\x08\\x0b\\x0c\\x0e-\\x1f\\x7f]|\\x21|[\\x23-\\x5b]|[\\x5d-\\x7e]|[\\x{00A0}-\\x{D7FF}\\x{F900}-\\x{FDCF}\\x{FDF0}-\\x{FFEF}])|(\\([\\x01-\\x09\\x0b\\x0c\\x0d-\\x7f]|[\\x{00A0}-\\x{D7FF}\\x{F900}-\\x{FDCF}\\x{FDF0}-\\x{FFEF}]))))*(((\\x20|\\x09)*(\\x0d\\x0a))?(\\x20|\\x09)+)?(\\x22)))@((([a-zA-Z]|\\d|[\\x{00A0}-\\x{D7FF}\\x{F900}-\\x{FDCF}\\x{FDF0}-\\x{FFEF}])|(([a-zA-Z]|\\d|[\\x{00A0}-\\x{D7FF}\\x{F900}-\\x{FDCF}\\x{FDF0}-\\x{FFEF}])([a-zA-Z]|\\d|-|\\.|_|~|[\\x{00A0}-\\x{D7FF}\\x{F900}-\\x{FDCF}\\x{FDF0}-\\x{FFEF}])*([a-zA-Z]|\\d|[\\x{00A0}-\\x{D7FF}\\x{F900}-\\x{FDCF}\\x{FDF0}-\\x{FFEF}])))\\.)+(([a-zA-Z]|[\\x{00A0}-\\x{D7FF}\\x{F900}-\\x{FDCF}\\x{FDF0}-\\x{FFEF}])|(([a-zA-Z]|[\\x{00A0}-\\x{D7FF}\\x{F900}-\\x{FDCF}\\x{FDF0}-\\x{FFEF}])([a-zA-Z]|\\d|-|\\.|_|~|[\\x{00A0}-\\x{D7FF}\\x{F900}-\\x{FDCF}\\x{FDF0}-\\x{FFEF}])*([a-zA-Z]|[\\x{00A0}-\\x{D7FF}\\x{F900}-\\x{FDCF}\\x{FDF0}-\\x{FFEF}])))\\.?$") + + // Regex credit: https://www.socketloop.com/tutorials/golang-validate-hostname + rxHostname = regexp.MustCompile(`^([a-zA-Z0-9]|[a-zA-Z0-9][a-zA-Z0-9\-]{0,61}[a-zA-Z0-9])(\.([a-zA-Z0-9]|[a-zA-Z0-9][a-zA-Z0-9\-]{0,61}[a-zA-Z0-9]))*$`) + + rxUUID = regexp.MustCompile("^[a-f0-9]{8}-[a-f0-9]{4}-[a-f0-9]{4}-[a-f0-9]{4}-[a-f0-9]{12}$") +) + +// Add adds a FormatChecker to the FormatCheckerChain +// The name used will be the value used for the format key in your json schema +func (c *FormatCheckerChain) Add(name string, f FormatChecker) *FormatCheckerChain { + c.formatters[name] = f + + return c +} + +// Remove deletes a FormatChecker from the FormatCheckerChain (if it exists) +func (c *FormatCheckerChain) Remove(name string) *FormatCheckerChain { + delete(c.formatters, name) + + return c +} + +// Has checks to see if the FormatCheckerChain holds a FormatChecker with the given name +func (c *FormatCheckerChain) Has(name string) bool { + _, ok := c.formatters[name] + + return ok +} + +// IsFormat will check an input against a FormatChecker with the given name +// to see if it is the correct format +func (c *FormatCheckerChain) IsFormat(name string, input interface{}) bool { + f, ok := c.formatters[name] + + if !ok { + return false + } + + return f.IsFormat(input) +} + +func (f EmailFormatChecker) IsFormat(input interface{}) bool { + + asString, ok := input.(string) + if ok == false { + return false + } + + return rxEmail.MatchString(asString) +} + +// Credit: https://github.com/asaskevich/govalidator +func (f IPV4FormatChecker) IsFormat(input interface{}) bool { + + asString, ok := input.(string) + if ok == false { + return false + } + + ip := net.ParseIP(asString) + return ip != nil && strings.Contains(asString, ".") +} + +// Credit: https://github.com/asaskevich/govalidator +func (f IPV6FormatChecker) IsFormat(input interface{}) bool { + + asString, ok := input.(string) + if ok == false { + return false + } + + ip := net.ParseIP(asString) + return ip != nil && strings.Contains(asString, ":") +} + +func (f DateTimeFormatChecker) IsFormat(input interface{}) bool { + + asString, ok := input.(string) + if ok == false { + return false + } + + formats := []string{ + "15:04:05", + "15:04:05Z07:00", + "2006-01-02", + time.RFC3339, + time.RFC3339Nano, + } + + for _, format := range formats { + if _, err := time.Parse(format, asString); err == nil { + return true + } + } + + return false +} + +func (f URIFormatChecker) IsFormat(input interface{}) bool { + + asString, ok := input.(string) + if ok == false { + return false + } + + u, err := url.Parse(asString) + if err != nil || u.Scheme == "" { + return false + } + + return true +} + +func (f URIReferenceFormatChecker) IsFormat(input interface{}) bool { + + asString, ok := input.(string) + if ok == false { + return false + } + + _, err := url.Parse(asString) + return err == nil +} + +func (f HostnameFormatChecker) IsFormat(input interface{}) bool { + + asString, ok := input.(string) + if ok == false { + return false + } + + return rxHostname.MatchString(asString) && len(asString) < 256 +} + +func (f UUIDFormatChecker) IsFormat(input interface{}) bool { + + asString, ok := input.(string) + if ok == false { + return false + } + + return rxUUID.MatchString(asString) +} + +// IsFormat implements FormatChecker interface. +func (f RegexFormatChecker) IsFormat(input interface{}) bool { + + asString, ok := input.(string) + if ok == false { + return false + } + + if asString == "" { + return true + } + _, err := regexp.Compile(asString) + if err != nil { + return false + } + return true +} diff --git a/vendor/github.com/xeipuuv/gojsonschema/internalLog.go b/vendor/github.com/xeipuuv/gojsonschema/internalLog.go new file mode 100644 index 00000000..4ef7a8d0 --- /dev/null +++ b/vendor/github.com/xeipuuv/gojsonschema/internalLog.go @@ -0,0 +1,37 @@ +// Copyright 2015 xeipuuv ( https://github.com/xeipuuv ) +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// author xeipuuv +// author-github https://github.com/xeipuuv +// author-mail xeipuuv@gmail.com +// +// repository-name gojsonschema +// repository-desc An implementation of JSON Schema, based on IETF's draft v4 - Go language. +// +// description Very simple log wrapper. +// Used for debugging/testing purposes. +// +// created 01-01-2015 + +package gojsonschema + +import ( + "log" +) + +const internalLogEnabled = false + +func internalLog(format string, v ...interface{}) { + log.Printf(format, v...) +} diff --git a/vendor/github.com/xeipuuv/gojsonschema/jsonContext.go b/vendor/github.com/xeipuuv/gojsonschema/jsonContext.go new file mode 100644 index 00000000..fcc8d9d6 --- /dev/null +++ b/vendor/github.com/xeipuuv/gojsonschema/jsonContext.go @@ -0,0 +1,72 @@ +// Copyright 2013 MongoDB, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// author tolsen +// author-github https://github.com/tolsen +// +// repository-name gojsonschema +// repository-desc An implementation of JSON Schema, based on IETF's draft v4 - Go language. +// +// description Implements a persistent (immutable w/ shared structure) singly-linked list of strings for the purpose of storing a json context +// +// created 04-09-2013 + +package gojsonschema + +import "bytes" + +// jsonContext implements a persistent linked-list of strings +type jsonContext struct { + head string + tail *jsonContext +} + +func newJsonContext(head string, tail *jsonContext) *jsonContext { + return &jsonContext{head, tail} +} + +// String displays the context in reverse. +// This plays well with the data structure's persistent nature with +// Cons and a json document's tree structure. +func (c *jsonContext) String(del ...string) string { + byteArr := make([]byte, 0, c.stringLen()) + buf := bytes.NewBuffer(byteArr) + c.writeStringToBuffer(buf, del) + + return buf.String() +} + +func (c *jsonContext) stringLen() int { + length := 0 + if c.tail != nil { + length = c.tail.stringLen() + 1 // add 1 for "." + } + + length += len(c.head) + return length +} + +func (c *jsonContext) writeStringToBuffer(buf *bytes.Buffer, del []string) { + if c.tail != nil { + c.tail.writeStringToBuffer(buf, del) + + if len(del) > 0 { + buf.WriteString(del[0]) + } else { + buf.WriteString(".") + } + } + + buf.WriteString(c.head) +} diff --git a/vendor/github.com/xeipuuv/gojsonschema/jsonLoader.go b/vendor/github.com/xeipuuv/gojsonschema/jsonLoader.go new file mode 100644 index 00000000..a77a81e4 --- /dev/null +++ b/vendor/github.com/xeipuuv/gojsonschema/jsonLoader.go @@ -0,0 +1,341 @@ +// Copyright 2015 xeipuuv ( https://github.com/xeipuuv ) +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// author xeipuuv +// author-github https://github.com/xeipuuv +// author-mail xeipuuv@gmail.com +// +// repository-name gojsonschema +// repository-desc An implementation of JSON Schema, based on IETF's draft v4 - Go language. +// +// description Different strategies to load JSON files. +// Includes References (file and HTTP), JSON strings and Go types. +// +// created 01-02-2015 + +package gojsonschema + +import ( + "bytes" + "encoding/json" + "errors" + "io" + "io/ioutil" + "net/http" + "os" + "path/filepath" + "runtime" + "strings" + + + "github.com/xeipuuv/gojsonreference" +) + +var osFS = osFileSystem(os.Open) + +// JSON loader interface + +type JSONLoader interface { + JsonSource() interface{} + LoadJSON() (interface{}, error) + JsonReference() (gojsonreference.JsonReference, error) + LoaderFactory() JSONLoaderFactory +} + +type JSONLoaderFactory interface { + New(source string) JSONLoader +} + +type DefaultJSONLoaderFactory struct { +} + +type FileSystemJSONLoaderFactory struct { + fs http.FileSystem +} + +func (d DefaultJSONLoaderFactory) New(source string) JSONLoader { + return &jsonReferenceLoader{ + fs: osFS, + source: source, + } +} + +func (f FileSystemJSONLoaderFactory) New(source string) JSONLoader { + return &jsonReferenceLoader{ + fs: f.fs, + source: source, + } +} + +// osFileSystem is a functional wrapper for os.Open that implements http.FileSystem. +type osFileSystem func(string) (*os.File, error) + +func (o osFileSystem) Open(name string) (http.File, error) { + return o(name) +} + +// JSON Reference loader +// references are used to load JSONs from files and HTTP + +type jsonReferenceLoader struct { + fs http.FileSystem + source string +} + +func (l *jsonReferenceLoader) JsonSource() interface{} { + return l.source +} + +func (l *jsonReferenceLoader) JsonReference() (gojsonreference.JsonReference, error) { + return gojsonreference.NewJsonReference(l.JsonSource().(string)) +} + +func (l *jsonReferenceLoader) LoaderFactory() JSONLoaderFactory { + return &FileSystemJSONLoaderFactory{ + fs: l.fs, + } +} + +// NewReferenceLoader returns a JSON reference loader using the given source and the local OS file system. +func NewReferenceLoader(source string) *jsonReferenceLoader { + return &jsonReferenceLoader{ + fs: osFS, + source: source, + } +} + +// NewReferenceLoaderFileSystem returns a JSON reference loader using the given source and file system. +func NewReferenceLoaderFileSystem(source string, fs http.FileSystem) *jsonReferenceLoader { + return &jsonReferenceLoader{ + fs: fs, + source: source, + } +} + +func (l *jsonReferenceLoader) LoadJSON() (interface{}, error) { + + var err error + + reference, err := gojsonreference.NewJsonReference(l.JsonSource().(string)) + if err != nil { + return nil, err + } + + refToUrl := reference + refToUrl.GetUrl().Fragment = "" + + var document interface{} + + if reference.HasFileScheme { + + filename := strings.Replace(refToUrl.GetUrl().Path, "file://", "", -1) + if runtime.GOOS == "windows" { + // on Windows, a file URL may have an extra leading slash, use slashes + // instead of backslashes, and have spaces escaped + if strings.HasPrefix(filename, "/") { + filename = filename[1:] + } + filename = filepath.FromSlash(filename) + } + + document, err = l.loadFromFile(filename) + if err != nil { + return nil, err + } + + } else { + + document, err = l.loadFromHTTP(refToUrl.String()) + if err != nil { + return nil, err + } + + } + + return document, nil + +} + +func (l *jsonReferenceLoader) loadFromHTTP(address string) (interface{}, error) { + + resp, err := http.Get(address) + if err != nil { + return nil, err + } + + // must return HTTP Status 200 OK + if resp.StatusCode != http.StatusOK { + return nil, errors.New(formatErrorDescription(Locale.HttpBadStatus(), ErrorDetails{"status": resp.Status})) + } + + bodyBuff, err := ioutil.ReadAll(resp.Body) + if err != nil { + return nil, err + } + + return decodeJsonUsingNumber(bytes.NewReader(bodyBuff)) + +} + +func (l *jsonReferenceLoader) loadFromFile(path string) (interface{}, error) { + f, err := l.fs.Open(path) + if err != nil { + return nil, err + } + defer f.Close() + + bodyBuff, err := ioutil.ReadAll(f) + if err != nil { + return nil, err + } + + return decodeJsonUsingNumber(bytes.NewReader(bodyBuff)) + +} + +// JSON string loader + +type jsonStringLoader struct { + source string +} + +func (l *jsonStringLoader) JsonSource() interface{} { + return l.source +} + +func (l *jsonStringLoader) JsonReference() (gojsonreference.JsonReference, error) { + return gojsonreference.NewJsonReference("#") +} + +func (l *jsonStringLoader) LoaderFactory() JSONLoaderFactory { + return &DefaultJSONLoaderFactory{} +} + +func NewStringLoader(source string) *jsonStringLoader { + return &jsonStringLoader{source: source} +} + +func (l *jsonStringLoader) LoadJSON() (interface{}, error) { + + return decodeJsonUsingNumber(strings.NewReader(l.JsonSource().(string))) + +} + +// JSON bytes loader + +type jsonBytesLoader struct { + source []byte +} + +func (l *jsonBytesLoader) JsonSource() interface{} { + return l.source +} + +func (l *jsonBytesLoader) JsonReference() (gojsonreference.JsonReference, error) { + return gojsonreference.NewJsonReference("#") +} + +func (l *jsonBytesLoader) LoaderFactory() JSONLoaderFactory { + return &DefaultJSONLoaderFactory{} +} + +func NewBytesLoader(source []byte) *jsonBytesLoader { + return &jsonBytesLoader{source: source} +} + +func (l *jsonBytesLoader) LoadJSON() (interface{}, error) { + return decodeJsonUsingNumber(bytes.NewReader(l.JsonSource().([]byte))) +} + +// JSON Go (types) loader +// used to load JSONs from the code as maps, interface{}, structs ... + +type jsonGoLoader struct { + source interface{} +} + +func (l *jsonGoLoader) JsonSource() interface{} { + return l.source +} + +func (l *jsonGoLoader) JsonReference() (gojsonreference.JsonReference, error) { + return gojsonreference.NewJsonReference("#") +} + +func (l *jsonGoLoader) LoaderFactory() JSONLoaderFactory { + return &DefaultJSONLoaderFactory{} +} + +func NewGoLoader(source interface{}) *jsonGoLoader { + return &jsonGoLoader{source: source} +} + +func (l *jsonGoLoader) LoadJSON() (interface{}, error) { + + // convert it to a compliant JSON first to avoid types "mismatches" + + jsonBytes, err := json.Marshal(l.JsonSource()) + if err != nil { + return nil, err + } + + return decodeJsonUsingNumber(bytes.NewReader(jsonBytes)) + +} + +type jsonIOLoader struct { + buf *bytes.Buffer +} + +func NewReaderLoader(source io.Reader) (*jsonIOLoader, io.Reader) { + buf := &bytes.Buffer{} + return &jsonIOLoader{buf: buf}, io.TeeReader(source, buf) +} + +func NewWriterLoader(source io.Writer) (*jsonIOLoader, io.Writer) { + buf := &bytes.Buffer{} + return &jsonIOLoader{buf: buf}, io.MultiWriter(source, buf) +} + +func (l *jsonIOLoader) JsonSource() interface{} { + return l.buf.String() +} + +func (l *jsonIOLoader) LoadJSON() (interface{}, error) { + return decodeJsonUsingNumber(l.buf) +} + +func (l *jsonIOLoader) JsonReference() (gojsonreference.JsonReference, error) { + return gojsonreference.NewJsonReference("#") +} + +func (l *jsonIOLoader) LoaderFactory() JSONLoaderFactory { + return &DefaultJSONLoaderFactory{} +} + +func decodeJsonUsingNumber(r io.Reader) (interface{}, error) { + + var document interface{} + + decoder := json.NewDecoder(r) + decoder.UseNumber() + + err := decoder.Decode(&document) + if err != nil { + return nil, err + } + + return document, nil + +} diff --git a/vendor/github.com/xeipuuv/gojsonschema/locales.go b/vendor/github.com/xeipuuv/gojsonschema/locales.go new file mode 100644 index 00000000..9446f848 --- /dev/null +++ b/vendor/github.com/xeipuuv/gojsonschema/locales.go @@ -0,0 +1,286 @@ +// Copyright 2015 xeipuuv ( https://github.com/xeipuuv ) +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// author xeipuuv +// author-github https://github.com/xeipuuv +// author-mail xeipuuv@gmail.com +// +// repository-name gojsonschema +// repository-desc An implementation of JSON Schema, based on IETF's draft v4 - Go language. +// +// description Contains const string and messages. +// +// created 01-01-2015 + +package gojsonschema + +type ( + // locale is an interface for defining custom error strings + locale interface { + Required() string + InvalidType() string + NumberAnyOf() string + NumberOneOf() string + NumberAllOf() string + NumberNot() string + MissingDependency() string + Internal() string + Enum() string + ArrayNotEnoughItems() string + ArrayNoAdditionalItems() string + ArrayMinItems() string + ArrayMaxItems() string + Unique() string + ArrayMinProperties() string + ArrayMaxProperties() string + AdditionalPropertyNotAllowed() string + InvalidPropertyPattern() string + StringGTE() string + StringLTE() string + DoesNotMatchPattern() string + DoesNotMatchFormat() string + MultipleOf() string + NumberGTE() string + NumberGT() string + NumberLTE() string + NumberLT() string + + // Schema validations + RegexPattern() string + GreaterThanZero() string + MustBeOfA() string + MustBeOfAn() string + CannotBeUsedWithout() string + CannotBeGT() string + MustBeOfType() string + MustBeValidRegex() string + MustBeValidFormat() string + MustBeGTEZero() string + KeyCannotBeGreaterThan() string + KeyItemsMustBeOfType() string + KeyItemsMustBeUnique() string + ReferenceMustBeCanonical() string + NotAValidType() string + Duplicated() string + HttpBadStatus() string + ParseError() string + + // ErrorFormat + ErrorFormat() string + } + + // DefaultLocale is the default locale for this package + DefaultLocale struct{} +) + +func (l DefaultLocale) Required() string { + return `{{.property}} is required` +} + +func (l DefaultLocale) InvalidType() string { + return `Invalid type. Expected: {{.expected}}, given: {{.given}}` +} + +func (l DefaultLocale) NumberAnyOf() string { + return `Must validate at least one schema (anyOf)` +} + +func (l DefaultLocale) NumberOneOf() string { + return `Must validate one and only one schema (oneOf)` +} + +func (l DefaultLocale) NumberAllOf() string { + return `Must validate all the schemas (allOf)` +} + +func (l DefaultLocale) NumberNot() string { + return `Must not validate the schema (not)` +} + +func (l DefaultLocale) MissingDependency() string { + return `Has a dependency on {{.dependency}}` +} + +func (l DefaultLocale) Internal() string { + return `Internal Error {{.error}}` +} + +func (l DefaultLocale) Enum() string { + return `{{.field}} must be one of the following: {{.allowed}}` +} + +func (l DefaultLocale) ArrayNoAdditionalItems() string { + return `No additional items allowed on array` +} + +func (l DefaultLocale) ArrayNotEnoughItems() string { + return `Not enough items on array to match positional list of schema` +} + +func (l DefaultLocale) ArrayMinItems() string { + return `Array must have at least {{.min}} items` +} + +func (l DefaultLocale) ArrayMaxItems() string { + return `Array must have at most {{.max}} items` +} + +func (l DefaultLocale) Unique() string { + return `{{.type}} items must be unique` +} + +func (l DefaultLocale) ArrayMinProperties() string { + return `Must have at least {{.min}} properties` +} + +func (l DefaultLocale) ArrayMaxProperties() string { + return `Must have at most {{.max}} properties` +} + +func (l DefaultLocale) AdditionalPropertyNotAllowed() string { + return `Additional property {{.property}} is not allowed` +} + +func (l DefaultLocale) InvalidPropertyPattern() string { + return `Property "{{.property}}" does not match pattern {{.pattern}}` +} + +func (l DefaultLocale) StringGTE() string { + return `String length must be greater than or equal to {{.min}}` +} + +func (l DefaultLocale) StringLTE() string { + return `String length must be less than or equal to {{.max}}` +} + +func (l DefaultLocale) DoesNotMatchPattern() string { + return `Does not match pattern '{{.pattern}}'` +} + +func (l DefaultLocale) DoesNotMatchFormat() string { + return `Does not match format '{{.format}}'` +} + +func (l DefaultLocale) MultipleOf() string { + return `Must be a multiple of {{.multiple}}` +} + +func (l DefaultLocale) NumberGTE() string { + return `Must be greater than or equal to {{.min}}` +} + +func (l DefaultLocale) NumberGT() string { + return `Must be greater than {{.min}}` +} + +func (l DefaultLocale) NumberLTE() string { + return `Must be less than or equal to {{.max}}` +} + +func (l DefaultLocale) NumberLT() string { + return `Must be less than {{.max}}` +} + +// Schema validators +func (l DefaultLocale) RegexPattern() string { + return `Invalid regex pattern '{{.pattern}}'` +} + +func (l DefaultLocale) GreaterThanZero() string { + return `{{.number}} must be strictly greater than 0` +} + +func (l DefaultLocale) MustBeOfA() string { + return `{{.x}} must be of a {{.y}}` +} + +func (l DefaultLocale) MustBeOfAn() string { + return `{{.x}} must be of an {{.y}}` +} + +func (l DefaultLocale) CannotBeUsedWithout() string { + return `{{.x}} cannot be used without {{.y}}` +} + +func (l DefaultLocale) CannotBeGT() string { + return `{{.x}} cannot be greater than {{.y}}` +} + +func (l DefaultLocale) MustBeOfType() string { + return `{{.key}} must be of type {{.type}}` +} + +func (l DefaultLocale) MustBeValidRegex() string { + return `{{.key}} must be a valid regex` +} + +func (l DefaultLocale) MustBeValidFormat() string { + return `{{.key}} must be a valid format {{.given}}` +} + +func (l DefaultLocale) MustBeGTEZero() string { + return `{{.key}} must be greater than or equal to 0` +} + +func (l DefaultLocale) KeyCannotBeGreaterThan() string { + return `{{.key}} cannot be greater than {{.y}}` +} + +func (l DefaultLocale) KeyItemsMustBeOfType() string { + return `{{.key}} items must be {{.type}}` +} + +func (l DefaultLocale) KeyItemsMustBeUnique() string { + return `{{.key}} items must be unique` +} + +func (l DefaultLocale) ReferenceMustBeCanonical() string { + return `Reference {{.reference}} must be canonical` +} + +func (l DefaultLocale) NotAValidType() string { + return `has a primitive type that is NOT VALID -- given: {{.given}} Expected valid values are:{{.expected}}` +} + +func (l DefaultLocale) Duplicated() string { + return `{{.type}} type is duplicated` +} + +func (l DefaultLocale) HttpBadStatus() string { + return `Could not read schema from HTTP, response status is {{.status}}` +} + +// Replacement options: field, description, context, value +func (l DefaultLocale) ErrorFormat() string { + return `{{.field}}: {{.description}}` +} + +//Parse error +func (l DefaultLocale) ParseError() string { + return `Expected: {{.expected}}, given: Invalid JSON` +} + +const ( + STRING_NUMBER = "number" + STRING_ARRAY_OF_STRINGS = "array of strings" + STRING_ARRAY_OF_SCHEMAS = "array of schemas" + STRING_SCHEMA = "valid schema" + STRING_SCHEMA_OR_ARRAY_OF_STRINGS = "schema or array of strings" + STRING_PROPERTIES = "properties" + STRING_DEPENDENCY = "dependency" + STRING_PROPERTY = "property" + STRING_UNDEFINED = "undefined" + STRING_CONTEXT_ROOT = "(root)" + STRING_ROOT_SCHEMA_PROPERTY = "(root)" +) diff --git a/vendor/github.com/xeipuuv/gojsonschema/result.go b/vendor/github.com/xeipuuv/gojsonschema/result.go new file mode 100644 index 00000000..6ad56ae8 --- /dev/null +++ b/vendor/github.com/xeipuuv/gojsonschema/result.go @@ -0,0 +1,172 @@ +// Copyright 2015 xeipuuv ( https://github.com/xeipuuv ) +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// author xeipuuv +// author-github https://github.com/xeipuuv +// author-mail xeipuuv@gmail.com +// +// repository-name gojsonschema +// repository-desc An implementation of JSON Schema, based on IETF's draft v4 - Go language. +// +// description Result and ResultError implementations. +// +// created 01-01-2015 + +package gojsonschema + +import ( + "fmt" + "strings" +) + +type ( + // ErrorDetails is a map of details specific to each error. + // While the values will vary, every error will contain a "field" value + ErrorDetails map[string]interface{} + + // ResultError is the interface that library errors must implement + ResultError interface { + Field() string + SetType(string) + Type() string + SetContext(*jsonContext) + Context() *jsonContext + SetDescription(string) + Description() string + SetValue(interface{}) + Value() interface{} + SetDetails(ErrorDetails) + Details() ErrorDetails + String() string + } + + // ResultErrorFields holds the fields for each ResultError implementation. + // ResultErrorFields implements the ResultError interface, so custom errors + // can be defined by just embedding this type + ResultErrorFields struct { + errorType string // A string with the type of error (i.e. invalid_type) + context *jsonContext // Tree like notation of the part that failed the validation. ex (root).a.b ... + description string // A human readable error message + value interface{} // Value given by the JSON file that is the source of the error + details ErrorDetails + } + + Result struct { + errors []ResultError + // Scores how well the validation matched. Useful in generating + // better error messages for anyOf and oneOf. + score int + } +) + +// Field outputs the field name without the root context +// i.e. firstName or person.firstName instead of (root).firstName or (root).person.firstName +func (v *ResultErrorFields) Field() string { + if p, ok := v.Details()["property"]; ok { + if str, isString := p.(string); isString { + return str + } + } + + return strings.TrimPrefix(v.context.String(), STRING_ROOT_SCHEMA_PROPERTY+".") +} + +func (v *ResultErrorFields) SetType(errorType string) { + v.errorType = errorType +} + +func (v *ResultErrorFields) Type() string { + return v.errorType +} + +func (v *ResultErrorFields) SetContext(context *jsonContext) { + v.context = context +} + +func (v *ResultErrorFields) Context() *jsonContext { + return v.context +} + +func (v *ResultErrorFields) SetDescription(description string) { + v.description = description +} + +func (v *ResultErrorFields) Description() string { + return v.description +} + +func (v *ResultErrorFields) SetValue(value interface{}) { + v.value = value +} + +func (v *ResultErrorFields) Value() interface{} { + return v.value +} + +func (v *ResultErrorFields) SetDetails(details ErrorDetails) { + v.details = details +} + +func (v *ResultErrorFields) Details() ErrorDetails { + return v.details +} + +func (v ResultErrorFields) String() string { + // as a fallback, the value is displayed go style + valueString := fmt.Sprintf("%v", v.value) + + // marshal the go value value to json + if v.value == nil { + valueString = TYPE_NULL + } else { + if vs, err := marshalToJsonString(v.value); err == nil { + if vs == nil { + valueString = TYPE_NULL + } else { + valueString = *vs + } + } + } + + return formatErrorDescription(Locale.ErrorFormat(), ErrorDetails{ + "context": v.context.String(), + "description": v.description, + "value": valueString, + "field": v.Field(), + }) +} + +func (v *Result) Valid() bool { + return len(v.errors) == 0 +} + +func (v *Result) Errors() []ResultError { + return v.errors +} + +func (v *Result) addError(err ResultError, context *jsonContext, value interface{}, details ErrorDetails) { + newError(err, context, value, Locale, details) + v.errors = append(v.errors, err) + v.score -= 2 // results in a net -1 when added to the +1 we get at the end of the validation function +} + +// Used to copy errors from a sub-schema to the main one +func (v *Result) mergeErrors(otherResult *Result) { + v.errors = append(v.errors, otherResult.Errors()...) + v.score += otherResult.score +} + +func (v *Result) incrementScore() { + v.score++ +} diff --git a/vendor/github.com/xeipuuv/gojsonschema/schema.go b/vendor/github.com/xeipuuv/gojsonschema/schema.go new file mode 100644 index 00000000..f1fbde38 --- /dev/null +++ b/vendor/github.com/xeipuuv/gojsonschema/schema.go @@ -0,0 +1,971 @@ +// Copyright 2015 xeipuuv ( https://github.com/xeipuuv ) +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// author xeipuuv +// author-github https://github.com/xeipuuv +// author-mail xeipuuv@gmail.com +// +// repository-name gojsonschema +// repository-desc An implementation of JSON Schema, based on IETF's draft v4 - Go language. +// +// description Defines Schema, the main entry to every subSchema. +// Contains the parsing logic and error checking. +// +// created 26-02-2013 + +package gojsonschema + +import ( + "errors" + "reflect" + "regexp" + "text/template" + + "github.com/xeipuuv/gojsonreference" +) + +var ( + // Locale is the default locale to use + // Library users can overwrite with their own implementation + Locale locale = DefaultLocale{} + + // ErrorTemplateFuncs allows you to define custom template funcs for use in localization. + ErrorTemplateFuncs template.FuncMap +) + +func NewSchema(l JSONLoader) (*Schema, error) { + ref, err := l.JsonReference() + if err != nil { + return nil, err + } + + d := Schema{} + d.pool = newSchemaPool(l.LoaderFactory()) + d.documentReference = ref + d.referencePool = newSchemaReferencePool() + + var spd *schemaPoolDocument + var doc interface{} + if ref.String() != "" { + // Get document from schema pool + spd, err = d.pool.GetDocument(d.documentReference) + if err != nil { + return nil, err + } + doc = spd.Document + + // Deal with fragment pointers + jsonPointer := ref.GetPointer() + doc, _, err = jsonPointer.Get(doc) + if err != nil { + return nil, err + } + } else { + // Load JSON directly + doc, err = l.LoadJSON() + if err != nil { + return nil, err + } + } + d.pool.SetStandaloneDocument(doc) + + err = d.parse(doc) + if err != nil { + return nil, err + } + + return &d, nil +} + +type Schema struct { + documentReference gojsonreference.JsonReference + rootSchema *subSchema + pool *schemaPool + referencePool *schemaReferencePool +} + +func (d *Schema) parse(document interface{}) error { + d.rootSchema = &subSchema{property: STRING_ROOT_SCHEMA_PROPERTY} + return d.parseSchema(document, d.rootSchema) +} + +func (d *Schema) SetRootSchemaName(name string) { + d.rootSchema.property = name +} + +// Parses a subSchema +// +// Pretty long function ( sorry :) )... but pretty straight forward, repetitive and boring +// Not much magic involved here, most of the job is to validate the key names and their values, +// then the values are copied into subSchema struct +// +func (d *Schema) parseSchema(documentNode interface{}, currentSchema *subSchema) error { + + if !isKind(documentNode, reflect.Map) { + return errors.New(formatErrorDescription( + Locale.ParseError(), + ErrorDetails{ + "expected": STRING_SCHEMA, + }, + )) + } + if currentSchema.parent == nil { + currentSchema.ref = &d.documentReference + currentSchema.id = &d.documentReference + } + + if currentSchema.id == nil && currentSchema.parent != nil { + currentSchema.id = currentSchema.parent.id + } + + m := documentNode.(map[string]interface{}) + + // id + if existsMapKey(m, KEY_ID) && !isKind(m[KEY_ID], reflect.String) { + return errors.New(formatErrorDescription( + Locale.InvalidType(), + ErrorDetails{ + "expected": TYPE_STRING, + "given": KEY_ID, + }, + )) + } + if k, ok := m[KEY_ID].(string); ok { + jsonReference, err := gojsonreference.NewJsonReference(k) + if err != nil { + return err + } + if currentSchema == d.rootSchema { + currentSchema.id = &jsonReference + } else { + ref, err := currentSchema.parent.id.Inherits(jsonReference) + if err != nil { + return err + } + currentSchema.id = ref + } + } + + // Add schema to document cache. The same id is passed down to subsequent + // subschemas, but as only the first and top one is used it will always reference + // the correct schema. Doing it once here prevents having + // to do this same step at every corner case. + d.referencePool.Add(currentSchema.id.String(), currentSchema) + + // $subSchema + if existsMapKey(m, KEY_SCHEMA) { + if !isKind(m[KEY_SCHEMA], reflect.String) { + return errors.New(formatErrorDescription( + Locale.InvalidType(), + ErrorDetails{ + "expected": TYPE_STRING, + "given": KEY_SCHEMA, + }, + )) + } + schemaRef := m[KEY_SCHEMA].(string) + schemaReference, err := gojsonreference.NewJsonReference(schemaRef) + currentSchema.subSchema = &schemaReference + if err != nil { + return err + } + } + + // $ref + if existsMapKey(m, KEY_REF) && !isKind(m[KEY_REF], reflect.String) { + return errors.New(formatErrorDescription( + Locale.InvalidType(), + ErrorDetails{ + "expected": TYPE_STRING, + "given": KEY_REF, + }, + )) + } + if k, ok := m[KEY_REF].(string); ok { + + jsonReference, err := gojsonreference.NewJsonReference(k) + if err != nil { + return err + } + + if jsonReference.HasFullUrl { + currentSchema.ref = &jsonReference + } else { + inheritedReference, err := currentSchema.id.Inherits(jsonReference) + if err != nil { + return err + } + currentSchema.ref = inheritedReference + } + if sch, ok := d.referencePool.Get(currentSchema.ref.String()); ok { + currentSchema.refSchema = sch + } else { + err := d.parseReference(documentNode, currentSchema) + + if err != nil { + return err + } + + return nil + } + } + + // definitions + if existsMapKey(m, KEY_DEFINITIONS) { + if isKind(m[KEY_DEFINITIONS], reflect.Map) { + currentSchema.definitions = make(map[string]*subSchema) + for dk, dv := range m[KEY_DEFINITIONS].(map[string]interface{}) { + if isKind(dv, reflect.Map) { + + ref, err := gojsonreference.NewJsonReference("#/" + KEY_DEFINITIONS + "/" + dk) + if err != nil { + return err + } + + newSchemaID, err := currentSchema.id.Inherits(ref) + if err != nil { + return err + } + newSchema := &subSchema{property: KEY_DEFINITIONS, parent: currentSchema, id: newSchemaID} + currentSchema.definitions[dk] = newSchema + + err = d.parseSchema(dv, newSchema) + + if err != nil { + return err + } + } else { + return errors.New(formatErrorDescription( + Locale.InvalidType(), + ErrorDetails{ + "expected": STRING_ARRAY_OF_SCHEMAS, + "given": KEY_DEFINITIONS, + }, + )) + } + } + } else { + return errors.New(formatErrorDescription( + Locale.InvalidType(), + ErrorDetails{ + "expected": STRING_ARRAY_OF_SCHEMAS, + "given": KEY_DEFINITIONS, + }, + )) + } + + } + + // title + if existsMapKey(m, KEY_TITLE) && !isKind(m[KEY_TITLE], reflect.String) { + return errors.New(formatErrorDescription( + Locale.InvalidType(), + ErrorDetails{ + "expected": TYPE_STRING, + "given": KEY_TITLE, + }, + )) + } + if k, ok := m[KEY_TITLE].(string); ok { + currentSchema.title = &k + } + + // description + if existsMapKey(m, KEY_DESCRIPTION) && !isKind(m[KEY_DESCRIPTION], reflect.String) { + return errors.New(formatErrorDescription( + Locale.InvalidType(), + ErrorDetails{ + "expected": TYPE_STRING, + "given": KEY_DESCRIPTION, + }, + )) + } + if k, ok := m[KEY_DESCRIPTION].(string); ok { + currentSchema.description = &k + } + + // type + if existsMapKey(m, KEY_TYPE) { + if isKind(m[KEY_TYPE], reflect.String) { + if k, ok := m[KEY_TYPE].(string); ok { + err := currentSchema.types.Add(k) + if err != nil { + return err + } + } + } else { + if isKind(m[KEY_TYPE], reflect.Slice) { + arrayOfTypes := m[KEY_TYPE].([]interface{}) + for _, typeInArray := range arrayOfTypes { + if reflect.ValueOf(typeInArray).Kind() != reflect.String { + return errors.New(formatErrorDescription( + Locale.InvalidType(), + ErrorDetails{ + "expected": TYPE_STRING + "/" + STRING_ARRAY_OF_STRINGS, + "given": KEY_TYPE, + }, + )) + } else { + currentSchema.types.Add(typeInArray.(string)) + } + } + + } else { + return errors.New(formatErrorDescription( + Locale.InvalidType(), + ErrorDetails{ + "expected": TYPE_STRING + "/" + STRING_ARRAY_OF_STRINGS, + "given": KEY_TYPE, + }, + )) + } + } + } + + // properties + if existsMapKey(m, KEY_PROPERTIES) { + err := d.parseProperties(m[KEY_PROPERTIES], currentSchema) + if err != nil { + return err + } + } + + // additionalProperties + if existsMapKey(m, KEY_ADDITIONAL_PROPERTIES) { + if isKind(m[KEY_ADDITIONAL_PROPERTIES], reflect.Bool) { + currentSchema.additionalProperties = m[KEY_ADDITIONAL_PROPERTIES].(bool) + } else if isKind(m[KEY_ADDITIONAL_PROPERTIES], reflect.Map) { + newSchema := &subSchema{property: KEY_ADDITIONAL_PROPERTIES, parent: currentSchema, ref: currentSchema.ref} + currentSchema.additionalProperties = newSchema + err := d.parseSchema(m[KEY_ADDITIONAL_PROPERTIES], newSchema) + if err != nil { + return errors.New(err.Error()) + } + } else { + return errors.New(formatErrorDescription( + Locale.InvalidType(), + ErrorDetails{ + "expected": TYPE_BOOLEAN + "/" + STRING_SCHEMA, + "given": KEY_ADDITIONAL_PROPERTIES, + }, + )) + } + } + + // patternProperties + if existsMapKey(m, KEY_PATTERN_PROPERTIES) { + if isKind(m[KEY_PATTERN_PROPERTIES], reflect.Map) { + patternPropertiesMap := m[KEY_PATTERN_PROPERTIES].(map[string]interface{}) + if len(patternPropertiesMap) > 0 { + currentSchema.patternProperties = make(map[string]*subSchema) + for k, v := range patternPropertiesMap { + _, err := regexp.MatchString(k, "") + if err != nil { + return errors.New(formatErrorDescription( + Locale.RegexPattern(), + ErrorDetails{"pattern": k}, + )) + } + newSchema := &subSchema{property: k, parent: currentSchema, ref: currentSchema.ref} + err = d.parseSchema(v, newSchema) + if err != nil { + return errors.New(err.Error()) + } + currentSchema.patternProperties[k] = newSchema + } + } + } else { + return errors.New(formatErrorDescription( + Locale.InvalidType(), + ErrorDetails{ + "expected": STRING_SCHEMA, + "given": KEY_PATTERN_PROPERTIES, + }, + )) + } + } + + // dependencies + if existsMapKey(m, KEY_DEPENDENCIES) { + err := d.parseDependencies(m[KEY_DEPENDENCIES], currentSchema) + if err != nil { + return err + } + } + + // items + if existsMapKey(m, KEY_ITEMS) { + if isKind(m[KEY_ITEMS], reflect.Slice) { + for _, itemElement := range m[KEY_ITEMS].([]interface{}) { + if isKind(itemElement, reflect.Map) { + newSchema := &subSchema{parent: currentSchema, property: KEY_ITEMS} + newSchema.ref = currentSchema.ref + currentSchema.AddItemsChild(newSchema) + err := d.parseSchema(itemElement, newSchema) + if err != nil { + return err + } + } else { + return errors.New(formatErrorDescription( + Locale.InvalidType(), + ErrorDetails{ + "expected": STRING_SCHEMA + "/" + STRING_ARRAY_OF_SCHEMAS, + "given": KEY_ITEMS, + }, + )) + } + currentSchema.itemsChildrenIsSingleSchema = false + } + } else if isKind(m[KEY_ITEMS], reflect.Map) { + newSchema := &subSchema{parent: currentSchema, property: KEY_ITEMS} + newSchema.ref = currentSchema.ref + currentSchema.AddItemsChild(newSchema) + err := d.parseSchema(m[KEY_ITEMS], newSchema) + if err != nil { + return err + } + currentSchema.itemsChildrenIsSingleSchema = true + } else { + return errors.New(formatErrorDescription( + Locale.InvalidType(), + ErrorDetails{ + "expected": STRING_SCHEMA + "/" + STRING_ARRAY_OF_SCHEMAS, + "given": KEY_ITEMS, + }, + )) + } + } + + // additionalItems + if existsMapKey(m, KEY_ADDITIONAL_ITEMS) { + if isKind(m[KEY_ADDITIONAL_ITEMS], reflect.Bool) { + currentSchema.additionalItems = m[KEY_ADDITIONAL_ITEMS].(bool) + } else if isKind(m[KEY_ADDITIONAL_ITEMS], reflect.Map) { + newSchema := &subSchema{property: KEY_ADDITIONAL_ITEMS, parent: currentSchema, ref: currentSchema.ref} + currentSchema.additionalItems = newSchema + err := d.parseSchema(m[KEY_ADDITIONAL_ITEMS], newSchema) + if err != nil { + return errors.New(err.Error()) + } + } else { + return errors.New(formatErrorDescription( + Locale.InvalidType(), + ErrorDetails{ + "expected": TYPE_BOOLEAN + "/" + STRING_SCHEMA, + "given": KEY_ADDITIONAL_ITEMS, + }, + )) + } + } + + // validation : number / integer + + if existsMapKey(m, KEY_MULTIPLE_OF) { + multipleOfValue := mustBeNumber(m[KEY_MULTIPLE_OF]) + if multipleOfValue == nil { + return errors.New(formatErrorDescription( + Locale.InvalidType(), + ErrorDetails{ + "expected": STRING_NUMBER, + "given": KEY_MULTIPLE_OF, + }, + )) + } + if *multipleOfValue <= 0 { + return errors.New(formatErrorDescription( + Locale.GreaterThanZero(), + ErrorDetails{"number": KEY_MULTIPLE_OF}, + )) + } + currentSchema.multipleOf = multipleOfValue + } + + if existsMapKey(m, KEY_MINIMUM) { + minimumValue := mustBeNumber(m[KEY_MINIMUM]) + if minimumValue == nil { + return errors.New(formatErrorDescription( + Locale.MustBeOfA(), + ErrorDetails{"x": KEY_MINIMUM, "y": STRING_NUMBER}, + )) + } + currentSchema.minimum = minimumValue + } + + if existsMapKey(m, KEY_EXCLUSIVE_MINIMUM) { + if isKind(m[KEY_EXCLUSIVE_MINIMUM], reflect.Bool) { + if currentSchema.minimum == nil { + return errors.New(formatErrorDescription( + Locale.CannotBeUsedWithout(), + ErrorDetails{"x": KEY_EXCLUSIVE_MINIMUM, "y": KEY_MINIMUM}, + )) + } + exclusiveMinimumValue := m[KEY_EXCLUSIVE_MINIMUM].(bool) + currentSchema.exclusiveMinimum = exclusiveMinimumValue + } else { + return errors.New(formatErrorDescription( + Locale.MustBeOfA(), + ErrorDetails{"x": KEY_EXCLUSIVE_MINIMUM, "y": TYPE_BOOLEAN}, + )) + } + } + + if existsMapKey(m, KEY_MAXIMUM) { + maximumValue := mustBeNumber(m[KEY_MAXIMUM]) + if maximumValue == nil { + return errors.New(formatErrorDescription( + Locale.MustBeOfA(), + ErrorDetails{"x": KEY_MAXIMUM, "y": STRING_NUMBER}, + )) + } + currentSchema.maximum = maximumValue + } + + if existsMapKey(m, KEY_EXCLUSIVE_MAXIMUM) { + if isKind(m[KEY_EXCLUSIVE_MAXIMUM], reflect.Bool) { + if currentSchema.maximum == nil { + return errors.New(formatErrorDescription( + Locale.CannotBeUsedWithout(), + ErrorDetails{"x": KEY_EXCLUSIVE_MAXIMUM, "y": KEY_MAXIMUM}, + )) + } + exclusiveMaximumValue := m[KEY_EXCLUSIVE_MAXIMUM].(bool) + currentSchema.exclusiveMaximum = exclusiveMaximumValue + } else { + return errors.New(formatErrorDescription( + Locale.MustBeOfA(), + ErrorDetails{"x": KEY_EXCLUSIVE_MAXIMUM, "y": STRING_NUMBER}, + )) + } + } + + if currentSchema.minimum != nil && currentSchema.maximum != nil { + if *currentSchema.minimum > *currentSchema.maximum { + return errors.New(formatErrorDescription( + Locale.CannotBeGT(), + ErrorDetails{"x": KEY_MINIMUM, "y": KEY_MAXIMUM}, + )) + } + } + + // validation : string + + if existsMapKey(m, KEY_MIN_LENGTH) { + minLengthIntegerValue := mustBeInteger(m[KEY_MIN_LENGTH]) + if minLengthIntegerValue == nil { + return errors.New(formatErrorDescription( + Locale.MustBeOfAn(), + ErrorDetails{"x": KEY_MIN_LENGTH, "y": TYPE_INTEGER}, + )) + } + if *minLengthIntegerValue < 0 { + return errors.New(formatErrorDescription( + Locale.MustBeGTEZero(), + ErrorDetails{"key": KEY_MIN_LENGTH}, + )) + } + currentSchema.minLength = minLengthIntegerValue + } + + if existsMapKey(m, KEY_MAX_LENGTH) { + maxLengthIntegerValue := mustBeInteger(m[KEY_MAX_LENGTH]) + if maxLengthIntegerValue == nil { + return errors.New(formatErrorDescription( + Locale.MustBeOfAn(), + ErrorDetails{"x": KEY_MAX_LENGTH, "y": TYPE_INTEGER}, + )) + } + if *maxLengthIntegerValue < 0 { + return errors.New(formatErrorDescription( + Locale.MustBeGTEZero(), + ErrorDetails{"key": KEY_MAX_LENGTH}, + )) + } + currentSchema.maxLength = maxLengthIntegerValue + } + + if currentSchema.minLength != nil && currentSchema.maxLength != nil { + if *currentSchema.minLength > *currentSchema.maxLength { + return errors.New(formatErrorDescription( + Locale.CannotBeGT(), + ErrorDetails{"x": KEY_MIN_LENGTH, "y": KEY_MAX_LENGTH}, + )) + } + } + + if existsMapKey(m, KEY_PATTERN) { + if isKind(m[KEY_PATTERN], reflect.String) { + regexpObject, err := regexp.Compile(m[KEY_PATTERN].(string)) + if err != nil { + return errors.New(formatErrorDescription( + Locale.MustBeValidRegex(), + ErrorDetails{"key": KEY_PATTERN}, + )) + } + currentSchema.pattern = regexpObject + } else { + return errors.New(formatErrorDescription( + Locale.MustBeOfA(), + ErrorDetails{"x": KEY_PATTERN, "y": TYPE_STRING}, + )) + } + } + + if existsMapKey(m, KEY_FORMAT) { + formatString, ok := m[KEY_FORMAT].(string) + if ok && FormatCheckers.Has(formatString) { + currentSchema.format = formatString + } + } + + // validation : object + + if existsMapKey(m, KEY_MIN_PROPERTIES) { + minPropertiesIntegerValue := mustBeInteger(m[KEY_MIN_PROPERTIES]) + if minPropertiesIntegerValue == nil { + return errors.New(formatErrorDescription( + Locale.MustBeOfAn(), + ErrorDetails{"x": KEY_MIN_PROPERTIES, "y": TYPE_INTEGER}, + )) + } + if *minPropertiesIntegerValue < 0 { + return errors.New(formatErrorDescription( + Locale.MustBeGTEZero(), + ErrorDetails{"key": KEY_MIN_PROPERTIES}, + )) + } + currentSchema.minProperties = minPropertiesIntegerValue + } + + if existsMapKey(m, KEY_MAX_PROPERTIES) { + maxPropertiesIntegerValue := mustBeInteger(m[KEY_MAX_PROPERTIES]) + if maxPropertiesIntegerValue == nil { + return errors.New(formatErrorDescription( + Locale.MustBeOfAn(), + ErrorDetails{"x": KEY_MAX_PROPERTIES, "y": TYPE_INTEGER}, + )) + } + if *maxPropertiesIntegerValue < 0 { + return errors.New(formatErrorDescription( + Locale.MustBeGTEZero(), + ErrorDetails{"key": KEY_MAX_PROPERTIES}, + )) + } + currentSchema.maxProperties = maxPropertiesIntegerValue + } + + if currentSchema.minProperties != nil && currentSchema.maxProperties != nil { + if *currentSchema.minProperties > *currentSchema.maxProperties { + return errors.New(formatErrorDescription( + Locale.KeyCannotBeGreaterThan(), + ErrorDetails{"key": KEY_MIN_PROPERTIES, "y": KEY_MAX_PROPERTIES}, + )) + } + } + + if existsMapKey(m, KEY_REQUIRED) { + if isKind(m[KEY_REQUIRED], reflect.Slice) { + requiredValues := m[KEY_REQUIRED].([]interface{}) + for _, requiredValue := range requiredValues { + if isKind(requiredValue, reflect.String) { + err := currentSchema.AddRequired(requiredValue.(string)) + if err != nil { + return err + } + } else { + return errors.New(formatErrorDescription( + Locale.KeyItemsMustBeOfType(), + ErrorDetails{"key": KEY_REQUIRED, "type": TYPE_STRING}, + )) + } + } + } else { + return errors.New(formatErrorDescription( + Locale.MustBeOfAn(), + ErrorDetails{"x": KEY_REQUIRED, "y": TYPE_ARRAY}, + )) + } + } + + // validation : array + + if existsMapKey(m, KEY_MIN_ITEMS) { + minItemsIntegerValue := mustBeInteger(m[KEY_MIN_ITEMS]) + if minItemsIntegerValue == nil { + return errors.New(formatErrorDescription( + Locale.MustBeOfAn(), + ErrorDetails{"x": KEY_MIN_ITEMS, "y": TYPE_INTEGER}, + )) + } + if *minItemsIntegerValue < 0 { + return errors.New(formatErrorDescription( + Locale.MustBeGTEZero(), + ErrorDetails{"key": KEY_MIN_ITEMS}, + )) + } + currentSchema.minItems = minItemsIntegerValue + } + + if existsMapKey(m, KEY_MAX_ITEMS) { + maxItemsIntegerValue := mustBeInteger(m[KEY_MAX_ITEMS]) + if maxItemsIntegerValue == nil { + return errors.New(formatErrorDescription( + Locale.MustBeOfAn(), + ErrorDetails{"x": KEY_MAX_ITEMS, "y": TYPE_INTEGER}, + )) + } + if *maxItemsIntegerValue < 0 { + return errors.New(formatErrorDescription( + Locale.MustBeGTEZero(), + ErrorDetails{"key": KEY_MAX_ITEMS}, + )) + } + currentSchema.maxItems = maxItemsIntegerValue + } + + if existsMapKey(m, KEY_UNIQUE_ITEMS) { + if isKind(m[KEY_UNIQUE_ITEMS], reflect.Bool) { + currentSchema.uniqueItems = m[KEY_UNIQUE_ITEMS].(bool) + } else { + return errors.New(formatErrorDescription( + Locale.MustBeOfA(), + ErrorDetails{"x": KEY_UNIQUE_ITEMS, "y": TYPE_BOOLEAN}, + )) + } + } + + // validation : all + + if existsMapKey(m, KEY_ENUM) { + if isKind(m[KEY_ENUM], reflect.Slice) { + for _, v := range m[KEY_ENUM].([]interface{}) { + err := currentSchema.AddEnum(v) + if err != nil { + return err + } + } + } else { + return errors.New(formatErrorDescription( + Locale.MustBeOfAn(), + ErrorDetails{"x": KEY_ENUM, "y": TYPE_ARRAY}, + )) + } + } + + // validation : subSchema + + if existsMapKey(m, KEY_ONE_OF) { + if isKind(m[KEY_ONE_OF], reflect.Slice) { + for _, v := range m[KEY_ONE_OF].([]interface{}) { + newSchema := &subSchema{property: KEY_ONE_OF, parent: currentSchema, ref: currentSchema.ref} + currentSchema.AddOneOf(newSchema) + err := d.parseSchema(v, newSchema) + if err != nil { + return err + } + } + } else { + return errors.New(formatErrorDescription( + Locale.MustBeOfAn(), + ErrorDetails{"x": KEY_ONE_OF, "y": TYPE_ARRAY}, + )) + } + } + + if existsMapKey(m, KEY_ANY_OF) { + if isKind(m[KEY_ANY_OF], reflect.Slice) { + for _, v := range m[KEY_ANY_OF].([]interface{}) { + newSchema := &subSchema{property: KEY_ANY_OF, parent: currentSchema, ref: currentSchema.ref} + currentSchema.AddAnyOf(newSchema) + err := d.parseSchema(v, newSchema) + if err != nil { + return err + } + } + } else { + return errors.New(formatErrorDescription( + Locale.MustBeOfAn(), + ErrorDetails{"x": KEY_ANY_OF, "y": TYPE_ARRAY}, + )) + } + } + + if existsMapKey(m, KEY_ALL_OF) { + if isKind(m[KEY_ALL_OF], reflect.Slice) { + for _, v := range m[KEY_ALL_OF].([]interface{}) { + newSchema := &subSchema{property: KEY_ALL_OF, parent: currentSchema, ref: currentSchema.ref} + currentSchema.AddAllOf(newSchema) + err := d.parseSchema(v, newSchema) + if err != nil { + return err + } + } + } else { + return errors.New(formatErrorDescription( + Locale.MustBeOfAn(), + ErrorDetails{"x": KEY_ANY_OF, "y": TYPE_ARRAY}, + )) + } + } + + if existsMapKey(m, KEY_NOT) { + if isKind(m[KEY_NOT], reflect.Map) { + newSchema := &subSchema{property: KEY_NOT, parent: currentSchema, ref: currentSchema.ref} + currentSchema.SetNot(newSchema) + err := d.parseSchema(m[KEY_NOT], newSchema) + if err != nil { + return err + } + } else { + return errors.New(formatErrorDescription( + Locale.MustBeOfAn(), + ErrorDetails{"x": KEY_NOT, "y": TYPE_OBJECT}, + )) + } + } + + return nil +} + +func (d *Schema) parseReference(documentNode interface{}, currentSchema *subSchema) error { + var ( + refdDocumentNode interface{} + dsp *schemaPoolDocument + err error + ) + jsonPointer := currentSchema.ref.GetPointer() + standaloneDocument := d.pool.GetStandaloneDocument() + + newSchema := &subSchema{property: KEY_REF, parent: currentSchema, ref: currentSchema.ref} + + if currentSchema.ref.HasFragmentOnly { + refdDocumentNode, _, err = jsonPointer.Get(standaloneDocument) + if err != nil { + return err + } + + } else { + dsp, err = d.pool.GetDocument(*currentSchema.ref) + if err != nil { + return err + } + newSchema.id = currentSchema.ref + + refdDocumentNode, _, err = jsonPointer.Get(dsp.Document) + + if err != nil { + return err + } + + } + + if !isKind(refdDocumentNode, reflect.Map) { + return errors.New(formatErrorDescription( + Locale.MustBeOfType(), + ErrorDetails{"key": STRING_SCHEMA, "type": TYPE_OBJECT}, + )) + } + + // returns the loaded referenced subSchema for the caller to update its current subSchema + newSchemaDocument := refdDocumentNode.(map[string]interface{}) + + err = d.parseSchema(newSchemaDocument, newSchema) + if err != nil { + return err + } + + currentSchema.refSchema = newSchema + + return nil + +} + +func (d *Schema) parseProperties(documentNode interface{}, currentSchema *subSchema) error { + + if !isKind(documentNode, reflect.Map) { + return errors.New(formatErrorDescription( + Locale.MustBeOfType(), + ErrorDetails{"key": STRING_PROPERTIES, "type": TYPE_OBJECT}, + )) + } + + m := documentNode.(map[string]interface{}) + for k := range m { + schemaProperty := k + newSchema := &subSchema{property: schemaProperty, parent: currentSchema, ref: currentSchema.ref} + currentSchema.AddPropertiesChild(newSchema) + err := d.parseSchema(m[k], newSchema) + if err != nil { + return err + } + } + + return nil +} + +func (d *Schema) parseDependencies(documentNode interface{}, currentSchema *subSchema) error { + + if !isKind(documentNode, reflect.Map) { + return errors.New(formatErrorDescription( + Locale.MustBeOfType(), + ErrorDetails{"key": KEY_DEPENDENCIES, "type": TYPE_OBJECT}, + )) + } + + m := documentNode.(map[string]interface{}) + currentSchema.dependencies = make(map[string]interface{}) + + for k := range m { + switch reflect.ValueOf(m[k]).Kind() { + + case reflect.Slice: + values := m[k].([]interface{}) + var valuesToRegister []string + + for _, value := range values { + if !isKind(value, reflect.String) { + return errors.New(formatErrorDescription( + Locale.MustBeOfType(), + ErrorDetails{ + "key": STRING_DEPENDENCY, + "type": STRING_SCHEMA_OR_ARRAY_OF_STRINGS, + }, + )) + } else { + valuesToRegister = append(valuesToRegister, value.(string)) + } + currentSchema.dependencies[k] = valuesToRegister + } + + case reflect.Map: + depSchema := &subSchema{property: k, parent: currentSchema, ref: currentSchema.ref} + err := d.parseSchema(m[k], depSchema) + if err != nil { + return err + } + currentSchema.dependencies[k] = depSchema + + default: + return errors.New(formatErrorDescription( + Locale.MustBeOfType(), + ErrorDetails{ + "key": STRING_DEPENDENCY, + "type": STRING_SCHEMA_OR_ARRAY_OF_STRINGS, + }, + )) + } + + } + + return nil +} diff --git a/vendor/github.com/xeipuuv/gojsonschema/schemaPool.go b/vendor/github.com/xeipuuv/gojsonschema/schemaPool.go new file mode 100644 index 00000000..ff9715f0 --- /dev/null +++ b/vendor/github.com/xeipuuv/gojsonschema/schemaPool.go @@ -0,0 +1,103 @@ +// Copyright 2015 xeipuuv ( https://github.com/xeipuuv ) +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// author xeipuuv +// author-github https://github.com/xeipuuv +// author-mail xeipuuv@gmail.com +// +// repository-name gojsonschema +// repository-desc An implementation of JSON Schema, based on IETF's draft v4 - Go language. +// +// description Defines resources pooling. +// Eases referencing and avoids downloading the same resource twice. +// +// created 26-02-2013 + +package gojsonschema + +import ( + "errors" + + "github.com/xeipuuv/gojsonreference" +) + +type schemaPoolDocument struct { + Document interface{} +} + +type schemaPool struct { + schemaPoolDocuments map[string]*schemaPoolDocument + standaloneDocument interface{} + jsonLoaderFactory JSONLoaderFactory +} + +func newSchemaPool(f JSONLoaderFactory) *schemaPool { + + p := &schemaPool{} + p.schemaPoolDocuments = make(map[string]*schemaPoolDocument) + p.standaloneDocument = nil + p.jsonLoaderFactory = f + + return p +} + +func (p *schemaPool) SetStandaloneDocument(document interface{}) { + p.standaloneDocument = document +} + +func (p *schemaPool) GetStandaloneDocument() (document interface{}) { + return p.standaloneDocument +} + +func (p *schemaPool) GetDocument(reference gojsonreference.JsonReference) (*schemaPoolDocument, error) { + + var ( + spd *schemaPoolDocument + ok bool + err error + ) + + if internalLogEnabled { + internalLog("Get Document ( %s )", reference.String()) + } + + // It is not possible to load anything that is not canonical... + if !reference.IsCanonical() { + return nil, errors.New(formatErrorDescription( + Locale.ReferenceMustBeCanonical(), + ErrorDetails{"reference": reference}, + )) + } + refToUrl := reference + refToUrl.GetUrl().Fragment = "" + + if spd, ok = p.schemaPoolDocuments[refToUrl.String()]; ok { + if internalLogEnabled { + internalLog(" From pool") + } + return spd, nil + } + + jsonReferenceLoader := p.jsonLoaderFactory.New(reference.String()) + document, err := jsonReferenceLoader.LoadJSON() + if err != nil { + return nil, err + } + + spd = &schemaPoolDocument{Document: document} + // add the document to the pool for potential later use + p.schemaPoolDocuments[refToUrl.String()] = spd + + return spd, nil +} diff --git a/vendor/github.com/xeipuuv/gojsonschema/schemaReferencePool.go b/vendor/github.com/xeipuuv/gojsonschema/schemaReferencePool.go new file mode 100644 index 00000000..6e5e1b5c --- /dev/null +++ b/vendor/github.com/xeipuuv/gojsonschema/schemaReferencePool.go @@ -0,0 +1,68 @@ +// Copyright 2015 xeipuuv ( https://github.com/xeipuuv ) +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// author xeipuuv +// author-github https://github.com/xeipuuv +// author-mail xeipuuv@gmail.com +// +// repository-name gojsonschema +// repository-desc An implementation of JSON Schema, based on IETF's draft v4 - Go language. +// +// description Pool of referenced schemas. +// +// created 25-06-2013 + +package gojsonschema + +import ( + "fmt" +) + +type schemaReferencePool struct { + documents map[string]*subSchema +} + +func newSchemaReferencePool() *schemaReferencePool { + + p := &schemaReferencePool{} + p.documents = make(map[string]*subSchema) + + return p +} + +func (p *schemaReferencePool) Get(ref string) (r *subSchema, o bool) { + + if internalLogEnabled { + internalLog(fmt.Sprintf("Schema Reference ( %s )", ref)) + } + + if sch, ok := p.documents[ref]; ok { + if internalLogEnabled { + internalLog(fmt.Sprintf(" From pool")) + } + return sch, true + } + + return nil, false +} + +func (p *schemaReferencePool) Add(ref string, sch *subSchema) { + + if internalLogEnabled { + internalLog(fmt.Sprintf("Add Schema Reference %s to pool", ref)) + } + if _, ok := p.documents[ref]; !ok { + p.documents[ref] = sch + } +} diff --git a/vendor/github.com/xeipuuv/gojsonschema/schemaType.go b/vendor/github.com/xeipuuv/gojsonschema/schemaType.go new file mode 100644 index 00000000..36b447a2 --- /dev/null +++ b/vendor/github.com/xeipuuv/gojsonschema/schemaType.go @@ -0,0 +1,83 @@ +// Copyright 2015 xeipuuv ( https://github.com/xeipuuv ) +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// author xeipuuv +// author-github https://github.com/xeipuuv +// author-mail xeipuuv@gmail.com +// +// repository-name gojsonschema +// repository-desc An implementation of JSON Schema, based on IETF's draft v4 - Go language. +// +// description Helper structure to handle schema types, and the combination of them. +// +// created 28-02-2013 + +package gojsonschema + +import ( + "errors" + "fmt" + "strings" +) + +type jsonSchemaType struct { + types []string +} + +// Is the schema typed ? that is containing at least one type +// When not typed, the schema does not need any type validation +func (t *jsonSchemaType) IsTyped() bool { + return len(t.types) > 0 +} + +func (t *jsonSchemaType) Add(etype string) error { + + if !isStringInSlice(JSON_TYPES, etype) { + return errors.New(formatErrorDescription(Locale.NotAValidType(), ErrorDetails{"given": "/" + etype + "/", "expected": JSON_TYPES})) + } + + if t.Contains(etype) { + return errors.New(formatErrorDescription(Locale.Duplicated(), ErrorDetails{"type": etype})) + } + + t.types = append(t.types, etype) + + return nil +} + +func (t *jsonSchemaType) Contains(etype string) bool { + + for _, v := range t.types { + if v == etype { + return true + } + } + + return false +} + +func (t *jsonSchemaType) String() string { + + if len(t.types) == 0 { + return STRING_UNDEFINED // should never happen + } + + // Displayed as a list [type1,type2,...] + if len(t.types) > 1 { + return fmt.Sprintf("[%s]", strings.Join(t.types, ",")) + } + + // Only one type: name only + return t.types[0] +} diff --git a/vendor/github.com/xeipuuv/gojsonschema/subSchema.go b/vendor/github.com/xeipuuv/gojsonschema/subSchema.go new file mode 100644 index 00000000..9961d92b --- /dev/null +++ b/vendor/github.com/xeipuuv/gojsonschema/subSchema.go @@ -0,0 +1,227 @@ +// Copyright 2015 xeipuuv ( https://github.com/xeipuuv ) +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// author xeipuuv +// author-github https://github.com/xeipuuv +// author-mail xeipuuv@gmail.com +// +// repository-name gojsonschema +// repository-desc An implementation of JSON Schema, based on IETF's draft v4 - Go language. +// +// description Defines the structure of a sub-subSchema. +// A sub-subSchema can contain other sub-schemas. +// +// created 27-02-2013 + +package gojsonschema + +import ( + "errors" + "regexp" + "strings" + + "github.com/xeipuuv/gojsonreference" +) + +const ( + KEY_SCHEMA = "$subSchema" + KEY_ID = "id" + KEY_REF = "$ref" + KEY_TITLE = "title" + KEY_DESCRIPTION = "description" + KEY_TYPE = "type" + KEY_ITEMS = "items" + KEY_ADDITIONAL_ITEMS = "additionalItems" + KEY_PROPERTIES = "properties" + KEY_PATTERN_PROPERTIES = "patternProperties" + KEY_ADDITIONAL_PROPERTIES = "additionalProperties" + KEY_DEFINITIONS = "definitions" + KEY_MULTIPLE_OF = "multipleOf" + KEY_MINIMUM = "minimum" + KEY_MAXIMUM = "maximum" + KEY_EXCLUSIVE_MINIMUM = "exclusiveMinimum" + KEY_EXCLUSIVE_MAXIMUM = "exclusiveMaximum" + KEY_MIN_LENGTH = "minLength" + KEY_MAX_LENGTH = "maxLength" + KEY_PATTERN = "pattern" + KEY_FORMAT = "format" + KEY_MIN_PROPERTIES = "minProperties" + KEY_MAX_PROPERTIES = "maxProperties" + KEY_DEPENDENCIES = "dependencies" + KEY_REQUIRED = "required" + KEY_MIN_ITEMS = "minItems" + KEY_MAX_ITEMS = "maxItems" + KEY_UNIQUE_ITEMS = "uniqueItems" + KEY_ENUM = "enum" + KEY_ONE_OF = "oneOf" + KEY_ANY_OF = "anyOf" + KEY_ALL_OF = "allOf" + KEY_NOT = "not" +) + +type subSchema struct { + + // basic subSchema meta properties + id *gojsonreference.JsonReference + title *string + description *string + + property string + + // Types associated with the subSchema + types jsonSchemaType + + // Reference url + ref *gojsonreference.JsonReference + // Schema referenced + refSchema *subSchema + // Json reference + subSchema *gojsonreference.JsonReference + + // hierarchy + parent *subSchema + definitions map[string]*subSchema + definitionsChildren []*subSchema + itemsChildren []*subSchema + itemsChildrenIsSingleSchema bool + propertiesChildren []*subSchema + + // validation : number / integer + multipleOf *float64 + maximum *float64 + exclusiveMaximum bool + minimum *float64 + exclusiveMinimum bool + + // validation : string + minLength *int + maxLength *int + pattern *regexp.Regexp + format string + + // validation : object + minProperties *int + maxProperties *int + required []string + + dependencies map[string]interface{} + additionalProperties interface{} + patternProperties map[string]*subSchema + + // validation : array + minItems *int + maxItems *int + uniqueItems bool + + additionalItems interface{} + + // validation : all + enum []string + + // validation : subSchema + oneOf []*subSchema + anyOf []*subSchema + allOf []*subSchema + not *subSchema +} + +func (s *subSchema) AddEnum(i interface{}) error { + + is, err := marshalToJsonString(i) + if err != nil { + return err + } + + if isStringInSlice(s.enum, *is) { + return errors.New(formatErrorDescription( + Locale.KeyItemsMustBeUnique(), + ErrorDetails{"key": KEY_ENUM}, + )) + } + + s.enum = append(s.enum, *is) + + return nil +} + +func (s *subSchema) ContainsEnum(i interface{}) (bool, error) { + + is, err := marshalToJsonString(i) + if err != nil { + return false, err + } + + return isStringInSlice(s.enum, *is), nil +} + +func (s *subSchema) AddOneOf(subSchema *subSchema) { + s.oneOf = append(s.oneOf, subSchema) +} + +func (s *subSchema) AddAllOf(subSchema *subSchema) { + s.allOf = append(s.allOf, subSchema) +} + +func (s *subSchema) AddAnyOf(subSchema *subSchema) { + s.anyOf = append(s.anyOf, subSchema) +} + +func (s *subSchema) SetNot(subSchema *subSchema) { + s.not = subSchema +} + +func (s *subSchema) AddRequired(value string) error { + + if isStringInSlice(s.required, value) { + return errors.New(formatErrorDescription( + Locale.KeyItemsMustBeUnique(), + ErrorDetails{"key": KEY_REQUIRED}, + )) + } + + s.required = append(s.required, value) + + return nil +} + +func (s *subSchema) AddDefinitionChild(child *subSchema) { + s.definitionsChildren = append(s.definitionsChildren, child) +} + +func (s *subSchema) AddItemsChild(child *subSchema) { + s.itemsChildren = append(s.itemsChildren, child) +} + +func (s *subSchema) AddPropertiesChild(child *subSchema) { + s.propertiesChildren = append(s.propertiesChildren, child) +} + +func (s *subSchema) PatternPropertiesString() string { + + if s.patternProperties == nil || len(s.patternProperties) == 0 { + return STRING_UNDEFINED // should never happen + } + + patternPropertiesKeySlice := []string{} + for pk := range s.patternProperties { + patternPropertiesKeySlice = append(patternPropertiesKeySlice, `"`+pk+`"`) + } + + if len(patternPropertiesKeySlice) == 1 { + return patternPropertiesKeySlice[0] + } + + return "[" + strings.Join(patternPropertiesKeySlice, ",") + "]" + +} diff --git a/vendor/github.com/xeipuuv/gojsonschema/types.go b/vendor/github.com/xeipuuv/gojsonschema/types.go new file mode 100644 index 00000000..952d22ef --- /dev/null +++ b/vendor/github.com/xeipuuv/gojsonschema/types.go @@ -0,0 +1,58 @@ +// Copyright 2015 xeipuuv ( https://github.com/xeipuuv ) +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// author xeipuuv +// author-github https://github.com/xeipuuv +// author-mail xeipuuv@gmail.com +// +// repository-name gojsonschema +// repository-desc An implementation of JSON Schema, based on IETF's draft v4 - Go language. +// +// description Contains const types for schema and JSON. +// +// created 28-02-2013 + +package gojsonschema + +const ( + TYPE_ARRAY = `array` + TYPE_BOOLEAN = `boolean` + TYPE_INTEGER = `integer` + TYPE_NUMBER = `number` + TYPE_NULL = `null` + TYPE_OBJECT = `object` + TYPE_STRING = `string` +) + +var JSON_TYPES []string +var SCHEMA_TYPES []string + +func init() { + JSON_TYPES = []string{ + TYPE_ARRAY, + TYPE_BOOLEAN, + TYPE_INTEGER, + TYPE_NUMBER, + TYPE_NULL, + TYPE_OBJECT, + TYPE_STRING} + + SCHEMA_TYPES = []string{ + TYPE_ARRAY, + TYPE_BOOLEAN, + TYPE_INTEGER, + TYPE_NUMBER, + TYPE_OBJECT, + TYPE_STRING} +} diff --git a/vendor/github.com/xeipuuv/gojsonschema/utils.go b/vendor/github.com/xeipuuv/gojsonschema/utils.go new file mode 100644 index 00000000..26cf75eb --- /dev/null +++ b/vendor/github.com/xeipuuv/gojsonschema/utils.go @@ -0,0 +1,208 @@ +// Copyright 2015 xeipuuv ( https://github.com/xeipuuv ) +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// author xeipuuv +// author-github https://github.com/xeipuuv +// author-mail xeipuuv@gmail.com +// +// repository-name gojsonschema +// repository-desc An implementation of JSON Schema, based on IETF's draft v4 - Go language. +// +// description Various utility functions. +// +// created 26-02-2013 + +package gojsonschema + +import ( + "encoding/json" + "fmt" + "math" + "reflect" + "strconv" +) + +func isKind(what interface{}, kind reflect.Kind) bool { + target := what + if isJsonNumber(what) { + // JSON Numbers are strings! + target = *mustBeNumber(what) + } + return reflect.ValueOf(target).Kind() == kind +} + +func existsMapKey(m map[string]interface{}, k string) bool { + _, ok := m[k] + return ok +} + +func isStringInSlice(s []string, what string) bool { + for i := range s { + if s[i] == what { + return true + } + } + return false +} + +func marshalToJsonString(value interface{}) (*string, error) { + + mBytes, err := json.Marshal(value) + if err != nil { + return nil, err + } + + sBytes := string(mBytes) + return &sBytes, nil +} + +func isJsonNumber(what interface{}) bool { + + switch what.(type) { + + case json.Number: + return true + } + + return false +} + +func checkJsonNumber(what interface{}) (isValidFloat64 bool, isValidInt64 bool, isValidInt32 bool) { + + jsonNumber := what.(json.Number) + + f64, errFloat64 := jsonNumber.Float64() + s64 := strconv.FormatFloat(f64, 'f', -1, 64) + _, errInt64 := strconv.ParseInt(s64, 10, 64) + + isValidFloat64 = errFloat64 == nil + isValidInt64 = errInt64 == nil + + _, errInt32 := strconv.ParseInt(s64, 10, 32) + isValidInt32 = isValidInt64 && errInt32 == nil + + return + +} + +// same as ECMA Number.MAX_SAFE_INTEGER and Number.MIN_SAFE_INTEGER +const ( + max_json_float = float64(1<<53 - 1) // 9007199254740991.0 2^53 - 1 + min_json_float = -float64(1<<53 - 1) //-9007199254740991.0 -2^53 - 1 +) + +func isFloat64AnInteger(f float64) bool { + + if math.IsNaN(f) || math.IsInf(f, 0) || f < min_json_float || f > max_json_float { + return false + } + + return f == float64(int64(f)) || f == float64(uint64(f)) +} + +func mustBeInteger(what interface{}) *int { + + if isJsonNumber(what) { + + number := what.(json.Number) + + _, _, isValidInt32 := checkJsonNumber(number) + + if isValidInt32 { + + int64Value, err := number.Int64() + if err != nil { + return nil + } + + int32Value := int(int64Value) + return &int32Value + + } else { + return nil + } + + } + + return nil +} + +func mustBeNumber(what interface{}) *float64 { + + if isJsonNumber(what) { + + number := what.(json.Number) + float64Value, err := number.Float64() + + if err == nil { + return &float64Value + } else { + return nil + } + + } + + return nil + +} + +// formats a number so that it is displayed as the smallest string possible +func resultErrorFormatJsonNumber(n json.Number) string { + + if int64Value, err := n.Int64(); err == nil { + return fmt.Sprintf("%d", int64Value) + } + + float64Value, _ := n.Float64() + + return fmt.Sprintf("%g", float64Value) +} + +// formats a number so that it is displayed as the smallest string possible +func resultErrorFormatNumber(n float64) string { + + if isFloat64AnInteger(n) { + return fmt.Sprintf("%d", int64(n)) + } + + return fmt.Sprintf("%g", n) +} + +func convertDocumentNode(val interface{}) interface{} { + + if lval, ok := val.([]interface{}); ok { + + res := []interface{}{} + for _, v := range lval { + res = append(res, convertDocumentNode(v)) + } + + return res + + } + + if mval, ok := val.(map[interface{}]interface{}); ok { + + res := map[string]interface{}{} + + for k, v := range mval { + res[k.(string)] = convertDocumentNode(v) + } + + return res + + } + + return val +} diff --git a/vendor/github.com/xeipuuv/gojsonschema/validation.go b/vendor/github.com/xeipuuv/gojsonschema/validation.go new file mode 100644 index 00000000..9afea251 --- /dev/null +++ b/vendor/github.com/xeipuuv/gojsonschema/validation.go @@ -0,0 +1,844 @@ +// Copyright 2015 xeipuuv ( https://github.com/xeipuuv ) +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// author xeipuuv +// author-github https://github.com/xeipuuv +// author-mail xeipuuv@gmail.com +// +// repository-name gojsonschema +// repository-desc An implementation of JSON Schema, based on IETF's draft v4 - Go language. +// +// description Extends Schema and subSchema, implements the validation phase. +// +// created 28-02-2013 + +package gojsonschema + +import ( + "encoding/json" + "reflect" + "regexp" + "strconv" + "strings" + "unicode/utf8" +) + +func Validate(ls JSONLoader, ld JSONLoader) (*Result, error) { + + var err error + + // load schema + + schema, err := NewSchema(ls) + if err != nil { + return nil, err + } + + // begine validation + + return schema.Validate(ld) + +} + +func (v *Schema) Validate(l JSONLoader) (*Result, error) { + + // load document + + root, err := l.LoadJSON() + if err != nil { + return nil, err + } + + // begin validation + + result := &Result{} + context := newJsonContext(STRING_CONTEXT_ROOT, nil) + v.rootSchema.validateRecursive(v.rootSchema, root, result, context) + + return result, nil + +} + +func (v *subSchema) subValidateWithContext(document interface{}, context *jsonContext) *Result { + result := &Result{} + v.validateRecursive(v, document, result, context) + return result +} + +// Walker function to validate the json recursively against the subSchema +func (v *subSchema) validateRecursive(currentSubSchema *subSchema, currentNode interface{}, result *Result, context *jsonContext) { + + if internalLogEnabled { + internalLog("validateRecursive %s", context.String()) + internalLog(" %v", currentNode) + } + + // Handle referenced schemas, returns directly when a $ref is found + if currentSubSchema.refSchema != nil { + v.validateRecursive(currentSubSchema.refSchema, currentNode, result, context) + return + } + + // Check for null value + if currentNode == nil { + if currentSubSchema.types.IsTyped() && !currentSubSchema.types.Contains(TYPE_NULL) { + result.addError( + new(InvalidTypeError), + context, + currentNode, + ErrorDetails{ + "expected": currentSubSchema.types.String(), + "given": TYPE_NULL, + }, + ) + return + } + + currentSubSchema.validateSchema(currentSubSchema, currentNode, result, context) + v.validateCommon(currentSubSchema, currentNode, result, context) + + } else { // Not a null value + + if isJsonNumber(currentNode) { + + value := currentNode.(json.Number) + + _, isValidInt64, _ := checkJsonNumber(value) + + validType := currentSubSchema.types.Contains(TYPE_NUMBER) || (isValidInt64 && currentSubSchema.types.Contains(TYPE_INTEGER)) + + if currentSubSchema.types.IsTyped() && !validType { + + givenType := TYPE_INTEGER + if !isValidInt64 { + givenType = TYPE_NUMBER + } + + result.addError( + new(InvalidTypeError), + context, + currentNode, + ErrorDetails{ + "expected": currentSubSchema.types.String(), + "given": givenType, + }, + ) + return + } + + currentSubSchema.validateSchema(currentSubSchema, value, result, context) + v.validateNumber(currentSubSchema, value, result, context) + v.validateCommon(currentSubSchema, value, result, context) + v.validateString(currentSubSchema, value, result, context) + + } else { + + rValue := reflect.ValueOf(currentNode) + rKind := rValue.Kind() + + switch rKind { + + // Slice => JSON array + + case reflect.Slice: + + if currentSubSchema.types.IsTyped() && !currentSubSchema.types.Contains(TYPE_ARRAY) { + result.addError( + new(InvalidTypeError), + context, + currentNode, + ErrorDetails{ + "expected": currentSubSchema.types.String(), + "given": TYPE_ARRAY, + }, + ) + return + } + + castCurrentNode := currentNode.([]interface{}) + + currentSubSchema.validateSchema(currentSubSchema, castCurrentNode, result, context) + + v.validateArray(currentSubSchema, castCurrentNode, result, context) + v.validateCommon(currentSubSchema, castCurrentNode, result, context) + + // Map => JSON object + + case reflect.Map: + if currentSubSchema.types.IsTyped() && !currentSubSchema.types.Contains(TYPE_OBJECT) { + result.addError( + new(InvalidTypeError), + context, + currentNode, + ErrorDetails{ + "expected": currentSubSchema.types.String(), + "given": TYPE_OBJECT, + }, + ) + return + } + + castCurrentNode, ok := currentNode.(map[string]interface{}) + if !ok { + castCurrentNode = convertDocumentNode(currentNode).(map[string]interface{}) + } + + currentSubSchema.validateSchema(currentSubSchema, castCurrentNode, result, context) + + v.validateObject(currentSubSchema, castCurrentNode, result, context) + v.validateCommon(currentSubSchema, castCurrentNode, result, context) + + for _, pSchema := range currentSubSchema.propertiesChildren { + nextNode, ok := castCurrentNode[pSchema.property] + if ok { + subContext := newJsonContext(pSchema.property, context) + v.validateRecursive(pSchema, nextNode, result, subContext) + } + } + + // Simple JSON values : string, number, boolean + + case reflect.Bool: + + if currentSubSchema.types.IsTyped() && !currentSubSchema.types.Contains(TYPE_BOOLEAN) { + result.addError( + new(InvalidTypeError), + context, + currentNode, + ErrorDetails{ + "expected": currentSubSchema.types.String(), + "given": TYPE_BOOLEAN, + }, + ) + return + } + + value := currentNode.(bool) + + currentSubSchema.validateSchema(currentSubSchema, value, result, context) + v.validateNumber(currentSubSchema, value, result, context) + v.validateCommon(currentSubSchema, value, result, context) + v.validateString(currentSubSchema, value, result, context) + + case reflect.String: + + if currentSubSchema.types.IsTyped() && !currentSubSchema.types.Contains(TYPE_STRING) { + result.addError( + new(InvalidTypeError), + context, + currentNode, + ErrorDetails{ + "expected": currentSubSchema.types.String(), + "given": TYPE_STRING, + }, + ) + return + } + + value := currentNode.(string) + + currentSubSchema.validateSchema(currentSubSchema, value, result, context) + v.validateNumber(currentSubSchema, value, result, context) + v.validateCommon(currentSubSchema, value, result, context) + v.validateString(currentSubSchema, value, result, context) + + } + + } + + } + + result.incrementScore() +} + +// Different kinds of validation there, subSchema / common / array / object / string... +func (v *subSchema) validateSchema(currentSubSchema *subSchema, currentNode interface{}, result *Result, context *jsonContext) { + + if internalLogEnabled { + internalLog("validateSchema %s", context.String()) + internalLog(" %v", currentNode) + } + + if len(currentSubSchema.anyOf) > 0 { + + validatedAnyOf := false + var bestValidationResult *Result + + for _, anyOfSchema := range currentSubSchema.anyOf { + if !validatedAnyOf { + validationResult := anyOfSchema.subValidateWithContext(currentNode, context) + validatedAnyOf = validationResult.Valid() + + if !validatedAnyOf && (bestValidationResult == nil || validationResult.score > bestValidationResult.score) { + bestValidationResult = validationResult + } + } + } + if !validatedAnyOf { + + result.addError(new(NumberAnyOfError), context, currentNode, ErrorDetails{}) + + if bestValidationResult != nil { + // add error messages of closest matching subSchema as + // that's probably the one the user was trying to match + result.mergeErrors(bestValidationResult) + } + } + } + + if len(currentSubSchema.oneOf) > 0 { + + nbValidated := 0 + var bestValidationResult *Result + + for _, oneOfSchema := range currentSubSchema.oneOf { + validationResult := oneOfSchema.subValidateWithContext(currentNode, context) + if validationResult.Valid() { + nbValidated++ + } else if nbValidated == 0 && (bestValidationResult == nil || validationResult.score > bestValidationResult.score) { + bestValidationResult = validationResult + } + } + + if nbValidated != 1 { + + result.addError(new(NumberOneOfError), context, currentNode, ErrorDetails{}) + + if nbValidated == 0 { + // add error messages of closest matching subSchema as + // that's probably the one the user was trying to match + result.mergeErrors(bestValidationResult) + } + } + + } + + if len(currentSubSchema.allOf) > 0 { + nbValidated := 0 + + for _, allOfSchema := range currentSubSchema.allOf { + validationResult := allOfSchema.subValidateWithContext(currentNode, context) + if validationResult.Valid() { + nbValidated++ + } + result.mergeErrors(validationResult) + } + + if nbValidated != len(currentSubSchema.allOf) { + result.addError(new(NumberAllOfError), context, currentNode, ErrorDetails{}) + } + } + + if currentSubSchema.not != nil { + validationResult := currentSubSchema.not.subValidateWithContext(currentNode, context) + if validationResult.Valid() { + result.addError(new(NumberNotError), context, currentNode, ErrorDetails{}) + } + } + + if currentSubSchema.dependencies != nil && len(currentSubSchema.dependencies) > 0 { + if isKind(currentNode, reflect.Map) { + for elementKey := range currentNode.(map[string]interface{}) { + if dependency, ok := currentSubSchema.dependencies[elementKey]; ok { + switch dependency := dependency.(type) { + + case []string: + for _, dependOnKey := range dependency { + if _, dependencyResolved := currentNode.(map[string]interface{})[dependOnKey]; !dependencyResolved { + result.addError( + new(MissingDependencyError), + context, + currentNode, + ErrorDetails{"dependency": dependOnKey}, + ) + } + } + + case *subSchema: + dependency.validateRecursive(dependency, currentNode, result, context) + + } + } + } + } + } + + result.incrementScore() +} + +func (v *subSchema) validateCommon(currentSubSchema *subSchema, value interface{}, result *Result, context *jsonContext) { + + if internalLogEnabled { + internalLog("validateCommon %s", context.String()) + internalLog(" %v", value) + } + + // enum: + if len(currentSubSchema.enum) > 0 { + has, err := currentSubSchema.ContainsEnum(value) + if err != nil { + result.addError(new(InternalError), context, value, ErrorDetails{"error": err}) + } + if !has { + result.addError( + new(EnumError), + context, + value, + ErrorDetails{ + "allowed": strings.Join(currentSubSchema.enum, ", "), + }, + ) + } + } + + result.incrementScore() +} + +func (v *subSchema) validateArray(currentSubSchema *subSchema, value []interface{}, result *Result, context *jsonContext) { + + if internalLogEnabled { + internalLog("validateArray %s", context.String()) + internalLog(" %v", value) + } + + nbValues := len(value) + + // TODO explain + if currentSubSchema.itemsChildrenIsSingleSchema { + for i := range value { + subContext := newJsonContext(strconv.Itoa(i), context) + validationResult := currentSubSchema.itemsChildren[0].subValidateWithContext(value[i], subContext) + result.mergeErrors(validationResult) + } + } else { + if currentSubSchema.itemsChildren != nil && len(currentSubSchema.itemsChildren) > 0 { + + nbItems := len(currentSubSchema.itemsChildren) + + // while we have both schemas and values, check them against each other + for i := 0; i != nbItems && i != nbValues; i++ { + subContext := newJsonContext(strconv.Itoa(i), context) + validationResult := currentSubSchema.itemsChildren[i].subValidateWithContext(value[i], subContext) + result.mergeErrors(validationResult) + } + + if nbItems < nbValues { + // we have less schemas than elements in the instance array, + // but that might be ok if "additionalItems" is specified. + + switch currentSubSchema.additionalItems.(type) { + case bool: + if !currentSubSchema.additionalItems.(bool) { + result.addError(new(ArrayNoAdditionalItemsError), context, value, ErrorDetails{}) + } + case *subSchema: + additionalItemSchema := currentSubSchema.additionalItems.(*subSchema) + for i := nbItems; i != nbValues; i++ { + subContext := newJsonContext(strconv.Itoa(i), context) + validationResult := additionalItemSchema.subValidateWithContext(value[i], subContext) + result.mergeErrors(validationResult) + } + } + } + } + } + + // minItems & maxItems + if currentSubSchema.minItems != nil { + if nbValues < int(*currentSubSchema.minItems) { + result.addError( + new(ArrayMinItemsError), + context, + value, + ErrorDetails{"min": *currentSubSchema.minItems}, + ) + } + } + if currentSubSchema.maxItems != nil { + if nbValues > int(*currentSubSchema.maxItems) { + result.addError( + new(ArrayMaxItemsError), + context, + value, + ErrorDetails{"max": *currentSubSchema.maxItems}, + ) + } + } + + // uniqueItems: + if currentSubSchema.uniqueItems { + var stringifiedItems []string + for _, v := range value { + vString, err := marshalToJsonString(v) + if err != nil { + result.addError(new(InternalError), context, value, ErrorDetails{"err": err}) + } + if isStringInSlice(stringifiedItems, *vString) { + result.addError( + new(ItemsMustBeUniqueError), + context, + value, + ErrorDetails{"type": TYPE_ARRAY}, + ) + } + stringifiedItems = append(stringifiedItems, *vString) + } + } + + result.incrementScore() +} + +func (v *subSchema) validateObject(currentSubSchema *subSchema, value map[string]interface{}, result *Result, context *jsonContext) { + + if internalLogEnabled { + internalLog("validateObject %s", context.String()) + internalLog(" %v", value) + } + + // minProperties & maxProperties: + if currentSubSchema.minProperties != nil { + if len(value) < int(*currentSubSchema.minProperties) { + result.addError( + new(ArrayMinPropertiesError), + context, + value, + ErrorDetails{"min": *currentSubSchema.minProperties}, + ) + } + } + if currentSubSchema.maxProperties != nil { + if len(value) > int(*currentSubSchema.maxProperties) { + result.addError( + new(ArrayMaxPropertiesError), + context, + value, + ErrorDetails{"max": *currentSubSchema.maxProperties}, + ) + } + } + + // required: + for _, requiredProperty := range currentSubSchema.required { + _, ok := value[requiredProperty] + if ok { + result.incrementScore() + } else { + result.addError( + new(RequiredError), + context, + value, + ErrorDetails{"property": requiredProperty}, + ) + } + } + + // additionalProperty & patternProperty: + if currentSubSchema.additionalProperties != nil { + + switch currentSubSchema.additionalProperties.(type) { + case bool: + + if !currentSubSchema.additionalProperties.(bool) { + + for pk := range value { + + found := false + for _, spValue := range currentSubSchema.propertiesChildren { + if pk == spValue.property { + found = true + } + } + + pp_has, pp_match := v.validatePatternProperty(currentSubSchema, pk, value[pk], result, context) + + if found { + + if pp_has && !pp_match { + result.addError( + new(AdditionalPropertyNotAllowedError), + context, + value[pk], + ErrorDetails{"property": pk}, + ) + } + + } else { + + if !pp_has || !pp_match { + result.addError( + new(AdditionalPropertyNotAllowedError), + context, + value[pk], + ErrorDetails{"property": pk}, + ) + } + + } + } + } + + case *subSchema: + + additionalPropertiesSchema := currentSubSchema.additionalProperties.(*subSchema) + for pk := range value { + + found := false + for _, spValue := range currentSubSchema.propertiesChildren { + if pk == spValue.property { + found = true + } + } + + pp_has, pp_match := v.validatePatternProperty(currentSubSchema, pk, value[pk], result, context) + + if found { + + if pp_has && !pp_match { + validationResult := additionalPropertiesSchema.subValidateWithContext(value[pk], context) + result.mergeErrors(validationResult) + } + + } else { + + if !pp_has || !pp_match { + validationResult := additionalPropertiesSchema.subValidateWithContext(value[pk], context) + result.mergeErrors(validationResult) + } + + } + + } + } + } else { + + for pk := range value { + + pp_has, pp_match := v.validatePatternProperty(currentSubSchema, pk, value[pk], result, context) + + if pp_has && !pp_match { + + result.addError( + new(InvalidPropertyPatternError), + context, + value[pk], + ErrorDetails{ + "property": pk, + "pattern": currentSubSchema.PatternPropertiesString(), + }, + ) + } + + } + } + + result.incrementScore() +} + +func (v *subSchema) validatePatternProperty(currentSubSchema *subSchema, key string, value interface{}, result *Result, context *jsonContext) (has bool, matched bool) { + + if internalLogEnabled { + internalLog("validatePatternProperty %s", context.String()) + internalLog(" %s %v", key, value) + } + + has = false + + validatedkey := false + + for pk, pv := range currentSubSchema.patternProperties { + if matches, _ := regexp.MatchString(pk, key); matches { + has = true + subContext := newJsonContext(key, context) + validationResult := pv.subValidateWithContext(value, subContext) + result.mergeErrors(validationResult) + if validationResult.Valid() { + validatedkey = true + } + } + } + + if !validatedkey { + return has, false + } + + result.incrementScore() + + return has, true +} + +func (v *subSchema) validateString(currentSubSchema *subSchema, value interface{}, result *Result, context *jsonContext) { + + // Ignore JSON numbers + if isJsonNumber(value) { + return + } + + // Ignore non strings + if !isKind(value, reflect.String) { + return + } + + if internalLogEnabled { + internalLog("validateString %s", context.String()) + internalLog(" %v", value) + } + + stringValue := value.(string) + + // minLength & maxLength: + if currentSubSchema.minLength != nil { + if utf8.RuneCount([]byte(stringValue)) < int(*currentSubSchema.minLength) { + result.addError( + new(StringLengthGTEError), + context, + value, + ErrorDetails{"min": *currentSubSchema.minLength}, + ) + } + } + if currentSubSchema.maxLength != nil { + if utf8.RuneCount([]byte(stringValue)) > int(*currentSubSchema.maxLength) { + result.addError( + new(StringLengthLTEError), + context, + value, + ErrorDetails{"max": *currentSubSchema.maxLength}, + ) + } + } + + // pattern: + if currentSubSchema.pattern != nil { + if !currentSubSchema.pattern.MatchString(stringValue) { + result.addError( + new(DoesNotMatchPatternError), + context, + value, + ErrorDetails{"pattern": currentSubSchema.pattern}, + ) + + } + } + + // format + if currentSubSchema.format != "" { + if !FormatCheckers.IsFormat(currentSubSchema.format, stringValue) { + result.addError( + new(DoesNotMatchFormatError), + context, + value, + ErrorDetails{"format": currentSubSchema.format}, + ) + } + } + + result.incrementScore() +} + +func (v *subSchema) validateNumber(currentSubSchema *subSchema, value interface{}, result *Result, context *jsonContext) { + + // Ignore non numbers + if !isJsonNumber(value) { + return + } + + if internalLogEnabled { + internalLog("validateNumber %s", context.String()) + internalLog(" %v", value) + } + + number := value.(json.Number) + float64Value, _ := number.Float64() + + // multipleOf: + if currentSubSchema.multipleOf != nil { + + if !isFloat64AnInteger(float64Value / *currentSubSchema.multipleOf) { + result.addError( + new(MultipleOfError), + context, + resultErrorFormatJsonNumber(number), + ErrorDetails{"multiple": *currentSubSchema.multipleOf}, + ) + } + } + + //maximum & exclusiveMaximum: + if currentSubSchema.maximum != nil { + if currentSubSchema.exclusiveMaximum { + if float64Value >= *currentSubSchema.maximum { + result.addError( + new(NumberLTError), + context, + resultErrorFormatJsonNumber(number), + ErrorDetails{ + "max": resultErrorFormatNumber(*currentSubSchema.maximum), + }, + ) + } + } else { + if float64Value > *currentSubSchema.maximum { + result.addError( + new(NumberLTEError), + context, + resultErrorFormatJsonNumber(number), + ErrorDetails{ + "max": resultErrorFormatNumber(*currentSubSchema.maximum), + }, + ) + } + } + } + + //minimum & exclusiveMinimum: + if currentSubSchema.minimum != nil { + if currentSubSchema.exclusiveMinimum { + if float64Value <= *currentSubSchema.minimum { + result.addError( + new(NumberGTError), + context, + resultErrorFormatJsonNumber(number), + ErrorDetails{ + "min": resultErrorFormatNumber(*currentSubSchema.minimum), + }, + ) + } + } else { + if float64Value < *currentSubSchema.minimum { + result.addError( + new(NumberGTEError), + context, + resultErrorFormatJsonNumber(number), + ErrorDetails{ + "min": resultErrorFormatNumber(*currentSubSchema.minimum), + }, + ) + } + } + } + + // format + if currentSubSchema.format != "" { + if !FormatCheckers.IsFormat(currentSubSchema.format, float64Value) { + result.addError( + new(DoesNotMatchFormatError), + context, + value, + ErrorDetails{"format": currentSubSchema.format}, + ) + } + } + + result.incrementScore() +} diff --git a/vendor/k8s.io/api/apps/v1beta2/generated.proto b/vendor/k8s.io/api/apps/v1beta2/generated.proto index dac5a5f6..d8fac403 100644 --- a/vendor/k8s.io/api/apps/v1beta2/generated.proto +++ b/vendor/k8s.io/api/apps/v1beta2/generated.proto @@ -31,6 +31,8 @@ import "k8s.io/apimachinery/pkg/util/intstr/generated.proto"; // Package-wide variables from generator "generated". option go_package = "v1beta2"; +// DEPRECATED - This group version of ControllerRevision is deprecated by apps/v1/ControllerRevision. See the +// release notes for more information. // ControllerRevision implements an immutable snapshot of state data. Clients // are responsible for serializing and deserializing the objects that contain // their internal state. @@ -63,6 +65,8 @@ message ControllerRevisionList { repeated ControllerRevision items = 2; } +// DEPRECATED - This group version of DaemonSet is deprecated by apps/v1/DaemonSet. See the release notes for +// more information. // DaemonSet represents the configuration of a daemon set. message DaemonSet { // Standard object's metadata. @@ -218,6 +222,8 @@ message DaemonSetUpdateStrategy { optional RollingUpdateDaemonSet rollingUpdate = 2; } +// DEPRECATED - This group version of Deployment is deprecated by apps/v1/Deployment. See the release notes for +// more information. // Deployment enables declarative updates for Pods and ReplicaSets. message Deployment { // Standard object metadata. @@ -362,6 +368,8 @@ message DeploymentStrategy { optional RollingUpdateDeployment rollingUpdate = 2; } +// DEPRECATED - This group version of ReplicaSet is deprecated by apps/v1/ReplicaSet. See the release notes for +// more information. // ReplicaSet ensures that a specified number of pod replicas are running at any given time. message ReplicaSet { // If the Labels of a ReplicaSet are empty, they are defaulted to @@ -574,6 +582,8 @@ message ScaleStatus { optional string targetSelector = 3; } +// DEPRECATED - This group version of StatefulSet is deprecated by apps/v1/StatefulSet. See the release notes for +// more information. // StatefulSet represents a set of pods with consistent identities. // Identities are defined as: // - Network: A single stable DNS and hostname. diff --git a/vendor/k8s.io/api/apps/v1beta2/types.go b/vendor/k8s.io/api/apps/v1beta2/types.go index fb54f95c..c8be2aac 100644 --- a/vendor/k8s.io/api/apps/v1beta2/types.go +++ b/vendor/k8s.io/api/apps/v1beta2/types.go @@ -82,6 +82,8 @@ type Scale struct { // +genclient:method=UpdateScale,verb=update,subresource=scale,input=Scale,result=Scale // +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object +// DEPRECATED - This group version of StatefulSet is deprecated by apps/v1/StatefulSet. See the release notes for +// more information. // StatefulSet represents a set of pods with consistent identities. // Identities are defined as: // - Network: A single stable DNS and hostname. @@ -294,6 +296,8 @@ type StatefulSetList struct { // +genclient // +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object +// DEPRECATED - This group version of Deployment is deprecated by apps/v1/Deployment. See the release notes for +// more information. // Deployment enables declarative updates for Pods and ReplicaSets. type Deployment struct { metav1.TypeMeta `json:",inline"` @@ -657,6 +661,8 @@ type DaemonSetCondition struct { // +genclient // +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object +// DEPRECATED - This group version of DaemonSet is deprecated by apps/v1/DaemonSet. See the release notes for +// more information. // DaemonSet represents the configuration of a daemon set. type DaemonSet struct { metav1.TypeMeta `json:",inline"` @@ -703,6 +709,8 @@ type DaemonSetList struct { // +genclient // +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object +// DEPRECATED - This group version of ReplicaSet is deprecated by apps/v1/ReplicaSet. See the release notes for +// more information. // ReplicaSet ensures that a specified number of pod replicas are running at any given time. type ReplicaSet struct { metav1.TypeMeta `json:",inline"` @@ -829,6 +837,8 @@ type ReplicaSetCondition struct { // +genclient // +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object +// DEPRECATED - This group version of ControllerRevision is deprecated by apps/v1/ControllerRevision. See the +// release notes for more information. // ControllerRevision implements an immutable snapshot of state data. Clients // are responsible for serializing and deserializing the objects that contain // their internal state. diff --git a/vendor/k8s.io/api/apps/v1beta2/types_swagger_doc_generated.go b/vendor/k8s.io/api/apps/v1beta2/types_swagger_doc_generated.go index 43864f4c..e2f133b5 100644 --- a/vendor/k8s.io/api/apps/v1beta2/types_swagger_doc_generated.go +++ b/vendor/k8s.io/api/apps/v1beta2/types_swagger_doc_generated.go @@ -28,7 +28,7 @@ package v1beta2 // AUTO-GENERATED FUNCTIONS START HERE var map_ControllerRevision = map[string]string{ - "": "ControllerRevision implements an immutable snapshot of state data. Clients are responsible for serializing and deserializing the objects that contain their internal state. Once a ControllerRevision has been successfully created, it can not be updated. The API Server will fail validation of all requests that attempt to mutate the Data field. ControllerRevisions may, however, be deleted. Note that, due to its use by both the DaemonSet and StatefulSet controllers for update and rollback, this object is beta. However, it may be subject to name and representation changes in future releases, and clients should not depend on its stability. It is primarily for internal use by controllers.", + "": "DEPRECATED - This group version of ControllerRevision is deprecated by apps/v1/ControllerRevision. See the release notes for more information. ControllerRevision implements an immutable snapshot of state data. Clients are responsible for serializing and deserializing the objects that contain their internal state. Once a ControllerRevision has been successfully created, it can not be updated. The API Server will fail validation of all requests that attempt to mutate the Data field. ControllerRevisions may, however, be deleted. Note that, due to its use by both the DaemonSet and StatefulSet controllers for update and rollback, this object is beta. However, it may be subject to name and representation changes in future releases, and clients should not depend on its stability. It is primarily for internal use by controllers.", "metadata": "Standard object's metadata. More info: https://git.k8s.io/community/contributors/devel/api-conventions.md#metadata", "data": "Data is the serialized representation of the state.", "revision": "Revision indicates the revision of the state represented by Data.", @@ -49,7 +49,7 @@ func (ControllerRevisionList) SwaggerDoc() map[string]string { } var map_DaemonSet = map[string]string{ - "": "DaemonSet represents the configuration of a daemon set.", + "": "DEPRECATED - This group version of DaemonSet is deprecated by apps/v1/DaemonSet. See the release notes for more information. DaemonSet represents the configuration of a daemon set.", "metadata": "Standard object's metadata. More info: https://git.k8s.io/community/contributors/devel/api-conventions.md#metadata", "spec": "The desired behavior of this daemon set. More info: https://git.k8s.io/community/contributors/devel/api-conventions.md#spec-and-status", "status": "The current status of this daemon set. This data may be out of date by some window of time. Populated by the system. Read-only. More info: https://git.k8s.io/community/contributors/devel/api-conventions.md#spec-and-status", @@ -124,7 +124,7 @@ func (DaemonSetUpdateStrategy) SwaggerDoc() map[string]string { } var map_Deployment = map[string]string{ - "": "Deployment enables declarative updates for Pods and ReplicaSets.", + "": "DEPRECATED - This group version of Deployment is deprecated by apps/v1/Deployment. See the release notes for more information. Deployment enables declarative updates for Pods and ReplicaSets.", "metadata": "Standard object metadata.", "spec": "Specification of the desired behavior of the Deployment.", "status": "Most recently observed status of the Deployment.", @@ -201,7 +201,7 @@ func (DeploymentStrategy) SwaggerDoc() map[string]string { } var map_ReplicaSet = map[string]string{ - "": "ReplicaSet ensures that a specified number of pod replicas are running at any given time.", + "": "DEPRECATED - This group version of ReplicaSet is deprecated by apps/v1/ReplicaSet. See the release notes for more information. ReplicaSet ensures that a specified number of pod replicas are running at any given time.", "metadata": "If the Labels of a ReplicaSet are empty, they are defaulted to be the same as the Pod(s) that the ReplicaSet manages. Standard object's metadata. More info: https://git.k8s.io/community/contributors/devel/api-conventions.md#metadata", "spec": "Spec defines the specification of the desired behavior of the ReplicaSet. More info: https://git.k8s.io/community/contributors/devel/api-conventions.md#spec-and-status", "status": "Status is the most recently observed status of the ReplicaSet. This data may be out of date by some window of time. Populated by the system. Read-only. More info: https://git.k8s.io/community/contributors/devel/api-conventions.md#spec-and-status", @@ -320,7 +320,7 @@ func (ScaleStatus) SwaggerDoc() map[string]string { } var map_StatefulSet = map[string]string{ - "": "StatefulSet represents a set of pods with consistent identities. Identities are defined as:\n - Network: A single stable DNS and hostname.\n - Storage: As many VolumeClaims as requested.\nThe StatefulSet guarantees that a given network identity will always map to the same storage identity.", + "": "DEPRECATED - This group version of StatefulSet is deprecated by apps/v1/StatefulSet. See the release notes for more information. StatefulSet represents a set of pods with consistent identities. Identities are defined as:\n - Network: A single stable DNS and hostname.\n - Storage: As many VolumeClaims as requested.\nThe StatefulSet guarantees that a given network identity will always map to the same storage identity.", "spec": "Spec defines the desired identities of pods in this set.", "status": "Status is the current status of Pods in this StatefulSet. This data may be out of date by some window of time.", } diff --git a/version/version.go b/version/version.go index 1a66ef29..7eaa12c4 100644 --- a/version/version.go +++ b/version/version.go @@ -1,4 +1,4 @@ package version // Version is the version of the build. -const Version = "1.9.2-dev" +const Version = "1.9.0-dev"