From f7d8f1315c6ce482952a0d4e4af27eb527a8b634 Mon Sep 17 00:00:00 2001 From: Vincent Batts Date: Thu, 17 Oct 2019 09:56:45 -0400 Subject: [PATCH 1/3] rpm: crafting an rpmspec for this tool Now you can `make srpm` or `make rpm`. Additionally, trying to make a way for copr to build this as well. We'll see if that is something. --- .copr/Makefile | 1 + .gitignore | 2 ++ BuildSourceImage.sh | 12 ++++++++- BuildSourceImage.spec | 43 +++++++++++++++++++++++++++++ Makefile | 63 ++++++++++++++++++++++++++++++++++++------- 5 files changed, 110 insertions(+), 11 deletions(-) create mode 120000 .copr/Makefile create mode 100644 BuildSourceImage.spec diff --git a/.copr/Makefile b/.copr/Makefile new file mode 120000 index 0000000..d0b0e8e --- /dev/null +++ b/.copr/Makefile @@ -0,0 +1 @@ +../Makefile \ No newline at end of file diff --git a/.gitignore b/.gitignore index 50554ee..34913a3 100644 --- a/.gitignore +++ b/.gitignore @@ -2,3 +2,5 @@ .build-container .testprep .validate +*.rpm +x86_64/ diff --git a/BuildSourceImage.sh b/BuildSourceImage.sh index 59b08f7..8bb713e 100755 --- a/BuildSourceImage.sh +++ b/BuildSourceImage.sh @@ -16,6 +16,8 @@ _usage() { _version echo "Usage: $(basename "$0") [-D] [-b ] [-c ] [-e ] [-r ] [-o ] [-i ] [-p ] [-l] [-d ]" echo "" + echo " Container Source Image tool" + echo "" echo -e " -b \tbase path for source image builds" echo -e " -c \tbuild context for the container image. Can be provided via CONTEXT_DIR env variable" echo -e " -e \textra src for the container image. Can be provided via EXTRA_SRC_DIR env variable" @@ -128,6 +130,14 @@ _tar() { fi } +_rpm_download() { + if [ "$(command -v yumdownloader)" != "" ] ; then + yumdownloader "${@}" + else + dnf download "${@}" + fi +} + # output things, only when $DEBUG is set _debug() { if [ -n "${DEBUG}" ] ; then @@ -841,7 +851,7 @@ sourcedriver_rpm_fetch() { rpm=${srcrpm%*.src.rpm} if [ ! -f "${out_dir}/${srcrpm}" ] ; then _debug "--> fetching ${srcrpm}" - dnf download \ + _rpm_download \ --quiet \ --installroot "${rootfs}" \ --release "${release}" \ diff --git a/BuildSourceImage.spec b/BuildSourceImage.spec new file mode 100644 index 0000000..879399e --- /dev/null +++ b/BuildSourceImage.spec @@ -0,0 +1,43 @@ +Name: BuildSourceImage +Version: 0.2 +Release: 1%{?dist} +Summary: Container Source Image tool + +Group: containers +License: GPLv2 +URL: https://github.com/containers/BuildSourceImage +Source0: BuildSourceImage.sh + +#BuildRequires: +Requires: jq +Requires: skopeo +Requires: findutils +Requires: file +%if 0%{?rhel} > 6 +Requires: yum-utils +%else +Requires: dnf-command(download) +%endif + +%description +%{summary}. + +%prep + + +%build + + +%install +%{__mkdir_p} %{buildroot}/%{_bindir} +%{__install} -T -m 0755 ${RPM_SOURCE_DIR}/BuildSourceImage.sh %{buildroot}/%{_bindir}/BuildSourceImage + + +%files +%doc ${RPM_SOURCE_DIR}/LICENSE ${RPM_SOURCE_DIR}/README.md +%{_bindir}/BuildSourceImage + + + +%changelog + diff --git a/Makefile b/Makefile index c38791d..f8b4969 100644 --- a/Makefile +++ b/Makefile @@ -1,14 +1,29 @@ -SRC := ./BuildSourceImage.sh -CTR_IMAGE := localhost/containers/buildsourceimage -CTR_ENGINE ?= podman -BATS_OPTS ?= -cleanfiles = +pkgname := BuildSourceImage +CTR_IMAGE := localhost/containers/buildsourceimage +CTR_ENGINE ?= podman +BATS_OPTS ?= +cleanfiles = # these are packages whose src.rpms are very small -srpm_urls = \ +srpm_urls = \ https://archive.kernel.org/centos-vault/7.0.1406/os/Source/SPackages/basesystem-10.0-7.el7.centos.src.rpm \ https://archive.kernel.org/centos-vault/7.0.1406/os/Source/SPackages/rootfiles-8.1-11.el7.src.rpm \ https://archive.kernel.org/centos-vault/7.0.1406/os/Source/SPackages/centos-bookmarks-7-1.el7.src.rpm -srpms = $(addprefix ./.testprep/srpms/,$(notdir $(rpms))) +srpms = $(addprefix ./.testprep/srpms/,$(notdir $(rpms))) + +spec ?= $(pkgname).spec +pwd := $(shell pwd) +NAME := $(shell rpmspec -q --qf "%{name}" $(spec)) +VERSION := $(shell rpmspec -q --qf "%{version}" $(spec)) +RELEASE := $(shell rpmspec -q --qf "%{release}" $(spec)) +ARCH := $(shell rpmspec -q --qf "%{arch}" $(spec)) +NVR := $(NAME)-$(VERSION)-$(RELEASE) +outdir ?= $(pwd) + +SHELL_SRC := ./BuildSourceImage.sh +DIST_FILES := \ + $(SHELL_SRC) \ + LICENSE \ + README.md export CTR_IMAGE export CTR_ENGINE @@ -18,13 +33,13 @@ all: validate validate: .validate cleanfiles += .validate -.validate: $(SRC) - shellcheck $(SRC) && touch $@ +.validate: $(SHELL_SRC) + shellcheck $(SHELL_SRC) && touch $@ build-container: .build-container cleanfiles += .build-container -.build-container: .validate Dockerfile $(SRC) +.build-container: .validate Dockerfile $(SHELL_SRC) @echo @echo "==> Building BuildSourceImage Container" $(CTR_ENGINE) build --quiet --file Dockerfile --tag $(CTR_IMAGE) . && touch $@ @@ -41,6 +56,34 @@ test-integration: .build-container .testprep @echo "==> Running integration tests" TMPDIR=$(realpath .testprep/tmp) bats $(BATS_OPTS) test/ +.PHONY: srpm +srpm: $(NVR).src.rpm + @echo $^ + +cleanfiles += $(NVR).src.rpm +$(NVR).src.rpm: $(spec) $(DIST_FILES) + rpmbuild \ + --define '_sourcedir $(pwd)' \ + --define '_specdir $(pwd)' \ + --define '_builddir $(pwd)' \ + --define '_srcrpmdir $(outdir)' \ + --define '_rpmdir $(outdir)' \ + --nodeps \ + -bs ./$(spec) + +.PHONY: rpm +rpm: $(ARCH)/$(NVR).$(ARCH).rpm + @echo $^ + +cleanfiles += $(ARCH)/$(NVR).$(ARCH).rpm +$(ARCH)/$(NVR).$(ARCH).rpm: $(spec) $(DIST_FILES) + rpmbuild \ + --define '_sourcedir $(pwd)' \ + --define '_specdir $(pwd)' \ + --define '_builddir $(pwd)' \ + --define '_srcrpmdir $(outdir)' \ + --define '_rpmdir $(outdir)' \ + -bb ./$(spec) clean: if [ -n "$(cleanfiles)" ] ; then rm -rf $(cleanfiles) ; fi From 3f6301a8a478371e52f25a5061e3035e5bbfd2d2 Mon Sep 17 00:00:00 2001 From: Vincent Batts Date: Thu, 17 Oct 2019 20:46:59 +0000 Subject: [PATCH 2/3] README: point to non-vbatts container repo Fixes #17 --- README.md | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 6fdb4ed..da5e1a0 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,5 @@ [![Build Status](https://travis-ci.org/containers/BuildSourceImage.svg?branch=master)](https://travis-ci.org/containers/BuildSourceImage) -[![Container Image Repository on Quay](https://quay.io/repository/vbatts/bsi/status "Container Image Repository on Quay")](https://quay.io/repository/vbatts/bsi) +[![Container Image Repository on Quay](https://quay.io/repository/ctrs/bsi/status "Container Image Repository on Quay")](https://quay.io/repository/ctrs/bsi) # BuildSourceImage @@ -32,9 +32,8 @@ Usage: BuildSourceImage.sh [-D] [-b ] [-c ] [-e ] [-r ] Nicely usable inside a container: ```bash -$> podman build -t containers/buildsourceimage . $> mkdir ./output/ -$> podman run -it -v $(pwd)/output/:/output/ -v $(pwd)/SRCRPMS/:/data/ -u $(id -u) containers/buildsourceimage -s /data/ +$> podman run -it -v $(pwd)/output/:/output/ -v $(pwd)/SRCRPMS/:/data/ -u $(id -u) quay.io/ctrs/bsi -s /data/ -o /output/ ``` ## Examples From 906951f906f5b20acea12ac20a79a682fe879e18 Mon Sep 17 00:00:00 2001 From: Vincent Batts Date: Mon, 11 Nov 2019 14:37:18 +0000 Subject: [PATCH 3/3] WIP --- BuildSourceImage.sh | 82 +-------------------------------- relocate.sh | 108 ++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 110 insertions(+), 80 deletions(-) create mode 100644 relocate.sh diff --git a/BuildSourceImage.sh b/BuildSourceImage.sh index 59b08f7..96a505b 100755 --- a/BuildSourceImage.sh +++ b/BuildSourceImage.sh @@ -39,7 +39,7 @@ _init() { set -o pipefail # check for tools we depend on - for cmd in jq skopeo dnf file find tar stat date ; do + for cmd in jq dnf file find tar stat date ; do if [ -z "$(command -v ${cmd})" ] ; then # TODO: maybe this could be individual checks so it can report # where to find the tools @@ -209,7 +209,7 @@ parse_img_tag() { # # an inline prefixer for containers/image tools -# +# XXX redo this to only validate for 'oci:...', otherwise bail ref_prefix() { local ref="${1}" local pfxs @@ -243,68 +243,6 @@ ref_src_img_tag() { echo -n "$(parse_img_tag "${ref}")""${source_image_suffix}" } -# -# call out to registry for the image reference's digest checksum -# -fetch_img_digest() { - local ref="${1}" - local dgst - local ret - - ## TODO: check for authfile, creds, and whether it's an insecure registry - dgst=$(skopeo inspect "$(ref_prefix "${ref}")" | jq .Digest | tr -d \") - ret=$? - if [ $ret -ne 0 ] ; then - echo "ERROR: check the image reference: ${ref}" >&2 - return $ret - fi - - echo -n "${dgst}" -} - -# -# pull down the image to an OCI layout -# arguments: image ref -# returns: path:tag to the OCI layout -# -# any commands should only output to stderr, so that the caller can receive the -# path reference to the OCI layout. -# -fetch_img() { - local ref="${1}" - local dst="${2}" - local base - local tag - local dgst - local from - local ret - - _mkdir_p "${dst}" - - base="$(parse_img_base "${ref}")" - tag="$(parse_img_tag "${ref}")" - dgst="$(parse_img_digest "${ref}")" - from="" - # skopeo currently only support _either_ tag _or_ digest, so we'll be specific. - if [ -n "${dgst}" ] ; then - from="$(ref_prefix "${base}")@${dgst}" - else - from="$(ref_prefix "${base}"):${tag}" - fi - - ## TODO: check for authfile, creds, and whether it's an insecure registry - ## destination name must have the image tag included (umoci expects it) - skopeo \ - copy \ - "${from}" \ - "oci:${dst}:${tag}" >&2 - ret=$? - if [ ${ret} -ne 0 ] ; then - return ${ret} - fi - echo -n "${dst}:${tag}" -} - # # upack_img # @@ -427,21 +365,6 @@ unpack_img_umoci() { return $ret } -# -# copy an image from one location to another -# -push_img() { - local src="${1}" - local dst="${2}" - - _debug "pushing image ${src} to ${dst}" - ## TODO: check for authfile, creds, and whether it's an insecure registry - skopeo copy --dest-tls-verify=false "$(ref_prefix "${src}")" "$(ref_prefix "${dst}")" # XXX for demo only - #skopeo copy "$(ref_prefix "${src}")" "$(ref_prefix "${dst}")" - ret=$? - return $ret -} - # # sets up a basic new OCI layout, for an image with the provided (or default 'latest') tag # @@ -1258,7 +1181,6 @@ main() { # TODO maybe look to a directory like /usr/libexec/BuildSourceImage/drivers/ for drop-ins to run _info "succesfully packed 'oci:${src_img_dir}:${src_img_tag}'" - _debug "$(skopeo inspect oci:"${src_img_dir}":"${src_img_tag}")" ## if an output directory is provided then save a copy to it if [ -n "${output_dir}" ] ; then diff --git a/relocate.sh b/relocate.sh new file mode 100644 index 0000000..8d52a87 --- /dev/null +++ b/relocate.sh @@ -0,0 +1,108 @@ +#!/bin/bash + +## depracted code for moving containers (what skopeo does) + +# +# copy an image from one location to another +# +push_img() { + local src="${1}" + local dst="${2}" + + _debug "pushing image ${src} to ${dst}" + ## TODO: check for authfile, creds, and whether it's an insecure registry + skopeo copy --dest-tls-verify=false "$(ref_prefix "${src}")" "$(ref_prefix "${dst}")" # XXX for demo only + #skopeo copy "$(ref_prefix "${src}")" "$(ref_prefix "${dst}")" + ret=$? + return $ret +} + + +# +# call out to registry for the image reference's digest checksum +# +fetch_img_digest() { + local ref="${1}" + local dgst + local ret + + ## TODO: check for authfile, creds, and whether it's an insecure registry + dgst=$(skopeo inspect "$(ref_prefix "${ref}")" | jq .Digest | tr -d \") + ret=$? + if [ $ret -ne 0 ] ; then + echo "ERROR: check the image reference: ${ref}" >&2 + return $ret + fi + + echo -n "${dgst}" +} + + +# +# an inline prefixer for containers/image tools +# +ref_prefix() { + local ref="${1}" + local pfxs + local ret + + # get the supported prefixes of the current version of skopeo + mapfile -t pfxs < <(skopeo copy --help | grep -A1 "Supported transports:" | grep -v "Supported transports" | sed 's/, /\n/g') + ret=$? + if [ ${ret} -ne 0 ] ; then + return ${ret} + fi + + for pfx in "${pfxs[@]}" ; do + if echo "${ref}" | grep -q "^${pfx}:" ; then + # break if we match a known prefix + echo "${ref}" + return 0 + fi + done + # else default + echo "docker://${ref}" +} + +# +# pull down the image to an OCI layout +# arguments: image ref +# returns: path:tag to the OCI layout +# +# any commands should only output to stderr, so that the caller can receive the +# path reference to the OCI layout. +# +fetch_img() { + local ref="${1}" + local dst="${2}" + local base + local tag + local dgst + local from + local ret + + _mkdir_p "${dst}" + + base="$(parse_img_base "${ref}")" + tag="$(parse_img_tag "${ref}")" + dgst="$(parse_img_digest "${ref}")" + from="" + # skopeo currently only support _either_ tag _or_ digest, so we'll be specific. + if [ -n "${dgst}" ] ; then + from="$(ref_prefix "${base}")@${dgst}" + else + from="$(ref_prefix "${base}"):${tag}" + fi + + ## TODO: check for authfile, creds, and whether it's an insecure registry + ## destination name must have the image tag included (umoci expects it) + skopeo \ + copy \ + "${from}" \ + "oci:${dst}:${tag}" >&2 + ret=$? + if [ ${ret} -ne 0 ] ; then + return ${ret} + fi + echo -n "${dst}:${tag}" +} \ No newline at end of file