Merge pull request #54 from vbatts/new_container_repo

README: point to non-vbatts container repo

.copr/Makefile

@@ -0,0 +1 @@
+../Makefile

.gitignore

@@ -2,3 +2,5 @@
 .build-container
 .testprep
 .validate
+*.rpm
+x86_64/

.travis.yml

@@ -1,4 +1,5 @@
 sudo: required
+dist: bionic
 
 services:
         - docker
@@ -7,8 +8,6 @@ env:
         - CTR_ENGINE=docker
 
 before_install:
-        - sudo add-apt-repository ppa:duggan/bats --yes
-        - sudo apt-get update -qq
         - sudo apt-get install -qq bats shellcheck
 
 script:

BuildSourceImage.sh

@@ -8,7 +8,7 @@ export source_image_suffix="-source"
 
 # output version string
 _version() {
-    echo "$(basename "${0}") version 0.1"
+    echo "$(basename "${0}") version 0.2.0-dev"
 }
 
 # output the cli usage and exit
@@ -16,6 +16,8 @@ _usage() {
     _version
     echo "Usage: $(basename "$0") [-D] [-b <path>] [-c <path>] [-e <path>] [-r <path>] [-o <path>] [-i <image>] [-p <image>] [-l] [-d <drivers>]"
     echo ""
+    echo "          Container Source Image tool"
+    echo ""
     echo -e "       -b <path>\tbase path for source image builds"
     echo -e "       -c <path>\tbuild context for the container image. Can be provided via CONTEXT_DIR env variable"
     echo -e "       -e <path>\textra src for the container image. Can be provided via EXTRA_SRC_DIR env variable"
@@ -28,6 +30,10 @@ _usage() {
     echo -e "       -D\t\tdebuging output. Can be set via DEBUG env variable"
     echo -e "       -h\t\tthis usage information"
     echo -e "       -v\t\tversion"
+    echo -e ""
+    echo -e "    Subcommands:"
+    echo -e "       unpack\tUnpack an OCI layout to a rootfs directory"
+    echo -e ""
 }
 
 # sanity checks on startup
@@ -44,6 +50,23 @@ _init() {
     done
 }
 
+# enable access to some of functions as subcommands!
+_subcommand() {
+    local command="${1}"
+    local ret
+
+    shift
+
+    case "${command}" in
+        unpack)
+            # (vb) i'd prefer this subcommand directly match the function name, but it isn't as pretty.
+            unpack_img "${@}"
+            ret=$?
+            exit "${ret}"
+            ;;
+    esac
+}
+
 # _is_sourced tests whether this script is being source, or executed directly
 _is_sourced() {
     # https://unix.stackexchange.com/a/215279
@@ -107,6 +130,14 @@ _tar() {
     fi
 }
 
+_rpm_download() {
+    if [ "$(command -v yumdownloader)" != "" ] ; then
+        yumdownloader "${@}"
+    else
+        dnf download "${@}"
+    fi
+}
+
 # output things, only when $DEBUG is set
 _debug() {
     if [ -n "${DEBUG}" ] ; then
@@ -292,6 +323,20 @@ unpack_img() {
     local unpack_dir="${2}"
     local ret
 
+    while getopts ":h" opts; do
+        case "${opts}" in
+            *)
+                echo "$0 unpack <oci layout path> <unpack path>"
+                return 1
+                ;;
+        esac
+    done
+    shift $((OPTIND-1))
+
+    if [ -z "${image_dir}" ] || [ -z "${unpack_dir}" ] ; then
+        _error "[unpack_img] blank arguments provided"
+    fi
+
     if [ -d "${unpack_dir}" ] ; then
         _rm_rf "${unpack_dir}"
     fi
@@ -336,9 +381,10 @@ unpack_img_bash() {
         return ${ret}
     fi
 
-    # Since we're landing the reference as an OCI layout, this mediaType is fairly predictable
-    # TODO don't always assume +gzip
-    layer_dgsts="$(jq '.layers[] | select(.mediaType == "application/vnd.oci.image.layer.v1.tar+gzip") | .digest' "${image_dir}"/blobs/"${mnfst_dgst/:/\/}" | tr -d \")"
+    # TODO this will need to be refactored when we start seeing +zstd layers.
+    # Then it will be better to no just get a list of digests, but maybe to
+    # iterate on each descriptor independently?
+    layer_dgsts="$(jq '.layers | map(select(.mediaType == "application/vnd.oci.image.layer.v1.tar+gzip"),select(.mediaType == "application/vnd.oci.image.layer.v1.tar"),select(.mediaType == "application/vnd.docker.image.rootfs.diff.tar.gzip")) | .[] | .digest' "${image_dir}"/blobs/"${mnfst_dgst/:/\/}" | tr -d \")"
     ret=$?
     if [ ${ret} -ne 0 ] ; then
         return ${ret}
@@ -410,12 +456,62 @@ push_img() {
 # sets up a basic new OCI layout, for an image with the provided (or default 'latest') tag
 #
 layout_new() {
+    local out_dir="${1}"
+    local image_tag="${2:-latest}"
+    local ret
+
+    if [ -n "$(command -v umoci)" ] ; then
+        layout_new_umoci "${out_dir}" "${image_tag}"
+        ret=$?
+        if [ ${ret} -ne 0 ] ; then
+            return ${ret}
+        fi
+    else
+        layout_new_bash "${out_dir}" "${image_tag}"
+        ret=$?
+        if [ ${ret} -ne 0 ] ; then
+            return ${ret}
+        fi
+    fi
+}
+
+#
+# sets up new OCI layout, using `umoci`
+#
+layout_new_umoci() {
+    local out_dir="${1}"
+    local image_tag="${2:-latest}"
+    local ret
+
+    # umoci expects the layout path to _not_ exist and will fail if it does exist
+    _rm_rf "${out_dir}"
+
+    umoci init --layout "${out_dir}"
+    ret=$?
+    if [ "${ret}" -ne 0 ] ; then
+        return "${ret}"
+    fi
+
+    # XXX currently does not support adding the rich annotations like I've done with the _bash
+    # https://github.com/openSUSE/umoci/issues/298
+    umoci new --image "${out_dir}:${image_tag}"
+    ret=$?
+    if [ "${ret}" -ne 0 ] ; then
+        return "${ret}"
+    fi
+}
+
+#
+# sets up new OCI layout, all with bash and jq
+#
+layout_new_bash() {
     local out_dir="${1}"
     local image_tag="${2:-latest}"
     local config
     local mnfst
     local config_sum
     local mnfst_sum
+    local ret
 
     _mkdir_p "${out_dir}/blobs/sha256"
     echo '{"imageLayoutVersion":"1.0.0"}' > "${out_dir}/oci-layout"
@@ -432,7 +528,15 @@ layout_new() {
 }
     '
     config_sum=$(echo "${config}" | jq -c | tr -d '\n' | sha256sum | awk '{ ORS=""; print $1 }')
+    ret=$?
+    if [ "${ret}" -ne 0 ] ; then
+        return "${ret}"
+    fi
     echo "${config}" | jq -c | tr -d '\n' > "${out_dir}/blobs/sha256/${config_sum}"
+    ret=$?
+    if [ "${ret}" -ne 0 ] ; then
+        return "${ret}"
+    fi
 
     mnfst='
 {
@@ -474,6 +578,72 @@ layout_new() {
 #   * tag used in the layout (default is 'latest')
 #
 layout_insert() {
+    local out_dir="${1}"
+    local artifact_path="${2}"
+    local tar_path="${3}"
+    local annotations_file="${4}"
+    local image_tag="${5:-latest}"
+    local ret
+
+    if [ -n "$(command -v umoci)" ] ; then
+        layout_insert_umoci "${out_dir}" "${artifact_path}" "${tar_path}" "${annotations_file}" "${image_tag}"
+        ret=$?
+        if [ ${ret} -ne 0 ] ; then
+            return ${ret}
+        fi
+    else
+        layout_insert_bash "${out_dir}" "${artifact_path}" "${tar_path}" "${annotations_file}" "${image_tag}"
+        ret=$?
+        if [ ${ret} -ne 0 ] ; then
+            return ${ret}
+        fi
+    fi
+}
+
+layout_insert_umoci() {
+    local out_dir="${1}"
+    local artifact_path="${2}"
+    local tar_path="${3}"
+    local annotations_file="${4}"
+    local image_tag="${5:-latest}"
+    local sum
+    local ret
+
+    # prep the blob path for inside the layer, so we can just copy that whole path in
+    tmpdir="$(_mktemp_d)"
+
+    # TODO account for "artifact_path" being a directory?
+    sum="$(sha256sum "${artifact_path}" | awk '{ print $1 }')"
+
+    _mkdir_p "${tmpdir}/blobs/sha256"
+    cp "${artifact_path}" "${tmpdir}/blobs/sha256/${sum}"
+    if [ "$(basename "${tar_path}")" == "$(basename "${artifact_path}")" ] ; then
+        _mkdir_p "${tmpdir}/$(dirname "${tar_path}")"
+        # TODO this symlink need to be relative path, not to `/blobs/...`
+        ln -s "/blobs/sha256/${sum}" "${tmpdir}/${tar_path}"
+    else
+        _mkdir_p "${tmpdir}/${tar_path}"
+        # TODO this symlink need to be relative path, not to `/blobs/...`
+        ln -s "/blobs/sha256/${sum}" "${tmpdir}/${tar_path}/$(basename "${artifact_path}")"
+    fi
+
+    # XXX currently does not support adding the rich annotations like I've done with the _bash
+    # https://github.com/openSUSE/umoci/issues/298
+    # XXX this insert operation can not disable compression
+    # https://github.com/openSUSE/umoci/issues/300
+    umoci insert \
+        --rootless \
+        --image "${out_dir}:${image_tag}" \
+        --history.created "$(_date_ns)" \
+        --history.comment "#(nop) $(_version) adding artifact: ${sum}" \
+        "${tmpdir}" "/"
+    ret=$?
+    if [ ${ret} -ne 0 ] ; then
+        return ${ret}
+    fi
+}
+
+layout_insert_bash() {
     local out_dir="${1}"
     local artifact_path="${2}"
     local tar_path="${3}"
@@ -517,11 +687,11 @@ layout_insert() {
     if [ "$(basename "${tar_path}")" == "$(basename "${artifact_path}")" ] ; then
         _mkdir_p "${tmpdir}/$(dirname "${tar_path}")"
         # TODO this symlink need to be relative path, not to `/blobs/...`
-        ln -s "/blobs/sha256/${sum}" "${tmpdir}/${tar_path}"
+        ln -s "../blobs/sha256/${sum}" "${tmpdir}/${tar_path}"
     else
         _mkdir_p "${tmpdir}/${tar_path}"
         # TODO this symlink need to be relative path, not to `/blobs/...`
-        ln -s "/blobs/sha256/${sum}" "${tmpdir}/${tar_path}/$(basename "${artifact_path}")"
+        ln -s "../blobs/sha256/${sum}" "${tmpdir}/${tar_path}/$(basename "${artifact_path}")"
     fi
     tmptar="$(_mktemp)"
 
@@ -542,7 +712,7 @@ layout_insert() {
     jq -c \
         --arg date "$(_date_ns)" \
         --arg tmptar_sum "sha256:${tmptar_sum}" \
-        --arg comment "#(nop) BuildSourceImage adding artifact: ${sum}" \
+        --arg comment "#(nop) $(_version) adding artifact: ${sum}" \
         '
         .created = $date
         | .rootfs.diff_ids += [ $tmptar_sum ]
@@ -681,7 +851,7 @@ sourcedriver_rpm_fetch() {
         rpm=${srcrpm%*.src.rpm}
         if [ ! -f "${out_dir}/${srcrpm}" ] ; then
             _debug "--> fetching ${srcrpm}"
-            dnf download \
+            _rpm_download \
                 --quiet \
                 --installroot "${rootfs}" \
                 --release "${release}" \
@@ -902,8 +1072,9 @@ main() {
     local work_dir
 
     _init "${@}"
+    _subcommand "${@}"
 
-    base_dir="$(pwd)/${ABV_NAME}"
+    base_dir="${BASE_DIR:-$(pwd)/${ABV_NAME}}"
     # using the bash builtin to parse
     while getopts ":hlvDi:c:s:e:o:b:d:p:" opts; do
         case "${opts}" in
@@ -996,6 +1167,10 @@ main() {
         # including its digest.
         if [ -z "${inspect_image_digest}" ] ; then
             inspect_image_digest="$(fetch_img_digest "$(parse_img_base "${input_inspect_image_ref}"):$(parse_img_tag "${input_inspect_image_ref}")")"
+            ret=$?
+            if [ ${ret} -ne 0 ] ; then
+                _error "failed to detect image digest"
+            fi
         fi
         _debug "inspect_image_digest: ${inspect_image_digest}"
 

BuildSourceImage.spec

@@ -0,0 +1,43 @@
+Name:		BuildSourceImage
+Version:	0.2
+Release:	1%{?dist}
+Summary:	Container Source Image tool
+
+Group:		containers
+License:	GPLv2
+URL:		https://github.com/containers/BuildSourceImage
+Source0:	BuildSourceImage.sh
+
+#BuildRequires:	
+Requires:	jq
+Requires:	skopeo
+Requires:	findutils
+Requires:	file
+%if 0%{?rhel} > 6
+Requires:	yum-utils
+%else
+Requires:	dnf-command(download)
+%endif
+
+%description
+%{summary}.
+
+%prep
+
+
+%build
+
+
+%install
+%{__mkdir_p} %{buildroot}/%{_bindir}
+%{__install} -T -m 0755 ${RPM_SOURCE_DIR}/BuildSourceImage.sh %{buildroot}/%{_bindir}/BuildSourceImage
+
+
+%files
+%doc ${RPM_SOURCE_DIR}/LICENSE ${RPM_SOURCE_DIR}/README.md
+%{_bindir}/BuildSourceImage
+
+
+
+%changelog
+

Dockerfile

@@ -4,8 +4,6 @@ RUN dnf install -y jq skopeo findutils file 'dnf-command(download)'
 
 COPY ./BuildSourceImage.sh /usr/local/bin/BuildSourceImage.sh
 
-RUN mkdir -p /output
-ENV OUTPUT_DIR=/output
-VOLUME /output
+ENV BASE_DIR=/tmp
 
-ENTRYPOINT ["/usr/local/bin/BuildSourceImage.sh", "-b", "/tmp/"]
+ENTRYPOINT ["/usr/local/bin/BuildSourceImage.sh"]

Makefile

@@ -1,14 +1,29 @@
-SRC := ./BuildSourceImage.sh
-CTR_IMAGE := localhost/containers/buildsourceimage
-CTR_ENGINE ?= podman
-BATS_OPTS ?=
-cleanfiles =
+pkgname		:= BuildSourceImage
+CTR_IMAGE	:= localhost/containers/buildsourceimage
+CTR_ENGINE	?= podman
+BATS_OPTS	?=
+cleanfiles	=
 # these are packages whose src.rpms are very small
-srpm_urls = \
+srpm_urls	= \
 	https://archive.kernel.org/centos-vault/7.0.1406/os/Source/SPackages/basesystem-10.0-7.el7.centos.src.rpm \
 	https://archive.kernel.org/centos-vault/7.0.1406/os/Source/SPackages/rootfiles-8.1-11.el7.src.rpm \
 	https://archive.kernel.org/centos-vault/7.0.1406/os/Source/SPackages/centos-bookmarks-7-1.el7.src.rpm
-srpms = $(addprefix ./.testprep/srpms/,$(notdir $(rpms)))
+srpms		= $(addprefix ./.testprep/srpms/,$(notdir $(rpms)))
+
+spec		?= $(pkgname).spec
+pwd		:= $(shell pwd)
+NAME		:= $(shell rpmspec -q --qf "%{name}" $(spec))
+VERSION		:= $(shell rpmspec -q --qf "%{version}" $(spec))
+RELEASE		:= $(shell rpmspec -q --qf "%{release}" $(spec))
+ARCH		:= $(shell rpmspec -q --qf "%{arch}" $(spec))
+NVR		:= $(NAME)-$(VERSION)-$(RELEASE)
+outdir		?= $(pwd)
+
+SHELL_SRC		:= ./BuildSourceImage.sh
+DIST_FILES		:= \
+	$(SHELL_SRC) \
+	LICENSE \
+	README.md
 
 export CTR_IMAGE
 export CTR_ENGINE
@@ -18,13 +33,13 @@ all: validate
 validate: .validate
 
 cleanfiles += .validate
-.validate: $(SRC)
-	shellcheck $(SRC) && touch $@
+.validate: $(SHELL_SRC)
+	shellcheck $(SHELL_SRC) && touch $@
 
 build-container: .build-container
 
 cleanfiles += .build-container
-.build-container: .validate Dockerfile $(SRC)
+.build-container: .validate Dockerfile $(SHELL_SRC)
 	@echo
 	@echo "==> Building BuildSourceImage Container"
 	$(CTR_ENGINE) build --quiet --file Dockerfile --tag $(CTR_IMAGE) . && touch $@
@@ -39,8 +54,36 @@ cleanfiles += .testprep $(srpms)
 test-integration: .build-container .testprep
 	@echo
 	@echo "==> Running integration tests"
-	TMPDIR=$(shell realpath .testprep/tmp) bats $(BATS_OPTS) test/
+	TMPDIR=$(realpath .testprep/tmp) bats $(BATS_OPTS) test/
 
+.PHONY: srpm
+srpm: $(NVR).src.rpm
+	@echo $^
+
+cleanfiles += $(NVR).src.rpm
+$(NVR).src.rpm: $(spec) $(DIST_FILES)
+	rpmbuild \
+                --define '_sourcedir $(pwd)' \
+                --define '_specdir $(pwd)' \
+                --define '_builddir $(pwd)' \
+                --define '_srcrpmdir $(outdir)' \
+                --define '_rpmdir $(outdir)' \
+                --nodeps \
+                -bs ./$(spec)
+
+.PHONY: rpm
+rpm: $(ARCH)/$(NVR).$(ARCH).rpm
+	@echo $^
+
+cleanfiles += $(ARCH)/$(NVR).$(ARCH).rpm
+$(ARCH)/$(NVR).$(ARCH).rpm: $(spec) $(DIST_FILES)
+	rpmbuild \
+                --define '_sourcedir $(pwd)' \
+                --define '_specdir $(pwd)' \
+                --define '_builddir $(pwd)' \
+                --define '_srcrpmdir $(outdir)' \
+                --define '_rpmdir $(outdir)' \
+                -bb ./$(spec)
 
 clean:
 	if [ -n "$(cleanfiles)" ] ; then rm -rf $(cleanfiles) ; fi

README.md

@@ -1,4 +1,5 @@
 [![Build Status](https://travis-ci.org/containers/BuildSourceImage.svg?branch=master)](https://travis-ci.org/containers/BuildSourceImage)
+[![Container Image Repository on Quay](https://quay.io/repository/ctrs/bsi/status "Container Image Repository on Quay")](https://quay.io/repository/ctrs/bsi)
 
 # BuildSourceImage
 
@@ -31,9 +32,8 @@ Usage: BuildSourceImage.sh [-D] [-b <path>] [-c <path>] [-e <path>] [-r <path>]
 Nicely usable inside a container:
 
 ```bash
-$> podman build -t containers/buildsourceimage .
 $> mkdir ./output/
-$> podman run -it -v $(pwd)/output/:/output/ -v $(pwd)/SRCRPMS/:/data/ -u $(id -u) containers/buildsourceimage -s /data/
+$> podman run -it -v $(pwd)/output/:/output/ -v $(pwd)/SRCRPMS/:/data/ -u $(id -u) quay.io/ctrs/bsi -s /data/ -o /output/
 ```
 
 ## Examples

developing.md

@@ -0,0 +1,33 @@
+# Developing
+
+## Requirements
+
+* `make`
+* `shellcheck` (package `ShellCheck` on fedora)
+* `bats`
+* `wget`
+* `podman` (or `docker`)
+* `jq`
+
+## Lint
+
+[ShellCheck](https://www.shellcheck.net/) is used to ensure the shell script is nice and tidy.
+
+```bash
+make validate
+```
+
+## Tests
+
+Testing is done with [`bats`](https://github.com/bats-core/bats-core).
+
+While it's possible to kick the tests by calling `bats ./test/`, many of the tests are written to use the script as built into a container image.
+If you are making local changes and have not rebuilt the container, then they will be missed.
+
+Best to kick off the build like:
+```bash
+make test-integration
+```
+This will rebuild the container if needed before running the tests.
+
+##

test/01-from_rpms.bats

@@ -7,7 +7,7 @@ load helpers
 	d=$(mktemp -d)
 	echo "temporary directory: ${d}"
 
-	run_ctr -v $(pwd)/.testprep/srpms/:/src:ro --mount type=bind,source=${d},destination=/output $CTR_IMAGE -s /src
+	run_ctr -v $(pwd)/.testprep/srpms/:/src:ro --mount type=bind,source=${d},destination=/output $CTR_IMAGE -s /src -o /output
 	[ "$status" -eq 0 ]
 	[[ ${lines[0]} =~ "[SrcImg][INFO] calling source collection drivers" ]]
 	# get the number of the last line
@@ -22,9 +22,9 @@ load helpers
 
 	# let's press that the files are predictable
 	[ "$(find ${d} -type f | wc -l)" -eq 7 ]
-	[ -f "${d}/blobs/sha256/3afb43699ea82a69b16efb215363604d9e4ffe16c9ace7e53df66663847309cf" ]
-	[ -f "${d}/blobs/sha256/7f4a50f05b7bd38017be8396b6320e1d2e6a05af097672e3ed23ef3df2ddeadb" ]
-	[ -f "${d}/blobs/sha256/8f4e610748f8b58a3297ecf78ecc8ff7b6420c3e559e3e20cad8ac178c6fe4e8" ]
+	[ -f "${d}/blobs/sha256/549ac1e4eb73e55781f39f4b8ee08c1158f1b1c1a523cf278d602386613e2f12" ]
+	[ -f "${d}/blobs/sha256/b5d5efc6c334cc52223eaea4ac046f21f089c3088b6abb4de027339e5e6dce4b" ]
+	[ -f "${d}/blobs/sha256/ce0608ce0a601a4cac453b0a0e181cac444027d800a26d5b44b80a74c6dc94e8" ]
 }
 
 @test "build from RPMS and push" {

test/02-from_image_ref.bats

@@ -3,12 +3,13 @@
 load helpers
 
 @test "Build from image reference" {
+	#skip "this takes like 20min ..."
     local d
     d=$(mktemp -d)
     echo "temporary directory: ${d}"
 
     ref="registry.fedoraproject.org/fedora-minimal"
-	run_ctr --mount type=bind,source=${d},destination=/output $CTR_IMAGE -i "${ref}"
+	run_ctr --mount type=bind,source=${d},destination=/output $CTR_IMAGE -i "${ref}" -o /output
 	[ "$status" -eq 0 ]
 	#echo ${lines[@]}
 	[[ ${lines[0]} =~ "Getting image source signatures" ]]

test/03-unpack.bats

@@ -0,0 +1,32 @@
+#!/usr/bin/env bats -t
+
+load helpers
+
+@test "unpack - no args" {
+	run_ctr $CTR_IMAGE unpack
+	[ "$status" -eq 1 ]
+	[[ ${lines[0]} =~ "[SrcImg][ERROR] [unpack_img] blank arguments provided" ]]
+}
+
+@test "unpack - Help" {
+	run_ctr $CTR_IMAGE unpack -h
+	[ "$status" -eq 1 ]
+	[[ ${lines[0]} =~ "BuildSourceImage.sh unpack <oci layout path> <unpack path>" ]]
+}
+
+@test "unpack - from a SRPM build" {
+	local d
+	local r
+
+	d=$(mktemp -d)
+	echo "temporary directories: output - ${d}"
+	run_ctr -v $(pwd)/.testprep/srpms/:/src:ro --mount type=bind,source=${d},destination=/output $CTR_IMAGE -s /src -o /output
+	[ "$status" -eq 0 ]
+	[ -f "${d}/index.json" ]
+
+	r=$(mktemp -d)
+	echo "temporary directories: unpacked - ${r}"
+	run_ctr --mount type=bind,source=${d},destination=/output -v ${r}:/unpacked/ $CTR_IMAGE unpack /output/ /unpacked/
+	[ "$(find ${r} -type f | wc -l)" -eq 3 ] # regular files
+	[ "$(find ${r} -type l | wc -l)" -eq 3 ] # and symlinks
+}

test/helpers.bash

@@ -1,3 +1,8 @@
+#!/bin/bash
+
+export CTR_IMAGE="${CTR_IMAGE:-localhost/containers/buildsourceimage}"
+export CTR_ENGINE="${CTR_ENGINE:-podman}"
+
 function run_ctr() {
 	run $CTR_ENGINE run --security-opt label=disable --rm "$@"
 }