diff --git a/.copr/Makefile b/.copr/Makefile new file mode 120000 index 0000000..d0b0e8e --- /dev/null +++ b/.copr/Makefile @@ -0,0 +1 @@ +../Makefile \ No newline at end of file diff --git a/.gitignore b/.gitignore index 50554ee..34913a3 100644 --- a/.gitignore +++ b/.gitignore @@ -2,3 +2,5 @@ .build-container .testprep .validate +*.rpm +x86_64/ diff --git a/.travis.yml b/.travis.yml index f13760e..39204a9 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,4 +1,5 @@ sudo: required +dist: bionic services: - docker @@ -7,8 +8,6 @@ env: - CTR_ENGINE=docker before_install: - - sudo add-apt-repository ppa:duggan/bats --yes - - sudo apt-get update -qq - sudo apt-get install -qq bats shellcheck script: diff --git a/BuildSourceImage.sh b/BuildSourceImage.sh index 4296dbd..8bb713e 100755 --- a/BuildSourceImage.sh +++ b/BuildSourceImage.sh @@ -8,7 +8,7 @@ export source_image_suffix="-source" # output version string _version() { - echo "$(basename "${0}") version 0.1" + echo "$(basename "${0}") version 0.2.0-dev" } # output the cli usage and exit @@ -16,6 +16,8 @@ _usage() { _version echo "Usage: $(basename "$0") [-D] [-b ] [-c ] [-e ] [-r ] [-o ] [-i ] [-p ] [-l] [-d ]" echo "" + echo " Container Source Image tool" + echo "" echo -e " -b \tbase path for source image builds" echo -e " -c \tbuild context for the container image. Can be provided via CONTEXT_DIR env variable" echo -e " -e \textra src for the container image. Can be provided via EXTRA_SRC_DIR env variable" @@ -28,6 +30,10 @@ _usage() { echo -e " -D\t\tdebuging output. Can be set via DEBUG env variable" echo -e " -h\t\tthis usage information" echo -e " -v\t\tversion" + echo -e "" + echo -e " Subcommands:" + echo -e " unpack\tUnpack an OCI layout to a rootfs directory" + echo -e "" } # sanity checks on startup @@ -44,6 +50,23 @@ _init() { done } +# enable access to some of functions as subcommands! +_subcommand() { + local command="${1}" + local ret + + shift + + case "${command}" in + unpack) + # (vb) i'd prefer this subcommand directly match the function name, but it isn't as pretty. + unpack_img "${@}" + ret=$? + exit "${ret}" + ;; + esac +} + # _is_sourced tests whether this script is being source, or executed directly _is_sourced() { # https://unix.stackexchange.com/a/215279 @@ -107,6 +130,14 @@ _tar() { fi } +_rpm_download() { + if [ "$(command -v yumdownloader)" != "" ] ; then + yumdownloader "${@}" + else + dnf download "${@}" + fi +} + # output things, only when $DEBUG is set _debug() { if [ -n "${DEBUG}" ] ; then @@ -292,6 +323,20 @@ unpack_img() { local unpack_dir="${2}" local ret + while getopts ":h" opts; do + case "${opts}" in + *) + echo "$0 unpack " + return 1 + ;; + esac + done + shift $((OPTIND-1)) + + if [ -z "${image_dir}" ] || [ -z "${unpack_dir}" ] ; then + _error "[unpack_img] blank arguments provided" + fi + if [ -d "${unpack_dir}" ] ; then _rm_rf "${unpack_dir}" fi @@ -336,9 +381,10 @@ unpack_img_bash() { return ${ret} fi - # Since we're landing the reference as an OCI layout, this mediaType is fairly predictable - # TODO don't always assume +gzip - layer_dgsts="$(jq '.layers[] | select(.mediaType == "application/vnd.oci.image.layer.v1.tar+gzip") | .digest' "${image_dir}"/blobs/"${mnfst_dgst/:/\/}" | tr -d \")" + # TODO this will need to be refactored when we start seeing +zstd layers. + # Then it will be better to no just get a list of digests, but maybe to + # iterate on each descriptor independently? + layer_dgsts="$(jq '.layers | map(select(.mediaType == "application/vnd.oci.image.layer.v1.tar+gzip"),select(.mediaType == "application/vnd.oci.image.layer.v1.tar"),select(.mediaType == "application/vnd.docker.image.rootfs.diff.tar.gzip")) | .[] | .digest' "${image_dir}"/blobs/"${mnfst_dgst/:/\/}" | tr -d \")" ret=$? if [ ${ret} -ne 0 ] ; then return ${ret} @@ -410,12 +456,62 @@ push_img() { # sets up a basic new OCI layout, for an image with the provided (or default 'latest') tag # layout_new() { + local out_dir="${1}" + local image_tag="${2:-latest}" + local ret + + if [ -n "$(command -v umoci)" ] ; then + layout_new_umoci "${out_dir}" "${image_tag}" + ret=$? + if [ ${ret} -ne 0 ] ; then + return ${ret} + fi + else + layout_new_bash "${out_dir}" "${image_tag}" + ret=$? + if [ ${ret} -ne 0 ] ; then + return ${ret} + fi + fi +} + +# +# sets up new OCI layout, using `umoci` +# +layout_new_umoci() { + local out_dir="${1}" + local image_tag="${2:-latest}" + local ret + + # umoci expects the layout path to _not_ exist and will fail if it does exist + _rm_rf "${out_dir}" + + umoci init --layout "${out_dir}" + ret=$? + if [ "${ret}" -ne 0 ] ; then + return "${ret}" + fi + + # XXX currently does not support adding the rich annotations like I've done with the _bash + # https://github.com/openSUSE/umoci/issues/298 + umoci new --image "${out_dir}:${image_tag}" + ret=$? + if [ "${ret}" -ne 0 ] ; then + return "${ret}" + fi +} + +# +# sets up new OCI layout, all with bash and jq +# +layout_new_bash() { local out_dir="${1}" local image_tag="${2:-latest}" local config local mnfst local config_sum local mnfst_sum + local ret _mkdir_p "${out_dir}/blobs/sha256" echo '{"imageLayoutVersion":"1.0.0"}' > "${out_dir}/oci-layout" @@ -432,7 +528,15 @@ layout_new() { } ' config_sum=$(echo "${config}" | jq -c | tr -d '\n' | sha256sum | awk '{ ORS=""; print $1 }') + ret=$? + if [ "${ret}" -ne 0 ] ; then + return "${ret}" + fi echo "${config}" | jq -c | tr -d '\n' > "${out_dir}/blobs/sha256/${config_sum}" + ret=$? + if [ "${ret}" -ne 0 ] ; then + return "${ret}" + fi mnfst=' { @@ -474,6 +578,72 @@ layout_new() { # * tag used in the layout (default is 'latest') # layout_insert() { + local out_dir="${1}" + local artifact_path="${2}" + local tar_path="${3}" + local annotations_file="${4}" + local image_tag="${5:-latest}" + local ret + + if [ -n "$(command -v umoci)" ] ; then + layout_insert_umoci "${out_dir}" "${artifact_path}" "${tar_path}" "${annotations_file}" "${image_tag}" + ret=$? + if [ ${ret} -ne 0 ] ; then + return ${ret} + fi + else + layout_insert_bash "${out_dir}" "${artifact_path}" "${tar_path}" "${annotations_file}" "${image_tag}" + ret=$? + if [ ${ret} -ne 0 ] ; then + return ${ret} + fi + fi +} + +layout_insert_umoci() { + local out_dir="${1}" + local artifact_path="${2}" + local tar_path="${3}" + local annotations_file="${4}" + local image_tag="${5:-latest}" + local sum + local ret + + # prep the blob path for inside the layer, so we can just copy that whole path in + tmpdir="$(_mktemp_d)" + + # TODO account for "artifact_path" being a directory? + sum="$(sha256sum "${artifact_path}" | awk '{ print $1 }')" + + _mkdir_p "${tmpdir}/blobs/sha256" + cp "${artifact_path}" "${tmpdir}/blobs/sha256/${sum}" + if [ "$(basename "${tar_path}")" == "$(basename "${artifact_path}")" ] ; then + _mkdir_p "${tmpdir}/$(dirname "${tar_path}")" + # TODO this symlink need to be relative path, not to `/blobs/...` + ln -s "/blobs/sha256/${sum}" "${tmpdir}/${tar_path}" + else + _mkdir_p "${tmpdir}/${tar_path}" + # TODO this symlink need to be relative path, not to `/blobs/...` + ln -s "/blobs/sha256/${sum}" "${tmpdir}/${tar_path}/$(basename "${artifact_path}")" + fi + + # XXX currently does not support adding the rich annotations like I've done with the _bash + # https://github.com/openSUSE/umoci/issues/298 + # XXX this insert operation can not disable compression + # https://github.com/openSUSE/umoci/issues/300 + umoci insert \ + --rootless \ + --image "${out_dir}:${image_tag}" \ + --history.created "$(_date_ns)" \ + --history.comment "#(nop) $(_version) adding artifact: ${sum}" \ + "${tmpdir}" "/" + ret=$? + if [ ${ret} -ne 0 ] ; then + return ${ret} + fi +} + +layout_insert_bash() { local out_dir="${1}" local artifact_path="${2}" local tar_path="${3}" @@ -517,11 +687,11 @@ layout_insert() { if [ "$(basename "${tar_path}")" == "$(basename "${artifact_path}")" ] ; then _mkdir_p "${tmpdir}/$(dirname "${tar_path}")" # TODO this symlink need to be relative path, not to `/blobs/...` - ln -s "/blobs/sha256/${sum}" "${tmpdir}/${tar_path}" + ln -s "../blobs/sha256/${sum}" "${tmpdir}/${tar_path}" else _mkdir_p "${tmpdir}/${tar_path}" # TODO this symlink need to be relative path, not to `/blobs/...` - ln -s "/blobs/sha256/${sum}" "${tmpdir}/${tar_path}/$(basename "${artifact_path}")" + ln -s "../blobs/sha256/${sum}" "${tmpdir}/${tar_path}/$(basename "${artifact_path}")" fi tmptar="$(_mktemp)" @@ -542,7 +712,7 @@ layout_insert() { jq -c \ --arg date "$(_date_ns)" \ --arg tmptar_sum "sha256:${tmptar_sum}" \ - --arg comment "#(nop) BuildSourceImage adding artifact: ${sum}" \ + --arg comment "#(nop) $(_version) adding artifact: ${sum}" \ ' .created = $date | .rootfs.diff_ids += [ $tmptar_sum ] @@ -681,7 +851,7 @@ sourcedriver_rpm_fetch() { rpm=${srcrpm%*.src.rpm} if [ ! -f "${out_dir}/${srcrpm}" ] ; then _debug "--> fetching ${srcrpm}" - dnf download \ + _rpm_download \ --quiet \ --installroot "${rootfs}" \ --release "${release}" \ @@ -902,8 +1072,9 @@ main() { local work_dir _init "${@}" + _subcommand "${@}" - base_dir="$(pwd)/${ABV_NAME}" + base_dir="${BASE_DIR:-$(pwd)/${ABV_NAME}}" # using the bash builtin to parse while getopts ":hlvDi:c:s:e:o:b:d:p:" opts; do case "${opts}" in @@ -996,6 +1167,10 @@ main() { # including its digest. if [ -z "${inspect_image_digest}" ] ; then inspect_image_digest="$(fetch_img_digest "$(parse_img_base "${input_inspect_image_ref}"):$(parse_img_tag "${input_inspect_image_ref}")")" + ret=$? + if [ ${ret} -ne 0 ] ; then + _error "failed to detect image digest" + fi fi _debug "inspect_image_digest: ${inspect_image_digest}" diff --git a/BuildSourceImage.spec b/BuildSourceImage.spec new file mode 100644 index 0000000..879399e --- /dev/null +++ b/BuildSourceImage.spec @@ -0,0 +1,43 @@ +Name: BuildSourceImage +Version: 0.2 +Release: 1%{?dist} +Summary: Container Source Image tool + +Group: containers +License: GPLv2 +URL: https://github.com/containers/BuildSourceImage +Source0: BuildSourceImage.sh + +#BuildRequires: +Requires: jq +Requires: skopeo +Requires: findutils +Requires: file +%if 0%{?rhel} > 6 +Requires: yum-utils +%else +Requires: dnf-command(download) +%endif + +%description +%{summary}. + +%prep + + +%build + + +%install +%{__mkdir_p} %{buildroot}/%{_bindir} +%{__install} -T -m 0755 ${RPM_SOURCE_DIR}/BuildSourceImage.sh %{buildroot}/%{_bindir}/BuildSourceImage + + +%files +%doc ${RPM_SOURCE_DIR}/LICENSE ${RPM_SOURCE_DIR}/README.md +%{_bindir}/BuildSourceImage + + + +%changelog + diff --git a/Dockerfile b/Dockerfile index a50044c..8c64206 100644 --- a/Dockerfile +++ b/Dockerfile @@ -4,8 +4,6 @@ RUN dnf install -y jq skopeo findutils file 'dnf-command(download)' COPY ./BuildSourceImage.sh /usr/local/bin/BuildSourceImage.sh -RUN mkdir -p /output -ENV OUTPUT_DIR=/output -VOLUME /output +ENV BASE_DIR=/tmp -ENTRYPOINT ["/usr/local/bin/BuildSourceImage.sh", "-b", "/tmp/"] +ENTRYPOINT ["/usr/local/bin/BuildSourceImage.sh"] diff --git a/Makefile b/Makefile index 69b388a..f8b4969 100644 --- a/Makefile +++ b/Makefile @@ -1,14 +1,29 @@ -SRC := ./BuildSourceImage.sh -CTR_IMAGE := localhost/containers/buildsourceimage -CTR_ENGINE ?= podman -BATS_OPTS ?= -cleanfiles = +pkgname := BuildSourceImage +CTR_IMAGE := localhost/containers/buildsourceimage +CTR_ENGINE ?= podman +BATS_OPTS ?= +cleanfiles = # these are packages whose src.rpms are very small -srpm_urls = \ +srpm_urls = \ https://archive.kernel.org/centos-vault/7.0.1406/os/Source/SPackages/basesystem-10.0-7.el7.centos.src.rpm \ https://archive.kernel.org/centos-vault/7.0.1406/os/Source/SPackages/rootfiles-8.1-11.el7.src.rpm \ https://archive.kernel.org/centos-vault/7.0.1406/os/Source/SPackages/centos-bookmarks-7-1.el7.src.rpm -srpms = $(addprefix ./.testprep/srpms/,$(notdir $(rpms))) +srpms = $(addprefix ./.testprep/srpms/,$(notdir $(rpms))) + +spec ?= $(pkgname).spec +pwd := $(shell pwd) +NAME := $(shell rpmspec -q --qf "%{name}" $(spec)) +VERSION := $(shell rpmspec -q --qf "%{version}" $(spec)) +RELEASE := $(shell rpmspec -q --qf "%{release}" $(spec)) +ARCH := $(shell rpmspec -q --qf "%{arch}" $(spec)) +NVR := $(NAME)-$(VERSION)-$(RELEASE) +outdir ?= $(pwd) + +SHELL_SRC := ./BuildSourceImage.sh +DIST_FILES := \ + $(SHELL_SRC) \ + LICENSE \ + README.md export CTR_IMAGE export CTR_ENGINE @@ -18,13 +33,13 @@ all: validate validate: .validate cleanfiles += .validate -.validate: $(SRC) - shellcheck $(SRC) && touch $@ +.validate: $(SHELL_SRC) + shellcheck $(SHELL_SRC) && touch $@ build-container: .build-container cleanfiles += .build-container -.build-container: .validate Dockerfile $(SRC) +.build-container: .validate Dockerfile $(SHELL_SRC) @echo @echo "==> Building BuildSourceImage Container" $(CTR_ENGINE) build --quiet --file Dockerfile --tag $(CTR_IMAGE) . && touch $@ @@ -39,8 +54,36 @@ cleanfiles += .testprep $(srpms) test-integration: .build-container .testprep @echo @echo "==> Running integration tests" - TMPDIR=$(shell realpath .testprep/tmp) bats $(BATS_OPTS) test/ + TMPDIR=$(realpath .testprep/tmp) bats $(BATS_OPTS) test/ +.PHONY: srpm +srpm: $(NVR).src.rpm + @echo $^ + +cleanfiles += $(NVR).src.rpm +$(NVR).src.rpm: $(spec) $(DIST_FILES) + rpmbuild \ + --define '_sourcedir $(pwd)' \ + --define '_specdir $(pwd)' \ + --define '_builddir $(pwd)' \ + --define '_srcrpmdir $(outdir)' \ + --define '_rpmdir $(outdir)' \ + --nodeps \ + -bs ./$(spec) + +.PHONY: rpm +rpm: $(ARCH)/$(NVR).$(ARCH).rpm + @echo $^ + +cleanfiles += $(ARCH)/$(NVR).$(ARCH).rpm +$(ARCH)/$(NVR).$(ARCH).rpm: $(spec) $(DIST_FILES) + rpmbuild \ + --define '_sourcedir $(pwd)' \ + --define '_specdir $(pwd)' \ + --define '_builddir $(pwd)' \ + --define '_srcrpmdir $(outdir)' \ + --define '_rpmdir $(outdir)' \ + -bb ./$(spec) clean: if [ -n "$(cleanfiles)" ] ; then rm -rf $(cleanfiles) ; fi diff --git a/README.md b/README.md index b3b7b7c..da5e1a0 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,5 @@ [![Build Status](https://travis-ci.org/containers/BuildSourceImage.svg?branch=master)](https://travis-ci.org/containers/BuildSourceImage) +[![Container Image Repository on Quay](https://quay.io/repository/ctrs/bsi/status "Container Image Repository on Quay")](https://quay.io/repository/ctrs/bsi) # BuildSourceImage @@ -31,9 +32,8 @@ Usage: BuildSourceImage.sh [-D] [-b ] [-c ] [-e ] [-r ] Nicely usable inside a container: ```bash -$> podman build -t containers/buildsourceimage . $> mkdir ./output/ -$> podman run -it -v $(pwd)/output/:/output/ -v $(pwd)/SRCRPMS/:/data/ -u $(id -u) containers/buildsourceimage -s /data/ +$> podman run -it -v $(pwd)/output/:/output/ -v $(pwd)/SRCRPMS/:/data/ -u $(id -u) quay.io/ctrs/bsi -s /data/ -o /output/ ``` ## Examples diff --git a/developing.md b/developing.md new file mode 100644 index 0000000..de7d310 --- /dev/null +++ b/developing.md @@ -0,0 +1,33 @@ +# Developing + +## Requirements + +* `make` +* `shellcheck` (package `ShellCheck` on fedora) +* `bats` +* `wget` +* `podman` (or `docker`) +* `jq` + +## Lint + +[ShellCheck](https://www.shellcheck.net/) is used to ensure the shell script is nice and tidy. + +```bash +make validate +``` + +## Tests + +Testing is done with [`bats`](https://github.com/bats-core/bats-core). + +While it's possible to kick the tests by calling `bats ./test/`, many of the tests are written to use the script as built into a container image. +If you are making local changes and have not rebuilt the container, then they will be missed. + +Best to kick off the build like: +```bash +make test-integration +``` +This will rebuild the container if needed before running the tests. + +## \ No newline at end of file diff --git a/test/01-from_rpms.bats b/test/01-from_rpms.bats index 07aa5d3..1f9d380 100644 --- a/test/01-from_rpms.bats +++ b/test/01-from_rpms.bats @@ -7,7 +7,7 @@ load helpers d=$(mktemp -d) echo "temporary directory: ${d}" - run_ctr -v $(pwd)/.testprep/srpms/:/src:ro --mount type=bind,source=${d},destination=/output $CTR_IMAGE -s /src + run_ctr -v $(pwd)/.testprep/srpms/:/src:ro --mount type=bind,source=${d},destination=/output $CTR_IMAGE -s /src -o /output [ "$status" -eq 0 ] [[ ${lines[0]} =~ "[SrcImg][INFO] calling source collection drivers" ]] # get the number of the last line @@ -22,9 +22,9 @@ load helpers # let's press that the files are predictable [ "$(find ${d} -type f | wc -l)" -eq 7 ] - [ -f "${d}/blobs/sha256/3afb43699ea82a69b16efb215363604d9e4ffe16c9ace7e53df66663847309cf" ] - [ -f "${d}/blobs/sha256/7f4a50f05b7bd38017be8396b6320e1d2e6a05af097672e3ed23ef3df2ddeadb" ] - [ -f "${d}/blobs/sha256/8f4e610748f8b58a3297ecf78ecc8ff7b6420c3e559e3e20cad8ac178c6fe4e8" ] + [ -f "${d}/blobs/sha256/549ac1e4eb73e55781f39f4b8ee08c1158f1b1c1a523cf278d602386613e2f12" ] + [ -f "${d}/blobs/sha256/b5d5efc6c334cc52223eaea4ac046f21f089c3088b6abb4de027339e5e6dce4b" ] + [ -f "${d}/blobs/sha256/ce0608ce0a601a4cac453b0a0e181cac444027d800a26d5b44b80a74c6dc94e8" ] } @test "build from RPMS and push" { diff --git a/test/02-from_image_ref.bats b/test/02-from_image_ref.bats index 38ce66c..b9f1ce3 100644 --- a/test/02-from_image_ref.bats +++ b/test/02-from_image_ref.bats @@ -3,12 +3,13 @@ load helpers @test "Build from image reference" { + #skip "this takes like 20min ..." local d d=$(mktemp -d) echo "temporary directory: ${d}" ref="registry.fedoraproject.org/fedora-minimal" - run_ctr --mount type=bind,source=${d},destination=/output $CTR_IMAGE -i "${ref}" + run_ctr --mount type=bind,source=${d},destination=/output $CTR_IMAGE -i "${ref}" -o /output [ "$status" -eq 0 ] #echo ${lines[@]} [[ ${lines[0]} =~ "Getting image source signatures" ]] diff --git a/test/03-unpack.bats b/test/03-unpack.bats new file mode 100644 index 0000000..ee540cf --- /dev/null +++ b/test/03-unpack.bats @@ -0,0 +1,32 @@ +#!/usr/bin/env bats -t + +load helpers + +@test "unpack - no args" { + run_ctr $CTR_IMAGE unpack + [ "$status" -eq 1 ] + [[ ${lines[0]} =~ "[SrcImg][ERROR] [unpack_img] blank arguments provided" ]] +} + +@test "unpack - Help" { + run_ctr $CTR_IMAGE unpack -h + [ "$status" -eq 1 ] + [[ ${lines[0]} =~ "BuildSourceImage.sh unpack " ]] +} + +@test "unpack - from a SRPM build" { + local d + local r + + d=$(mktemp -d) + echo "temporary directories: output - ${d}" + run_ctr -v $(pwd)/.testprep/srpms/:/src:ro --mount type=bind,source=${d},destination=/output $CTR_IMAGE -s /src -o /output + [ "$status" -eq 0 ] + [ -f "${d}/index.json" ] + + r=$(mktemp -d) + echo "temporary directories: unpacked - ${r}" + run_ctr --mount type=bind,source=${d},destination=/output -v ${r}:/unpacked/ $CTR_IMAGE unpack /output/ /unpacked/ + [ "$(find ${r} -type f | wc -l)" -eq 3 ] # regular files + [ "$(find ${r} -type l | wc -l)" -eq 3 ] # and symlinks +} diff --git a/test/helpers.bash b/test/helpers.bash index 67428d9..8dbb1c1 100644 --- a/test/helpers.bash +++ b/test/helpers.bash @@ -1,3 +1,8 @@ +#!/bin/bash + +export CTR_IMAGE="${CTR_IMAGE:-localhost/containers/buildsourceimage}" +export CTR_ENGINE="${CTR_ENGINE:-podman}" + function run_ctr() { run $CTR_ENGINE run --security-opt label=disable --rm "$@" }