From 9726e11de56db896a49356cca912308c2ed54a24 Mon Sep 17 00:00:00 2001 From: Vincent Batts Date: Sat, 21 Sep 2019 01:35:53 -0400 Subject: [PATCH 01/14] BuildSourceImage: bump the version Fixes #36 Signed-off-by: Vincent Batts --- BuildSourceImage.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/BuildSourceImage.sh b/BuildSourceImage.sh index 4296dbd..f75a3ef 100755 --- a/BuildSourceImage.sh +++ b/BuildSourceImage.sh @@ -8,7 +8,7 @@ export source_image_suffix="-source" # output version string _version() { - echo "$(basename "${0}") version 0.1" + echo "$(basename "${0}") version 0.2.0-dev" } # output the cli usage and exit From 778c9b47a5ea6368680e9681624752c9ed48379f Mon Sep 17 00:00:00 2001 From: Vincent Batts Date: Sat, 21 Sep 2019 02:12:07 -0400 Subject: [PATCH 02/14] README: add the quay build status Signed-off-by: Vincent Batts --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index b3b7b7c..6fdb4ed 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,5 @@ [![Build Status](https://travis-ci.org/containers/BuildSourceImage.svg?branch=master)](https://travis-ci.org/containers/BuildSourceImage) +[![Container Image Repository on Quay](https://quay.io/repository/vbatts/bsi/status "Container Image Repository on Quay")](https://quay.io/repository/vbatts/bsi) # BuildSourceImage From 19b9ecc797d93465a9d5ee40089d93256b0eedd7 Mon Sep 17 00:00:00 2001 From: Vincent Batts Date: Thu, 26 Sep 2019 12:20:46 -0400 Subject: [PATCH 03/14] BuildSourceImage: use umoci for OCI layout, if present Signed-off-by: Vincent Batts --- BuildSourceImage.sh | 58 +++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 58 insertions(+) diff --git a/BuildSourceImage.sh b/BuildSourceImage.sh index f75a3ef..e607c08 100755 --- a/BuildSourceImage.sh +++ b/BuildSourceImage.sh @@ -410,12 +410,62 @@ push_img() { # sets up a basic new OCI layout, for an image with the provided (or default 'latest') tag # layout_new() { + local out_dir="${1}" + local image_tag="${2:-latest}" + local ret + + if [ -n "$(command -v umoci)" ] ; then + layout_new_umoci "${out_dir}" "${image_tag}" + ret=$? + if [ ${ret} -ne 0 ] ; then + return ${ret} + fi + else + layout_new_bash "${out_dir}" "${image_tag}" + ret=$? + if [ ${ret} -ne 0 ] ; then + return ${ret} + fi + fi +} + +# +# sets up new OCI layout, using `umoci` +# +layout_new_umoci() { + local out_dir="${1}" + local image_tag="${2:-latest}" + local ret + + # umoci expects the layout path to _not_ exist and will fail if it does exist + _rm_rf "${out_dir}" + + umoci init --layout "${out_dir}" + ret=$? + if [ "${ret}" -ne 0 ] ; then + return "${ret}" + fi + + # XXX currently does not support adding the rich annotations like I've done with the _bash + # https://github.com/openSUSE/umoci/issues/298 + umoci new --image "${out_dir}:${image_tag}" + ret=$? + if [ "${ret}" -ne 0 ] ; then + return "${ret}" + fi +} + +# +# sets up new OCI layout, all with bash and jq +# +layout_new_bash() { local out_dir="${1}" local image_tag="${2:-latest}" local config local mnfst local config_sum local mnfst_sum + local ret _mkdir_p "${out_dir}/blobs/sha256" echo '{"imageLayoutVersion":"1.0.0"}' > "${out_dir}/oci-layout" @@ -432,7 +482,15 @@ layout_new() { } ' config_sum=$(echo "${config}" | jq -c | tr -d '\n' | sha256sum | awk '{ ORS=""; print $1 }') + ret=$? + if [ "${ret}" -ne 0 ] ; then + return "${ret}" + fi echo "${config}" | jq -c | tr -d '\n' > "${out_dir}/blobs/sha256/${config_sum}" + ret=$? + if [ "${ret}" -ne 0 ] ; then + return "${ret}" + fi mnfst=' { From c03dcdeb76b922fe20b1f21117c498f17f5aa283 Mon Sep 17 00:00:00 2001 From: Vincent Batts Date: Thu, 26 Sep 2019 12:53:20 -0400 Subject: [PATCH 04/14] BuildSourceImage: insert layer with umoci, if present Signed-off-by: Vincent Batts --- BuildSourceImage.sh | 68 ++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 67 insertions(+), 1 deletion(-) diff --git a/BuildSourceImage.sh b/BuildSourceImage.sh index e607c08..6034768 100755 --- a/BuildSourceImage.sh +++ b/BuildSourceImage.sh @@ -532,6 +532,72 @@ layout_new_bash() { # * tag used in the layout (default is 'latest') # layout_insert() { + local out_dir="${1}" + local artifact_path="${2}" + local tar_path="${3}" + local annotations_file="${4}" + local image_tag="${5:-latest}" + local ret + + if [ -n "$(command -v umoci)" ] ; then + layout_insert_umoci "${out_dir}" "${artifact_path}" "${tar_path}" "${annotations_file}" "${image_tag}" + ret=$? + if [ ${ret} -ne 0 ] ; then + return ${ret} + fi + else + layout_insert_bash "${out_dir}" "${artifact_path}" "${tar_path}" "${annotations_file}" "${image_tag}" + ret=$? + if [ ${ret} -ne 0 ] ; then + return ${ret} + fi + fi +} + +layout_insert_umoci() { + local out_dir="${1}" + local artifact_path="${2}" + local tar_path="${3}" + local annotations_file="${4}" + local image_tag="${5:-latest}" + local sum + local ret + + # prep the blob path for inside the layer, so we can just copy that whole path in + tmpdir="$(_mktemp_d)" + + # TODO account for "artifact_path" being a directory? + sum="$(sha256sum "${artifact_path}" | awk '{ print $1 }')" + + _mkdir_p "${tmpdir}/blobs/sha256" + cp "${artifact_path}" "${tmpdir}/blobs/sha256/${sum}" + if [ "$(basename "${tar_path}")" == "$(basename "${artifact_path}")" ] ; then + _mkdir_p "${tmpdir}/$(dirname "${tar_path}")" + # TODO this symlink need to be relative path, not to `/blobs/...` + ln -s "/blobs/sha256/${sum}" "${tmpdir}/${tar_path}" + else + _mkdir_p "${tmpdir}/${tar_path}" + # TODO this symlink need to be relative path, not to `/blobs/...` + ln -s "/blobs/sha256/${sum}" "${tmpdir}/${tar_path}/$(basename "${artifact_path}")" + fi + + # XXX currently does not support adding the rich annotations like I've done with the _bash + # https://github.com/openSUSE/umoci/issues/298 + # XXX this insert operation can not disable compression + # https://github.com/openSUSE/umoci/issues/300 + umoci insert \ + --rootless \ + --image "${out_dir}:${image_tag}" \ + --history.created "$(_date_ns)" \ + --history.comment "#(nop) $(_version) adding artifact: ${sum}" \ + "${tmpdir}" "/" + ret=$? + if [ ${ret} -ne 0 ] ; then + return ${ret} + fi +} + +layout_insert_bash() { local out_dir="${1}" local artifact_path="${2}" local tar_path="${3}" @@ -600,7 +666,7 @@ layout_insert() { jq -c \ --arg date "$(_date_ns)" \ --arg tmptar_sum "sha256:${tmptar_sum}" \ - --arg comment "#(nop) BuildSourceImage adding artifact: ${sum}" \ + --arg comment "#(nop) $(_version) adding artifact: ${sum}" \ ' .created = $date | .rootfs.diff_ids += [ $tmptar_sum ] From a3585314603efb419caa906678fed7ba2913e033 Mon Sep 17 00:00:00 2001 From: Vincent Batts Date: Thu, 26 Sep 2019 13:03:12 -0400 Subject: [PATCH 05/14] Dockerfile: no default output directory Just let the user provide this if wanted Signed-off-by: Vincent Batts --- Dockerfile | 4 ---- test/01-from_rpms.bats | 2 +- test/02-from_image_ref.bats | 2 +- 3 files changed, 2 insertions(+), 6 deletions(-) diff --git a/Dockerfile b/Dockerfile index a50044c..0714352 100644 --- a/Dockerfile +++ b/Dockerfile @@ -4,8 +4,4 @@ RUN dnf install -y jq skopeo findutils file 'dnf-command(download)' COPY ./BuildSourceImage.sh /usr/local/bin/BuildSourceImage.sh -RUN mkdir -p /output -ENV OUTPUT_DIR=/output -VOLUME /output - ENTRYPOINT ["/usr/local/bin/BuildSourceImage.sh", "-b", "/tmp/"] diff --git a/test/01-from_rpms.bats b/test/01-from_rpms.bats index 07aa5d3..df9660e 100644 --- a/test/01-from_rpms.bats +++ b/test/01-from_rpms.bats @@ -7,7 +7,7 @@ load helpers d=$(mktemp -d) echo "temporary directory: ${d}" - run_ctr -v $(pwd)/.testprep/srpms/:/src:ro --mount type=bind,source=${d},destination=/output $CTR_IMAGE -s /src + run_ctr -v $(pwd)/.testprep/srpms/:/src:ro --mount type=bind,source=${d},destination=/output $CTR_IMAGE -s /src -o /output [ "$status" -eq 0 ] [[ ${lines[0]} =~ "[SrcImg][INFO] calling source collection drivers" ]] # get the number of the last line diff --git a/test/02-from_image_ref.bats b/test/02-from_image_ref.bats index 38ce66c..74595d2 100644 --- a/test/02-from_image_ref.bats +++ b/test/02-from_image_ref.bats @@ -8,7 +8,7 @@ load helpers echo "temporary directory: ${d}" ref="registry.fedoraproject.org/fedora-minimal" - run_ctr --mount type=bind,source=${d},destination=/output $CTR_IMAGE -i "${ref}" + run_ctr --mount type=bind,source=${d},destination=/output $CTR_IMAGE -i "${ref}" -o /output [ "$status" -eq 0 ] #echo ${lines[@]} [[ ${lines[0]} =~ "Getting image source signatures" ]] From 1a85b05a327369f352e4a8d6c4dad8d243901a43 Mon Sep 17 00:00:00 2001 From: Vincent Batts Date: Thu, 26 Sep 2019 15:09:40 -0400 Subject: [PATCH 06/14] Makefile: no need to shell to realpath Fixes #41 Signed-off-by: Vincent Batts --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 69b388a..c38791d 100644 --- a/Makefile +++ b/Makefile @@ -39,7 +39,7 @@ cleanfiles += .testprep $(srpms) test-integration: .build-container .testprep @echo @echo "==> Running integration tests" - TMPDIR=$(shell realpath .testprep/tmp) bats $(BATS_OPTS) test/ + TMPDIR=$(realpath .testprep/tmp) bats $(BATS_OPTS) test/ clean: From d7fcf047aca84fb86673e4a17e3ed0190c4c0965 Mon Sep 17 00:00:00 2001 From: Valentin Rothberg Date: Tue, 15 Oct 2019 15:59:34 +0200 Subject: [PATCH 07/14] travis: use bionic and the default bats Signed-off-by: Valentin Rothberg --- .travis.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index f13760e..39204a9 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,4 +1,5 @@ sudo: required +dist: bionic services: - docker @@ -7,8 +8,6 @@ env: - CTR_ENGINE=docker before_install: - - sudo add-apt-repository ppa:duggan/bats --yes - - sudo apt-get update -qq - sudo apt-get install -qq bats shellcheck script: From cb1dd4ca214057578e18fac9f77b6037807a7255 Mon Sep 17 00:00:00 2001 From: Vincent Batts Date: Wed, 9 Oct 2019 14:01:22 -0400 Subject: [PATCH 08/14] BuildSourceImage: catch return code and exit Signed-off-by: Vincent Batts --- BuildSourceImage.sh | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/BuildSourceImage.sh b/BuildSourceImage.sh index 6034768..d0db1ca 100755 --- a/BuildSourceImage.sh +++ b/BuildSourceImage.sh @@ -1120,6 +1120,10 @@ main() { # including its digest. if [ -z "${inspect_image_digest}" ] ; then inspect_image_digest="$(fetch_img_digest "$(parse_img_base "${input_inspect_image_ref}"):$(parse_img_tag "${input_inspect_image_ref}")")" + ret=$? + if [ ${ret} -ne 0 ] ; then + _error "failed to detect image digest" + fi fi _debug "inspect_image_digest: ${inspect_image_digest}" From 8e438c7a6da833f0a3a80a7ef82c315f465ea3e9 Mon Sep 17 00:00:00 2001 From: Vincent Batts Date: Thu, 26 Sep 2019 16:19:13 -0400 Subject: [PATCH 09/14] BuildSourceImage: expose "unpack" as subcommand This may be a pattern we can do for some of the more useful functions. But for now, folks can build, push, etc. and once they skopeo copy an image to their host, they can ```bash ./BuildSourceImage unpack ``` and it will be nicely exposed in the container build too Also included, exporting the defaults into the bats test for easier direct running. Perhaps we can move the srpm caching into bats too, so the Makefile target isn't strictly required. Signed-off-by: Vincent Batts --- BuildSourceImage.sh | 45 +++++++++++++++++++++++++++++++++---- Dockerfile | 4 +++- test/02-from_image_ref.bats | 1 + test/03-unpack.bats | 32 ++++++++++++++++++++++++++ test/helpers.bash | 5 +++++ 5 files changed, 82 insertions(+), 5 deletions(-) create mode 100644 test/03-unpack.bats diff --git a/BuildSourceImage.sh b/BuildSourceImage.sh index 6034768..7484776 100755 --- a/BuildSourceImage.sh +++ b/BuildSourceImage.sh @@ -28,6 +28,10 @@ _usage() { echo -e " -D\t\tdebuging output. Can be set via DEBUG env variable" echo -e " -h\t\tthis usage information" echo -e " -v\t\tversion" + echo -e "" + echo -e " Subcommands:" + echo -e " unpack\tUnpack an OCI layout to a rootfs directory" + echo -e "" } # sanity checks on startup @@ -44,6 +48,23 @@ _init() { done } +# enable access to some of functions as subcommands! +_subcommand() { + local command="${1}" + local ret + + shift + + case "${command}" in + unpack) + # (vb) i'd prefer this subcommand directly match the function name, but it isn't as pretty. + unpack_img "${@}" + ret=$? + exit "${ret}" + ;; + esac +} + # _is_sourced tests whether this script is being source, or executed directly _is_sourced() { # https://unix.stackexchange.com/a/215279 @@ -292,6 +313,20 @@ unpack_img() { local unpack_dir="${2}" local ret + while getopts ":h" opts; do + case "${opts}" in + *) + echo "$0 unpack " + return 1 + ;; + esac + done + shift $((OPTIND-1)) + + if [ -z "${image_dir}" ] || [ -z "${unpack_dir}" ] ; then + _error "[unpack_img] blank arguments provided" + fi + if [ -d "${unpack_dir}" ] ; then _rm_rf "${unpack_dir}" fi @@ -336,9 +371,10 @@ unpack_img_bash() { return ${ret} fi - # Since we're landing the reference as an OCI layout, this mediaType is fairly predictable - # TODO don't always assume +gzip - layer_dgsts="$(jq '.layers[] | select(.mediaType == "application/vnd.oci.image.layer.v1.tar+gzip") | .digest' "${image_dir}"/blobs/"${mnfst_dgst/:/\/}" | tr -d \")" + # TODO this will need to be refactored when we start seeing +zstd layers. + # Then it will be better to no just get a list of digests, but maybe to + # iterate on each descriptor independently? + layer_dgsts="$(jq '.layers | map(select(.mediaType == "application/vnd.oci.image.layer.v1.tar+gzip"),select(.mediaType == "application/vnd.oci.image.layer.v1.tar"),select(.mediaType == "application/vnd.docker.image.rootfs.diff.tar.gzip")) | .[] | .digest' "${image_dir}"/blobs/"${mnfst_dgst/:/\/}" | tr -d \")" ret=$? if [ ${ret} -ne 0 ] ; then return ${ret} @@ -1026,8 +1062,9 @@ main() { local work_dir _init "${@}" + _subcommand "${@}" - base_dir="$(pwd)/${ABV_NAME}" + base_dir="${BASE_DIR:-$(pwd)/${ABV_NAME}}" # using the bash builtin to parse while getopts ":hlvDi:c:s:e:o:b:d:p:" opts; do case "${opts}" in diff --git a/Dockerfile b/Dockerfile index 0714352..8c64206 100644 --- a/Dockerfile +++ b/Dockerfile @@ -4,4 +4,6 @@ RUN dnf install -y jq skopeo findutils file 'dnf-command(download)' COPY ./BuildSourceImage.sh /usr/local/bin/BuildSourceImage.sh -ENTRYPOINT ["/usr/local/bin/BuildSourceImage.sh", "-b", "/tmp/"] +ENV BASE_DIR=/tmp + +ENTRYPOINT ["/usr/local/bin/BuildSourceImage.sh"] diff --git a/test/02-from_image_ref.bats b/test/02-from_image_ref.bats index 74595d2..b9f1ce3 100644 --- a/test/02-from_image_ref.bats +++ b/test/02-from_image_ref.bats @@ -3,6 +3,7 @@ load helpers @test "Build from image reference" { + #skip "this takes like 20min ..." local d d=$(mktemp -d) echo "temporary directory: ${d}" diff --git a/test/03-unpack.bats b/test/03-unpack.bats new file mode 100644 index 0000000..ee540cf --- /dev/null +++ b/test/03-unpack.bats @@ -0,0 +1,32 @@ +#!/usr/bin/env bats -t + +load helpers + +@test "unpack - no args" { + run_ctr $CTR_IMAGE unpack + [ "$status" -eq 1 ] + [[ ${lines[0]} =~ "[SrcImg][ERROR] [unpack_img] blank arguments provided" ]] +} + +@test "unpack - Help" { + run_ctr $CTR_IMAGE unpack -h + [ "$status" -eq 1 ] + [[ ${lines[0]} =~ "BuildSourceImage.sh unpack " ]] +} + +@test "unpack - from a SRPM build" { + local d + local r + + d=$(mktemp -d) + echo "temporary directories: output - ${d}" + run_ctr -v $(pwd)/.testprep/srpms/:/src:ro --mount type=bind,source=${d},destination=/output $CTR_IMAGE -s /src -o /output + [ "$status" -eq 0 ] + [ -f "${d}/index.json" ] + + r=$(mktemp -d) + echo "temporary directories: unpacked - ${r}" + run_ctr --mount type=bind,source=${d},destination=/output -v ${r}:/unpacked/ $CTR_IMAGE unpack /output/ /unpacked/ + [ "$(find ${r} -type f | wc -l)" -eq 3 ] # regular files + [ "$(find ${r} -type l | wc -l)" -eq 3 ] # and symlinks +} diff --git a/test/helpers.bash b/test/helpers.bash index 67428d9..8dbb1c1 100644 --- a/test/helpers.bash +++ b/test/helpers.bash @@ -1,3 +1,8 @@ +#!/bin/bash + +export CTR_IMAGE="${CTR_IMAGE:-localhost/containers/buildsourceimage}" +export CTR_ENGINE="${CTR_ENGINE:-podman}" + function run_ctr() { run $CTR_ENGINE run --security-opt label=disable --rm "$@" } From fd4c31be2fec0809a070709f1ebf44557a995c2d Mon Sep 17 00:00:00 2001 From: Vincent Batts Date: Tue, 15 Oct 2019 13:08:08 +0000 Subject: [PATCH 10/14] *: add a doc for development --- developing.md | 33 +++++++++++++++++++++++++++++++++ 1 file changed, 33 insertions(+) create mode 100644 developing.md diff --git a/developing.md b/developing.md new file mode 100644 index 0000000..de7d310 --- /dev/null +++ b/developing.md @@ -0,0 +1,33 @@ +# Developing + +## Requirements + +* `make` +* `shellcheck` (package `ShellCheck` on fedora) +* `bats` +* `wget` +* `podman` (or `docker`) +* `jq` + +## Lint + +[ShellCheck](https://www.shellcheck.net/) is used to ensure the shell script is nice and tidy. + +```bash +make validate +``` + +## Tests + +Testing is done with [`bats`](https://github.com/bats-core/bats-core). + +While it's possible to kick the tests by calling `bats ./test/`, many of the tests are written to use the script as built into a container image. +If you are making local changes and have not rebuilt the container, then they will be missed. + +Best to kick off the build like: +```bash +make test-integration +``` +This will rebuild the container if needed before running the tests. + +## \ No newline at end of file From cd0bcd3c6b7c134f9039c6e91a76405f69b43a95 Mon Sep 17 00:00:00 2001 From: Vincent Batts Date: Thu, 26 Sep 2019 15:50:55 -0400 Subject: [PATCH 11/14] BuildSourceImage: naive relative symlink path Fixes #44 This does not account for the `${artifact_path}` potentially being nested. Signed-off-by: Vincent Batts --- BuildSourceImage.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/BuildSourceImage.sh b/BuildSourceImage.sh index d0db1ca..7b9ad34 100755 --- a/BuildSourceImage.sh +++ b/BuildSourceImage.sh @@ -641,11 +641,11 @@ layout_insert_bash() { if [ "$(basename "${tar_path}")" == "$(basename "${artifact_path}")" ] ; then _mkdir_p "${tmpdir}/$(dirname "${tar_path}")" # TODO this symlink need to be relative path, not to `/blobs/...` - ln -s "/blobs/sha256/${sum}" "${tmpdir}/${tar_path}" + ln -s "../blobs/sha256/${sum}" "${tmpdir}/${tar_path}" else _mkdir_p "${tmpdir}/${tar_path}" # TODO this symlink need to be relative path, not to `/blobs/...` - ln -s "/blobs/sha256/${sum}" "${tmpdir}/${tar_path}/$(basename "${artifact_path}")" + ln -s "../blobs/sha256/${sum}" "${tmpdir}/${tar_path}/$(basename "${artifact_path}")" fi tmptar="$(_mktemp)" From a958ccbc922b1aff98e8327463f113c026236b0a Mon Sep 17 00:00:00 2001 From: Vincent Batts Date: Tue, 15 Oct 2019 20:32:35 +0000 Subject: [PATCH 12/14] test: fixing the symlink path, changes the checksum of the tar layer --- test/01-from_rpms.bats | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/test/01-from_rpms.bats b/test/01-from_rpms.bats index df9660e..1f9d380 100644 --- a/test/01-from_rpms.bats +++ b/test/01-from_rpms.bats @@ -22,9 +22,9 @@ load helpers # let's press that the files are predictable [ "$(find ${d} -type f | wc -l)" -eq 7 ] - [ -f "${d}/blobs/sha256/3afb43699ea82a69b16efb215363604d9e4ffe16c9ace7e53df66663847309cf" ] - [ -f "${d}/blobs/sha256/7f4a50f05b7bd38017be8396b6320e1d2e6a05af097672e3ed23ef3df2ddeadb" ] - [ -f "${d}/blobs/sha256/8f4e610748f8b58a3297ecf78ecc8ff7b6420c3e559e3e20cad8ac178c6fe4e8" ] + [ -f "${d}/blobs/sha256/549ac1e4eb73e55781f39f4b8ee08c1158f1b1c1a523cf278d602386613e2f12" ] + [ -f "${d}/blobs/sha256/b5d5efc6c334cc52223eaea4ac046f21f089c3088b6abb4de027339e5e6dce4b" ] + [ -f "${d}/blobs/sha256/ce0608ce0a601a4cac453b0a0e181cac444027d800a26d5b44b80a74c6dc94e8" ] } @test "build from RPMS and push" { From f7d8f1315c6ce482952a0d4e4af27eb527a8b634 Mon Sep 17 00:00:00 2001 From: Vincent Batts Date: Thu, 17 Oct 2019 09:56:45 -0400 Subject: [PATCH 13/14] rpm: crafting an rpmspec for this tool Now you can `make srpm` or `make rpm`. Additionally, trying to make a way for copr to build this as well. We'll see if that is something. --- .copr/Makefile | 1 + .gitignore | 2 ++ BuildSourceImage.sh | 12 ++++++++- BuildSourceImage.spec | 43 +++++++++++++++++++++++++++++ Makefile | 63 ++++++++++++++++++++++++++++++++++++------- 5 files changed, 110 insertions(+), 11 deletions(-) create mode 120000 .copr/Makefile create mode 100644 BuildSourceImage.spec diff --git a/.copr/Makefile b/.copr/Makefile new file mode 120000 index 0000000..d0b0e8e --- /dev/null +++ b/.copr/Makefile @@ -0,0 +1 @@ +../Makefile \ No newline at end of file diff --git a/.gitignore b/.gitignore index 50554ee..34913a3 100644 --- a/.gitignore +++ b/.gitignore @@ -2,3 +2,5 @@ .build-container .testprep .validate +*.rpm +x86_64/ diff --git a/BuildSourceImage.sh b/BuildSourceImage.sh index 59b08f7..8bb713e 100755 --- a/BuildSourceImage.sh +++ b/BuildSourceImage.sh @@ -16,6 +16,8 @@ _usage() { _version echo "Usage: $(basename "$0") [-D] [-b ] [-c ] [-e ] [-r ] [-o ] [-i ] [-p ] [-l] [-d ]" echo "" + echo " Container Source Image tool" + echo "" echo -e " -b \tbase path for source image builds" echo -e " -c \tbuild context for the container image. Can be provided via CONTEXT_DIR env variable" echo -e " -e \textra src for the container image. Can be provided via EXTRA_SRC_DIR env variable" @@ -128,6 +130,14 @@ _tar() { fi } +_rpm_download() { + if [ "$(command -v yumdownloader)" != "" ] ; then + yumdownloader "${@}" + else + dnf download "${@}" + fi +} + # output things, only when $DEBUG is set _debug() { if [ -n "${DEBUG}" ] ; then @@ -841,7 +851,7 @@ sourcedriver_rpm_fetch() { rpm=${srcrpm%*.src.rpm} if [ ! -f "${out_dir}/${srcrpm}" ] ; then _debug "--> fetching ${srcrpm}" - dnf download \ + _rpm_download \ --quiet \ --installroot "${rootfs}" \ --release "${release}" \ diff --git a/BuildSourceImage.spec b/BuildSourceImage.spec new file mode 100644 index 0000000..879399e --- /dev/null +++ b/BuildSourceImage.spec @@ -0,0 +1,43 @@ +Name: BuildSourceImage +Version: 0.2 +Release: 1%{?dist} +Summary: Container Source Image tool + +Group: containers +License: GPLv2 +URL: https://github.com/containers/BuildSourceImage +Source0: BuildSourceImage.sh + +#BuildRequires: +Requires: jq +Requires: skopeo +Requires: findutils +Requires: file +%if 0%{?rhel} > 6 +Requires: yum-utils +%else +Requires: dnf-command(download) +%endif + +%description +%{summary}. + +%prep + + +%build + + +%install +%{__mkdir_p} %{buildroot}/%{_bindir} +%{__install} -T -m 0755 ${RPM_SOURCE_DIR}/BuildSourceImage.sh %{buildroot}/%{_bindir}/BuildSourceImage + + +%files +%doc ${RPM_SOURCE_DIR}/LICENSE ${RPM_SOURCE_DIR}/README.md +%{_bindir}/BuildSourceImage + + + +%changelog + diff --git a/Makefile b/Makefile index c38791d..f8b4969 100644 --- a/Makefile +++ b/Makefile @@ -1,14 +1,29 @@ -SRC := ./BuildSourceImage.sh -CTR_IMAGE := localhost/containers/buildsourceimage -CTR_ENGINE ?= podman -BATS_OPTS ?= -cleanfiles = +pkgname := BuildSourceImage +CTR_IMAGE := localhost/containers/buildsourceimage +CTR_ENGINE ?= podman +BATS_OPTS ?= +cleanfiles = # these are packages whose src.rpms are very small -srpm_urls = \ +srpm_urls = \ https://archive.kernel.org/centos-vault/7.0.1406/os/Source/SPackages/basesystem-10.0-7.el7.centos.src.rpm \ https://archive.kernel.org/centos-vault/7.0.1406/os/Source/SPackages/rootfiles-8.1-11.el7.src.rpm \ https://archive.kernel.org/centos-vault/7.0.1406/os/Source/SPackages/centos-bookmarks-7-1.el7.src.rpm -srpms = $(addprefix ./.testprep/srpms/,$(notdir $(rpms))) +srpms = $(addprefix ./.testprep/srpms/,$(notdir $(rpms))) + +spec ?= $(pkgname).spec +pwd := $(shell pwd) +NAME := $(shell rpmspec -q --qf "%{name}" $(spec)) +VERSION := $(shell rpmspec -q --qf "%{version}" $(spec)) +RELEASE := $(shell rpmspec -q --qf "%{release}" $(spec)) +ARCH := $(shell rpmspec -q --qf "%{arch}" $(spec)) +NVR := $(NAME)-$(VERSION)-$(RELEASE) +outdir ?= $(pwd) + +SHELL_SRC := ./BuildSourceImage.sh +DIST_FILES := \ + $(SHELL_SRC) \ + LICENSE \ + README.md export CTR_IMAGE export CTR_ENGINE @@ -18,13 +33,13 @@ all: validate validate: .validate cleanfiles += .validate -.validate: $(SRC) - shellcheck $(SRC) && touch $@ +.validate: $(SHELL_SRC) + shellcheck $(SHELL_SRC) && touch $@ build-container: .build-container cleanfiles += .build-container -.build-container: .validate Dockerfile $(SRC) +.build-container: .validate Dockerfile $(SHELL_SRC) @echo @echo "==> Building BuildSourceImage Container" $(CTR_ENGINE) build --quiet --file Dockerfile --tag $(CTR_IMAGE) . && touch $@ @@ -41,6 +56,34 @@ test-integration: .build-container .testprep @echo "==> Running integration tests" TMPDIR=$(realpath .testprep/tmp) bats $(BATS_OPTS) test/ +.PHONY: srpm +srpm: $(NVR).src.rpm + @echo $^ + +cleanfiles += $(NVR).src.rpm +$(NVR).src.rpm: $(spec) $(DIST_FILES) + rpmbuild \ + --define '_sourcedir $(pwd)' \ + --define '_specdir $(pwd)' \ + --define '_builddir $(pwd)' \ + --define '_srcrpmdir $(outdir)' \ + --define '_rpmdir $(outdir)' \ + --nodeps \ + -bs ./$(spec) + +.PHONY: rpm +rpm: $(ARCH)/$(NVR).$(ARCH).rpm + @echo $^ + +cleanfiles += $(ARCH)/$(NVR).$(ARCH).rpm +$(ARCH)/$(NVR).$(ARCH).rpm: $(spec) $(DIST_FILES) + rpmbuild \ + --define '_sourcedir $(pwd)' \ + --define '_specdir $(pwd)' \ + --define '_builddir $(pwd)' \ + --define '_srcrpmdir $(outdir)' \ + --define '_rpmdir $(outdir)' \ + -bb ./$(spec) clean: if [ -n "$(cleanfiles)" ] ; then rm -rf $(cleanfiles) ; fi From 3f6301a8a478371e52f25a5061e3035e5bbfd2d2 Mon Sep 17 00:00:00 2001 From: Vincent Batts Date: Thu, 17 Oct 2019 20:46:59 +0000 Subject: [PATCH 14/14] README: point to non-vbatts container repo Fixes #17 --- README.md | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 6fdb4ed..da5e1a0 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,5 @@ [![Build Status](https://travis-ci.org/containers/BuildSourceImage.svg?branch=master)](https://travis-ci.org/containers/BuildSourceImage) -[![Container Image Repository on Quay](https://quay.io/repository/vbatts/bsi/status "Container Image Repository on Quay")](https://quay.io/repository/vbatts/bsi) +[![Container Image Repository on Quay](https://quay.io/repository/ctrs/bsi/status "Container Image Repository on Quay")](https://quay.io/repository/ctrs/bsi) # BuildSourceImage @@ -32,9 +32,8 @@ Usage: BuildSourceImage.sh [-D] [-b ] [-c ] [-e ] [-r ] Nicely usable inside a container: ```bash -$> podman build -t containers/buildsourceimage . $> mkdir ./output/ -$> podman run -it -v $(pwd)/output/:/output/ -v $(pwd)/SRCRPMS/:/data/ -u $(id -u) containers/buildsourceimage -s /data/ +$> podman run -it -v $(pwd)/output/:/output/ -v $(pwd)/SRCRPMS/:/data/ -u $(id -u) quay.io/ctrs/bsi -s /data/ -o /output/ ``` ## Examples