diff --git a/.copr/Makefile b/.copr/Makefile deleted file mode 120000 index d0b0e8e..0000000 --- a/.copr/Makefile +++ /dev/null @@ -1 +0,0 @@ -../Makefile \ No newline at end of file diff --git a/.gitignore b/.gitignore index 34913a3..50554ee 100644 --- a/.gitignore +++ b/.gitignore @@ -2,5 +2,3 @@ .build-container .testprep .validate -*.rpm -x86_64/ diff --git a/.travis.yml b/.travis.yml index 39204a9..f13760e 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,5 +1,4 @@ sudo: required -dist: bionic services: - docker @@ -8,6 +7,8 @@ env: - CTR_ENGINE=docker before_install: + - sudo add-apt-repository ppa:duggan/bats --yes + - sudo apt-get update -qq - sudo apt-get install -qq bats shellcheck script: diff --git a/BuildSourceImage.sh b/BuildSourceImage.sh index 8bb713e..4296dbd 100755 --- a/BuildSourceImage.sh +++ b/BuildSourceImage.sh @@ -8,7 +8,7 @@ export source_image_suffix="-source" # output version string _version() { - echo "$(basename "${0}") version 0.2.0-dev" + echo "$(basename "${0}") version 0.1" } # output the cli usage and exit @@ -16,8 +16,6 @@ _usage() { _version echo "Usage: $(basename "$0") [-D] [-b ] [-c ] [-e ] [-r ] [-o ] [-i ] [-p ] [-l] [-d ]" echo "" - echo " Container Source Image tool" - echo "" echo -e " -b \tbase path for source image builds" echo -e " -c \tbuild context for the container image. Can be provided via CONTEXT_DIR env variable" echo -e " -e \textra src for the container image. Can be provided via EXTRA_SRC_DIR env variable" @@ -30,10 +28,6 @@ _usage() { echo -e " -D\t\tdebuging output. Can be set via DEBUG env variable" echo -e " -h\t\tthis usage information" echo -e " -v\t\tversion" - echo -e "" - echo -e " Subcommands:" - echo -e " unpack\tUnpack an OCI layout to a rootfs directory" - echo -e "" } # sanity checks on startup @@ -50,23 +44,6 @@ _init() { done } -# enable access to some of functions as subcommands! -_subcommand() { - local command="${1}" - local ret - - shift - - case "${command}" in - unpack) - # (vb) i'd prefer this subcommand directly match the function name, but it isn't as pretty. - unpack_img "${@}" - ret=$? - exit "${ret}" - ;; - esac -} - # _is_sourced tests whether this script is being source, or executed directly _is_sourced() { # https://unix.stackexchange.com/a/215279 @@ -130,14 +107,6 @@ _tar() { fi } -_rpm_download() { - if [ "$(command -v yumdownloader)" != "" ] ; then - yumdownloader "${@}" - else - dnf download "${@}" - fi -} - # output things, only when $DEBUG is set _debug() { if [ -n "${DEBUG}" ] ; then @@ -323,20 +292,6 @@ unpack_img() { local unpack_dir="${2}" local ret - while getopts ":h" opts; do - case "${opts}" in - *) - echo "$0 unpack " - return 1 - ;; - esac - done - shift $((OPTIND-1)) - - if [ -z "${image_dir}" ] || [ -z "${unpack_dir}" ] ; then - _error "[unpack_img] blank arguments provided" - fi - if [ -d "${unpack_dir}" ] ; then _rm_rf "${unpack_dir}" fi @@ -381,10 +336,9 @@ unpack_img_bash() { return ${ret} fi - # TODO this will need to be refactored when we start seeing +zstd layers. - # Then it will be better to no just get a list of digests, but maybe to - # iterate on each descriptor independently? - layer_dgsts="$(jq '.layers | map(select(.mediaType == "application/vnd.oci.image.layer.v1.tar+gzip"),select(.mediaType == "application/vnd.oci.image.layer.v1.tar"),select(.mediaType == "application/vnd.docker.image.rootfs.diff.tar.gzip")) | .[] | .digest' "${image_dir}"/blobs/"${mnfst_dgst/:/\/}" | tr -d \")" + # Since we're landing the reference as an OCI layout, this mediaType is fairly predictable + # TODO don't always assume +gzip + layer_dgsts="$(jq '.layers[] | select(.mediaType == "application/vnd.oci.image.layer.v1.tar+gzip") | .digest' "${image_dir}"/blobs/"${mnfst_dgst/:/\/}" | tr -d \")" ret=$? if [ ${ret} -ne 0 ] ; then return ${ret} @@ -456,62 +410,12 @@ push_img() { # sets up a basic new OCI layout, for an image with the provided (or default 'latest') tag # layout_new() { - local out_dir="${1}" - local image_tag="${2:-latest}" - local ret - - if [ -n "$(command -v umoci)" ] ; then - layout_new_umoci "${out_dir}" "${image_tag}" - ret=$? - if [ ${ret} -ne 0 ] ; then - return ${ret} - fi - else - layout_new_bash "${out_dir}" "${image_tag}" - ret=$? - if [ ${ret} -ne 0 ] ; then - return ${ret} - fi - fi -} - -# -# sets up new OCI layout, using `umoci` -# -layout_new_umoci() { - local out_dir="${1}" - local image_tag="${2:-latest}" - local ret - - # umoci expects the layout path to _not_ exist and will fail if it does exist - _rm_rf "${out_dir}" - - umoci init --layout "${out_dir}" - ret=$? - if [ "${ret}" -ne 0 ] ; then - return "${ret}" - fi - - # XXX currently does not support adding the rich annotations like I've done with the _bash - # https://github.com/openSUSE/umoci/issues/298 - umoci new --image "${out_dir}:${image_tag}" - ret=$? - if [ "${ret}" -ne 0 ] ; then - return "${ret}" - fi -} - -# -# sets up new OCI layout, all with bash and jq -# -layout_new_bash() { local out_dir="${1}" local image_tag="${2:-latest}" local config local mnfst local config_sum local mnfst_sum - local ret _mkdir_p "${out_dir}/blobs/sha256" echo '{"imageLayoutVersion":"1.0.0"}' > "${out_dir}/oci-layout" @@ -528,15 +432,7 @@ layout_new_bash() { } ' config_sum=$(echo "${config}" | jq -c | tr -d '\n' | sha256sum | awk '{ ORS=""; print $1 }') - ret=$? - if [ "${ret}" -ne 0 ] ; then - return "${ret}" - fi echo "${config}" | jq -c | tr -d '\n' > "${out_dir}/blobs/sha256/${config_sum}" - ret=$? - if [ "${ret}" -ne 0 ] ; then - return "${ret}" - fi mnfst=' { @@ -578,72 +474,6 @@ layout_new_bash() { # * tag used in the layout (default is 'latest') # layout_insert() { - local out_dir="${1}" - local artifact_path="${2}" - local tar_path="${3}" - local annotations_file="${4}" - local image_tag="${5:-latest}" - local ret - - if [ -n "$(command -v umoci)" ] ; then - layout_insert_umoci "${out_dir}" "${artifact_path}" "${tar_path}" "${annotations_file}" "${image_tag}" - ret=$? - if [ ${ret} -ne 0 ] ; then - return ${ret} - fi - else - layout_insert_bash "${out_dir}" "${artifact_path}" "${tar_path}" "${annotations_file}" "${image_tag}" - ret=$? - if [ ${ret} -ne 0 ] ; then - return ${ret} - fi - fi -} - -layout_insert_umoci() { - local out_dir="${1}" - local artifact_path="${2}" - local tar_path="${3}" - local annotations_file="${4}" - local image_tag="${5:-latest}" - local sum - local ret - - # prep the blob path for inside the layer, so we can just copy that whole path in - tmpdir="$(_mktemp_d)" - - # TODO account for "artifact_path" being a directory? - sum="$(sha256sum "${artifact_path}" | awk '{ print $1 }')" - - _mkdir_p "${tmpdir}/blobs/sha256" - cp "${artifact_path}" "${tmpdir}/blobs/sha256/${sum}" - if [ "$(basename "${tar_path}")" == "$(basename "${artifact_path}")" ] ; then - _mkdir_p "${tmpdir}/$(dirname "${tar_path}")" - # TODO this symlink need to be relative path, not to `/blobs/...` - ln -s "/blobs/sha256/${sum}" "${tmpdir}/${tar_path}" - else - _mkdir_p "${tmpdir}/${tar_path}" - # TODO this symlink need to be relative path, not to `/blobs/...` - ln -s "/blobs/sha256/${sum}" "${tmpdir}/${tar_path}/$(basename "${artifact_path}")" - fi - - # XXX currently does not support adding the rich annotations like I've done with the _bash - # https://github.com/openSUSE/umoci/issues/298 - # XXX this insert operation can not disable compression - # https://github.com/openSUSE/umoci/issues/300 - umoci insert \ - --rootless \ - --image "${out_dir}:${image_tag}" \ - --history.created "$(_date_ns)" \ - --history.comment "#(nop) $(_version) adding artifact: ${sum}" \ - "${tmpdir}" "/" - ret=$? - if [ ${ret} -ne 0 ] ; then - return ${ret} - fi -} - -layout_insert_bash() { local out_dir="${1}" local artifact_path="${2}" local tar_path="${3}" @@ -687,11 +517,11 @@ layout_insert_bash() { if [ "$(basename "${tar_path}")" == "$(basename "${artifact_path}")" ] ; then _mkdir_p "${tmpdir}/$(dirname "${tar_path}")" # TODO this symlink need to be relative path, not to `/blobs/...` - ln -s "../blobs/sha256/${sum}" "${tmpdir}/${tar_path}" + ln -s "/blobs/sha256/${sum}" "${tmpdir}/${tar_path}" else _mkdir_p "${tmpdir}/${tar_path}" # TODO this symlink need to be relative path, not to `/blobs/...` - ln -s "../blobs/sha256/${sum}" "${tmpdir}/${tar_path}/$(basename "${artifact_path}")" + ln -s "/blobs/sha256/${sum}" "${tmpdir}/${tar_path}/$(basename "${artifact_path}")" fi tmptar="$(_mktemp)" @@ -712,7 +542,7 @@ layout_insert_bash() { jq -c \ --arg date "$(_date_ns)" \ --arg tmptar_sum "sha256:${tmptar_sum}" \ - --arg comment "#(nop) $(_version) adding artifact: ${sum}" \ + --arg comment "#(nop) BuildSourceImage adding artifact: ${sum}" \ ' .created = $date | .rootfs.diff_ids += [ $tmptar_sum ] @@ -851,7 +681,7 @@ sourcedriver_rpm_fetch() { rpm=${srcrpm%*.src.rpm} if [ ! -f "${out_dir}/${srcrpm}" ] ; then _debug "--> fetching ${srcrpm}" - _rpm_download \ + dnf download \ --quiet \ --installroot "${rootfs}" \ --release "${release}" \ @@ -1072,9 +902,8 @@ main() { local work_dir _init "${@}" - _subcommand "${@}" - base_dir="${BASE_DIR:-$(pwd)/${ABV_NAME}}" + base_dir="$(pwd)/${ABV_NAME}" # using the bash builtin to parse while getopts ":hlvDi:c:s:e:o:b:d:p:" opts; do case "${opts}" in @@ -1167,10 +996,6 @@ main() { # including its digest. if [ -z "${inspect_image_digest}" ] ; then inspect_image_digest="$(fetch_img_digest "$(parse_img_base "${input_inspect_image_ref}"):$(parse_img_tag "${input_inspect_image_ref}")")" - ret=$? - if [ ${ret} -ne 0 ] ; then - _error "failed to detect image digest" - fi fi _debug "inspect_image_digest: ${inspect_image_digest}" diff --git a/BuildSourceImage.spec b/BuildSourceImage.spec deleted file mode 100644 index 879399e..0000000 --- a/BuildSourceImage.spec +++ /dev/null @@ -1,43 +0,0 @@ -Name: BuildSourceImage -Version: 0.2 -Release: 1%{?dist} -Summary: Container Source Image tool - -Group: containers -License: GPLv2 -URL: https://github.com/containers/BuildSourceImage -Source0: BuildSourceImage.sh - -#BuildRequires: -Requires: jq -Requires: skopeo -Requires: findutils -Requires: file -%if 0%{?rhel} > 6 -Requires: yum-utils -%else -Requires: dnf-command(download) -%endif - -%description -%{summary}. - -%prep - - -%build - - -%install -%{__mkdir_p} %{buildroot}/%{_bindir} -%{__install} -T -m 0755 ${RPM_SOURCE_DIR}/BuildSourceImage.sh %{buildroot}/%{_bindir}/BuildSourceImage - - -%files -%doc ${RPM_SOURCE_DIR}/LICENSE ${RPM_SOURCE_DIR}/README.md -%{_bindir}/BuildSourceImage - - - -%changelog - diff --git a/Dockerfile b/Dockerfile index 8c64206..a50044c 100644 --- a/Dockerfile +++ b/Dockerfile @@ -4,6 +4,8 @@ RUN dnf install -y jq skopeo findutils file 'dnf-command(download)' COPY ./BuildSourceImage.sh /usr/local/bin/BuildSourceImage.sh -ENV BASE_DIR=/tmp +RUN mkdir -p /output +ENV OUTPUT_DIR=/output +VOLUME /output -ENTRYPOINT ["/usr/local/bin/BuildSourceImage.sh"] +ENTRYPOINT ["/usr/local/bin/BuildSourceImage.sh", "-b", "/tmp/"] diff --git a/Makefile b/Makefile index f8b4969..69b388a 100644 --- a/Makefile +++ b/Makefile @@ -1,29 +1,14 @@ -pkgname := BuildSourceImage -CTR_IMAGE := localhost/containers/buildsourceimage -CTR_ENGINE ?= podman -BATS_OPTS ?= -cleanfiles = +SRC := ./BuildSourceImage.sh +CTR_IMAGE := localhost/containers/buildsourceimage +CTR_ENGINE ?= podman +BATS_OPTS ?= +cleanfiles = # these are packages whose src.rpms are very small -srpm_urls = \ +srpm_urls = \ https://archive.kernel.org/centos-vault/7.0.1406/os/Source/SPackages/basesystem-10.0-7.el7.centos.src.rpm \ https://archive.kernel.org/centos-vault/7.0.1406/os/Source/SPackages/rootfiles-8.1-11.el7.src.rpm \ https://archive.kernel.org/centos-vault/7.0.1406/os/Source/SPackages/centos-bookmarks-7-1.el7.src.rpm -srpms = $(addprefix ./.testprep/srpms/,$(notdir $(rpms))) - -spec ?= $(pkgname).spec -pwd := $(shell pwd) -NAME := $(shell rpmspec -q --qf "%{name}" $(spec)) -VERSION := $(shell rpmspec -q --qf "%{version}" $(spec)) -RELEASE := $(shell rpmspec -q --qf "%{release}" $(spec)) -ARCH := $(shell rpmspec -q --qf "%{arch}" $(spec)) -NVR := $(NAME)-$(VERSION)-$(RELEASE) -outdir ?= $(pwd) - -SHELL_SRC := ./BuildSourceImage.sh -DIST_FILES := \ - $(SHELL_SRC) \ - LICENSE \ - README.md +srpms = $(addprefix ./.testprep/srpms/,$(notdir $(rpms))) export CTR_IMAGE export CTR_ENGINE @@ -33,13 +18,13 @@ all: validate validate: .validate cleanfiles += .validate -.validate: $(SHELL_SRC) - shellcheck $(SHELL_SRC) && touch $@ +.validate: $(SRC) + shellcheck $(SRC) && touch $@ build-container: .build-container cleanfiles += .build-container -.build-container: .validate Dockerfile $(SHELL_SRC) +.build-container: .validate Dockerfile $(SRC) @echo @echo "==> Building BuildSourceImage Container" $(CTR_ENGINE) build --quiet --file Dockerfile --tag $(CTR_IMAGE) . && touch $@ @@ -54,36 +39,8 @@ cleanfiles += .testprep $(srpms) test-integration: .build-container .testprep @echo @echo "==> Running integration tests" - TMPDIR=$(realpath .testprep/tmp) bats $(BATS_OPTS) test/ + TMPDIR=$(shell realpath .testprep/tmp) bats $(BATS_OPTS) test/ -.PHONY: srpm -srpm: $(NVR).src.rpm - @echo $^ - -cleanfiles += $(NVR).src.rpm -$(NVR).src.rpm: $(spec) $(DIST_FILES) - rpmbuild \ - --define '_sourcedir $(pwd)' \ - --define '_specdir $(pwd)' \ - --define '_builddir $(pwd)' \ - --define '_srcrpmdir $(outdir)' \ - --define '_rpmdir $(outdir)' \ - --nodeps \ - -bs ./$(spec) - -.PHONY: rpm -rpm: $(ARCH)/$(NVR).$(ARCH).rpm - @echo $^ - -cleanfiles += $(ARCH)/$(NVR).$(ARCH).rpm -$(ARCH)/$(NVR).$(ARCH).rpm: $(spec) $(DIST_FILES) - rpmbuild \ - --define '_sourcedir $(pwd)' \ - --define '_specdir $(pwd)' \ - --define '_builddir $(pwd)' \ - --define '_srcrpmdir $(outdir)' \ - --define '_rpmdir $(outdir)' \ - -bb ./$(spec) clean: if [ -n "$(cleanfiles)" ] ; then rm -rf $(cleanfiles) ; fi diff --git a/README.md b/README.md index da5e1a0..b3b7b7c 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,4 @@ [![Build Status](https://travis-ci.org/containers/BuildSourceImage.svg?branch=master)](https://travis-ci.org/containers/BuildSourceImage) -[![Container Image Repository on Quay](https://quay.io/repository/ctrs/bsi/status "Container Image Repository on Quay")](https://quay.io/repository/ctrs/bsi) # BuildSourceImage @@ -32,8 +31,9 @@ Usage: BuildSourceImage.sh [-D] [-b ] [-c ] [-e ] [-r ] Nicely usable inside a container: ```bash +$> podman build -t containers/buildsourceimage . $> mkdir ./output/ -$> podman run -it -v $(pwd)/output/:/output/ -v $(pwd)/SRCRPMS/:/data/ -u $(id -u) quay.io/ctrs/bsi -s /data/ -o /output/ +$> podman run -it -v $(pwd)/output/:/output/ -v $(pwd)/SRCRPMS/:/data/ -u $(id -u) containers/buildsourceimage -s /data/ ``` ## Examples diff --git a/developing.md b/developing.md deleted file mode 100644 index de7d310..0000000 --- a/developing.md +++ /dev/null @@ -1,33 +0,0 @@ -# Developing - -## Requirements - -* `make` -* `shellcheck` (package `ShellCheck` on fedora) -* `bats` -* `wget` -* `podman` (or `docker`) -* `jq` - -## Lint - -[ShellCheck](https://www.shellcheck.net/) is used to ensure the shell script is nice and tidy. - -```bash -make validate -``` - -## Tests - -Testing is done with [`bats`](https://github.com/bats-core/bats-core). - -While it's possible to kick the tests by calling `bats ./test/`, many of the tests are written to use the script as built into a container image. -If you are making local changes and have not rebuilt the container, then they will be missed. - -Best to kick off the build like: -```bash -make test-integration -``` -This will rebuild the container if needed before running the tests. - -## \ No newline at end of file diff --git a/test/01-from_rpms.bats b/test/01-from_rpms.bats index 1f9d380..07aa5d3 100644 --- a/test/01-from_rpms.bats +++ b/test/01-from_rpms.bats @@ -7,7 +7,7 @@ load helpers d=$(mktemp -d) echo "temporary directory: ${d}" - run_ctr -v $(pwd)/.testprep/srpms/:/src:ro --mount type=bind,source=${d},destination=/output $CTR_IMAGE -s /src -o /output + run_ctr -v $(pwd)/.testprep/srpms/:/src:ro --mount type=bind,source=${d},destination=/output $CTR_IMAGE -s /src [ "$status" -eq 0 ] [[ ${lines[0]} =~ "[SrcImg][INFO] calling source collection drivers" ]] # get the number of the last line @@ -22,9 +22,9 @@ load helpers # let's press that the files are predictable [ "$(find ${d} -type f | wc -l)" -eq 7 ] - [ -f "${d}/blobs/sha256/549ac1e4eb73e55781f39f4b8ee08c1158f1b1c1a523cf278d602386613e2f12" ] - [ -f "${d}/blobs/sha256/b5d5efc6c334cc52223eaea4ac046f21f089c3088b6abb4de027339e5e6dce4b" ] - [ -f "${d}/blobs/sha256/ce0608ce0a601a4cac453b0a0e181cac444027d800a26d5b44b80a74c6dc94e8" ] + [ -f "${d}/blobs/sha256/3afb43699ea82a69b16efb215363604d9e4ffe16c9ace7e53df66663847309cf" ] + [ -f "${d}/blobs/sha256/7f4a50f05b7bd38017be8396b6320e1d2e6a05af097672e3ed23ef3df2ddeadb" ] + [ -f "${d}/blobs/sha256/8f4e610748f8b58a3297ecf78ecc8ff7b6420c3e559e3e20cad8ac178c6fe4e8" ] } @test "build from RPMS and push" { diff --git a/test/02-from_image_ref.bats b/test/02-from_image_ref.bats index b9f1ce3..38ce66c 100644 --- a/test/02-from_image_ref.bats +++ b/test/02-from_image_ref.bats @@ -3,13 +3,12 @@ load helpers @test "Build from image reference" { - #skip "this takes like 20min ..." local d d=$(mktemp -d) echo "temporary directory: ${d}" ref="registry.fedoraproject.org/fedora-minimal" - run_ctr --mount type=bind,source=${d},destination=/output $CTR_IMAGE -i "${ref}" -o /output + run_ctr --mount type=bind,source=${d},destination=/output $CTR_IMAGE -i "${ref}" [ "$status" -eq 0 ] #echo ${lines[@]} [[ ${lines[0]} =~ "Getting image source signatures" ]] diff --git a/test/03-unpack.bats b/test/03-unpack.bats deleted file mode 100644 index ee540cf..0000000 --- a/test/03-unpack.bats +++ /dev/null @@ -1,32 +0,0 @@ -#!/usr/bin/env bats -t - -load helpers - -@test "unpack - no args" { - run_ctr $CTR_IMAGE unpack - [ "$status" -eq 1 ] - [[ ${lines[0]} =~ "[SrcImg][ERROR] [unpack_img] blank arguments provided" ]] -} - -@test "unpack - Help" { - run_ctr $CTR_IMAGE unpack -h - [ "$status" -eq 1 ] - [[ ${lines[0]} =~ "BuildSourceImage.sh unpack " ]] -} - -@test "unpack - from a SRPM build" { - local d - local r - - d=$(mktemp -d) - echo "temporary directories: output - ${d}" - run_ctr -v $(pwd)/.testprep/srpms/:/src:ro --mount type=bind,source=${d},destination=/output $CTR_IMAGE -s /src -o /output - [ "$status" -eq 0 ] - [ -f "${d}/index.json" ] - - r=$(mktemp -d) - echo "temporary directories: unpacked - ${r}" - run_ctr --mount type=bind,source=${d},destination=/output -v ${r}:/unpacked/ $CTR_IMAGE unpack /output/ /unpacked/ - [ "$(find ${r} -type f | wc -l)" -eq 3 ] # regular files - [ "$(find ${r} -type l | wc -l)" -eq 3 ] # and symlinks -} diff --git a/test/helpers.bash b/test/helpers.bash index 8dbb1c1..67428d9 100644 --- a/test/helpers.bash +++ b/test/helpers.bash @@ -1,8 +1,3 @@ -#!/bin/bash - -export CTR_IMAGE="${CTR_IMAGE:-localhost/containers/buildsourceimage}" -export CTR_ENGINE="${CTR_ENGINE:-podman}" - function run_ctr() { run $CTR_ENGINE run --security-opt label=disable --rm "$@" }