Compare commits

..

No commits in common. "master" and "v0.1.0" have entirely different histories.

13 changed files with 33 additions and 365 deletions

View file

@ -1 +0,0 @@
../Makefile

2
.gitignore vendored
View file

@ -2,5 +2,3 @@
.build-container
.testprep
.validate
*.rpm
x86_64/

View file

@ -1,5 +1,4 @@
sudo: required
dist: bionic
services:
- docker
@ -8,6 +7,8 @@ env:
- CTR_ENGINE=docker
before_install:
- sudo add-apt-repository ppa:duggan/bats --yes
- sudo apt-get update -qq
- sudo apt-get install -qq bats shellcheck
script:

View file

@ -8,7 +8,7 @@ export source_image_suffix="-source"
# output version string
_version() {
echo "$(basename "${0}") version 0.2.0-dev"
echo "$(basename "${0}") version 0.1"
}
# output the cli usage and exit
@ -16,8 +16,6 @@ _usage() {
_version
echo "Usage: $(basename "$0") [-D] [-b <path>] [-c <path>] [-e <path>] [-r <path>] [-o <path>] [-i <image>] [-p <image>] [-l] [-d <drivers>]"
echo ""
echo " Container Source Image tool"
echo ""
echo -e " -b <path>\tbase path for source image builds"
echo -e " -c <path>\tbuild context for the container image. Can be provided via CONTEXT_DIR env variable"
echo -e " -e <path>\textra src for the container image. Can be provided via EXTRA_SRC_DIR env variable"
@ -30,10 +28,6 @@ _usage() {
echo -e " -D\t\tdebuging output. Can be set via DEBUG env variable"
echo -e " -h\t\tthis usage information"
echo -e " -v\t\tversion"
echo -e ""
echo -e " Subcommands:"
echo -e " unpack\tUnpack an OCI layout to a rootfs directory"
echo -e ""
}
# sanity checks on startup
@ -50,23 +44,6 @@ _init() {
done
}
# enable access to some of functions as subcommands!
_subcommand() {
local command="${1}"
local ret
shift
case "${command}" in
unpack)
# (vb) i'd prefer this subcommand directly match the function name, but it isn't as pretty.
unpack_img "${@}"
ret=$?
exit "${ret}"
;;
esac
}
# _is_sourced tests whether this script is being source, or executed directly
_is_sourced() {
# https://unix.stackexchange.com/a/215279
@ -130,14 +107,6 @@ _tar() {
fi
}
_rpm_download() {
if [ "$(command -v yumdownloader)" != "" ] ; then
yumdownloader "${@}"
else
dnf download "${@}"
fi
}
# output things, only when $DEBUG is set
_debug() {
if [ -n "${DEBUG}" ] ; then
@ -323,20 +292,6 @@ unpack_img() {
local unpack_dir="${2}"
local ret
while getopts ":h" opts; do
case "${opts}" in
*)
echo "$0 unpack <oci layout path> <unpack path>"
return 1
;;
esac
done
shift $((OPTIND-1))
if [ -z "${image_dir}" ] || [ -z "${unpack_dir}" ] ; then
_error "[unpack_img] blank arguments provided"
fi
if [ -d "${unpack_dir}" ] ; then
_rm_rf "${unpack_dir}"
fi
@ -381,10 +336,9 @@ unpack_img_bash() {
return ${ret}
fi
# TODO this will need to be refactored when we start seeing +zstd layers.
# Then it will be better to no just get a list of digests, but maybe to
# iterate on each descriptor independently?
layer_dgsts="$(jq '.layers | map(select(.mediaType == "application/vnd.oci.image.layer.v1.tar+gzip"),select(.mediaType == "application/vnd.oci.image.layer.v1.tar"),select(.mediaType == "application/vnd.docker.image.rootfs.diff.tar.gzip")) | .[] | .digest' "${image_dir}"/blobs/"${mnfst_dgst/:/\/}" | tr -d \")"
# Since we're landing the reference as an OCI layout, this mediaType is fairly predictable
# TODO don't always assume +gzip
layer_dgsts="$(jq '.layers[] | select(.mediaType == "application/vnd.oci.image.layer.v1.tar+gzip") | .digest' "${image_dir}"/blobs/"${mnfst_dgst/:/\/}" | tr -d \")"
ret=$?
if [ ${ret} -ne 0 ] ; then
return ${ret}
@ -456,62 +410,12 @@ push_img() {
# sets up a basic new OCI layout, for an image with the provided (or default 'latest') tag
#
layout_new() {
local out_dir="${1}"
local image_tag="${2:-latest}"
local ret
if [ -n "$(command -v umoci)" ] ; then
layout_new_umoci "${out_dir}" "${image_tag}"
ret=$?
if [ ${ret} -ne 0 ] ; then
return ${ret}
fi
else
layout_new_bash "${out_dir}" "${image_tag}"
ret=$?
if [ ${ret} -ne 0 ] ; then
return ${ret}
fi
fi
}
#
# sets up new OCI layout, using `umoci`
#
layout_new_umoci() {
local out_dir="${1}"
local image_tag="${2:-latest}"
local ret
# umoci expects the layout path to _not_ exist and will fail if it does exist
_rm_rf "${out_dir}"
umoci init --layout "${out_dir}"
ret=$?
if [ "${ret}" -ne 0 ] ; then
return "${ret}"
fi
# XXX currently does not support adding the rich annotations like I've done with the _bash
# https://github.com/openSUSE/umoci/issues/298
umoci new --image "${out_dir}:${image_tag}"
ret=$?
if [ "${ret}" -ne 0 ] ; then
return "${ret}"
fi
}
#
# sets up new OCI layout, all with bash and jq
#
layout_new_bash() {
local out_dir="${1}"
local image_tag="${2:-latest}"
local config
local mnfst
local config_sum
local mnfst_sum
local ret
_mkdir_p "${out_dir}/blobs/sha256"
echo '{"imageLayoutVersion":"1.0.0"}' > "${out_dir}/oci-layout"
@ -528,15 +432,7 @@ layout_new_bash() {
}
'
config_sum=$(echo "${config}" | jq -c | tr -d '\n' | sha256sum | awk '{ ORS=""; print $1 }')
ret=$?
if [ "${ret}" -ne 0 ] ; then
return "${ret}"
fi
echo "${config}" | jq -c | tr -d '\n' > "${out_dir}/blobs/sha256/${config_sum}"
ret=$?
if [ "${ret}" -ne 0 ] ; then
return "${ret}"
fi
mnfst='
{
@ -578,72 +474,6 @@ layout_new_bash() {
# * tag used in the layout (default is 'latest')
#
layout_insert() {
local out_dir="${1}"
local artifact_path="${2}"
local tar_path="${3}"
local annotations_file="${4}"
local image_tag="${5:-latest}"
local ret
if [ -n "$(command -v umoci)" ] ; then
layout_insert_umoci "${out_dir}" "${artifact_path}" "${tar_path}" "${annotations_file}" "${image_tag}"
ret=$?
if [ ${ret} -ne 0 ] ; then
return ${ret}
fi
else
layout_insert_bash "${out_dir}" "${artifact_path}" "${tar_path}" "${annotations_file}" "${image_tag}"
ret=$?
if [ ${ret} -ne 0 ] ; then
return ${ret}
fi
fi
}
layout_insert_umoci() {
local out_dir="${1}"
local artifact_path="${2}"
local tar_path="${3}"
local annotations_file="${4}"
local image_tag="${5:-latest}"
local sum
local ret
# prep the blob path for inside the layer, so we can just copy that whole path in
tmpdir="$(_mktemp_d)"
# TODO account for "artifact_path" being a directory?
sum="$(sha256sum "${artifact_path}" | awk '{ print $1 }')"
_mkdir_p "${tmpdir}/blobs/sha256"
cp "${artifact_path}" "${tmpdir}/blobs/sha256/${sum}"
if [ "$(basename "${tar_path}")" == "$(basename "${artifact_path}")" ] ; then
_mkdir_p "${tmpdir}/$(dirname "${tar_path}")"
# TODO this symlink need to be relative path, not to `/blobs/...`
ln -s "/blobs/sha256/${sum}" "${tmpdir}/${tar_path}"
else
_mkdir_p "${tmpdir}/${tar_path}"
# TODO this symlink need to be relative path, not to `/blobs/...`
ln -s "/blobs/sha256/${sum}" "${tmpdir}/${tar_path}/$(basename "${artifact_path}")"
fi
# XXX currently does not support adding the rich annotations like I've done with the _bash
# https://github.com/openSUSE/umoci/issues/298
# XXX this insert operation can not disable compression
# https://github.com/openSUSE/umoci/issues/300
umoci insert \
--rootless \
--image "${out_dir}:${image_tag}" \
--history.created "$(_date_ns)" \
--history.comment "#(nop) $(_version) adding artifact: ${sum}" \
"${tmpdir}" "/"
ret=$?
if [ ${ret} -ne 0 ] ; then
return ${ret}
fi
}
layout_insert_bash() {
local out_dir="${1}"
local artifact_path="${2}"
local tar_path="${3}"
@ -687,11 +517,11 @@ layout_insert_bash() {
if [ "$(basename "${tar_path}")" == "$(basename "${artifact_path}")" ] ; then
_mkdir_p "${tmpdir}/$(dirname "${tar_path}")"
# TODO this symlink need to be relative path, not to `/blobs/...`
ln -s "../blobs/sha256/${sum}" "${tmpdir}/${tar_path}"
ln -s "/blobs/sha256/${sum}" "${tmpdir}/${tar_path}"
else
_mkdir_p "${tmpdir}/${tar_path}"
# TODO this symlink need to be relative path, not to `/blobs/...`
ln -s "../blobs/sha256/${sum}" "${tmpdir}/${tar_path}/$(basename "${artifact_path}")"
ln -s "/blobs/sha256/${sum}" "${tmpdir}/${tar_path}/$(basename "${artifact_path}")"
fi
tmptar="$(_mktemp)"
@ -712,7 +542,7 @@ layout_insert_bash() {
jq -c \
--arg date "$(_date_ns)" \
--arg tmptar_sum "sha256:${tmptar_sum}" \
--arg comment "#(nop) $(_version) adding artifact: ${sum}" \
--arg comment "#(nop) BuildSourceImage adding artifact: ${sum}" \
'
.created = $date
| .rootfs.diff_ids += [ $tmptar_sum ]
@ -851,7 +681,7 @@ sourcedriver_rpm_fetch() {
rpm=${srcrpm%*.src.rpm}
if [ ! -f "${out_dir}/${srcrpm}" ] ; then
_debug "--> fetching ${srcrpm}"
_rpm_download \
dnf download \
--quiet \
--installroot "${rootfs}" \
--release "${release}" \
@ -1072,9 +902,8 @@ main() {
local work_dir
_init "${@}"
_subcommand "${@}"
base_dir="${BASE_DIR:-$(pwd)/${ABV_NAME}}"
base_dir="$(pwd)/${ABV_NAME}"
# using the bash builtin to parse
while getopts ":hlvDi:c:s:e:o:b:d:p:" opts; do
case "${opts}" in
@ -1167,10 +996,6 @@ main() {
# including its digest.
if [ -z "${inspect_image_digest}" ] ; then
inspect_image_digest="$(fetch_img_digest "$(parse_img_base "${input_inspect_image_ref}"):$(parse_img_tag "${input_inspect_image_ref}")")"
ret=$?
if [ ${ret} -ne 0 ] ; then
_error "failed to detect image digest"
fi
fi
_debug "inspect_image_digest: ${inspect_image_digest}"

View file

@ -1,43 +0,0 @@
Name: BuildSourceImage
Version: 0.2
Release: 1%{?dist}
Summary: Container Source Image tool
Group: containers
License: GPLv2
URL: https://github.com/containers/BuildSourceImage
Source0: BuildSourceImage.sh
#BuildRequires:
Requires: jq
Requires: skopeo
Requires: findutils
Requires: file
%if 0%{?rhel} > 6
Requires: yum-utils
%else
Requires: dnf-command(download)
%endif
%description
%{summary}.
%prep
%build
%install
%{__mkdir_p} %{buildroot}/%{_bindir}
%{__install} -T -m 0755 ${RPM_SOURCE_DIR}/BuildSourceImage.sh %{buildroot}/%{_bindir}/BuildSourceImage
%files
%doc ${RPM_SOURCE_DIR}/LICENSE ${RPM_SOURCE_DIR}/README.md
%{_bindir}/BuildSourceImage
%changelog

View file

@ -4,6 +4,8 @@ RUN dnf install -y jq skopeo findutils file 'dnf-command(download)'
COPY ./BuildSourceImage.sh /usr/local/bin/BuildSourceImage.sh
ENV BASE_DIR=/tmp
RUN mkdir -p /output
ENV OUTPUT_DIR=/output
VOLUME /output
ENTRYPOINT ["/usr/local/bin/BuildSourceImage.sh"]
ENTRYPOINT ["/usr/local/bin/BuildSourceImage.sh", "-b", "/tmp/"]

View file

@ -1,29 +1,14 @@
pkgname := BuildSourceImage
CTR_IMAGE := localhost/containers/buildsourceimage
CTR_ENGINE ?= podman
BATS_OPTS ?=
cleanfiles =
SRC := ./BuildSourceImage.sh
CTR_IMAGE := localhost/containers/buildsourceimage
CTR_ENGINE ?= podman
BATS_OPTS ?=
cleanfiles =
# these are packages whose src.rpms are very small
srpm_urls = \
srpm_urls = \
https://archive.kernel.org/centos-vault/7.0.1406/os/Source/SPackages/basesystem-10.0-7.el7.centos.src.rpm \
https://archive.kernel.org/centos-vault/7.0.1406/os/Source/SPackages/rootfiles-8.1-11.el7.src.rpm \
https://archive.kernel.org/centos-vault/7.0.1406/os/Source/SPackages/centos-bookmarks-7-1.el7.src.rpm
srpms = $(addprefix ./.testprep/srpms/,$(notdir $(rpms)))
spec ?= $(pkgname).spec
pwd := $(shell pwd)
NAME := $(shell rpmspec -q --qf "%{name}" $(spec))
VERSION := $(shell rpmspec -q --qf "%{version}" $(spec))
RELEASE := $(shell rpmspec -q --qf "%{release}" $(spec))
ARCH := $(shell rpmspec -q --qf "%{arch}" $(spec))
NVR := $(NAME)-$(VERSION)-$(RELEASE)
outdir ?= $(pwd)
SHELL_SRC := ./BuildSourceImage.sh
DIST_FILES := \
$(SHELL_SRC) \
LICENSE \
README.md
srpms = $(addprefix ./.testprep/srpms/,$(notdir $(rpms)))
export CTR_IMAGE
export CTR_ENGINE
@ -33,13 +18,13 @@ all: validate
validate: .validate
cleanfiles += .validate
.validate: $(SHELL_SRC)
shellcheck $(SHELL_SRC) && touch $@
.validate: $(SRC)
shellcheck $(SRC) && touch $@
build-container: .build-container
cleanfiles += .build-container
.build-container: .validate Dockerfile $(SHELL_SRC)
.build-container: .validate Dockerfile $(SRC)
@echo
@echo "==> Building BuildSourceImage Container"
$(CTR_ENGINE) build --quiet --file Dockerfile --tag $(CTR_IMAGE) . && touch $@
@ -54,36 +39,8 @@ cleanfiles += .testprep $(srpms)
test-integration: .build-container .testprep
@echo
@echo "==> Running integration tests"
TMPDIR=$(realpath .testprep/tmp) bats $(BATS_OPTS) test/
TMPDIR=$(shell realpath .testprep/tmp) bats $(BATS_OPTS) test/
.PHONY: srpm
srpm: $(NVR).src.rpm
@echo $^
cleanfiles += $(NVR).src.rpm
$(NVR).src.rpm: $(spec) $(DIST_FILES)
rpmbuild \
--define '_sourcedir $(pwd)' \
--define '_specdir $(pwd)' \
--define '_builddir $(pwd)' \
--define '_srcrpmdir $(outdir)' \
--define '_rpmdir $(outdir)' \
--nodeps \
-bs ./$(spec)
.PHONY: rpm
rpm: $(ARCH)/$(NVR).$(ARCH).rpm
@echo $^
cleanfiles += $(ARCH)/$(NVR).$(ARCH).rpm
$(ARCH)/$(NVR).$(ARCH).rpm: $(spec) $(DIST_FILES)
rpmbuild \
--define '_sourcedir $(pwd)' \
--define '_specdir $(pwd)' \
--define '_builddir $(pwd)' \
--define '_srcrpmdir $(outdir)' \
--define '_rpmdir $(outdir)' \
-bb ./$(spec)
clean:
if [ -n "$(cleanfiles)" ] ; then rm -rf $(cleanfiles) ; fi

View file

@ -1,5 +1,4 @@
[![Build Status](https://travis-ci.org/containers/BuildSourceImage.svg?branch=master)](https://travis-ci.org/containers/BuildSourceImage)
[![Container Image Repository on Quay](https://quay.io/repository/ctrs/bsi/status "Container Image Repository on Quay")](https://quay.io/repository/ctrs/bsi)
# BuildSourceImage
@ -32,8 +31,9 @@ Usage: BuildSourceImage.sh [-D] [-b <path>] [-c <path>] [-e <path>] [-r <path>]
Nicely usable inside a container:
```bash
$> podman build -t containers/buildsourceimage .
$> mkdir ./output/
$> podman run -it -v $(pwd)/output/:/output/ -v $(pwd)/SRCRPMS/:/data/ -u $(id -u) quay.io/ctrs/bsi -s /data/ -o /output/
$> podman run -it -v $(pwd)/output/:/output/ -v $(pwd)/SRCRPMS/:/data/ -u $(id -u) containers/buildsourceimage -s /data/
```
## Examples

View file

@ -1,33 +0,0 @@
# Developing
## Requirements
* `make`
* `shellcheck` (package `ShellCheck` on fedora)
* `bats`
* `wget`
* `podman` (or `docker`)
* `jq`
## Lint
[ShellCheck](https://www.shellcheck.net/) is used to ensure the shell script is nice and tidy.
```bash
make validate
```
## Tests
Testing is done with [`bats`](https://github.com/bats-core/bats-core).
While it's possible to kick the tests by calling `bats ./test/`, many of the tests are written to use the script as built into a container image.
If you are making local changes and have not rebuilt the container, then they will be missed.
Best to kick off the build like:
```bash
make test-integration
```
This will rebuild the container if needed before running the tests.
##

View file

@ -7,7 +7,7 @@ load helpers
d=$(mktemp -d)
echo "temporary directory: ${d}"
run_ctr -v $(pwd)/.testprep/srpms/:/src:ro --mount type=bind,source=${d},destination=/output $CTR_IMAGE -s /src -o /output
run_ctr -v $(pwd)/.testprep/srpms/:/src:ro --mount type=bind,source=${d},destination=/output $CTR_IMAGE -s /src
[ "$status" -eq 0 ]
[[ ${lines[0]} =~ "[SrcImg][INFO] calling source collection drivers" ]]
# get the number of the last line
@ -22,9 +22,9 @@ load helpers
# let's press that the files are predictable
[ "$(find ${d} -type f | wc -l)" -eq 7 ]
[ -f "${d}/blobs/sha256/549ac1e4eb73e55781f39f4b8ee08c1158f1b1c1a523cf278d602386613e2f12" ]
[ -f "${d}/blobs/sha256/b5d5efc6c334cc52223eaea4ac046f21f089c3088b6abb4de027339e5e6dce4b" ]
[ -f "${d}/blobs/sha256/ce0608ce0a601a4cac453b0a0e181cac444027d800a26d5b44b80a74c6dc94e8" ]
[ -f "${d}/blobs/sha256/3afb43699ea82a69b16efb215363604d9e4ffe16c9ace7e53df66663847309cf" ]
[ -f "${d}/blobs/sha256/7f4a50f05b7bd38017be8396b6320e1d2e6a05af097672e3ed23ef3df2ddeadb" ]
[ -f "${d}/blobs/sha256/8f4e610748f8b58a3297ecf78ecc8ff7b6420c3e559e3e20cad8ac178c6fe4e8" ]
}
@test "build from RPMS and push" {

View file

@ -3,13 +3,12 @@
load helpers
@test "Build from image reference" {
#skip "this takes like 20min ..."
local d
d=$(mktemp -d)
echo "temporary directory: ${d}"
ref="registry.fedoraproject.org/fedora-minimal"
run_ctr --mount type=bind,source=${d},destination=/output $CTR_IMAGE -i "${ref}" -o /output
run_ctr --mount type=bind,source=${d},destination=/output $CTR_IMAGE -i "${ref}"
[ "$status" -eq 0 ]
#echo ${lines[@]}
[[ ${lines[0]} =~ "Getting image source signatures" ]]

View file

@ -1,32 +0,0 @@
#!/usr/bin/env bats -t
load helpers
@test "unpack - no args" {
run_ctr $CTR_IMAGE unpack
[ "$status" -eq 1 ]
[[ ${lines[0]} =~ "[SrcImg][ERROR] [unpack_img] blank arguments provided" ]]
}
@test "unpack - Help" {
run_ctr $CTR_IMAGE unpack -h
[ "$status" -eq 1 ]
[[ ${lines[0]} =~ "BuildSourceImage.sh unpack <oci layout path> <unpack path>" ]]
}
@test "unpack - from a SRPM build" {
local d
local r
d=$(mktemp -d)
echo "temporary directories: output - ${d}"
run_ctr -v $(pwd)/.testprep/srpms/:/src:ro --mount type=bind,source=${d},destination=/output $CTR_IMAGE -s /src -o /output
[ "$status" -eq 0 ]
[ -f "${d}/index.json" ]
r=$(mktemp -d)
echo "temporary directories: unpacked - ${r}"
run_ctr --mount type=bind,source=${d},destination=/output -v ${r}:/unpacked/ $CTR_IMAGE unpack /output/ /unpacked/
[ "$(find ${r} -type f | wc -l)" -eq 3 ] # regular files
[ "$(find ${r} -type l | wc -l)" -eq 3 ] # and symlinks
}

View file

@ -1,8 +1,3 @@
#!/bin/bash
export CTR_IMAGE="${CTR_IMAGE:-localhost/containers/buildsourceimage}"
export CTR_ENGINE="${CTR_ENGINE:-podman}"
function run_ctr() {
run $CTR_ENGINE run --security-opt label=disable --rm "$@"
}