diff --git a/BuildSourceImage.sh b/BuildSourceImage.sh index 4fcb55b..6992c64 100755 --- a/BuildSourceImage.sh +++ b/BuildSourceImage.sh @@ -1,82 +1,1116 @@ -#!/bin/sh +#!/bin/bash -set -e +# This script builds a Source Image via "drivers" to collect source -# This script requires an OCI IMAGE Name to pull. -# The script generates a SOURCE Image based on the OCI Image -# Script must be executed on the same OS or newer as the image. -if test $# -lt 2 ; then - echo Usage: $(basename $0) IMAGE CONTEXT_DIR [EXTRA_SRC_DIR] +export ABV_NAME="SrcImg" +# TODO maybe a flag for this? +export source_image_suffix="-source" + +# output version string +_version() { + echo "$(basename "${0}") version 0.1" +} + +# output the cli usage and exit +_usage() { + _version + echo "Usage: $(basename "$0") [-D] [-b ] [-c ] [-e ] [-r ] [-o ] [-i ] [-p ] [-l] [-d ]" + echo "" + echo -e " -b \tbase path for source image builds" + echo -e " -c \tbuild context for the container image. Can be provided via CONTEXT_DIR env variable" + echo -e " -e \textra src for the container image. Can be provided via EXTRA_SRC_DIR env variable" + echo -e " -s \tdirectory of SRPMS to add. Can be provided via SRPM_DIR env variable" + echo -e " -o \toutput the OCI image to path. Can be provided via OUTPUT_DIR env variable" + echo -e " -d \tenumerate specific source drivers to run" + echo -e " -l\t\tlist the source drivers available" + echo -e " -i \timage reference to fetch and inspect its rootfs to derive sources" + echo -e " -p \tpush source image to specified reference after build" + echo -e " -D\t\tdebuging output. Can be set via DEBUG env variable" + echo -e " -h\t\tthis usage information" + echo -e " -v\t\tversion" exit 1 -fi -export IMAGE=$1 -export SRC_RPM_DIR=$(pwd)/SRCRPMS -export SRC_IMAGE=$1-src -export CONTEXT_DIR=$2 -export EXTRA_SRC_DIR=$3 -export IMAGE_CTR=$(buildah from ${IMAGE}) -export IMAGE_MNT=$(buildah mount ${IMAGE_CTR}) -# -# From the executable image, get the RELEASEVER of the image -# -RELEASE=$(rpm -q --queryformat "%{VERSION}\n" --root $IMAGE_MNT -f /etc/os-release) -# -# From the executable image, list the SRC RPMS used to build the image -# -SRC_RPMS=$(rpm -qa --root ${IMAGE_MNT} --queryformat '%{SOURCERPM}\n' | grep -v '^gpg-pubkey' | sort -u) -buildah umount ${IMAGE_CTR} -buildah rm ${IMAGE_CTR} +} + +# sanity checks on startup +_init() { + set -o pipefail + + # check for tools we depend on + for cmd in jq skopeo dnf file find tar stat date ; do + if [ -z "$(command -v ${cmd})" ] ; then + # TODO: maybe this could be individual checks so it can report + # where to find the tools + _error "please install package to provide '${cmd}'" + fi + done +} + +# _is_sourced tests whether this script is being source, or executed directly +_is_sourced() { + # https://unix.stackexchange.com/a/215279 + # thanks @tianon + [ "${FUNCNAME[${#FUNCNAME[@]} - 1]}" == 'source' ] +} + +# count $character $string +_count_char_in_string() { + c="${2//[^${1}]}" + echo -n ${#c} +} + +# size of file/directory in bytes +_size() { + du -b "${1}" | awk '{ ORS=""; print $1 }' +} + +# date timestamp in RFC 3339, to the nanosecond, but slightly golang style ... +_date_ns() { + date --rfc-3339=ns | tr ' ' 'T' | tr -d '\n' +} + +# local `mktemp -d` +_mktemp_d() { + local v + v=$(mktemp -d "${TMPDIR:-/tmp}/${ABV_NAME}.XXXXXX") + _debug "mktemp -d --> ${v}" + echo "${v}" +} + +# local `mktemp` +_mktemp() { + local v + v=$(mktemp "${TMPDIR:-/tmp}/${ABV_NAME}.XXXXXX") + _debug "mktemp --> ${v}" + echo "${v}" +} + +# local rm -rf +_rm_rf() { + _debug "rm -rf ${*}" + rm -rf "${@}" +} + +# local mkdir -p +_mkdir_p() { + if [ -n "${DEBUG}" ] ; then + mkdir -vp "${@}" + else + mkdir -p "${@}" + fi +} + +# local tar +_tar() { + if [ -n "${DEBUG}" ] ; then + tar -v "${@}" + else + tar "${@}" + fi +} + +# output things, only when $DEBUG is set +_debug() { + if [ -n "${DEBUG}" ] ; then + echo "[${ABV_NAME}][DEBUG] ${*}" >&2 + fi +} + +# general echo but with prefix +_info() { + echo "[${ABV_NAME}][INFO] ${*}" +} + +_warn() { + echo "[${ABV_NAME}][WARN] ${*}" >&2 +} + +# general echo but with prefix +_error() { + echo "[${ABV_NAME}][ERROR] ${*}" >&2 + exit 1 +} # -# For each SRC_RPMS used to build the executable image, download the SRC RPM -# and generate a layer in the SRC RPM. +# parse the OCI image reference, accounting for: +# * transport name +# * presence or lack of transport port number +# * presence or lack of digest +# * presence or lack of image tag # -mkdir -p ${SRC_RPM_DIR} -pushd ${SRC_RPM_DIR} > /dev/null -export SRC_CTR=$(buildah from scratch) -for srpm in ${SRC_RPMS}; do - if [ ! -f ${srpm} ]; then - RPM=$(echo ${srpm} | sed 's/.src.rpm$//g') - dnf download --release $RELEASE --source $RPM || continue + +# +# return the image reference's digest, if any +# +parse_img_digest() { + local ref="${1}" + local digest="" + if [ "$(_count_char_in_string '@' "${ref}")" -gt 0 ] ; then + digest="${ref##*@}" # the digest after the "@" fi - echo "Adding ${srpm}" - touch --date=@`rpm -q --qf '%{buildtime}' ${srpm}` ${srpm} - buildah add ${SRC_CTR} ${srpm} /RPMS/ - buildah config --created-by "/bin/sh -c #(nop) ADD file:$(sha256sum ${srpm} | cut -f1 -d' ') in /RPMS" ${SRC_CTR} - export IMG=$(buildah commit --omit-timestamp --disable-compression --rm ${SRC_CTR}) - export SRC_CTR=$(buildah from ${IMG}) -done -popd > /dev/null + echo -n "${digest}" +} + +# +# determine image base name (without tag or digest) +# +parse_img_base() { + local ref="${1%@*}" # just the portion before the digest "@" + local base="${ref}" # default base is their reference + local last_word="" # splitting up their reference to get the last word/chunk + last_word="$(echo "${ref}" | tr '/' '\n' | tail -1 )" + if [ "$(_count_char_in_string ':' "${last_word}")" -gt 0 ] ; then + # which means everything before it is the base image name, **including + # transport (which could have a port delineation), and even a URI like network ports. + base="$(echo "${ref}" | rev | cut -d : -f 2 | rev )" + fi + echo -n "${base}" +} + +# +# determine, or guess, the image tag from the provided image reference +# +parse_img_tag() { + local ref="${1%@*}" # just the portion before the digest "@" + local tag="latest" # default tag + + if [ -z "${ref}" ] ; then + echo -n "${tag}" + return 0 + fi + + local last_word="" # splitting up their reference to get the last word/chunk + last_word="$(echo "${ref}" | tr '/' '\n' | tail -1 )" + if [ "$(_count_char_in_string ':' "${last_word}")" -gt 0 ] ; then + # if there are colons in the last segment after '/', then get that tag name + tag="${last_word#*:}" # this parameter expansion removes the prefix pattern before the ':' + fi + echo -n "${tag}" +} + +# +# an inline prefixer for containers/image tools +# +ref_prefix() { + local ref="${1}" + local pfxs + local ret + + # get the supported prefixes of the current version of skopeo + mapfile -t pfxs < <(skopeo copy --help | grep -A1 "Supported transports:" | grep -v "Supported transports" | sed 's/, /\n/g') + ret=$? + if [ ${ret} -ne 0 ] ; then + return ${ret} + fi + + for pfx in "${pfxs[@]}" ; do + if echo "${ref}" | grep -q "^${pfx}:" ; then + # break if we match a known prefix + echo "${ref}" + return 0 + fi + done + # else default + echo "docker://${ref}" +} + +# +# an inline namer for the source image +# Initially this is a tagging convention (which if we try estesp/manifest-tool +# can be directly mapped into a manifest-list/image-index). +# +ref_src_img_tag() { + local ref="${1}" + echo -n "$(parse_img_tag "${ref}")""${source_image_suffix}" +} + +# +# call out to registry for the image reference's digest checksum +# +fetch_img_digest() { + local ref="${1}" + local dgst + local ret + + ## TODO: check for authfile, creds, and whether it's an insecure registry + dgst=$(skopeo inspect "$(ref_prefix "${ref}")" | jq .Digest | tr -d \") + ret=$? + if [ $ret -ne 0 ] ; then + echo "ERROR: check the image reference: ${ref}" >&2 + return $ret + fi + + echo -n "${dgst}" +} + +# +# pull down the image to an OCI layout +# arguments: image ref +# returns: path:tag to the OCI layout +# +# any commands should only output to stderr, so that the caller can receive the +# path reference to the OCI layout. +# +fetch_img() { + local ref="${1}" + local dst="${2}" + local base + local tag + local dgst + local from + local ret + + _mkdir_p "${dst}" + + base="$(parse_img_base "${ref}")" + tag="$(parse_img_tag "${ref}")" + dgst="$(parse_img_digest "${ref}")" + from="" + # skopeo currently only support _either_ tag _or_ digest, so we'll be specific. + if [ -n "${dgst}" ] ; then + from="$(ref_prefix "${base}")@${dgst}" + else + from="$(ref_prefix "${base}"):${tag}" + fi + + ## TODO: check for authfile, creds, and whether it's an insecure registry + ## destination name must have the image tag included (umoci expects it) + skopeo \ + copy \ + "${from}" \ + "oci:${dst}:${tag}" >&2 + ret=$? + if [ ${ret} -ne 0 ] ; then + return ${ret} + fi + echo -n "${dst}:${tag}" +} + +# +# upack_img +# +unpack_img() { + local image_dir="${1}" + local unpack_dir="${2}" + local ret + + if [ -d "${unpack_dir}" ] ; then + _rm_rf "${unpack_dir}" + fi + + if [ -n "$(command -v umoci)" ] ; then + # can be done as non-root (even in a non-root container) + unpack_img_umoci "${image_dir}" "${unpack_dir}" + ret=$? + if [ ${ret} -ne 0 ] ; then + return ${ret} + fi + else + # can be done as non-root (even in a non-root container) + unpack_img_bash "${image_dir}" "${unpack_dir}" + ret=$? + if [ ${ret} -ne 0 ] ; then + return ${ret} + fi + fi +} + +# +# unpack an image layout using only jq and bash +# +unpack_img_bash() { + local image_dir="${1}" + local unpack_dir="${2}" + local mnfst_dgst + local layer_dgsts + local ret + + _debug "unpacking with bash+jq" + + # for compat with umoci (which wants the image tag as well) + if echo "${image_dir}" | grep -q ":" ; then + image_dir="${image_dir%:*}" + fi + + mnfst_dgst="$(jq '.manifests[0].digest' "${image_dir}"/index.json | tr -d \")" + ret=$? + if [ ${ret} -ne 0 ] ; then + return ${ret} + fi + + # Since we're landing the reference as an OCI layout, this mediaType is fairly predictable + # TODO don't always assume +gzip + layer_dgsts="$(jq '.layers[] | select(.mediaType == "application/vnd.oci.image.layer.v1.tar+gzip") | .digest' "${image_dir}"/blobs/"${mnfst_dgst/:/\/}" | tr -d \")" + ret=$? + if [ ${ret} -ne 0 ] ; then + return ${ret} + fi + + _mkdir_p "${unpack_dir}/rootfs" + for dgst in ${layer_dgsts} ; do + path="${image_dir}/blobs/${dgst/:/\/}" + tmp_file=$(_mktemp) + zcat "${path}" | _tar -t > "$tmp_file" + + # look for '.wh.' entries. They must be removed from the rootfs + # _before_ extracting the archive, then the .wh. entries themselves + # need to not remain afterwards + grep '\.wh\.' "${tmp_file}" | while read -r wh_path ; do + # if `some/path/.wh.foo` then `rm -rf `${unpack_dir}/some/path/foo` + # if `some/path/.wh..wh..opq` then `rm -rf `${unpack_dir}/some/path/*` + if [ "$(basename "${wh_path}")" == ".wh..wh..opq" ] ; then + _rm_rf "${unpack_dir}/rootfs/$(dirname "${wh_path}")/*" + elif basename "${wh_path}" | grep -qe '^\.wh\.' ; then + name=$(basename "${wh_path}" | sed -e 's/^\.wh\.//') + _rm_rf "${unpack_dir}/rootfs/$(dirname "${wh_path}")/${name}" + fi + done + + _info "[unpacking] layer ${dgst}" + # unpack layer to rootfs (without whiteouts) + zcat "${path}" | _tar --restrict --no-xattr --no-acls --no-selinux --exclude='*.wh.*' -x -C "${unpack_dir}/rootfs" + ret=$? + if [ ${ret} -ne 0 ] ; then + return ${ret} + fi + + # some of the directories get unpacked as 0555, so removing them gives an EPERM + find "${unpack_dir}" -type d -exec chmod 0755 "{}" \; + done +} + +# +# unpack using umoci +# +unpack_img_umoci() { + local image_dir="${1}" + local unpack_dir="${2}" + + _debug "unpacking with umoci" + # always assume we're not root I reckon + umoci unpack --rootless --image "${image_dir}" "${unpack_dir}" >&2 + ret=$? + return $ret +} + +# +# copy an image from one location to another +# +push_img() { + local src="${1}" + local dst="${2}" + + _debug "pushing image ${src} to ${dst}" + ## TODO: check for authfile, creds, and whether it's an insecure registry + skopeo copy --dest-tls-verify=false "$(ref_prefix "${src}")" "$(ref_prefix "${dst}")" # XXX for demo only + #skopeo copy "$(ref_prefix "${src}")" "$(ref_prefix "${dst}")" + ret=$? + return $ret +} + +# +# sets up a basic new OCI layout, for an image with the provided (or default 'latest') tag +# +layout_new() { + local out_dir="${1}" + local image_tag="${2:-latest}" + local config + local mnfst + local config_sum + local mnfst_sum + + _mkdir_p "${out_dir}/blobs/sha256" + echo '{"imageLayoutVersion":"1.0.0"}' > "${out_dir}/oci-layout" + config=' +{ + "created": "'$(_date_ns)'", + "architecture": "amd64", + "os": "linux", + "config": {}, + "rootfs": { + "type": "layers", + "diff_ids": [] + } +} + ' + config_sum=$(echo "${config}" | jq -c | tr -d '\n' | sha256sum | awk '{ ORS=""; print $1 }') + echo "${config}" | jq -c | tr -d '\n' > "${out_dir}/blobs/sha256/${config_sum}" + + mnfst=' +{ + "schemaVersion": 2, + "config": { + "mediaType": "application/vnd.oci.image.config.v1+json", + "digest": "sha256:'"${config_sum}"'", + "size": '"$(_size "${out_dir}"/blobs/sha256/"${config_sum}")"' + }, + "layers": [] +} + ' + mnfst_sum=$(echo "${mnfst}" | jq -c | tr -d '\n' | sha256sum | awk '{ ORS=""; print $1 }') + echo "${mnfst}" | jq -c | tr -d '\n' > "${out_dir}/blobs/sha256/${mnfst_sum}" + + echo ' +{ + "schemaVersion": 2, + "manifests": [ + { + "mediaType": "application/vnd.oci.image.manifest.v1+json", + "digest": "sha256:'"${mnfst_sum}"'", + "size": '"$(_size "${out_dir}"/blobs/sha256/"${mnfst_sum}")"', + "annotations": { + "org.opencontainers.image.ref.name": "'"${image_tag}"'" + } + } + ] +} + ' | jq -c | tr -d '\n' > "${out_dir}/index.json" +} + +# call this for every artifact, to insert it into an OCI layout +# args: +# * a path to the layout +# * a path to the artifact +# * the path inside the tar +# * json file to slurp in as annotations for this layer's OCI descriptor +# * tag used in the layout (default is 'latest') +# +layout_insert() { + local out_dir="${1}" + local artifact_path="${2}" + local tar_path="${3}" + local annotations_file="${4}" + local image_tag="${5:-latest}" + local mnfst_list + local mnfst_dgst + local mnfst + local tmpdir + local sum + local tmptar + local tmptar_sum + local tmptar_size + local config_sum + local tmpconfig + local tmpconfig_sum + local tmpconfig_size + local tmpmnfst + local tmpmnfst_sum + local tmpmnfst_size + local tmpmnfst_list + + mnfst_list="${out_dir}/index.json" + # get the digest to the manifest + test -f "${mnfst_list}" || return 1 + mnfst_dgst="$(jq --arg tag "${image_tag}" ' + .manifests[] + | select(.annotations."org.opencontainers.image.ref.name" == $tag ) + | .digest + ' "${mnfst_list}" | tr -d \" | tr -d '\n' )" + mnfst="${out_dir}/blobs/${mnfst_dgst/:/\/}" + test -f "${mnfst}" || return 1 + + # make tar of new object + tmpdir="$(_mktemp_d)" + # TODO account for "artifact_path" being a directory? + sum="$(sha256sum "${artifact_path}" | awk '{ print $1 }')" + # making a blob store in the layer + _mkdir_p "${tmpdir}/blobs/sha256" + cp "${artifact_path}" "${tmpdir}/blobs/sha256/${sum}" + if [ "$(basename "${tar_path}")" == "$(basename "${artifact_path}")" ] ; then + _mkdir_p "${tmpdir}/$(dirname "${tar_path}")" + # TODO this symlink need to be relative path, not to `/blobs/...` + ln -s "/blobs/sha256/${sum}" "${tmpdir}/${tar_path}" + else + _mkdir_p "${tmpdir}/${tar_path}" + # TODO this symlink need to be relative path, not to `/blobs/...` + ln -s "/blobs/sha256/${sum}" "${tmpdir}/${tar_path}/$(basename "${artifact_path}")" + fi + tmptar="$(_mktemp)" + + # zero all the things for as consistent blobs as possible + _tar -C "${tmpdir}" --mtime=@0 --owner=0 --group=0 --mode='a+rw' --no-xattrs --no-selinux --no-acls -cf "${tmptar}" . + _rm_rf "${tmpdir}" + + # checksum tar and move to blobs/sha256/$checksum + tmptar_sum="$(sha256sum "${tmptar}" | awk '{ ORS=""; print $1 }')" + tmptar_size="$(_size "${tmptar}")" + mv "${tmptar}" "${out_dir}/blobs/sha256/${tmptar_sum}" + + # find and read the prior config, mapped from the manifest + config_sum="$(jq '.config.digest' "${mnfst}" | tr -d \")" + + # use `jq` to append to prior config + tmpconfig="$(_mktemp)" + jq -c \ + --arg date "$(_date_ns)" \ + --arg tmptar_sum "sha256:${tmptar_sum}" \ + --arg comment "#(nop) BuildSourceImage adding artifact: ${sum}" \ + ' + .created = $date + | .rootfs.diff_ids += [ $tmptar_sum ] + | .history += [ + { + "created": $date, + "created_by": $comment + } + ] + ' "${out_dir}/blobs/${config_sum/:/\/}" > "${tmpconfig}" + _rm_rf "${out_dir}/blobs/${config_sum/:/\/}" + + # rename the config blob to its new checksum + tmpconfig_sum="$(sha256sum "${tmpconfig}" | awk '{ ORS=""; print $1 }')" + tmpconfig_size="$(_size "${tmpconfig}")" + mv "${tmpconfig}" "${out_dir}/blobs/sha256/${tmpconfig_sum}" + + # append layers list in the manifest, and its new config mapping + tmpmnfst="$(_mktemp)" + jq -c \ + --arg tmpconfig_sum "sha256:${tmpconfig_sum}" \ + --arg tmpconfig_size "${tmpconfig_size}" \ + --arg tmptar_sum "sha256:${tmptar_sum}" \ + --arg tmptar_size "${tmptar_size}" \ + --arg artifact "$(basename "${artifact_path}")" \ + --arg sum "sha256:${sum}" \ + --slurpfile annotations_slup "${annotations_file}" \ + ' + .config.digest = $tmpconfig_sum + | .config.size = ($tmpconfig_size|tonumber) + | { + "com.redhat.layer.type": "source", + "com.redhat.layer.content": $artifact, + "com.redhat.layer.content.checksum": $sum + } + $annotations_slup[0] as $annotations_merge + | .layers += [ + { + "mediaType": "application/vnd.oci.image.layer.v1.tar", + "size": ($tmptar_size|tonumber), + "digest": $tmptar_sum, + "annotations": $annotations_merge + } + ] + ' "${mnfst}" > "${tmpmnfst}" + ret=$? + if [ $ret -ne 0 ] ; then + return 1 + fi + _rm_rf "${mnfst}" + + # rename the manifest blob to its new checksum + tmpmnfst_sum="$(sha256sum "${tmpmnfst}" | awk '{ ORS=""; print $1 }')" + tmpmnfst_size="$(_size "${tmpmnfst}")" + mv "${tmpmnfst}" "${out_dir}/blobs/sha256/${tmpmnfst_sum}" + + # map the mnfst_list to the new mnfst checksum + tmpmnfst_list="$(_mktemp)" + jq -c \ + --arg tag "${image_tag}" \ + --arg tmpmnfst_sum "sha256:${tmpmnfst_sum}" \ + --arg tmpmnfst_size "${tmpmnfst_size}" \ + ' + [(.manifests[] | select(.annotations."org.opencontainers.image.ref.name" != $tag) )] as $manifests_reduced + | [ + { + "mediaType": "application/vnd.oci.image.manifest.v1+json", + "digest": $tmpmnfst_sum, + "size": ($tmpmnfst_size|tonumber), + "annotations": { + "com.redhat.image.type": "source", + "org.opencontainers.image.ref.name": $tag + } + } + ] as $manifests_new + | .manifests = $manifests_reduced + $manifests_new + ' "${mnfst_list}" > "${tmpmnfst_list}" + ret=$? + if [ $ret -ne 0 ] ; then + return 1 + fi + mv "${tmpmnfst_list}" "${mnfst_list}" +} + + +# +# Source Collection Drivers +# +# presently just bash functions. *notice* prefix the function name as `sourcedriver_` +# May become a ${ABV_NAME}/drivers.d/ +# +# Arguments: +# * image ref +# * path to inspect +# * output path for source (specifc to this driver) +# * output path for JSON file of source's annotations +# +# The JSON of source annotations is the key to discovering the source artifact +# to be added and including rich metadata about that archive into the final +# image. +# The name of each JSON file is appending '.json' to the artifact's name. So if +# you have `foo-1.0.src.rpm` then there MUST be a corresponding +# `foo-1.0.src.rpm.json`. +# The data structure in this annotation is just a dict/hashmap, with key/val +# according to +# https://github.com/opencontainers/image-spec/blob/master/annotations.md +# + +# +# driver to determine and fetch source rpms, based on the rootfs +# +sourcedriver_rpm_fetch() { + local self="${0#sourcedriver_*}" + local ref="${1}" + local rootfs="${2}" + local out_dir="${3}" + local manifest_dir="${4}" + local release + local rpm + local srcrpm_buildtime + local srcrpm_pkgid + local srcrpm_name + local srcrpm_version + local srcrpm_epoch + local srcrpm_release + local mimetype + + # Get the RELEASEVER from the image + release=$(rpm -q --queryformat "%{VERSION}\n" --root "${rootfs}" -f /etc/os-release) + + # From the rootfs of the works image, build out the src rpms to operate over + for srcrpm in $(rpm -qa --root "${rootfs}" --queryformat '%{SOURCERPM}\n' | grep -v '^gpg-pubkey' | sort -u) ; do + if [ "${srcrpm}" == "(none)" ] ; then + continue + fi + + rpm=${srcrpm%*.src.rpm} + if [ ! -f "${out_dir}/${srcrpm}" ] ; then + _debug "--> fetching ${srcrpm}" + dnf download \ + --quiet \ + --installroot "${rootfs}" \ + --release "${release}" \ + --destdir "${out_dir}" \ + --source \ + "${rpm}" + ret=$? + if [ $ret -ne 0 ] ; then + _warn "failed to fetch ${srcrpm}" + continue + fi + else + _debug "--> using cached ${srcrpm}" + fi + + # TODO one day, check and confirm with %{sourcepkgid} + # https://bugzilla.redhat.com/show_bug.cgi?id=1741715 + #rpm_sourcepkgid=$(rpm -q --root ${rootfs} --queryformat '%{sourcepkgid}' "${rpm}") + srcrpm_buildtime=$(rpm -qp --qf '%{buildtime}' "${out_dir}"/"${srcrpm}" ) + srcrpm_pkgid=$(rpm -qp --qf '%{pkgid}' "${out_dir}"/"${srcrpm}" ) + srcrpm_name=$(rpm -qp --qf '%{name}' "${out_dir}"/"${srcrpm}" ) + srcrpm_version=$(rpm -qp --qf '%{version}' "${out_dir}"/"${srcrpm}" ) + srcrpm_epoch=$(rpm -qp --qf '%{epoch}' "${out_dir}"/"${srcrpm}" ) + srcrpm_release=$(rpm -qp --qf '%{release}' "${out_dir}"/"${srcrpm}" ) + mimetype="$(file --brief --mime-type "${out_dir}"/"${srcrpm}")" + jq \ + -n \ + --arg filename "${srcrpm}" \ + --arg name "${srcrpm_name}" \ + --arg version "${srcrpm_version}" \ + --arg epoch "${srcrpm_epoch}" \ + --arg release "${srcrpm_release}" \ + --arg buildtime "${srcrpm_buildtime}" \ + --arg mimetype "${mimetype}" \ + ' + { + "source.artifact.filename": $filename, + "source.artifact.name": $name, + "source.artifact.version": $version, + "source.artifact.epoch": $epoch, + "source.artifact.release": $release, + "source.artifact.mimetype": $mimetype, + "source.artifact.buildtime": $buildtime + } + ' \ + > "${manifest_dir}/${srcrpm}.json" + ret=$? + if [ $ret -ne 0 ] ; then + return 1 + fi + done +} + +# +# driver to only package rpms from a provided rpm directory +# (koji use-case) +# +sourcedriver_rpm_dir() { + local self="${0#sourcedriver_*}" + local ref="${1}" + local rootfs="${2}" + local out_dir="${3}" + local manifest_dir="${4}" + local srcrpm_buildtime + local srcrpm_pkgid + local srcrpm_name + local srcrpm_version + local srcrpm_epoch + local srcrpm_release + local mimetype + + if [ -n "${SRPM_DIR}" ]; then + _debug "[$self] writing to $out_dir and $manifest_dir" + find "${SRPM_DIR}" -type f -name '*src.rpm' | while read -r srcrpm ; do + cp "${srcrpm}" "${out_dir}" + srcrpm="$(basename "${srcrpm}")" + _debug "[$self] --> ${srcrpm}" + srcrpm_buildtime=$(rpm -qp --qf '%{buildtime}' "${out_dir}"/"${srcrpm}" ) + srcrpm_pkgid=$(rpm -qp --qf '%{pkgid}' "${out_dir}"/"${srcrpm}" ) + srcrpm_name=$(rpm -qp --qf '%{name}' "${out_dir}"/"${srcrpm}" ) + srcrpm_version=$(rpm -qp --qf '%{version}' "${out_dir}"/"${srcrpm}" ) + srcrpm_epoch=$(rpm -qp --qf '%{epoch}' "${out_dir}"/"${srcrpm}" ) + srcrpm_release=$(rpm -qp --qf '%{release}' "${out_dir}"/"${srcrpm}" ) + mimetype="$(file --brief --mime-type "${out_dir}"/"${srcrpm}")" + jq \ + -n \ + --arg filename "${srcrpm}" \ + --arg name "${srcrpm_name}" \ + --arg version "${srcrpm_version}" \ + --arg epoch "${srcrpm_epoch}" \ + --arg release "${srcrpm_release}" \ + --arg buildtime "${srcrpm_buildtime}" \ + --arg mimetype "${mimetype}" \ + --arg pkgid "${srcrpm_pkgid}" \ + ' + { + "source.artifact.filename": $filename, + "source.artifact.name": $name, + "source.artifact.version": $version, + "source.artifact.epoch": $version, + "source.artifact.release": $release, + "source.artifact.mimetype": $mimetype, + "source.artifact.pkgid": $pkgid, + "source.artifact.buildtime": $buildtime + } + ' \ + > "${manifest_dir}/${srcrpm}.json" + ret=$? + if [ $ret -ne 0 ] ; then + return 1 + fi + done + fi +} + # # If the caller specified a context directory, -# add it to the CONTEXT DIR in SRC IMAGE # -if [ ! -z "${CONTEXT_DIR}" ]; then - CONTEXT_DIR=$(cd ${CONTEXT_DIR}; pwd) - buildah add ${SRC_CTR} ${CONTEXT_DIR} /CONTEXT - buildah config --created-by "/bin/sh -c #(nop) ADD file:$(cd ${CONTEXT_DIR}; tar cf - . | sha256sum -| cut -f1 -d' ') in /CONTEXT" ${SRC_CTR} - export IMG=$(buildah commit --omit-timestamp --rm ${SRC_CTR}) - export SRC_CTR=$(buildah from ${IMG}) -fi +# slightly special driver, as it has a flag/env passed in, that it uses +# +sourcedriver_context_dir() { + local self="${0#sourcedriver_*}" + local ref="${1}" + local rootfs="${2}" + local out_dir="${3}" + local manifest_dir="${4}" + local tarname + local mimetype + local source_info + + if [ -n "${CONTEXT_DIR}" ]; then + _debug "$self: writing to $out_dir and $manifest_dir" + tarname="context.tar" + _tar -C "${CONTEXT_DIR}" \ + --mtime=@0 --owner=0 --group=0 --mode='a+rw' --no-xattrs --no-selinux --no-acls \ + -cf "${out_dir}/${tarname}" . + mimetype="$(file --brief --mime-type "${out_dir}"/"${tarname}")" + source_info="${manifest_dir}/${tarname}.json" + jq \ + -n \ + --arg name "${tarname}" \ + --arg mimetype "${mimetype}" \ + ' + { + "source.artifact.name": $name, + "source.artifact.mimetype": $mimetype + } + ' \ + > "${source_info}" + ret=$? + if [ $ret -ne 0 ] ; then + return 1 + fi + fi +} # -# If the caller specified a extra directory, -# add it to the CONTEXT DIR in SRC IMAGE +# If the caller specified a extra directory # -if [ ! -z "${EXTRA_SRC_DIR}" ]; then - buildah add ${SRC_CTR} ${EXTRA_SRC_DIR} /EXTRA - buildah config --created-by "/bin/sh -c #(nop) ADD file:$(cd ${EXTRA_SRC_DIR}; tar cf - . | sha256sum -| cut -f1 -d' ') in /CONTEXT" ${SRC_CTR} - export IMG=$(buildah commit --omit-timestamp --rm ${EXTRA_SRC_CTR}) - export SRC_CTR=$(buildah from ${IMG}) -fi - -# Cleanup and remove source container -buildah rm ${SRC_CTR} - +# slightly special driver, as it has a flag/env passed in, that it uses # -# Add the final name to our image -# -buildah tag $IMG $SRC_IMAGE +sourcedriver_extra_src_dir() { + local self="${0#sourcedriver_*}" + local ref="${1}" + local rootfs="${2}" + local out_dir="${3}" + local manifest_dir="${4}" + local tarname + local mimetype + local source_info -# Push SRC_IMAGE to Registry -# buildah push $SRC_IMAGE REGISTRY_NAME/$SRC_IMAGE + if [ -n "${EXTRA_SRC_DIR}" ]; then + _debug "$self: writing to $out_dir and $manifest_dir" + tarname="extra-src.tar" + _tar -C "${EXTRA_SRC_DIR}" \ + --mtime=@0 --owner=0 --group=0 --mode='a+rw' --no-xattrs --no-selinux --no-acls \ + -cf "${out_dir}/${tarname}" . + mimetype="$(file --brief --mime-type "${out_dir}"/"${tarname}")" + source_info="${manifest_dir}/${tarname}.json" + jq \ + -n \ + --arg name "${tarname}" \ + --arg mimetype "${mimetype}" \ + ' + { + "source.artifact.name": $name, + "source.artifact.mimetype": $mimetype + } + ' \ + > "${source_info}" + ret=$? + if [ $ret -ne 0 ] ; then + return 1 + fi + fi +} + + +main() { + local base_dir + local input_context_dir + local input_extra_src_dir + local input_inspect_image_ref + local input_srpm_dir + local drivers + local image_ref + local img_layout + local list_drivers + local output_dir + local push_image_ref + local ret + local rootfs + local src_dir + local src_img_dir + local src_img_tag + local src_name + local unpack_dir + local work_dir + + _init "${@}" + + base_dir="$(pwd)/${ABV_NAME}" + # using the bash builtin to parse + while getopts ":hlvDi:c:s:e:o:b:d:p:" opts; do + case "${opts}" in + b) + base_dir="${OPTARG}" + ;; + c) + input_context_dir=${OPTARG} + ;; + e) + input_extra_src_dir=${OPTARG} + ;; + d) + drivers=${OPTARG} + ;; + h) + _usage + ;; + i) + input_inspect_image_ref=${OPTARG} + ;; + l) + list_drivers=1 + ;; + o) + output_dir=${OPTARG} + ;; + p) + push_image_ref=${OPTARG} + ;; + s) + input_srpm_dir=${OPTARG} + ;; + v) + _version + exit 0 + ;; + D) + export DEBUG=1 + ;; + *) + _usage + ;; + esac + done + shift $((OPTIND-1)) + + if [ -n "${list_drivers}" ] ; then + set | grep '^sourcedriver_.* () ' | tr -d ' ()' + exit 0 + fi + + # "local" variables are not set in `env`, but are seen in `set` + if [ "$(set | grep -c '^input_')" -eq 0 ] ; then + _error "provide an input (example: $(basename "${0}") -i docker.io/centos -e ./my-sources/ )" + fi + + # These three variables are slightly special, in that they're globals that + # specific drivers will expect. + export CONTEXT_DIR="${CONTEXT_DIR:-$input_context_dir}" + export EXTRA_SRC_DIR="${EXTRA_SRC_DIR:-$input_extra_src_dir}" + export SRPM_DIR="${SRPM_DIR:-$input_srpm_dir}" + + output_dir="${OUTPUT_DIR:-$output_dir}" + + export TMPDIR="${base_dir}/tmp" + if [ -d "${TMPDIR}" ] ; then + _rm_rf "${TMPDIR}" + fi + _mkdir_p "${TMPDIR}" + + # setup rootfs to be inspected (if any) + rootfs="" + image_ref="" + src_dir="" + work_dir="${base_dir}/work" + if [ -n "${input_inspect_image_ref}" ] ; then + _debug "Image Reference provided: ${input_inspect_image_ref}" + _debug "Image Reference base: $(parse_img_base "${input_inspect_image_ref}")" + _debug "Image Reference tag: $(parse_img_tag "${input_inspect_image_ref}")" + + inspect_image_digest="$(parse_img_digest "${input_inspect_image_ref}")" + # determine missing digest before fetch, so that we fetch the precise image + # including its digest. + if [ -z "${inspect_image_digest}" ] ; then + inspect_image_digest="$(fetch_img_digest "$(parse_img_base "${input_inspect_image_ref}"):$(parse_img_tag "${input_inspect_image_ref}")")" + fi + _debug "inspect_image_digest: ${inspect_image_digest}" + + img_layout="" + # if inspect and fetch image, then to an OCI layout dir + if [ ! -d "${work_dir}/layouts/${inspect_image_digest/:/\/}" ] ; then + # we'll store the image to a path based on its digest, that it can be reused + img_layout="$(fetch_img "$(parse_img_base "${input_inspect_image_ref}")":"$(parse_img_tag "${input_inspect_image_ref}")"@"${inspect_image_digest}" "${work_dir}"/layouts/"${inspect_image_digest/:/\/}" )" + ret=$? + if [ ${ret} -ne 0 ] ; then + _error "failed to copy image: $(parse_img_base "${input_inspect_image_ref}"):$(parse_img_tag "${input_inspect_image_ref}")@${inspect_image_digest}" + fi + else + img_layout="${work_dir}/layouts/${inspect_image_digest/:/\/}:$(parse_img_tag "${input_inspect_image_ref}")" + fi + _debug "image layout: ${img_layout}" + + # unpack or reuse fetched image + unpack_dir="${work_dir}/unpacked/${inspect_image_digest/:/\/}" + if [ -d "${unpack_dir}" ] ; then + _rm_rf "${unpack_dir}" + fi + unpack_img "${img_layout}" "${unpack_dir}" + ret=$? + if [ ${ret} -ne 0 ] ; then + return ${ret} + fi + + rootfs="${unpack_dir}/rootfs" + image_ref="$(parse_img_base "${input_inspect_image_ref}"):$(parse_img_tag "${input_inspect_image_ref}")@${inspect_image_digest}" + src_dir="${base_dir}/src/${inspect_image_digest/:/\/}" + work_dir="${base_dir}/work/${inspect_image_digest/:/\/}" + _info "inspecting image reference ${image_ref}" + else + # if we're not fething an image, then this is basically a nop + rootfs="$(_mktemp_d)" + image_ref="scratch" + src_dir="$(_mktemp_d)" + work_dir="$(_mktemp_d)" + fi + _debug "image layout: ${img_layout}" + + # setup rootfs, from that OCI layout + local unpack_dir="${work_dir}/unpacked/${IMAGE_DIGEST/:/\/}" + if [ ! -d "${unpack_dir}" ] ; then + unpack_img "${img_layout}" "${unpack_dir}" + fi + _debug "unpacked dir: ${unpack_dir}" + _debug "rootfs dir: ${rootfs}" + + # clear prior driver's info about source to insert into Source Image + _rm_rf "${work_dir}/driver" + + if [ -n "${drivers}" ] ; then + # clean up the args passed by the caller ... + drivers="$(echo "${drivers}" | tr ',' ' '| tr '\n' ' ')" + else + drivers="$(set | grep '^sourcedriver_.* () ' | tr -d ' ()' | tr '\n' ' ')" + fi + + # Prep the OCI layout for the source image + src_img_dir="$(_mktemp_d)" + src_img_tag="latest-source" # XXX this tag needs to be a reference to the image built from + layout_new "${src_img_dir}" "${src_img_tag}" + + # iterate on the drivers + #for driver in sourcedriver_rpm_fetch ; do + for driver in ${drivers} ; do + _info "calling $driver" + _mkdir_p "${src_dir}/${driver#sourcedriver_*}" + _mkdir_p "${work_dir}/driver/${driver#sourcedriver_*}" + $driver \ + "${image_ref}" \ + "${rootfs}" \ + "${src_dir}/${driver#sourcedriver_*}" \ + "${work_dir}/driver/${driver#sourcedriver_*}" + ret=$? + if [ $ret -ne 0 ] ; then + _error "$driver failed" + fi + + # walk the driver output to determine layers to be added + find "${work_dir}/driver/${driver#sourcedriver_*}" -type f -name '*.json' | while read -r src_json ; do + src_name=$(basename "${src_json}" .json) + layout_insert \ + "${src_img_dir}" \ + "${src_dir}/${driver#sourcedriver_*}/${src_name}" \ + "/${driver#sourcedriver_*}/${src_name}" \ + "${src_json}" \ + "${src_img_tag}" + ret=$? + if [ $ret -ne 0 ] ; then + # TODO probably just _error here to exit + _warn "failed to insert layout layer for ${src_name}" + fi + done + done + + _info "packed 'oci:$src_img_dir:${src_img_tag}'" + + # TODO maybe look to a directory like /usr/libexec/BuildSourceImage/drivers/ for drop-ins to run + + _info "succesfully packed 'oci:${src_img_dir}:${src_img_tag}'" + _debug "$(skopeo inspect oci:"${src_img_dir}":"${src_img_tag}")" + + ## if an output directory is provided then save a copy to it + if [ -n "${output_dir}" ] ; then + _mkdir_p "${output_dir}" + # XXX this $input_inspect_image_ref currently relies on the user passing in the `-i` flag + push_img "oci:$src_img_dir:${src_img_tag}" "oci:$output_dir:$(ref_src_img_tag "$(parse_img_tag "${input_inspect_image_ref}")")" + fi + + if [ -n "${push_image_ref}" ] ; then + # XXX may have to parse this reference to ensure it is valid, and that it has a `-source` tag + push_img "oci:$src_img_dir:${src_img_tag}" "${push_image_ref}" + fi + +} + +# only exec main if this is being called (this way we can source and test the functions) +_is_sourced || main "${@}" + +# vim:set shiftwidth=4 softtabstop=4 expandtab: diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..94cbed7 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,14 @@ +FROM fedora + +RUN dnf install -y jq skopeo findutils file 'dnf-command(download)' + +COPY ./BuildSourceImage.sh /usr/local/bin/BuildSourceImage.sh + +RUN mkdir -p /output +ENV OUTPUT_DIR=/output +VOLUME /output + +ENV SRC_DIR=/src +VOLUME /src + +ENTRYPOINT ["/usr/local/bin/BuildSourceImage.sh"]