From f5db5f704f779824c1f27db27545d281ea412a40 Mon Sep 17 00:00:00 2001 From: Vincent Batts Date: Wed, 14 Aug 2019 11:15:46 -0400 Subject: [PATCH 01/17] Dockerfile and export image if OUTPUT_DIR Signed-off-by: Vincent Batts --- BuildSourceImage.sh | 9 +++++++++ Dockerfile | 9 +++++++++ 2 files changed, 18 insertions(+) create mode 100644 Dockerfile diff --git a/BuildSourceImage.sh b/BuildSourceImage.sh index 4fcb55b..480e0f5 100755 --- a/BuildSourceImage.sh +++ b/BuildSourceImage.sh @@ -78,5 +78,14 @@ buildah rm ${SRC_CTR} # buildah tag $IMG $SRC_IMAGE +## if an output directory is provided then save a copy to it +## XXX(vbatts) this is not working inside a container like `quay.io/buildah/stable` yet +if [ -n "${OUTPUT_DIR}" ] ; then + mkdir -p "${OUTPUT_DIR}" + skopeo copy containers-storage:"${IMG}" oci:"${OUTPUT_DIR}"/ +fi + # Push SRC_IMAGE to Registry # buildah push $SRC_IMAGE REGISTRY_NAME/$SRC_IMAGE + +# vim:set shiftwidth=4 softtabstop=4 expandtab: diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..a851ab8 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,9 @@ +FROM quay.io/buildah/stable +RUN dnf install -y skopeo && \ + dnf clean all && \ + mkdir -p /output +COPY . /usr/local/bin/ +VOLUME /var/lib/container +VOLUME /output +ENV OUTPUT_DIR=/output +ENTRYPOINT ["/usr/local/bin/BuildSourceImage.sh"] From 56fd7c4de9cfa3ca2bdfbfb70e0a12170a4a18b8 Mon Sep 17 00:00:00 2001 From: Vincent Batts Date: Mon, 19 Aug 2019 17:35:25 -0400 Subject: [PATCH 02/17] *: big rewrite _(this is a WIP)_ for the requirements to not be so bound to RPM only, breaking collection of sources out into "plugins". Currently not much more sophisticated than just iterating through bash functions that expect 3 argugments. rootfs of the inspectee; output path for the sources collected; manifest path. The idea for the manifest is still loose, but how best to follow up the plugin which collected source, to then have the rich metadata available to attach to the individual "layers" that are produced for each source component. for the requirement of most contrained non-root container running this script as non-root, we'll switch away from buildah for now. This ought to just be skopeo copy, unpacking, inspecting, json, and skopeo copy. No mounting, no namespacing, etc. for the sake of writing unit tests (i.e. `bats`), break the script into a way that it can be sourced and run the functions individually. Only act like the whole script when run directly. If `umoci` is available it will get used for some functions (like unpacking), but it is not required and will be attempted with jq and bash otherwise. Signed-off-by: Vincent Batts --- BuildSourceImage.sh | 825 ++++++++++++++++++++++++++++++++++++++++---- Dockerfile | 26 +- 2 files changed, 782 insertions(+), 69 deletions(-) diff --git a/BuildSourceImage.sh b/BuildSourceImage.sh index 480e0f5..05ed203 100755 --- a/BuildSourceImage.sh +++ b/BuildSourceImage.sh @@ -1,43 +1,778 @@ -#!/bin/sh - -set -e - +#!/bin/bash # This script requires an OCI IMAGE Name to pull. # The script generates a SOURCE Image based on the OCI Image # Script must be executed on the same OS or newer as the image. -if test $# -lt 2 ; then - echo Usage: $(basename $0) IMAGE CONTEXT_DIR [EXTRA_SRC_DIR] + +export ABV_NAME="SrcImg" +# TODO maybe a flag for this? +export source_image_suffix="-source" + + +_usage() { + echo "Usage: $(basename $0) [-b ] [-c ] [-e ] [-o ] [-p] IMAGE" + echo "" + echo -e " -b \tbase path for source image builds" + echo -e " -c \tbuild context for the container image. Can be provided via CONTEXT_DIR env variable" + echo -e " -e \textra src for the container image. Can be provided via EXTRA_SRC_DIR env variable" + echo -e " -o \toutput the OCI image to path. Can be provided via OUTPUT_DIR env variable" + echo -e " -d \toutput the OCI image to path. Can be provided via OUTPUT_DIR env variable" + echo -e " -l\t\tlist the source drivers available" + echo -e " -p\t\tpush source image after build" + echo -e " -D\t\tdebuging output. Can be set via DEBUG env variable" exit 1 +} + +# +# sanity checks on startup +# +_init() { + if [ $# -lt 1 ] ; then + _usage + fi + + set -o pipefail + + # check for tools we depend on + for cmd in jq skopeo dnf file find ; do + if [ -z "$(command -v ${cmd})" ] ; then + # TODO: maybe this could be individual checks so it can report + # where to find the tools + echo "ERROR: please install '${cmd}' package" + fi + done +} + +_is_sourced() { + # https://unix.stackexchange.com/a/215279 + # thanks @tianon + [ "${FUNCNAME[${#FUNCNAME[@]} - 1]}" == 'source' ] +} + +# count $character $string +_count() { + #expr $(echo "${2}" | tr "${1}" '\n' | wc -l) - 1 + c="${2//[^${1}]}" + echo -n ${#c} +} + +# size of file in bytes +_size() { + local file="${1}" + stat -c "%s" "${file}" | tr -d '\n' +} + +# date timestamp in RFC 3339, to the nanosecond +_date_ns() { + date --rfc-3339=ns | tr -d '\n' +} + +# local `mktemp -d` +_mktemp_d() { + mktemp -d "${TMPDIR:-/tmp}/${ABV_NAME}.XXXXXX" +} + +# local `mktemp` +_mktemp() { + mktemp "${TMPDIR:-/tmp}/${ABV_NAME}.XXXXXX" +} + +# local rm -rf +_rm_rf() { + debug "rm -rf $@" + #rm -rf $@ +} + +# local tar +_tar() { + if [ -n "${DEBUG}" ] ; then + tar -v $@ + else + tar $@ + fi +} + +# +# output things, only when $DEBUG is set +# +debug() { + if [ -n "${DEBUG}" ] ; then + echo "[${ABV_NAME}][DEBUG] ${@}" + fi +} + +# +# general echo but with prefix +# +info() { + echo "[${ABV_NAME}][INFO] ${@}" +} + +# +# parse the OCI image reference, accounting for: +# * transport name +# * presence or lack of transport port number +# * presence or lack of digest +# * presence or lack of image tag +# + +# +# return the image reference's digest, if any +# +parse_img_digest() { + local ref="${1}" + local digest="" + if [ "$(_count '@' ${ref})" -gt 0 ] ; then + digest="${ref##*@}" # the digest after the "@" + fi + echo -n "${digest}" +} + +# +# determine image base name (without tag or digest) +# +parse_img_base() { + local ref="${1%@*}" # just the portion before the digest "@" + local base="${ref}" default to the same + if [ "$(_count ':' $(echo ${ref} | tr '/' '\n' | tail -1 ))" -gt 0 ] ; then + # which means everything before it is the base image name, **including + # transport (which could have a port delineation), and even a URI + base="$(echo ${ref} | rev | cut -d : -f 2 | rev )" + fi + echo -n "${base}" +} + +# +# determine, or guess, the image tag from the provided image reference +# +parse_img_tag() { + local ref="${1%@*}" # just the portion before the digest "@" + local tag="latest" # default tag + if [ "$(_count ':' $(echo ${ref} | tr '/' '\n' | tail -1 ))" -gt 0 ] ; then + # if there are colons in the last segment after '/', then get that tag name + tag="$(echo ${ref} | tr '/' '\n' | tail -1 | cut -d : -f 2 )" + fi + echo -n "${tag}" +} + +# +# an inline prefixer for containers/image tools +# +ref_prefix() { + local ref="${1}" + + # get the supported prefixes of the current version of skopeo + IFS=", " + local pfxs=( $(skopeo copy --help | grep -A1 "Supported transports:" | grep -v "Supported transports") ) + unset IFS + + for pfx in ${pfxs[@]} ; do + if echo ${ref} | grep -q "^${pfx}:" ; then + # break when we match + echo ${ref} + return 0 + fi + done + # else default + echo "docker://${ref}" +} + +# +# an inline namer for the source image +# Initially this is a tagging convention (which if we try estesp/manifest-tool +# can be directly mapped into a manifest-list/image-index). +# +ref_src_img_tag() { + local ref="${1}" + echo -n "$(parse_img_tag ${ref})${source_image_suffix}" +} + +# +# call out to registry for the image reference's digest checksum +# +fetch_img_digest() { + local ref="${1}" + ## TODO: check for authfile, creds, and whether it's an insecure registry + local dgst=$(skopeo inspect "$(ref_prefix ${ref})" | jq .Digest | tr -d \") + local ret=$? + if [ $ret -ne 0 ] ; then + echo "ERROR: check the image reference: ${ref}" >&2 + return $ret + fi + + echo -n "${dgst}" +} + +# +# pull down the image to an OCI layout +# arguments: image ref +# returns: path:tag to the OCI layout +# +# any commands should only output to stderr, so that the caller can receive the +# path reference to the OCI layout. +# +fetch_img() { + local ref="${1}" + local dst="${2}" + + mkdir -p "${dst}" + + local base="$(parse_img_base ${ref})" + local tag="$(parse_img_tag ${ref})" + local dgst="$(parse_img_digest ${ref})" + local from="" + # skopeo currently only support _either_ tag _or_ digest, so we'll be specific. + if [ -n "${dgst}" ] ; then + from="$(ref_prefix ${base})@${dgst}" + else + from="$(ref_prefix ${base}):${tag}" + fi + + ## TODO: check for authfile, creds, and whether it's an insecure registry + ## destination name must have the image tag included (umoci expects it) + skopeo \ + copy \ + "${from}" \ + "oci:${dst}:${tag}" >&2 + echo -n "${dst}:${tag}" +} + +# +# upack_img +# +unpack_img() { + local image_dir="${1}" + local unpack_dir="${2}" + + if [ -d "${unpack_dir}" ] ; then + _rm_rf "${unpack_dir}" + fi + + # TODO perhaps if uid == 0 and podman is present then we can try it? + if [ -z "$(command -v umoci)" ] ; then + # can be done as non-root (even in a non-root container) + unpack_img_umoci "${image_dir}" "${unpack_dir}" + else + # can be done as non-root (even in a non-root container) + unpack_img_bash "${image_dir}" "${unpack_dir}" + fi +} + +# +# unpack an image layout using only jq and bash +# +unpack_img_bash() { + local image_dir="${1}" + local unpack_dir="${2}" + + local mnfst_dgst="$(cat "${image_dir}"/index.json | jq '.manifests[0].digest' | tr -d \" )" + + # Since we're landing the reference as an OCI layout, this mediaType is fairly predictable + # TODO don't always assume +gzip + layer_dgsts="$(cat ${image_dir}/blobs/${mnfst_dgst/:/\/} | \ + jq '.layers[] | select(.mediaType == "application/vnd.oci.image.layer.v1.tar+gzip") | .digest' | tr -d \")" + + mkdir -vp "${unpack_dir}" + for dgst in ${layer_dgsts} ; do + path="${image_dir}/blobs/${dgst/:/\/}" + tmp_file=$(_mktemp_d) + zcat "${path}" | _tar -t > $tmp_file # TODO cleanup these files + + # look for '.wh.' entries. They must be removed from the rootfs + # _before_ extracting the archive, then the .wh. entries themselves + # need to not remain afterwards + grep '\.wh\.' "${tmp_file}" | while read line ; do + # if `some/path/.wh.foo` then `rm -rf `${unpack_dir}/some/path/foo` + # if `some/path/.wh..wh..opq` then `rm -rf `${unpack_dir}/some/path/*` + if [ "$(basename ${line})" == ".wh..wh..opq" ] ; then + _rm_rf "${unpack_dir}/$(dirname ${line})/*" + elif basename "${line}" | grep -qe '^\.wh\.' ; then + name=$(basename "${line}" | sed -e 's/^\.wh\.//') + _rm_rf "${unpack_dir}/$(dirname ${line})/${name}" + fi + done + + info "[unpacking] layer ${dgst}" + # unpack layer to rootfs (without whiteouts) + zcat "${path}" | _tar --restrict --no-xattr --no-acls --no-selinux --exclude='*.wh.*' -x -C "${unpack_dir}" + + # some of the directories get unpacked as 0555, so removing them gives an EPERM + find "${unpack_dir}" -type d -exec chmod 0755 "{}" \; + done +} + +# +# unpack using umoci +# +unpack_img_umoci() { + local image_dir="${1}" + local unpack_dir="${2}" + + debug "unpackging with umoci" + # always assume we're not root I reckon + umoci unpack --rootless --image "${image_dir}" "${unpack_dir}" >&2 +} + +# TODO this is not worked out yet +push_img() { + local ref="${1}" + local path="${2}" + + ## TODO: check for authfile, creds, and whether it's an insecure registry + skopeo copy "oci:${path}:$(ref_src_img_tag ${ref})" "$(ref_prefix ${ref})" +} + +# +# sets up a basic new OCI layout, for an image with the provided (or default 'latest') tag +# +layout_new() { + local out_dir="${1}" + local image_tag="${2:-latest}" + + mkdir -p "${out_dir}/blobs/sha256" + echo '{"imageLayoutVersion":"1.0.0"}' > "${out_dir}/oci-layout" + local config=' +{ + "created": "'$(_date_ns)'", + "architecture": "amd64", + "os": "linux", + "config": {}, + "rootfs": { + "type": "layers", + "diff_ids": [] + } +} + ' + local config_sum=$(echo "${config}" | jq -c | tr -d '\n' | sha256sum | awk '{ print $1 }' | tr -d '\n') + echo "${config}" | jq -c | tr -d '\n' > "${out_dir}/blobs/sha256/${config_sum}" + + local mnfst=' +{ + "schemaVersion": 2, + "config": { + "mediaType": "application/vnd.oci.image.config.v1+json", + "digest": "sha256:'"${config_sum}"'", + "size": '"$(_size ${out_dir}/blobs/sha256/${config_sum})"' + }, + "layers": [] +} + ' + local mnfst_sum=$(echo "${mnfst}" | jq -c | tr -d '\n' | sha256sum | awk '{ print $1 }' | tr -d '\n') + echo "${mnfst}" | jq -c | tr -d '\n' > "${out_dir}/blobs/sha256/${mnfst_sum}" + + echo ' +{ + "schemaVersion": 2, + "manifests": [ + { + "mediaType": "application/vnd.oci.image.manifest.v1+json", + "digest": "sha256:'"${mnfst_sum}"'", + "size": '"$(_size ${out_dir}/blobs/sha256/${mnfst_sum})"', + "annotations": { + "org.opencontainers.image.ref.name": "'"${image_tag}"'" + } + } + ] +} + ' | jq -c | tr -d '\n' > "${out_dir}/index.json" +} + +# TODO this is not finished yet +# call this for every artifact, to insert it into an OCI layout +# args: +# * a path to the layout +# * a path to the artifact +# * the path inside the tar +# * tag used in the layout (default is 'latest') +# +layout_insert() { + local out_dir="${1}" + local artifact_path="${2}" + local tar_path="${3}" + local image_tag="${4:-latest}" + + local mnfst_list="${out_dir}/index.json" + # get the digest to the manifest + test -f "${mnfst_list}" || return 1 + local mnfst_dgst="$(cat ${mnfst_list} | jq ' + .manifests[] + | select(.annotations."org.opencontainers.image.ref.name" == "'${image_tag}'") + | .digest + ' | tr -d \" | tr -d '\n' )" + local mnfst="${out_dir}/blobs/${mnfst_dgst/:/\/}" + test -f "${mnfst}" || return 1 + + # make tar of new object + local tmpdir="$(_mktemp_d)" + # TODO account for "artifact_path" being a directory? + local sum="$(sha256sum ${artifact_path} | awk '{ print $1 }')" + # making a blob store in the layer + mkdir -p "${tmpdir}/blobs/sha256" + cp "${artifact_path}" "${tmpdir}/blobs/sha256/${sum}" + if [ "$(basename ${tar_path})" == "$(basename ${artifact_path})" ] ; then + mkdir -p "${tmpdir}/$(dirname ${tar_path})" + # TODO this symlink need to be relative path, not to `/blobs/...` + ln -s "/blobs/sha256/${sum}" "${tmpdir}/${tar_path}" + else + mkdir -p "${tmpdir}/${tar_path}" + # TODO this symlink need to be relative path, not to `/blobs/...` + ln -s "/blobs/sha256/${sum}" "${tmpdir}/${tar_path}/$(basename ${artifact_path})" + fi + local tmptar="$(_mktemp)" + + # zero all the things for as consistent blobs as possible + _tar -C "${tmpdir}" --mtime=@0 --owner=0 --group=0 --mode='a+rw' --no-xattrs --no-selinux --no-acls -cf "${tmptar}" . + _rm_rf "${tmpdir}" + + # checksum tar and move to blobs/sha256/$checksum + local tmptar_sum="$(sha256sum ${tmptar} | awk '{ print $1 }')" + local tmptar_size="$(_size ${tmptar})" + mv "${tmptar}" "${out_dir}/blobs/sha256/${tmptar_sum}" + + # find and read the prior config, mapped from the manifest + local config_sum="$(jq '.config.digest' "${mnfst}" | tr -d \")" + + # use `jq` to append to prior config + local tmpconfig="$(_mktemp)" + cat "${out_dir}/blobs/${config_sum/:/\/}" | jq -c \ + --arg date "$(_date_ns)" \ + --arg tmptar_sum "${tmptar_sum}" \ + --arg sum "${sum}" \ + ' + .created = "$date" + | .rootfs.diff_ids = .rootfs.diff_ids + [ + "sha256:$tmptar_sum" + ] + | .history = .history + [ + { + "created": "$date", + "created_by": "#(nop) BuildSourceImage adding artifact: $sum" + } + ] + ' > "${tmpconfig}" + _rm_rf "${out_dir}/blobs/${config_sum/:/\/}" + + # rename the config blob to its new checksum + local tmpconfig_sum="$(sha256sum ${tmpconfig} | awk '{ print $1 }')" + local tmpconfig_size="$(_size ${tmpconfig})" + mv "${tmpconfig}" "${out_dir}/blobs/sha256/${tmpconfig_sum}" + + # append layers list in the manifest, and its new config mapping + local tmpmnfst="$(_mktemp)" + cat "${mnfst}" | jq -c \ + --arg tmpconfig_sum "${tmpconfig_sum}" \ + --arg tmpconfig_size "${tmpconfig_size}" \ + --arg tmptar_sum "${tmptar_sum}" \ + --arg tmptar_size "${tmptar_size}" \ + --arg artifact "$(basename ${artifact_path})" \ + --arg sum "${sum}" \ + ' + .config.digest = "sha256:$tmpconfig_sum" + | .config.size = $tmpconfig_size + | .layers = .layers + [ + { + "mediaType": "application/vnd.oci.image.layer.v1.tar", + "size": $tmptar_size, + "digest": "sha256:$tmptar_sum", + "annotations": { + "com.redhat.layer.type": "source", + "com.redhat.layer.content": "$artifact", + "com.redhat.layer.content.checksum": "sha256:$sum" + } + } + ] + ' > "${tmpmnfst}" + _rm_rf "${mnfst}" + + # rename the manifest blob to its new checksum + local tmpmnfst_sum="$(sha256sum ${tmpmnfst} | awk '{ print $1 }')" + local tmpmnfst_size="$(_size ${tmpmnfst})" + mv "${tmpmnfst}" "${out_dir}/blobs/sha256/${tmpmnfst_sum}" + + # map the mnfst_list to the new mnfst checksum + local tmpmnfst_list="$(_mktemp)" + cat "${mnfst_list}" | jq -c \ + --arg tag "${image_tag}" \ + --arg tmpmnfst_sum "${tmpmnfst_sum}" \ + --arg tmpmnfst_size "${tmpmnfst_size}" \ + ' + .manifests = [(.manifests[] | select(.annotations."org.opencontainers.image.ref.name" != "$tag") )] + + [ + { + "mediaType": "application/vnd.oci.image.manifest.v1+json", + "digest": "sha256:$tmpmnfst_sum", + "size": $tmpmnfst_size, + "annotations": { + "com.redhat.image.type": "source" + "org.opencontainers.image.ref.name": "$tag" + } + } + ] + ' > "${tmpmnfst_list}" + mv "${tmpmnfst_list}" "${mnfst_list}" +} + + +# +# Source Collection Drivers +# +# presently just bash functions. *notice* prefix the function name as `sourcedriver_` +# May become a ${ABV_NAME}/drivers.d/ +# +# Arguments: +# * image ref +# * path to inspect +# * output path for source (specifc to this driver) +# * output path for source json metadata (this addresses the files to be added and it's metadata) +# +# TODO TBD this does not account for how to pack/layer the sources collected here. +# This was my thought for outputing a `source.json` file, which is not a +# standard, but could be an array of metadata about _each_ object that should +# be packed. +# + +# +# driver to determine and fetch source rpms, based on the rootfs +# +sourcedriver_rpm_fetch() { + local ref="${1}" + local rootfs="${2}" + local out_dir="${3}" + local manifest_dir="${4}" + + # Get the RELEASEVER from the image + local release=$(rpm -q --queryformat "%{VERSION}\n" --root ${rootfs} -f /etc/os-release) + + # From the rootfs of the works image, build out the src rpms to operate over + for srcrpm in $(rpm -qa --root ${rootfs} --queryformat '%{SOURCERPM}\n' | grep -v '^gpg-pubkey' | sort -u) ; do + local rpm=${srcrpm%*.src.rpm} + if [ ! -f "${out_dir}/${srcrpm}" ] ; then + info "--> fetching ${srcrpm}" + # XXX i wonder if all the srcrpms could be downloaded at once, + # rather than serial. This would require building a new list of + # files that are not present in ${out_dir}. + dnf download \ + --quiet \ + --installroot "${rootfs}" \ + --release "${release}" \ + --destdir "${out_dir}" \ + --source \ + "${rpm}" || continue + else + info "--> using cached ${srcrpm}" + fi + + # XXX one day, check and confirm with %{sourcepkgid} + # https://bugzilla.redhat.com/show_bug.cgi?id=1741715 + #local rpm_sourcepkgid=$(rpm -q --root ${rootfs} --queryformat '%{sourcepkgid}' "${rpm}") + local srcrpm_buildtime=$( rpm -qp --qf '%{buildtime}' ${out_dir}/${srcrpm} ) + local srcrpm_pkgid=$( rpm -qp --qf '%{pkgid}' ${out_dir}/${srcrpm} ) + touch --date=@${srcrpm_buildtime} ${out_dir}/${srcrpm} + local mimetype="$(file --brief --mime-type ${out_dir}/${srcrpm})" + + local source_info="${manifest_dir}/${srcrpm}.json" + jq \ + -n \ + --arg name ${srcrpm} \ + --arg buildtime "${srcrpm_buildtime}" \ + --arg mimetype "${mimetype}" \ + ' + { + "name" : $name, + "annotations": { + "source.mediaType": $mimetype, + "source.mediaType": $mimetype, + "source.artifact.buildtime": $buildtime + } + } + ' \ + > "${source_info}" + done +} + +# +# If the caller specified a context directory, +# +# slightly special driver, as it has a flag/env passed in, that it uses +# +sourcedriver_context_dir() { + local ref="${1}" + local rootfs="${2}" + local out_dir="${3}" + local manifest_dir="${4}" + +if [ -n "${CONTEXT_DIR}" ]; then + context_dir=$(cd ${CONTEXT_DIR}; pwd) + buildah add ${SRC_CTR} ${context_dir} /CONTEXT + buildah config --created-by "/bin/sh -c #(nop) ADD file:$(cd ${context_dir}; _tar -cf - . | sha256sum -| cut -f1 -d' ') in /CONTEXT" ${SRC_CTR} + export IMG=$(buildah commit --omit-timestamp --rm ${SRC_CTR}) + export SRC_CTR=$(buildah from ${IMG}) fi -export IMAGE=$1 -export SRC_RPM_DIR=$(pwd)/SRCRPMS -export SRC_IMAGE=$1-src -export CONTEXT_DIR=$2 -export EXTRA_SRC_DIR=$3 -export IMAGE_CTR=$(buildah from ${IMAGE}) -export IMAGE_MNT=$(buildah mount ${IMAGE_CTR}) +} + # -# From the executable image, get the RELEASEVER of the image +# If the caller specified a extra directory # -RELEASE=$(rpm -q --queryformat "%{VERSION}\n" --root $IMAGE_MNT -f /etc/os-release) +# slightly special driver, as it has a flag/env passed in, that it uses # -# From the executable image, list the SRC RPMS used to build the image -# -SRC_RPMS=$(rpm -qa --root ${IMAGE_MNT} --queryformat '%{SOURCERPM}\n' | grep -v '^gpg-pubkey' | sort -u) -buildah umount ${IMAGE_CTR} -buildah rm ${IMAGE_CTR} +sourcedriver_extra_src_dir() { + local ref="${1}" + local rootfs="${2}" + local out_dir="${3}" + local manifest_dir="${4}" + + if [ -n "${EXTRA_SRC_DIR}" ]; then + fi +} + + +main() { + _init ${@} + + local base_dir="$(pwd)/${ABV_NAME}" + # using the bash builtin to parse + while getopts ":hplDc:e:o:b:d:" opts; do + case "${opts}" in + b) + base_dir="${OPTARG}" + ;; + c) + local context_dir=${OPTARG} + ;; + e) + local extra_src_dir=${OPTARG} + ;; + o) + local output_dir=${OPTARG} + ;; + d) + drivers=${OPTARG} + ;; + l) + list_drivers=1 + ;; + p) + push=1 + ;; + D) + export DEBUG=1 + ;; + h) + _usage + ;; + *) + _usage + ;; + esac + done + shift $((OPTIND-1)) + + if [ -n "${list_drivers}" ] ; then + set | grep '^sourcedriver_.* () ' | tr -d ' ()' + exit 0 + fi + + export CONTEXT_DIR="${CONTEXT_DIR:-$context_dir}" + export EXTRA_SRC_DIR="${EXTRA_SRC_DIR:-$extra_src_dir}" + + local output_dir="${OUTPUT_DIR:-$output_dir}" + local src_dir="${base_dir}/src" + local work_dir="${base_dir}/work" + + export TMPDIR="${work_dir}/tmp" + if [ -d "${TMPDIR}" ] ; then + _rm_rf "${TMPDIR}" + fi + mkdir -p "${TMPDIR}" + + IMAGE_REF="${1}" + debug "IMAGE_REF: ${IMAGE_REF}" + + IMAGE_REF_BASE="$(parse_img_base ${IMAGE_REF})" + debug "IMAGE_REF_BASE: ${IMAGE_REF_BASE}" + + IMAGE_TAG="$(parse_img_tag ${IMAGE_REF})" + debug "IMAGE_TAG: ${IMAGE_TAG}" + + IMAGE_DIGEST="$(parse_img_digest ${IMAGE_REF})" + # determine missing digest before fetch, so that we fetch the precise image + # including its digest. + if [ -z "${IMAGE_DIGEST}" ] ; then + IMAGE_DIGEST="$(fetch_img_digest ${IMAGE_REF_BASE}:${IMAGE_TAG})" + fi + debug "IMAGE_DIGEST: ${IMAGE_DIGEST}" + + # if inspect and fetch image, then to an OCI layout dir + if [ ! -d "${work_dir}/layouts/${IMAGE_DIGEST/:/\/}" ] ; then + # we'll store the image to a path based on its digest, that it can be reused + img_layout="$(fetch_img ${IMAGE_REF_BASE}:${IMAGE_TAG}@${IMAGE_DIGEST} ${work_dir}/layouts/${IMAGE_DIGEST/:/\/} )" + else + img_layout="${work_dir}/layouts/${IMAGE_DIGEST/:/\/}:${IMAGE_TAG}" + fi + debug "image layout: ${img_layout}" + + # setup rootfs, from that OCI layout + local unpack_dir="${work_dir}/unpacked/${IMAGE_DIGEST/:/\/}" + if [ ! -d "${unpack_dir}" ] ; then + unpack_img ${img_layout} ${unpack_dir} + fi + debug "unpacked dir: ${unpack_dir}" + + # clear prior driver's info about source to insert into Source Image + _rm_rf "${work_dir}/driver/${IMAGE_DIGEST/:/\/}" + + if [ -n "${drivers}" ] ; then + # clean up the args passed by the caller ... + drivers="$(echo ${drivers} | tr ',' ' '| tr '\n' ' ')" + else + drivers="$(set | grep '^sourcedriver_.* () ' | tr -d ' ()' | tr '\n' ' ')" + fi + # iterate on the drivers + #for driver in sourcedriver_rpm_fetch ; do + for driver in ${drivers} ; do + info "calling $driver" + mkdir -vp "${src_dir}/${IMAGE_DIGEST/:/\/}/${driver#sourcedriver_*}" + mkdir -vp "${work_dir}/driver/${IMAGE_DIGEST/:/\/}/${driver#sourcedriver_*}" + $driver \ + "${IMAGE_REF_BASE}:${IMAGE_TAG}@${IMAGE_DIGEST}" \ + "${unpack_dir}/rootfs" \ + "${src_dir}/${IMAGE_DIGEST/:/\/}/${driver#sourcedriver_*}" \ + "${work_dir}/driver/${IMAGE_DIGEST/:/\/}/${driver#sourcedriver_*}" + + # TODO walk the driver output to determine layers to be added + # find "${work_dir}/driver/${IMAGE_DIGEST/:/\/}/${driver#sourcedriver_*}" -type f -name '*.json' + done + + # TODO maybe look to a directory like /usr/libexec/BuildSourceImage/drivers/ for drop-ins to run + + # TODO commit the image + # This is going to be a hand craft of composing these layers using just bash and jq + +echo "bailing here for now" +return 0 + + ## if an output directory is provided then save a copy to it + if [ -n "${output_dir}" ] ; then + mkdir -p "${output_dir}" + # XXX WIP + push_img $src_img_dir "oci:$output_dir:$(ref_src_img_tag ${IMAGE_TAG})" + fi + + if [ -n "${push}" ] ; then + # XXX WIP + push_img $src_dir $IMAGE_REF + fi + # # For each SRC_RPMS used to build the executable image, download the SRC RPM # and generate a layer in the SRC RPM. # -mkdir -p ${SRC_RPM_DIR} pushd ${SRC_RPM_DIR} > /dev/null export SRC_CTR=$(buildah from scratch) -for srpm in ${SRC_RPMS}; do - if [ ! -f ${srpm} ]; then - RPM=$(echo ${srpm} | sed 's/.src.rpm$//g') - dnf download --release $RELEASE --source $RPM || continue +for i in ${SRC_RPMS}; do + if [ ! -f $i ]; then + RPM=$(echo $i | sed 's/.src.rpm$//g') + dnf download --release $RELEASE --source $RPM || continue # TODO: perhaps log failures somewhere fi echo "Adding ${srpm}" touch --date=@`rpm -q --qf '%{buildtime}' ${srpm}` ${srpm} @@ -47,45 +782,11 @@ for srpm in ${SRC_RPMS}; do export SRC_CTR=$(buildah from ${IMG}) done popd > /dev/null -# -# If the caller specified a context directory, -# add it to the CONTEXT DIR in SRC IMAGE -# -if [ ! -z "${CONTEXT_DIR}" ]; then - CONTEXT_DIR=$(cd ${CONTEXT_DIR}; pwd) - buildah add ${SRC_CTR} ${CONTEXT_DIR} /CONTEXT - buildah config --created-by "/bin/sh -c #(nop) ADD file:$(cd ${CONTEXT_DIR}; tar cf - . | sha256sum -| cut -f1 -d' ') in /CONTEXT" ${SRC_CTR} - export IMG=$(buildah commit --omit-timestamp --rm ${SRC_CTR}) - export SRC_CTR=$(buildah from ${IMG}) -fi -# -# If the caller specified a extra directory, -# add it to the CONTEXT DIR in SRC IMAGE -# -if [ ! -z "${EXTRA_SRC_DIR}" ]; then - buildah add ${SRC_CTR} ${EXTRA_SRC_DIR} /EXTRA - buildah config --created-by "/bin/sh -c #(nop) ADD file:$(cd ${EXTRA_SRC_DIR}; tar cf - . | sha256sum -| cut -f1 -d' ') in /CONTEXT" ${SRC_CTR} - export IMG=$(buildah commit --omit-timestamp --rm ${EXTRA_SRC_CTR}) - export SRC_CTR=$(buildah from ${IMG}) -fi -# Cleanup and remove source container -buildah rm ${SRC_CTR} +} -# -# Add the final name to our image -# -buildah tag $IMG $SRC_IMAGE - -## if an output directory is provided then save a copy to it -## XXX(vbatts) this is not working inside a container like `quay.io/buildah/stable` yet -if [ -n "${OUTPUT_DIR}" ] ; then - mkdir -p "${OUTPUT_DIR}" - skopeo copy containers-storage:"${IMG}" oci:"${OUTPUT_DIR}"/ -fi - -# Push SRC_IMAGE to Registry -# buildah push $SRC_IMAGE REGISTRY_NAME/$SRC_IMAGE +# only exec main if this is being called (this way we can source and test the functions) +_is_sourced || main ${@} # vim:set shiftwidth=4 softtabstop=4 expandtab: diff --git a/Dockerfile b/Dockerfile index a851ab8..3dfcb19 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,9 +1,21 @@ -FROM quay.io/buildah/stable -RUN dnf install -y skopeo && \ - dnf clean all && \ - mkdir -p /output +FROM fedora + +RUN dnf install -y skopeo golang jq git +RUN dnf install -y make findutils +ENV GOPATH=/usr/share/gocode +ENV GOBIN=/usr/local/bin +RUN git clone https://github.com/openSUSE/umoci $GOPATH/src/github.com/openSUSE/umoci +RUN cd $GOPATH/src/github.com/openSUSE/umoci && \ + make && \ + mv umoci /usr/local/bin + COPY . /usr/local/bin/ -VOLUME /var/lib/container -VOLUME /output + +RUN mkdir -p /output ENV OUTPUT_DIR=/output -ENTRYPOINT ["/usr/local/bin/BuildSourceImage.sh"] +VOLUME /output + +ENV SRC_DIR=/src +VOLUME /src + +#ENTRYPOINT ["/usr/local/bin/BuildSourceImage.sh"] From 0d103e778f878f3f94c0c75e4d8b1cfab136abec Mon Sep 17 00:00:00 2001 From: Vincent Batts Date: Wed, 4 Sep 2019 16:48:52 -0400 Subject: [PATCH 03/17] BuildSourceImage: readying for the koji use case for koji, there will be no fetching of the image, nor fetching of RPMS using `dnf download --source ...`. They'll just provide a directory of SRPMS used (perhaps as a composite of architectures). And just add these in similar fashion as when fetching. This used with the flag to specify which drivers to run will allow tailoring to just that use-case. Signed-off-by: Vincent Batts --- BuildSourceImage.sh | 169 ++++++++++++++++++++++++++++++-------------- 1 file changed, 117 insertions(+), 52 deletions(-) diff --git a/BuildSourceImage.sh b/BuildSourceImage.sh index 05ed203..d3d9147 100755 --- a/BuildSourceImage.sh +++ b/BuildSourceImage.sh @@ -14,6 +14,7 @@ _usage() { echo -e " -b \tbase path for source image builds" echo -e " -c \tbuild context for the container image. Can be provided via CONTEXT_DIR env variable" echo -e " -e \textra src for the container image. Can be provided via EXTRA_SRC_DIR env variable" + echo -e " -r \tdirectory of RPMS to add. Can be provided via RPM_DIR env variable" echo -e " -o \toutput the OCI image to path. Can be provided via OUTPUT_DIR env variable" echo -e " -d \toutput the OCI image to path. Can be provided via OUTPUT_DIR env variable" echo -e " -l\t\tlist the source drivers available" @@ -78,7 +79,7 @@ _mktemp() { # local rm -rf _rm_rf() { - debug "rm -rf $@" + _debug "rm -rf $@" #rm -rf $@ } @@ -91,22 +92,24 @@ _tar() { fi } -# # output things, only when $DEBUG is set -# -debug() { +_debug() { if [ -n "${DEBUG}" ] ; then echo "[${ABV_NAME}][DEBUG] ${@}" fi } -# # general echo but with prefix -# -info() { +_info() { echo "[${ABV_NAME}][INFO] ${@}" } +# general echo but with prefix +_error() { + echo "[${ABV_NAME}][ERROR] ${@}" >&2 + exit 1 +} + # # parse the OCI image reference, accounting for: # * transport name @@ -291,7 +294,7 @@ unpack_img_bash() { fi done - info "[unpacking] layer ${dgst}" + _info "[unpacking] layer ${dgst}" # unpack layer to rootfs (without whiteouts) zcat "${path}" | _tar --restrict --no-xattr --no-acls --no-selinux --exclude='*.wh.*' -x -C "${unpack_dir}" @@ -307,7 +310,7 @@ unpack_img_umoci() { local image_dir="${1}" local unpack_dir="${2}" - debug "unpackging with umoci" + _debug "unpackging with umoci" # always assume we're not root I reckon umoci unpack --rootless --image "${image_dir}" "${unpack_dir}" >&2 } @@ -393,9 +396,9 @@ layout_insert() { local mnfst_list="${out_dir}/index.json" # get the digest to the manifest test -f "${mnfst_list}" || return 1 - local mnfst_dgst="$(cat ${mnfst_list} | jq ' + local mnfst_dgst="$(cat ${mnfst_list} | jq --arg tag "${image_tag}" ' .manifests[] - | select(.annotations."org.opencontainers.image.ref.name" == "'${image_tag}'") + | select(.annotations."org.opencontainers.image.ref.name" == $tag ) | .digest ' | tr -d \" | tr -d '\n' )" local mnfst="${out_dir}/blobs/${mnfst_dgst/:/\/}" @@ -435,17 +438,17 @@ layout_insert() { local tmpconfig="$(_mktemp)" cat "${out_dir}/blobs/${config_sum/:/\/}" | jq -c \ --arg date "$(_date_ns)" \ - --arg tmptar_sum "${tmptar_sum}" \ - --arg sum "${sum}" \ + --arg tmptar_sum "sha256:${tmptar_sum}" \ + --arg comment "#(nop) BuildSourceImage adding artifact: ${sum}" \ ' - .created = "$date" + .created = $date | .rootfs.diff_ids = .rootfs.diff_ids + [ - "sha256:$tmptar_sum" + sha256:$tmptar_sum ] | .history = .history + [ { - "created": "$date", - "created_by": "#(nop) BuildSourceImage adding artifact: $sum" + "created": $date, + "created_by": $comment } ] ' > "${tmpconfig}" @@ -459,24 +462,24 @@ layout_insert() { # append layers list in the manifest, and its new config mapping local tmpmnfst="$(_mktemp)" cat "${mnfst}" | jq -c \ - --arg tmpconfig_sum "${tmpconfig_sum}" \ + --arg tmpconfig_sum "sha256:${tmpconfig_sum}" \ --arg tmpconfig_size "${tmpconfig_size}" \ - --arg tmptar_sum "${tmptar_sum}" \ + --arg tmptar_sum "sha256:${tmptar_sum}" \ --arg tmptar_size "${tmptar_size}" \ --arg artifact "$(basename ${artifact_path})" \ - --arg sum "${sum}" \ + --arg sum "sha256:${sum}" \ ' - .config.digest = "sha256:$tmpconfig_sum" + .config.digest = sha256:$tmpconfig_sum | .config.size = $tmpconfig_size | .layers = .layers + [ { "mediaType": "application/vnd.oci.image.layer.v1.tar", "size": $tmptar_size, - "digest": "sha256:$tmptar_sum", + "digest": $tmptar_sum, "annotations": { "com.redhat.layer.type": "source", - "com.redhat.layer.content": "$artifact", - "com.redhat.layer.content.checksum": "sha256:$sum" + "com.redhat.layer.content": $artifact, + "com.redhat.layer.content.checksum": $sum } } ] @@ -534,6 +537,7 @@ layout_insert() { # driver to determine and fetch source rpms, based on the rootfs # sourcedriver_rpm_fetch() { + local self="${0#sourcedriver_*}" local ref="${1}" local rootfs="${2}" local out_dir="${3}" @@ -546,10 +550,7 @@ sourcedriver_rpm_fetch() { for srcrpm in $(rpm -qa --root ${rootfs} --queryformat '%{SOURCERPM}\n' | grep -v '^gpg-pubkey' | sort -u) ; do local rpm=${srcrpm%*.src.rpm} if [ ! -f "${out_dir}/${srcrpm}" ] ; then - info "--> fetching ${srcrpm}" - # XXX i wonder if all the srcrpms could be downloaded at once, - # rather than serial. This would require building a new list of - # files that are not present in ${out_dir}. + _info "--> fetching ${srcrpm}" dnf download \ --quiet \ --installroot "${rootfs}" \ @@ -558,7 +559,7 @@ sourcedriver_rpm_fetch() { --source \ "${rpm}" || continue else - info "--> using cached ${srcrpm}" + _info "--> using cached ${srcrpm}" fi # XXX one day, check and confirm with %{sourcepkgid} @@ -572,41 +573,75 @@ sourcedriver_rpm_fetch() { local source_info="${manifest_dir}/${srcrpm}.json" jq \ -n \ - --arg name ${srcrpm} \ + --arg name "${srcrpm}" \ --arg buildtime "${srcrpm_buildtime}" \ --arg mimetype "${mimetype}" \ ' - { - "name" : $name, - "annotations": { - "source.mediaType": $mimetype, - "source.mediaType": $mimetype, - "source.artifact.buildtime": $buildtime - } + { + "source.artifact.name": $name, + "source.artifact.mimetype": $mimetype, + "source.artifact.buildtime": $buildtime } ' \ > "${source_info}" + ret=$? + if [ $ret -ne 0 ] ; then + return 1 + fi done } +# +# driver to only package rpms from a provided rpm directory +# +sourcedriver_rpm_dir() { + local self="${0#sourcedriver_*}" + local ref="${1}" + local rootfs="${2}" + local out_dir="${3}" + local manifest_dir="${4}" + + if [ -n "${RPM_DIR}" ]; then + _debug "$self: writing to $out_dir and $manifest_dir" + fi +} + # # If the caller specified a context directory, # # slightly special driver, as it has a flag/env passed in, that it uses # sourcedriver_context_dir() { + local self="${0#sourcedriver_*}" local ref="${1}" local rootfs="${2}" local out_dir="${3}" local manifest_dir="${4}" -if [ -n "${CONTEXT_DIR}" ]; then - context_dir=$(cd ${CONTEXT_DIR}; pwd) - buildah add ${SRC_CTR} ${context_dir} /CONTEXT - buildah config --created-by "/bin/sh -c #(nop) ADD file:$(cd ${context_dir}; _tar -cf - . | sha256sum -| cut -f1 -d' ') in /CONTEXT" ${SRC_CTR} - export IMG=$(buildah commit --omit-timestamp --rm ${SRC_CTR}) - export SRC_CTR=$(buildah from ${IMG}) -fi + if [ -n "${CONTEXT_DIR}" ]; then + _debug "$self: writing to $out_dir and $manifest_dir" + local tarname="context.tar" + _tar -C "${CONTEXT_DIR}" \ + --mtime=@0 --owner=0 --group=0 --mode='a+rw' --no-xattrs --no-selinux --no-acls \ + -cf "${out_dir}/${tarname}" . + local mimetype="$(file --brief --mime-type ${out_dir}/${tarname})" + local source_info="${manifest_dir}/${tarname}.json" + jq \ + -n \ + --arg name "${tarname}" \ + --arg mimetype "${mimetype}" \ + ' + { + "source.artifact.name": $name, + "source.artifact.mimetype": $mimetype + } + ' \ + > "${source_info}" + ret=$? + if [ $ret -ne 0 ] ; then + return 1 + fi + fi } # @@ -615,12 +650,35 @@ fi # slightly special driver, as it has a flag/env passed in, that it uses # sourcedriver_extra_src_dir() { + local self="${0#sourcedriver_*}" local ref="${1}" local rootfs="${2}" local out_dir="${3}" local manifest_dir="${4}" if [ -n "${EXTRA_SRC_DIR}" ]; then + _debug "$self: writing to $out_dir and $manifest_dir" + local tarname="extra-src.tar" + _tar -C "${EXTRA_SRC_DIR}" \ + --mtime=@0 --owner=0 --group=0 --mode='a+rw' --no-xattrs --no-selinux --no-acls \ + -cf "${out_dir}/${tarname}" . + local mimetype="$(file --brief --mime-type ${out_dir}/${tarname})" + local source_info="${manifest_dir}/${tarname}.json" + jq \ + -n \ + --arg name "${tarname}" \ + --arg mimetype "${mimetype}" \ + ' + { + "source.artifact.name": $name, + "source.artifact.mimetype": $mimetype + } + ' \ + > "${source_info}" + ret=$? + if [ $ret -ne 0 ] ; then + return 1 + fi fi } @@ -630,7 +688,7 @@ main() { local base_dir="$(pwd)/${ABV_NAME}" # using the bash builtin to parse - while getopts ":hplDc:e:o:b:d:" opts; do + while getopts ":hplDc:r:e:o:b:d:" opts; do case "${opts}" in b) base_dir="${OPTARG}" @@ -641,6 +699,9 @@ main() { e) local extra_src_dir=${OPTARG} ;; + r) + local rpm_dir=${OPTARG} + ;; o) local output_dir=${OPTARG} ;; @@ -673,6 +734,7 @@ main() { export CONTEXT_DIR="${CONTEXT_DIR:-$context_dir}" export EXTRA_SRC_DIR="${EXTRA_SRC_DIR:-$extra_src_dir}" + export RPM_DIR="${RPM_DIR:-$rpm_dir}" local output_dir="${OUTPUT_DIR:-$output_dir}" local src_dir="${base_dir}/src" @@ -685,13 +747,13 @@ main() { mkdir -p "${TMPDIR}" IMAGE_REF="${1}" - debug "IMAGE_REF: ${IMAGE_REF}" + _debug "IMAGE_REF: ${IMAGE_REF}" IMAGE_REF_BASE="$(parse_img_base ${IMAGE_REF})" - debug "IMAGE_REF_BASE: ${IMAGE_REF_BASE}" + _debug "IMAGE_REF_BASE: ${IMAGE_REF_BASE}" IMAGE_TAG="$(parse_img_tag ${IMAGE_REF})" - debug "IMAGE_TAG: ${IMAGE_TAG}" + _debug "IMAGE_TAG: ${IMAGE_TAG}" IMAGE_DIGEST="$(parse_img_digest ${IMAGE_REF})" # determine missing digest before fetch, so that we fetch the precise image @@ -699,7 +761,7 @@ main() { if [ -z "${IMAGE_DIGEST}" ] ; then IMAGE_DIGEST="$(fetch_img_digest ${IMAGE_REF_BASE}:${IMAGE_TAG})" fi - debug "IMAGE_DIGEST: ${IMAGE_DIGEST}" + _debug "IMAGE_DIGEST: ${IMAGE_DIGEST}" # if inspect and fetch image, then to an OCI layout dir if [ ! -d "${work_dir}/layouts/${IMAGE_DIGEST/:/\/}" ] ; then @@ -708,14 +770,14 @@ main() { else img_layout="${work_dir}/layouts/${IMAGE_DIGEST/:/\/}:${IMAGE_TAG}" fi - debug "image layout: ${img_layout}" + _debug "image layout: ${img_layout}" # setup rootfs, from that OCI layout local unpack_dir="${work_dir}/unpacked/${IMAGE_DIGEST/:/\/}" if [ ! -d "${unpack_dir}" ] ; then unpack_img ${img_layout} ${unpack_dir} fi - debug "unpacked dir: ${unpack_dir}" + _debug "unpacked dir: ${unpack_dir}" # clear prior driver's info about source to insert into Source Image _rm_rf "${work_dir}/driver/${IMAGE_DIGEST/:/\/}" @@ -729,7 +791,7 @@ main() { # iterate on the drivers #for driver in sourcedriver_rpm_fetch ; do for driver in ${drivers} ; do - info "calling $driver" + _info "calling $driver" mkdir -vp "${src_dir}/${IMAGE_DIGEST/:/\/}/${driver#sourcedriver_*}" mkdir -vp "${work_dir}/driver/${IMAGE_DIGEST/:/\/}/${driver#sourcedriver_*}" $driver \ @@ -737,6 +799,9 @@ main() { "${unpack_dir}/rootfs" \ "${src_dir}/${IMAGE_DIGEST/:/\/}/${driver#sourcedriver_*}" \ "${work_dir}/driver/${IMAGE_DIGEST/:/\/}/${driver#sourcedriver_*}" + if [ $? -ne 0 ] ; then + _error "$driver failed" + fi # TODO walk the driver output to determine layers to be added # find "${work_dir}/driver/${IMAGE_DIGEST/:/\/}/${driver#sourcedriver_*}" -type f -name '*.json' From b7e49bc1c687c9b1a30955007dc9111299c7c127 Mon Sep 17 00:00:00 2001 From: Vincent Batts Date: Thu, 5 Sep 2019 07:15:14 -0400 Subject: [PATCH 04/17] awk: don't print the newline. Removes the need for a `tr` Signed-off-by: Vincent Batts --- BuildSourceImage.sh | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/BuildSourceImage.sh b/BuildSourceImage.sh index d3d9147..6bcbe51 100755 --- a/BuildSourceImage.sh +++ b/BuildSourceImage.sh @@ -345,7 +345,7 @@ layout_new() { } } ' - local config_sum=$(echo "${config}" | jq -c | tr -d '\n' | sha256sum | awk '{ print $1 }' | tr -d '\n') + local config_sum=$(echo "${config}" | jq -c | tr -d '\n' | sha256sum | awk '{ ORS=""; print $1 }') echo "${config}" | jq -c | tr -d '\n' > "${out_dir}/blobs/sha256/${config_sum}" local mnfst=' @@ -359,7 +359,7 @@ layout_new() { "layers": [] } ' - local mnfst_sum=$(echo "${mnfst}" | jq -c | tr -d '\n' | sha256sum | awk '{ print $1 }' | tr -d '\n') + local mnfst_sum=$(echo "${mnfst}" | jq -c | tr -d '\n' | sha256sum | awk '{ ORS=""; print $1 }') echo "${mnfst}" | jq -c | tr -d '\n' > "${out_dir}/blobs/sha256/${mnfst_sum}" echo ' @@ -427,7 +427,7 @@ layout_insert() { _rm_rf "${tmpdir}" # checksum tar and move to blobs/sha256/$checksum - local tmptar_sum="$(sha256sum ${tmptar} | awk '{ print $1 }')" + local tmptar_sum="$(sha256sum ${tmptar} | awk '{ ORS=""; print $1 }')" local tmptar_size="$(_size ${tmptar})" mv "${tmptar}" "${out_dir}/blobs/sha256/${tmptar_sum}" @@ -455,7 +455,7 @@ layout_insert() { _rm_rf "${out_dir}/blobs/${config_sum/:/\/}" # rename the config blob to its new checksum - local tmpconfig_sum="$(sha256sum ${tmpconfig} | awk '{ print $1 }')" + local tmpconfig_sum="$(sha256sum ${tmpconfig} | awk '{ ORS=""; print $1 }')" local tmpconfig_size="$(_size ${tmpconfig})" mv "${tmpconfig}" "${out_dir}/blobs/sha256/${tmpconfig_sum}" @@ -487,7 +487,7 @@ layout_insert() { _rm_rf "${mnfst}" # rename the manifest blob to its new checksum - local tmpmnfst_sum="$(sha256sum ${tmpmnfst} | awk '{ print $1 }')" + local tmpmnfst_sum="$(sha256sum ${tmpmnfst} | awk '{ ORS=""; print $1 }')" local tmpmnfst_size="$(_size ${tmpmnfst})" mv "${tmpmnfst}" "${out_dir}/blobs/sha256/${tmpmnfst_sum}" From 977a003d16296b7edc26babe5a325b6c0c5a3912 Mon Sep 17 00:00:00 2001 From: Vincent Batts Date: Thu, 5 Sep 2019 17:10:39 -0400 Subject: [PATCH 05/17] *: tying it all together this makes everything a flag now. This is not 100% Signed-off-by: Vincent Batts --- BuildSourceImage.sh | 383 ++++++++++++++++++++++++++++---------------- 1 file changed, 246 insertions(+), 137 deletions(-) diff --git a/BuildSourceImage.sh b/BuildSourceImage.sh index 6bcbe51..416c95b 100755 --- a/BuildSourceImage.sh +++ b/BuildSourceImage.sh @@ -1,7 +1,6 @@ #!/bin/bash -# This script requires an OCI IMAGE Name to pull. -# The script generates a SOURCE Image based on the OCI Image -# Script must be executed on the same OS or newer as the image. + +# This script builds a Source Image via "drivers" to collect source export ABV_NAME="SrcImg" # TODO maybe a flag for this? @@ -9,7 +8,7 @@ export source_image_suffix="-source" _usage() { - echo "Usage: $(basename $0) [-b ] [-c ] [-e ] [-o ] [-p] IMAGE" + echo "Usage: $(basename $0) [-D] [-b ] [-c ] [-e ] [-r ] [-o ] [-i ] [-p ] [-l] [-d ]" echo "" echo -e " -b \tbase path for source image builds" echo -e " -c \tbuild context for the container image. Can be provided via CONTEXT_DIR env variable" @@ -18,7 +17,8 @@ _usage() { echo -e " -o \toutput the OCI image to path. Can be provided via OUTPUT_DIR env variable" echo -e " -d \toutput the OCI image to path. Can be provided via OUTPUT_DIR env variable" echo -e " -l\t\tlist the source drivers available" - echo -e " -p\t\tpush source image after build" + echo -e " -i \timage reference to fetch and inspect its rootfs" + echo -e " -p \tpush source image to reference after build" echo -e " -D\t\tdebuging output. Can be set via DEBUG env variable" exit 1 } @@ -27,18 +27,14 @@ _usage() { # sanity checks on startup # _init() { - if [ $# -lt 1 ] ; then - _usage - fi - set -o pipefail # check for tools we depend on - for cmd in jq skopeo dnf file find ; do + for cmd in jq skopeo dnf file find tar stat date ; do if [ -z "$(command -v ${cmd})" ] ; then # TODO: maybe this could be individual checks so it can report # where to find the tools - echo "ERROR: please install '${cmd}' package" + echo "ERROR: please install package to provide '${cmd}'" fi done } @@ -80,7 +76,16 @@ _mktemp() { # local rm -rf _rm_rf() { _debug "rm -rf $@" - #rm -rf $@ + rm -rf $@ +} + +# local mkdir -p +_mkdir_p() { + if [ -n "${DEBUG}" ] ; then + mkdir -vp $@ + else + mkdir -p $@ + fi } # local tar @@ -104,6 +109,10 @@ _info() { echo "[${ABV_NAME}][INFO] ${@}" } +_warn() { + echo "[${ABV_NAME}][WARN] ${@}" >&2 +} + # general echo but with prefix _error() { echo "[${ABV_NAME}][ERROR] ${@}" >&2 @@ -150,6 +159,12 @@ parse_img_base() { parse_img_tag() { local ref="${1%@*}" # just the portion before the digest "@" local tag="latest" # default tag + + if [ -z "${ref}" ] ; then + echo -n "${tag}" + return 0 + fi + if [ "$(_count ':' $(echo ${ref} | tr '/' '\n' | tail -1 ))" -gt 0 ] ; then # if there are colons in the last segment after '/', then get that tag name tag="$(echo ${ref} | tr '/' '\n' | tail -1 | cut -d : -f 2 )" @@ -217,7 +232,7 @@ fetch_img() { local ref="${1}" local dst="${2}" - mkdir -p "${dst}" + _mkdir_p "${dst}" local base="$(parse_img_base ${ref})" local tag="$(parse_img_tag ${ref})" @@ -267,6 +282,11 @@ unpack_img_bash() { local image_dir="${1}" local unpack_dir="${2}" + # for compat with umoci (which wants the image tag as well) + if echo "${image_dir}" | grep -q ":" ; then + image_dir="${image_dir%:*}" + fi + local mnfst_dgst="$(cat "${image_dir}"/index.json | jq '.manifests[0].digest' | tr -d \" )" # Since we're landing the reference as an OCI layout, this mediaType is fairly predictable @@ -274,10 +294,10 @@ unpack_img_bash() { layer_dgsts="$(cat ${image_dir}/blobs/${mnfst_dgst/:/\/} | \ jq '.layers[] | select(.mediaType == "application/vnd.oci.image.layer.v1.tar+gzip") | .digest' | tr -d \")" - mkdir -vp "${unpack_dir}" + _mkdir_p "${unpack_dir}/rootfs" for dgst in ${layer_dgsts} ; do path="${image_dir}/blobs/${dgst/:/\/}" - tmp_file=$(_mktemp_d) + tmp_file=$(_mktemp) zcat "${path}" | _tar -t > $tmp_file # TODO cleanup these files # look for '.wh.' entries. They must be removed from the rootfs @@ -287,16 +307,16 @@ unpack_img_bash() { # if `some/path/.wh.foo` then `rm -rf `${unpack_dir}/some/path/foo` # if `some/path/.wh..wh..opq` then `rm -rf `${unpack_dir}/some/path/*` if [ "$(basename ${line})" == ".wh..wh..opq" ] ; then - _rm_rf "${unpack_dir}/$(dirname ${line})/*" + _rm_rf "${unpack_dir}/rootfs/$(dirname ${line})/*" elif basename "${line}" | grep -qe '^\.wh\.' ; then name=$(basename "${line}" | sed -e 's/^\.wh\.//') - _rm_rf "${unpack_dir}/$(dirname ${line})/${name}" + _rm_rf "${unpack_dir}/rootfs/$(dirname ${line})/${name}" fi done _info "[unpacking] layer ${dgst}" # unpack layer to rootfs (without whiteouts) - zcat "${path}" | _tar --restrict --no-xattr --no-acls --no-selinux --exclude='*.wh.*' -x -C "${unpack_dir}" + zcat "${path}" | _tar --restrict --no-xattr --no-acls --no-selinux --exclude='*.wh.*' -x -C "${unpack_dir}/rootfs" # some of the directories get unpacked as 0555, so removing them gives an EPERM find "${unpack_dir}" -type d -exec chmod 0755 "{}" \; @@ -331,7 +351,7 @@ layout_new() { local out_dir="${1}" local image_tag="${2:-latest}" - mkdir -p "${out_dir}/blobs/sha256" + _mkdir_p "${out_dir}/blobs/sha256" echo '{"imageLayoutVersion":"1.0.0"}' > "${out_dir}/oci-layout" local config=' { @@ -379,19 +399,20 @@ layout_new() { ' | jq -c | tr -d '\n' > "${out_dir}/index.json" } -# TODO this is not finished yet # call this for every artifact, to insert it into an OCI layout # args: # * a path to the layout # * a path to the artifact # * the path inside the tar +# * json file to slurp in as annotations for this layer's OCI descriptor # * tag used in the layout (default is 'latest') # layout_insert() { local out_dir="${1}" local artifact_path="${2}" local tar_path="${3}" - local image_tag="${4:-latest}" + local annotations_file="${4}" + local image_tag="${5:-latest}" local mnfst_list="${out_dir}/index.json" # get the digest to the manifest @@ -409,14 +430,14 @@ layout_insert() { # TODO account for "artifact_path" being a directory? local sum="$(sha256sum ${artifact_path} | awk '{ print $1 }')" # making a blob store in the layer - mkdir -p "${tmpdir}/blobs/sha256" + _mkdir_p "${tmpdir}/blobs/sha256" cp "${artifact_path}" "${tmpdir}/blobs/sha256/${sum}" if [ "$(basename ${tar_path})" == "$(basename ${artifact_path})" ] ; then - mkdir -p "${tmpdir}/$(dirname ${tar_path})" + _mkdir_p "${tmpdir}/$(dirname ${tar_path})" # TODO this symlink need to be relative path, not to `/blobs/...` ln -s "/blobs/sha256/${sum}" "${tmpdir}/${tar_path}" else - mkdir -p "${tmpdir}/${tar_path}" + _mkdir_p "${tmpdir}/${tar_path}" # TODO this symlink need to be relative path, not to `/blobs/...` ln -s "/blobs/sha256/${sum}" "${tmpdir}/${tar_path}/$(basename ${artifact_path})" fi @@ -442,10 +463,8 @@ layout_insert() { --arg comment "#(nop) BuildSourceImage adding artifact: ${sum}" \ ' .created = $date - | .rootfs.diff_ids = .rootfs.diff_ids + [ - sha256:$tmptar_sum - ] - | .history = .history + [ + | .rootfs.diff_ids += [ $tmptar_sum ] + | .history += [ { "created": $date, "created_by": $comment @@ -468,22 +487,28 @@ layout_insert() { --arg tmptar_size "${tmptar_size}" \ --arg artifact "$(basename ${artifact_path})" \ --arg sum "sha256:${sum}" \ + --slurpfile annotations_slup "${annotations_file}" \ ' - .config.digest = sha256:$tmpconfig_sum - | .config.size = $tmpconfig_size - | .layers = .layers + [ + .config.digest = $tmpconfig_sum + | .config.size = ($tmpconfig_size|tonumber) + | { + "com.redhat.layer.type": "source", + "com.redhat.layer.content": $artifact, + "com.redhat.layer.content.checksum": $sum + } + $annotations_slup[0] as $annotations_merge + | .layers += [ { "mediaType": "application/vnd.oci.image.layer.v1.tar", - "size": $tmptar_size, + "size": ($tmptar_size|tonumber), "digest": $tmptar_sum, - "annotations": { - "com.redhat.layer.type": "source", - "com.redhat.layer.content": $artifact, - "com.redhat.layer.content.checksum": $sum - } + "annotations": $annotations_merge } ] ' > "${tmpmnfst}" + ret=$? + if [ $ret -ne 0 ] ; then + return 1 + fi _rm_rf "${mnfst}" # rename the manifest blob to its new checksum @@ -495,22 +520,27 @@ layout_insert() { local tmpmnfst_list="$(_mktemp)" cat "${mnfst_list}" | jq -c \ --arg tag "${image_tag}" \ - --arg tmpmnfst_sum "${tmpmnfst_sum}" \ + --arg tmpmnfst_sum "sha256:${tmpmnfst_sum}" \ --arg tmpmnfst_size "${tmpmnfst_size}" \ ' - .manifests = [(.manifests[] | select(.annotations."org.opencontainers.image.ref.name" != "$tag") )] - + [ + [(.manifests[] | select(.annotations."org.opencontainers.image.ref.name" != $tag) )] as $manifests_reduced + | [ { "mediaType": "application/vnd.oci.image.manifest.v1+json", - "digest": "sha256:$tmpmnfst_sum", - "size": $tmpmnfst_size, + "digest": $tmpmnfst_sum, + "size": ($tmpmnfst_size|tonumber), "annotations": { - "com.redhat.image.type": "source" - "org.opencontainers.image.ref.name": "$tag" + "com.redhat.image.type": "source", + "org.opencontainers.image.ref.name": $tag } } - ] + ] as $manifests_new + | .manifests = $manifests_reduced + $manifests_new ' > "${tmpmnfst_list}" + ret=$? + if [ $ret -ne 0 ] ; then + return 1 + fi mv "${tmpmnfst_list}" "${mnfst_list}" } @@ -525,12 +555,17 @@ layout_insert() { # * image ref # * path to inspect # * output path for source (specifc to this driver) -# * output path for source json metadata (this addresses the files to be added and it's metadata) +# * output path for JSON file of source's annotations # -# TODO TBD this does not account for how to pack/layer the sources collected here. -# This was my thought for outputing a `source.json` file, which is not a -# standard, but could be an array of metadata about _each_ object that should -# be packed. +# The JSON of source annotations is the key to discovering the source artifact +# to be added and including rich metadata about that archive into the final +# image. +# The name of each JSON file is appending '.json' to the artifact's name. So if +# you have `foo-1.0.src.rpm` then there MUST be a corresponding +# `foo-1.0.src.rpm.json`. +# The data structure in this annotation is just a dict/hashmap, with key/val +# according to +# https://github.com/opencontainers/image-spec/blob/master/annotations.md # # @@ -548,6 +583,10 @@ sourcedriver_rpm_fetch() { # From the rootfs of the works image, build out the src rpms to operate over for srcrpm in $(rpm -qa --root ${rootfs} --queryformat '%{SOURCERPM}\n' | grep -v '^gpg-pubkey' | sort -u) ; do + if [ "${srcrpm}" == "(none)" ] ; then + continue + fi + local rpm=${srcrpm%*.src.rpm} if [ ! -f "${out_dir}/${srcrpm}" ] ; then _info "--> fetching ${srcrpm}" @@ -557,7 +596,12 @@ sourcedriver_rpm_fetch() { --release "${release}" \ --destdir "${out_dir}" \ --source \ - "${rpm}" || continue + "${rpm}" + ret=$? + if [ $ret -ne 0 ] ; then + _warn "failed to fetch ${srcrpm}" + continue + fi else _info "--> using cached ${srcrpm}" fi @@ -565,25 +609,34 @@ sourcedriver_rpm_fetch() { # XXX one day, check and confirm with %{sourcepkgid} # https://bugzilla.redhat.com/show_bug.cgi?id=1741715 #local rpm_sourcepkgid=$(rpm -q --root ${rootfs} --queryformat '%{sourcepkgid}' "${rpm}") - local srcrpm_buildtime=$( rpm -qp --qf '%{buildtime}' ${out_dir}/${srcrpm} ) - local srcrpm_pkgid=$( rpm -qp --qf '%{pkgid}' ${out_dir}/${srcrpm} ) - touch --date=@${srcrpm_buildtime} ${out_dir}/${srcrpm} + local srcrpm_buildtime=$(rpm -qp --qf '%{buildtime}' ${out_dir}/${srcrpm} ) + local srcrpm_pkgid=$(rpm -qp --qf '%{pkgid}' ${out_dir}/${srcrpm} ) + local srcrpm_name=$(rpm -qp --qf '%{name}' ${out_dir}/${srcrpm} ) + local srcrpm_version=$(rpm -qp --qf '%{version}' ${out_dir}/${srcrpm} ) + local srcrpm_epoch=$(rpm -qp --qf '%{epoch}' ${out_dir}/${srcrpm} ) + local srcrpm_release=$(rpm -qp --qf '%{release}' ${out_dir}/${srcrpm} ) local mimetype="$(file --brief --mime-type ${out_dir}/${srcrpm})" - - local source_info="${manifest_dir}/${srcrpm}.json" jq \ -n \ - --arg name "${srcrpm}" \ + --arg filename "${srcrpm}" \ + --arg name "${srcrpm_name}" \ + --arg version "${srcrpm_version}" \ + --arg epoch "${srcrpm_epoch}" \ + --arg release "${srcrpm_release}" \ --arg buildtime "${srcrpm_buildtime}" \ --arg mimetype "${mimetype}" \ ' { + "source.artifact.filename": $filename, "source.artifact.name": $name, + "source.artifact.version": $version, + "source.artifact.epoch": $epoch, + "source.artifact.release": $release, "source.artifact.mimetype": $mimetype, "source.artifact.buildtime": $buildtime } ' \ - > "${source_info}" + > "${manifest_dir}/${srcrpm}.json" ret=$? if [ $ret -ne 0 ] ; then return 1 @@ -593,6 +646,7 @@ sourcedriver_rpm_fetch() { # # driver to only package rpms from a provided rpm directory +# (koji use-case) # sourcedriver_rpm_dir() { local self="${0#sourcedriver_*}" @@ -603,6 +657,41 @@ sourcedriver_rpm_dir() { if [ -n "${RPM_DIR}" ]; then _debug "$self: writing to $out_dir and $manifest_dir" + find "${RPM_DIR}" -type f -name '*src.rpm' | while read srcrpm ; do + cp "${srcrpm}" "${out_dir}" + local srcrpm_buildtime=$(rpm -qp --qf '%{buildtime}' ${out_dir}/${srcrpm} ) + local srcrpm_pkgid=$(rpm -qp --qf '%{pkgid}' ${out_dir}/${srcrpm} ) + local srcrpm_name=$(rpm -qp --qf '%{name}' ${out_dir}/${srcrpm} ) + local srcrpm_version=$(rpm -qp --qf '%{version}' ${out_dir}/${srcrpm} ) + local srcrpm_epoch=$(rpm -qp --qf '%{epoch}' ${out_dir}/${srcrpm} ) + local srcrpm_release=$(rpm -qp --qf '%{release}' ${out_dir}/${srcrpm} ) + local mimetype="$(file --brief --mime-type ${out_dir}/${srcrpm})" + jq \ + -n \ + --arg filename "${srcrpm}" \ + --arg name "${srcrpm_name}" \ + --arg version "${srcrpm_version}" \ + --arg epoch "${srcrpm_epoch}" \ + --arg release "${srcrpm_release}" \ + --arg buildtime "${srcrpm_buildtime}" \ + --arg mimetype "${mimetype}" \ + ' + { + "source.artifact.filename": $filename, + "source.artifact.name": $name, + "source.artifact.version": $version, + "source.artifact.epoch": $version, + "source.artifact.release": $release, + "source.artifact.mimetype": $mimetype, + "source.artifact.buildtime": $buildtime + } + ' \ + > "${manifest_dir}/${srcrpm}.json" + ret=$? + if [ $ret -ne 0 ] ; then + return 1 + fi + done fi } @@ -688,7 +777,7 @@ main() { local base_dir="$(pwd)/${ABV_NAME}" # using the bash builtin to parse - while getopts ":hplDc:r:e:o:b:d:" opts; do + while getopts ":hlDi:c:r:e:o:b:d:p:" opts; do case "${opts}" in b) base_dir="${OPTARG}" @@ -699,27 +788,30 @@ main() { e) local extra_src_dir=${OPTARG} ;; - r) - local rpm_dir=${OPTARG} + d) + local drivers=${OPTARG} + ;; + h) + _usage + ;; + i) + local inspect_image_ref=${OPTARG} + ;; + l) + local list_drivers=1 ;; o) local output_dir=${OPTARG} ;; - d) - drivers=${OPTARG} - ;; - l) - list_drivers=1 - ;; p) - push=1 + local push_image_ref${OPTARG} + ;; + r) + local rpm_dir=${OPTARG} ;; D) export DEBUG=1 ;; - h) - _usage - ;; *) _usage ;; @@ -732,43 +824,66 @@ main() { exit 0 fi + # These three variables are slightly special, in that they're globals that + # specific drivers will expect. export CONTEXT_DIR="${CONTEXT_DIR:-$context_dir}" export EXTRA_SRC_DIR="${EXTRA_SRC_DIR:-$extra_src_dir}" export RPM_DIR="${RPM_DIR:-$rpm_dir}" local output_dir="${OUTPUT_DIR:-$output_dir}" - local src_dir="${base_dir}/src" - local work_dir="${base_dir}/work" - export TMPDIR="${work_dir}/tmp" + export TMPDIR="${base_dir}/tmp" if [ -d "${TMPDIR}" ] ; then _rm_rf "${TMPDIR}" fi - mkdir -p "${TMPDIR}" + _mkdir_p "${TMPDIR}" - IMAGE_REF="${1}" - _debug "IMAGE_REF: ${IMAGE_REF}" + # setup rootfs to be inspected (if any) + local rootfs="" + local image_ref="" + local src_dir="" + local work_dir="${base_dir}/work" + if [ -n "${inspect_image_ref}" ] ; then + _debug "Image Reference provided: ${inspect_image_ref}" + _debug "Image Reference base: $(parse_img_base ${inspect_image_ref})" + _debug "Image Reference tag: $(parse_img_tag ${inspect_image_ref})" - IMAGE_REF_BASE="$(parse_img_base ${IMAGE_REF})" - _debug "IMAGE_REF_BASE: ${IMAGE_REF_BASE}" + inspect_image_digest="$(parse_img_digest ${inspect_image_ref})" + # determine missing digest before fetch, so that we fetch the precise image + # including its digest. + if [ -z "${inspect_image_digest}" ] ; then + inspect_image_digest="$(fetch_img_digest $(parse_img_base ${inspect_image_ref}):$(parse_img_tag ${inspect_image_ref}))" + fi + _debug "inspect_image_digest: ${inspect_image_digest}" - IMAGE_TAG="$(parse_img_tag ${IMAGE_REF})" - _debug "IMAGE_TAG: ${IMAGE_TAG}" + local img_layout="" + # if inspect and fetch image, then to an OCI layout dir + if [ ! -d "${work_dir}/layouts/${inspect_image_digest/:/\/}" ] ; then + # we'll store the image to a path based on its digest, that it can be reused + img_layout="$(fetch_img $(parse_img_base ${inspect_image_ref}):$(parse_img_tag ${inspect_image_ref})@${inspect_image_digest} ${work_dir}/layouts/${inspect_image_digest/:/\/} )" + else + img_layout="${work_dir}/layouts/${inspect_image_digest/:/\/}:$(parse_img_tag ${inspect_image_ref})" + fi + _debug "image layout: ${img_layout}" - IMAGE_DIGEST="$(parse_img_digest ${IMAGE_REF})" - # determine missing digest before fetch, so that we fetch the precise image - # including its digest. - if [ -z "${IMAGE_DIGEST}" ] ; then - IMAGE_DIGEST="$(fetch_img_digest ${IMAGE_REF_BASE}:${IMAGE_TAG})" - fi - _debug "IMAGE_DIGEST: ${IMAGE_DIGEST}" + # unpack or reuse fetched image + local unpack_dir="${work_dir}/unpacked/${inspect_image_digest/:/\/}" + if [ -d "${unpack_dir}" ] ; then + _rm_rf "${unpack_dir}" + fi + unpack_img ${img_layout} ${unpack_dir} - # if inspect and fetch image, then to an OCI layout dir - if [ ! -d "${work_dir}/layouts/${IMAGE_DIGEST/:/\/}" ] ; then - # we'll store the image to a path based on its digest, that it can be reused - img_layout="$(fetch_img ${IMAGE_REF_BASE}:${IMAGE_TAG}@${IMAGE_DIGEST} ${work_dir}/layouts/${IMAGE_DIGEST/:/\/} )" + rootfs="${unpack_dir}/rootfs" + image_ref="$(parse_img_base ${inspect_image_ref}):$(parse_img_tag ${inspect_image_ref})@${inspect_image_digest}" + src_dir="${base_dir}/src/${inspect_image_digest/:/\/}" + work_dir="${base_dir}/work/${inspect_image_digest/:/\/}" + _info "inspecting image reference ${image_ref}" else - img_layout="${work_dir}/layouts/${IMAGE_DIGEST/:/\/}:${IMAGE_TAG}" + # if we're not fething an image, then this is basically a nop + rootfs="$(_mktemp_d)" + image_ref="scratch" + src_dir="$(_mktemp_d)" + work_dir="$(_mktemp_d)" fi _debug "image layout: ${img_layout}" @@ -778,9 +893,10 @@ main() { unpack_img ${img_layout} ${unpack_dir} fi _debug "unpacked dir: ${unpack_dir}" + _debug "rootfs dir: ${rootfs}" # clear prior driver's info about source to insert into Source Image - _rm_rf "${work_dir}/driver/${IMAGE_DIGEST/:/\/}" + _rm_rf "${work_dir}/driver" if [ -n "${drivers}" ] ; then # clean up the args passed by the caller ... @@ -788,67 +904,60 @@ main() { else drivers="$(set | grep '^sourcedriver_.* () ' | tr -d ' ()' | tr '\n' ' ')" fi + + # Prep the OCI layout for the source image + local src_img_dir="$(_mktemp_d)" + local src_img_tag="latest-source" #XXX this tag needs to be a reference to the image built from + layout_new "${src_img_dir}" "${src_img_tag}" + # iterate on the drivers #for driver in sourcedriver_rpm_fetch ; do for driver in ${drivers} ; do _info "calling $driver" - mkdir -vp "${src_dir}/${IMAGE_DIGEST/:/\/}/${driver#sourcedriver_*}" - mkdir -vp "${work_dir}/driver/${IMAGE_DIGEST/:/\/}/${driver#sourcedriver_*}" + _mkdir_p "${src_dir}/${driver#sourcedriver_*}" + _mkdir_p "${work_dir}/driver/${driver#sourcedriver_*}" $driver \ - "${IMAGE_REF_BASE}:${IMAGE_TAG}@${IMAGE_DIGEST}" \ - "${unpack_dir}/rootfs" \ - "${src_dir}/${IMAGE_DIGEST/:/\/}/${driver#sourcedriver_*}" \ - "${work_dir}/driver/${IMAGE_DIGEST/:/\/}/${driver#sourcedriver_*}" - if [ $? -ne 0 ] ; then + "${image_ref}" \ + "${rootfs}" \ + "${src_dir}/${driver#sourcedriver_*}" \ + "${work_dir}/driver/${driver#sourcedriver_*}" + local ret=$? + if [ $ret -ne 0 ] ; then _error "$driver failed" fi - # TODO walk the driver output to determine layers to be added - # find "${work_dir}/driver/${IMAGE_DIGEST/:/\/}/${driver#sourcedriver_*}" -type f -name '*.json' + # walk the driver output to determine layers to be added + find "${work_dir}/driver/${driver#sourcedriver_*}" -type f -name '*.json' | while read src_json ; do + local src_name=$(basename "${src_json}" .json) + layout_insert \ + "${src_img_dir}" \ + "${src_dir}/${driver#sourcedriver_*}/${src_name}" \ + "/${driver#sourcedriver_*}/${src_name}" \ + "${src_json}" \ + "${src_img_tag}" + ret=$? + if [ $ret -ne 0 ] ; then + # TODO probably just _error here to exit + _warn "failed to insert layout layer for ${src_name}" + fi + done done + _info "packed 'oci:$src_img_dir:${src_img_tag}'" + # TODO maybe look to a directory like /usr/libexec/BuildSourceImage/drivers/ for drop-ins to run - # TODO commit the image - # This is going to be a hand craft of composing these layers using just bash and jq - -echo "bailing here for now" -return 0 - ## if an output directory is provided then save a copy to it if [ -n "${output_dir}" ] ; then - mkdir -p "${output_dir}" - # XXX WIP - push_img $src_img_dir "oci:$output_dir:$(ref_src_img_tag ${IMAGE_TAG})" + _mkdir_p "${output_dir}" + push_img "oci:$src_img_dir:${src_img_tag}" "oci:$output_dir:$(ref_src_img_tag $(parse_img_tag ${inspect_image_ref}))" fi - if [ -n "${push}" ] ; then - # XXX WIP - push_img $src_dir $IMAGE_REF + if [ -n "${push_image_ref}" ] ; then + # XXX may have to parse this reference to ensure it is valid, and that it has a `-source` tag + push_img "oci:$src_img_dir:${src_img_tag}" "${push_image_ref}" fi - -# -# For each SRC_RPMS used to build the executable image, download the SRC RPM -# and generate a layer in the SRC RPM. -# -pushd ${SRC_RPM_DIR} > /dev/null -export SRC_CTR=$(buildah from scratch) -for i in ${SRC_RPMS}; do - if [ ! -f $i ]; then - RPM=$(echo $i | sed 's/.src.rpm$//g') - dnf download --release $RELEASE --source $RPM || continue # TODO: perhaps log failures somewhere - fi - echo "Adding ${srpm}" - touch --date=@`rpm -q --qf '%{buildtime}' ${srpm}` ${srpm} - buildah add ${SRC_CTR} ${srpm} /RPMS/ - buildah config --created-by "/bin/sh -c #(nop) ADD file:$(sha256sum ${srpm} | cut -f1 -d' ') in /RPMS" ${SRC_CTR} - export IMG=$(buildah commit --omit-timestamp --disable-compression --rm ${SRC_CTR}) - export SRC_CTR=$(buildah from ${IMG}) -done -popd > /dev/null - - } # only exec main if this is being called (this way we can source and test the functions) From adcad3d78a314306ee9a79b3e37c5ee046287190 Mon Sep 17 00:00:00 2001 From: Vincent Batts Date: Thu, 5 Sep 2019 22:02:02 -0400 Subject: [PATCH 06/17] BuildSourceImage: looks like golang ts of RFC 3339 is special Signed-off-by: Vincent Batts --- BuildSourceImage.sh | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/BuildSourceImage.sh b/BuildSourceImage.sh index 416c95b..542cb13 100755 --- a/BuildSourceImage.sh +++ b/BuildSourceImage.sh @@ -58,9 +58,9 @@ _size() { stat -c "%s" "${file}" | tr -d '\n' } -# date timestamp in RFC 3339, to the nanosecond +# date timestamp in RFC 3339, to the nanosecond, but slightly golang style ... _date_ns() { - date --rfc-3339=ns | tr -d '\n' + date --rfc-3339=ns | tr ' ' 'T' | tr -d '\n' } # local `mktemp -d` @@ -947,6 +947,9 @@ main() { # TODO maybe look to a directory like /usr/libexec/BuildSourceImage/drivers/ for drop-ins to run + _info "succesfully packed 'oci:$src_img_dir:${src_img_tag}'" + _debug "$(skopeo inspect oci:$src_img_dir:${src_img_tag})" + ## if an output directory is provided then save a copy to it if [ -n "${output_dir}" ] ; then _mkdir_p "${output_dir}" From 9078c53d4ce79a6b5f6e214e2b89a2270580afcd Mon Sep 17 00:00:00 2001 From: Vincent Batts Date: Thu, 5 Sep 2019 22:09:06 -0400 Subject: [PATCH 07/17] BuildSourceImage: ensure that build from only SRPMS works Signed-off-by: Vincent Batts --- BuildSourceImage.sh | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/BuildSourceImage.sh b/BuildSourceImage.sh index 542cb13..1589b59 100755 --- a/BuildSourceImage.sh +++ b/BuildSourceImage.sh @@ -656,9 +656,11 @@ sourcedriver_rpm_dir() { local manifest_dir="${4}" if [ -n "${RPM_DIR}" ]; then - _debug "$self: writing to $out_dir and $manifest_dir" + _debug "[$self] writing to $out_dir and $manifest_dir" find "${RPM_DIR}" -type f -name '*src.rpm' | while read srcrpm ; do cp "${srcrpm}" "${out_dir}" + srcrpm="$(basename ${srcrpm})" + _debug "[$self] --> ${srcrpm}" local srcrpm_buildtime=$(rpm -qp --qf '%{buildtime}' ${out_dir}/${srcrpm} ) local srcrpm_pkgid=$(rpm -qp --qf '%{pkgid}' ${out_dir}/${srcrpm} ) local srcrpm_name=$(rpm -qp --qf '%{name}' ${out_dir}/${srcrpm} ) From 89bfe5bf20f159e2befeff4c283535ddb4821056 Mon Sep 17 00:00:00 2001 From: Vincent Batts Date: Thu, 5 Sep 2019 22:30:38 -0400 Subject: [PATCH 08/17] BuildSourceImage: basic function of pushing the image afterward Signed-off-by: Vincent Batts --- BuildSourceImage.sh | 22 +++++++++++++++------- 1 file changed, 15 insertions(+), 7 deletions(-) diff --git a/BuildSourceImage.sh b/BuildSourceImage.sh index 1589b59..e303b96 100755 --- a/BuildSourceImage.sh +++ b/BuildSourceImage.sh @@ -333,15 +333,22 @@ unpack_img_umoci() { _debug "unpackging with umoci" # always assume we're not root I reckon umoci unpack --rootless --image "${image_dir}" "${unpack_dir}" >&2 + ret=$? + return $ret } -# TODO this is not worked out yet +# +# copy an image from one location to another +# push_img() { - local ref="${1}" - local path="${2}" + local src="${1}" + local dst="${2}" ## TODO: check for authfile, creds, and whether it's an insecure registry - skopeo copy "oci:${path}:$(ref_src_img_tag ${ref})" "$(ref_prefix ${ref})" + skopeo copy --dest-tls-verify=false "$(ref_prefix ${src})" "$(ref_prefix ${dst})" # XXX for demo only + #skopeo copy "$(ref_prefix ${src})" "$(ref_prefix ${dst})" + ret=$? + return $ret } # @@ -589,7 +596,7 @@ sourcedriver_rpm_fetch() { local rpm=${srcrpm%*.src.rpm} if [ ! -f "${out_dir}/${srcrpm}" ] ; then - _info "--> fetching ${srcrpm}" + _debug "--> fetching ${srcrpm}" dnf download \ --quiet \ --installroot "${rootfs}" \ @@ -603,7 +610,7 @@ sourcedriver_rpm_fetch() { continue fi else - _info "--> using cached ${srcrpm}" + _debug "--> using cached ${srcrpm}" fi # XXX one day, check and confirm with %{sourcepkgid} @@ -806,7 +813,7 @@ main() { local output_dir=${OPTARG} ;; p) - local push_image_ref${OPTARG} + local push_image_ref=${OPTARG} ;; r) local rpm_dir=${OPTARG} @@ -955,6 +962,7 @@ main() { ## if an output directory is provided then save a copy to it if [ -n "${output_dir}" ] ; then _mkdir_p "${output_dir}" + # XXX this $inspect_image_ref currently relies on the user passing in the `-i` flag push_img "oci:$src_img_dir:${src_img_tag}" "oci:$output_dir:$(ref_src_img_tag $(parse_img_tag ${inspect_image_ref}))" fi From b0940d1f8705f999570e0b78065ea154be11d421 Mon Sep 17 00:00:00 2001 From: Vincent Batts Date: Fri, 6 Sep 2019 08:38:31 -0400 Subject: [PATCH 09/17] Dockerfile: trim down to the bash rewrite Signed-off-by: Vincent Batts --- Dockerfile | 11 ++--------- 1 file changed, 2 insertions(+), 9 deletions(-) diff --git a/Dockerfile b/Dockerfile index 3dfcb19..6948a5b 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,13 +1,6 @@ FROM fedora -RUN dnf install -y skopeo golang jq git -RUN dnf install -y make findutils -ENV GOPATH=/usr/share/gocode -ENV GOBIN=/usr/local/bin -RUN git clone https://github.com/openSUSE/umoci $GOPATH/src/github.com/openSUSE/umoci -RUN cd $GOPATH/src/github.com/openSUSE/umoci && \ - make && \ - mv umoci /usr/local/bin +RUN dnf install -y jq skopeo findutils file COPY . /usr/local/bin/ @@ -18,4 +11,4 @@ VOLUME /output ENV SRC_DIR=/src VOLUME /src -#ENTRYPOINT ["/usr/local/bin/BuildSourceImage.sh"] +ENTRYPOINT ["/usr/local/bin/BuildSourceImage.sh"] From a4444e2e9083118ca5ed051d0969dee5f4880d1f Mon Sep 17 00:00:00 2001 From: Vincent Batts Date: Fri, 6 Sep 2019 16:23:02 -0400 Subject: [PATCH 10/17] BuildSourceImage: lint cleanup reported by `shellcheck -a` more info at https://www.shellcheck.net/wiki/ Signed-off-by: Vincent Batts --- BuildSourceImage.sh | 359 ++++++++++++++++++++++++++------------------ 1 file changed, 217 insertions(+), 142 deletions(-) diff --git a/BuildSourceImage.sh b/BuildSourceImage.sh index e303b96..b9f57b7 100755 --- a/BuildSourceImage.sh +++ b/BuildSourceImage.sh @@ -8,14 +8,14 @@ export source_image_suffix="-source" _usage() { - echo "Usage: $(basename $0) [-D] [-b ] [-c ] [-e ] [-r ] [-o ] [-i ] [-p ] [-l] [-d ]" + echo "Usage: $(basename "$0") [-D] [-b ] [-c ] [-e ] [-r ] [-o ] [-i ] [-p ] [-l] [-d ]" echo "" echo -e " -b \tbase path for source image builds" echo -e " -c \tbuild context for the container image. Can be provided via CONTEXT_DIR env variable" echo -e " -e \textra src for the container image. Can be provided via EXTRA_SRC_DIR env variable" echo -e " -r \tdirectory of RPMS to add. Can be provided via RPM_DIR env variable" echo -e " -o \toutput the OCI image to path. Can be provided via OUTPUT_DIR env variable" - echo -e " -d \toutput the OCI image to path. Can be provided via OUTPUT_DIR env variable" + echo -e " -d \tenumerate specific source drivers to run" echo -e " -l\t\tlist the source drivers available" echo -e " -i \timage reference to fetch and inspect its rootfs" echo -e " -p \tpush source image to reference after build" @@ -75,47 +75,47 @@ _mktemp() { # local rm -rf _rm_rf() { - _debug "rm -rf $@" - rm -rf $@ + _debug "rm -rf ${*}" + rm -rf "${@}" } # local mkdir -p _mkdir_p() { if [ -n "${DEBUG}" ] ; then - mkdir -vp $@ + mkdir -vp "${@}" else - mkdir -p $@ + mkdir -p "${@}" fi } # local tar _tar() { if [ -n "${DEBUG}" ] ; then - tar -v $@ + tar -v "${@}" else - tar $@ + tar "${@}" fi } # output things, only when $DEBUG is set _debug() { if [ -n "${DEBUG}" ] ; then - echo "[${ABV_NAME}][DEBUG] ${@}" + echo "[${ABV_NAME}][DEBUG] ${*}" fi } # general echo but with prefix _info() { - echo "[${ABV_NAME}][INFO] ${@}" + echo "[${ABV_NAME}][INFO] ${*}" } _warn() { - echo "[${ABV_NAME}][WARN] ${@}" >&2 + echo "[${ABV_NAME}][WARN] ${*}" >&2 } # general echo but with prefix _error() { - echo "[${ABV_NAME}][ERROR] ${@}" >&2 + echo "[${ABV_NAME}][ERROR] ${*}" >&2 exit 1 } @@ -133,7 +133,7 @@ _error() { parse_img_digest() { local ref="${1}" local digest="" - if [ "$(_count '@' ${ref})" -gt 0 ] ; then + if [ "$(_count '@' "${ref}")" -gt 0 ] ; then digest="${ref##*@}" # the digest after the "@" fi echo -n "${digest}" @@ -144,11 +144,13 @@ parse_img_digest() { # parse_img_base() { local ref="${1%@*}" # just the portion before the digest "@" - local base="${ref}" default to the same - if [ "$(_count ':' $(echo ${ref} | tr '/' '\n' | tail -1 ))" -gt 0 ] ; then + local base="${ref}" # default base is their reference + local last_word="" # splitting up their reference to get the last word/chunk + last_word="$(echo "${ref}" | tr '/' '\n' | tail -1 )" + if [ "$(_count ':' "${last_word}")" -gt 0 ] ; then # which means everything before it is the base image name, **including - # transport (which could have a port delineation), and even a URI - base="$(echo ${ref} | rev | cut -d : -f 2 | rev )" + # transport (which could have a port delineation), and even a URI like network ports. + base="$(echo "${ref}" | rev | cut -d : -f 2 | rev )" fi echo -n "${base}" } @@ -165,9 +167,11 @@ parse_img_tag() { return 0 fi - if [ "$(_count ':' $(echo ${ref} | tr '/' '\n' | tail -1 ))" -gt 0 ] ; then + local last_word="" # splitting up their reference to get the last word/chunk + last_word="$(echo "${ref}" | tr '/' '\n' | tail -1 )" + if [ "$(_count ':' "${last_word}")" -gt 0 ] ; then # if there are colons in the last segment after '/', then get that tag name - tag="$(echo ${ref} | tr '/' '\n' | tail -1 | cut -d : -f 2 )" + tag="${last_word#*:}" # this parameter expansion removes the prefix pattern before the ':' fi echo -n "${tag}" } @@ -177,16 +181,15 @@ parse_img_tag() { # ref_prefix() { local ref="${1}" + local pfxs # get the supported prefixes of the current version of skopeo - IFS=", " - local pfxs=( $(skopeo copy --help | grep -A1 "Supported transports:" | grep -v "Supported transports") ) - unset IFS + mapfile -t pfxs < <(skopeo copy --help | grep -A1 "Supported transports:" | grep -v "Supported transports" | sed 's/, /\n/g') - for pfx in ${pfxs[@]} ; do - if echo ${ref} | grep -q "^${pfx}:" ; then - # break when we match - echo ${ref} + for pfx in "${pfxs[@]}" ; do + if echo "${ref}" | grep -q "^${pfx}:" ; then + # break if we match a known prefix + echo "${ref}" return 0 fi done @@ -201,7 +204,7 @@ ref_prefix() { # ref_src_img_tag() { local ref="${1}" - echo -n "$(parse_img_tag ${ref})${source_image_suffix}" + echo -n "$(parse_img_tag "${ref}")""${source_image_suffix}" } # @@ -209,9 +212,12 @@ ref_src_img_tag() { # fetch_img_digest() { local ref="${1}" + local dgst + local ret + ## TODO: check for authfile, creds, and whether it's an insecure registry - local dgst=$(skopeo inspect "$(ref_prefix ${ref})" | jq .Digest | tr -d \") - local ret=$? + dgst=$(skopeo inspect "$(ref_prefix "${ref}")" | jq .Digest | tr -d \") + ret=$? if [ $ret -ne 0 ] ; then echo "ERROR: check the image reference: ${ref}" >&2 return $ret @@ -231,18 +237,22 @@ fetch_img_digest() { fetch_img() { local ref="${1}" local dst="${2}" + local base + local tag + local dgst + local from _mkdir_p "${dst}" - local base="$(parse_img_base ${ref})" - local tag="$(parse_img_tag ${ref})" - local dgst="$(parse_img_digest ${ref})" - local from="" + base="$(parse_img_base "${ref}")" + tag="$(parse_img_tag "${ref}")" + dgst="$(parse_img_digest "${ref}")" + from="" # skopeo currently only support _either_ tag _or_ digest, so we'll be specific. if [ -n "${dgst}" ] ; then - from="$(ref_prefix ${base})@${dgst}" + from="$(ref_prefix "${base}")@${dgst}" else - from="$(ref_prefix ${base}):${tag}" + from="$(ref_prefix "${base}"):${tag}" fi ## TODO: check for authfile, creds, and whether it's an insecure registry @@ -281,36 +291,37 @@ unpack_img() { unpack_img_bash() { local image_dir="${1}" local unpack_dir="${2}" + local mnfst_dgst + local layer_dgsts # for compat with umoci (which wants the image tag as well) if echo "${image_dir}" | grep -q ":" ; then image_dir="${image_dir%:*}" fi - local mnfst_dgst="$(cat "${image_dir}"/index.json | jq '.manifests[0].digest' | tr -d \" )" + mnfst_dgst="$(jq '.manifests[0].digest' "${image_dir}"/index.json | tr -d \")" # Since we're landing the reference as an OCI layout, this mediaType is fairly predictable # TODO don't always assume +gzip - layer_dgsts="$(cat ${image_dir}/blobs/${mnfst_dgst/:/\/} | \ - jq '.layers[] | select(.mediaType == "application/vnd.oci.image.layer.v1.tar+gzip") | .digest' | tr -d \")" + layer_dgsts="$(jq '.layers[] | select(.mediaType == "application/vnd.oci.image.layer.v1.tar+gzip") | .digest' "${image_dir}"/blobs/"${mnfst_dgst/:/\/}" | tr -d \")" _mkdir_p "${unpack_dir}/rootfs" for dgst in ${layer_dgsts} ; do path="${image_dir}/blobs/${dgst/:/\/}" tmp_file=$(_mktemp) - zcat "${path}" | _tar -t > $tmp_file # TODO cleanup these files + zcat "${path}" | _tar -t > "$tmp_file" # TODO cleanup these files # look for '.wh.' entries. They must be removed from the rootfs # _before_ extracting the archive, then the .wh. entries themselves # need to not remain afterwards - grep '\.wh\.' "${tmp_file}" | while read line ; do + grep '\.wh\.' "${tmp_file}" | while read -r wh_path ; do # if `some/path/.wh.foo` then `rm -rf `${unpack_dir}/some/path/foo` # if `some/path/.wh..wh..opq` then `rm -rf `${unpack_dir}/some/path/*` - if [ "$(basename ${line})" == ".wh..wh..opq" ] ; then - _rm_rf "${unpack_dir}/rootfs/$(dirname ${line})/*" - elif basename "${line}" | grep -qe '^\.wh\.' ; then - name=$(basename "${line}" | sed -e 's/^\.wh\.//') - _rm_rf "${unpack_dir}/rootfs/$(dirname ${line})/${name}" + if [ "$(basename "${wh_path}")" == ".wh..wh..opq" ] ; then + _rm_rf "${unpack_dir}/rootfs/$(dirname "${wh_path}")/*" + elif basename "${wh_path}" | grep -qe '^\.wh\.' ; then + name=$(basename "${wh_path}" | sed -e 's/^\.wh\.//') + _rm_rf "${unpack_dir}/rootfs/$(dirname "${wh_path}")/${name}" fi done @@ -345,8 +356,8 @@ push_img() { local dst="${2}" ## TODO: check for authfile, creds, and whether it's an insecure registry - skopeo copy --dest-tls-verify=false "$(ref_prefix ${src})" "$(ref_prefix ${dst})" # XXX for demo only - #skopeo copy "$(ref_prefix ${src})" "$(ref_prefix ${dst})" + skopeo copy --dest-tls-verify=false "$(ref_prefix "${src}")" "$(ref_prefix "${dst}")" # XXX for demo only + #skopeo copy "$(ref_prefix "${src}")" "$(ref_prefix "${dst}")" ret=$? return $ret } @@ -357,10 +368,14 @@ push_img() { layout_new() { local out_dir="${1}" local image_tag="${2:-latest}" + local config + local mnfst + local config_sum + local mnfst_sum _mkdir_p "${out_dir}/blobs/sha256" echo '{"imageLayoutVersion":"1.0.0"}' > "${out_dir}/oci-layout" - local config=' + config=' { "created": "'$(_date_ns)'", "architecture": "amd64", @@ -372,21 +387,21 @@ layout_new() { } } ' - local config_sum=$(echo "${config}" | jq -c | tr -d '\n' | sha256sum | awk '{ ORS=""; print $1 }') + config_sum=$(echo "${config}" | jq -c | tr -d '\n' | sha256sum | awk '{ ORS=""; print $1 }') echo "${config}" | jq -c | tr -d '\n' > "${out_dir}/blobs/sha256/${config_sum}" - local mnfst=' + mnfst=' { "schemaVersion": 2, "config": { "mediaType": "application/vnd.oci.image.config.v1+json", "digest": "sha256:'"${config_sum}"'", - "size": '"$(_size ${out_dir}/blobs/sha256/${config_sum})"' + "size": '"$(_size "${out_dir}"/blobs/sha256/"${config_sum}")"' }, "layers": [] } ' - local mnfst_sum=$(echo "${mnfst}" | jq -c | tr -d '\n' | sha256sum | awk '{ ORS=""; print $1 }') + mnfst_sum=$(echo "${mnfst}" | jq -c | tr -d '\n' | sha256sum | awk '{ ORS=""; print $1 }') echo "${mnfst}" | jq -c | tr -d '\n' > "${out_dir}/blobs/sha256/${mnfst_sum}" echo ' @@ -396,7 +411,7 @@ layout_new() { { "mediaType": "application/vnd.oci.image.manifest.v1+json", "digest": "sha256:'"${mnfst_sum}"'", - "size": '"$(_size ${out_dir}/blobs/sha256/${mnfst_sum})"', + "size": '"$(_size "${out_dir}"/blobs/sha256/"${mnfst_sum}")"', "annotations": { "org.opencontainers.image.ref.name": "'"${image_tag}"'" } @@ -420,51 +435,67 @@ layout_insert() { local tar_path="${3}" local annotations_file="${4}" local image_tag="${5:-latest}" + local mnfst_list + local mnfst_dgst + local mnfst + local tmpdir + local sum + local tmptar + local tmptar_sum + local tmptar_size + local config_sum + local tmpconfig + local tmpconfig_sum + local tmpconfig_size + local tmpmnfst + local tmpmnfst_sum + local tmpmnfst_size + local tmpmnfst_list - local mnfst_list="${out_dir}/index.json" + mnfst_list="${out_dir}/index.json" # get the digest to the manifest test -f "${mnfst_list}" || return 1 - local mnfst_dgst="$(cat ${mnfst_list} | jq --arg tag "${image_tag}" ' + mnfst_dgst="$(jq --arg tag "${image_tag}" ' .manifests[] | select(.annotations."org.opencontainers.image.ref.name" == $tag ) | .digest - ' | tr -d \" | tr -d '\n' )" - local mnfst="${out_dir}/blobs/${mnfst_dgst/:/\/}" + ' "${mnfst_list}" | tr -d \" | tr -d '\n' )" + mnfst="${out_dir}/blobs/${mnfst_dgst/:/\/}" test -f "${mnfst}" || return 1 # make tar of new object - local tmpdir="$(_mktemp_d)" + tmpdir="$(_mktemp_d)" # TODO account for "artifact_path" being a directory? - local sum="$(sha256sum ${artifact_path} | awk '{ print $1 }')" + sum="$(sha256sum "${artifact_path}" | awk '{ print $1 }')" # making a blob store in the layer _mkdir_p "${tmpdir}/blobs/sha256" cp "${artifact_path}" "${tmpdir}/blobs/sha256/${sum}" - if [ "$(basename ${tar_path})" == "$(basename ${artifact_path})" ] ; then - _mkdir_p "${tmpdir}/$(dirname ${tar_path})" + if [ "$(basename "${tar_path}")" == "$(basename "${artifact_path}")" ] ; then + _mkdir_p "${tmpdir}/$(dirname "${tar_path}")" # TODO this symlink need to be relative path, not to `/blobs/...` ln -s "/blobs/sha256/${sum}" "${tmpdir}/${tar_path}" else _mkdir_p "${tmpdir}/${tar_path}" # TODO this symlink need to be relative path, not to `/blobs/...` - ln -s "/blobs/sha256/${sum}" "${tmpdir}/${tar_path}/$(basename ${artifact_path})" + ln -s "/blobs/sha256/${sum}" "${tmpdir}/${tar_path}/$(basename "${artifact_path}")" fi - local tmptar="$(_mktemp)" + tmptar="$(_mktemp)" # zero all the things for as consistent blobs as possible _tar -C "${tmpdir}" --mtime=@0 --owner=0 --group=0 --mode='a+rw' --no-xattrs --no-selinux --no-acls -cf "${tmptar}" . _rm_rf "${tmpdir}" # checksum tar and move to blobs/sha256/$checksum - local tmptar_sum="$(sha256sum ${tmptar} | awk '{ ORS=""; print $1 }')" - local tmptar_size="$(_size ${tmptar})" + tmptar_sum="$(sha256sum "${tmptar}" | awk '{ ORS=""; print $1 }')" + tmptar_size="$(_size "${tmptar}")" mv "${tmptar}" "${out_dir}/blobs/sha256/${tmptar_sum}" # find and read the prior config, mapped from the manifest - local config_sum="$(jq '.config.digest' "${mnfst}" | tr -d \")" + config_sum="$(jq '.config.digest' "${mnfst}" | tr -d \")" # use `jq` to append to prior config - local tmpconfig="$(_mktemp)" - cat "${out_dir}/blobs/${config_sum/:/\/}" | jq -c \ + tmpconfig="$(_mktemp)" + jq -c \ --arg date "$(_date_ns)" \ --arg tmptar_sum "sha256:${tmptar_sum}" \ --arg comment "#(nop) BuildSourceImage adding artifact: ${sum}" \ @@ -477,22 +508,22 @@ layout_insert() { "created_by": $comment } ] - ' > "${tmpconfig}" + ' "${out_dir}/blobs/${config_sum/:/\/}" > "${tmpconfig}" _rm_rf "${out_dir}/blobs/${config_sum/:/\/}" # rename the config blob to its new checksum - local tmpconfig_sum="$(sha256sum ${tmpconfig} | awk '{ ORS=""; print $1 }')" - local tmpconfig_size="$(_size ${tmpconfig})" + tmpconfig_sum="$(sha256sum "${tmpconfig}" | awk '{ ORS=""; print $1 }')" + tmpconfig_size="$(_size "${tmpconfig}")" mv "${tmpconfig}" "${out_dir}/blobs/sha256/${tmpconfig_sum}" # append layers list in the manifest, and its new config mapping - local tmpmnfst="$(_mktemp)" - cat "${mnfst}" | jq -c \ + tmpmnfst="$(_mktemp)" + jq -c \ --arg tmpconfig_sum "sha256:${tmpconfig_sum}" \ --arg tmpconfig_size "${tmpconfig_size}" \ --arg tmptar_sum "sha256:${tmptar_sum}" \ --arg tmptar_size "${tmptar_size}" \ - --arg artifact "$(basename ${artifact_path})" \ + --arg artifact "$(basename "${artifact_path}")" \ --arg sum "sha256:${sum}" \ --slurpfile annotations_slup "${annotations_file}" \ ' @@ -511,7 +542,7 @@ layout_insert() { "annotations": $annotations_merge } ] - ' > "${tmpmnfst}" + ' "${mnfst}" > "${tmpmnfst}" ret=$? if [ $ret -ne 0 ] ; then return 1 @@ -519,13 +550,13 @@ layout_insert() { _rm_rf "${mnfst}" # rename the manifest blob to its new checksum - local tmpmnfst_sum="$(sha256sum ${tmpmnfst} | awk '{ ORS=""; print $1 }')" - local tmpmnfst_size="$(_size ${tmpmnfst})" + tmpmnfst_sum="$(sha256sum "${tmpmnfst}" | awk '{ ORS=""; print $1 }')" + tmpmnfst_size="$(_size "${tmpmnfst}")" mv "${tmpmnfst}" "${out_dir}/blobs/sha256/${tmpmnfst_sum}" # map the mnfst_list to the new mnfst checksum - local tmpmnfst_list="$(_mktemp)" - cat "${mnfst_list}" | jq -c \ + tmpmnfst_list="$(_mktemp)" + jq -c \ --arg tag "${image_tag}" \ --arg tmpmnfst_sum "sha256:${tmpmnfst_sum}" \ --arg tmpmnfst_size "${tmpmnfst_size}" \ @@ -543,7 +574,7 @@ layout_insert() { } ] as $manifests_new | .manifests = $manifests_reduced + $manifests_new - ' > "${tmpmnfst_list}" + ' "${mnfst_list}" > "${tmpmnfst_list}" ret=$? if [ $ret -ne 0 ] ; then return 1 @@ -584,17 +615,26 @@ sourcedriver_rpm_fetch() { local rootfs="${2}" local out_dir="${3}" local manifest_dir="${4}" + local release + local rpm + local srcrpm_buildtime + local srcrpm_pkgid + local srcrpm_name + local srcrpm_version + local srcrpm_epoch + local srcrpm_release + local mimetype # Get the RELEASEVER from the image - local release=$(rpm -q --queryformat "%{VERSION}\n" --root ${rootfs} -f /etc/os-release) + release=$(rpm -q --queryformat "%{VERSION}\n" --root "${rootfs}" -f /etc/os-release) # From the rootfs of the works image, build out the src rpms to operate over - for srcrpm in $(rpm -qa --root ${rootfs} --queryformat '%{SOURCERPM}\n' | grep -v '^gpg-pubkey' | sort -u) ; do + for srcrpm in $(rpm -qa --root "${rootfs}" --queryformat '%{SOURCERPM}\n' | grep -v '^gpg-pubkey' | sort -u) ; do if [ "${srcrpm}" == "(none)" ] ; then continue fi - local rpm=${srcrpm%*.src.rpm} + rpm=${srcrpm%*.src.rpm} if [ ! -f "${out_dir}/${srcrpm}" ] ; then _debug "--> fetching ${srcrpm}" dnf download \ @@ -613,16 +653,16 @@ sourcedriver_rpm_fetch() { _debug "--> using cached ${srcrpm}" fi - # XXX one day, check and confirm with %{sourcepkgid} + # TODO one day, check and confirm with %{sourcepkgid} # https://bugzilla.redhat.com/show_bug.cgi?id=1741715 - #local rpm_sourcepkgid=$(rpm -q --root ${rootfs} --queryformat '%{sourcepkgid}' "${rpm}") - local srcrpm_buildtime=$(rpm -qp --qf '%{buildtime}' ${out_dir}/${srcrpm} ) - local srcrpm_pkgid=$(rpm -qp --qf '%{pkgid}' ${out_dir}/${srcrpm} ) - local srcrpm_name=$(rpm -qp --qf '%{name}' ${out_dir}/${srcrpm} ) - local srcrpm_version=$(rpm -qp --qf '%{version}' ${out_dir}/${srcrpm} ) - local srcrpm_epoch=$(rpm -qp --qf '%{epoch}' ${out_dir}/${srcrpm} ) - local srcrpm_release=$(rpm -qp --qf '%{release}' ${out_dir}/${srcrpm} ) - local mimetype="$(file --brief --mime-type ${out_dir}/${srcrpm})" + #rpm_sourcepkgid=$(rpm -q --root ${rootfs} --queryformat '%{sourcepkgid}' "${rpm}") + srcrpm_buildtime=$(rpm -qp --qf '%{buildtime}' "${out_dir}"/"${srcrpm}" ) + srcrpm_pkgid=$(rpm -qp --qf '%{pkgid}' "${out_dir}"/"${srcrpm}" ) + srcrpm_name=$(rpm -qp --qf '%{name}' "${out_dir}"/"${srcrpm}" ) + srcrpm_version=$(rpm -qp --qf '%{version}' "${out_dir}"/"${srcrpm}" ) + srcrpm_epoch=$(rpm -qp --qf '%{epoch}' "${out_dir}"/"${srcrpm}" ) + srcrpm_release=$(rpm -qp --qf '%{release}' "${out_dir}"/"${srcrpm}" ) + mimetype="$(file --brief --mime-type "${out_dir}"/"${srcrpm}")" jq \ -n \ --arg filename "${srcrpm}" \ @@ -661,20 +701,27 @@ sourcedriver_rpm_dir() { local rootfs="${2}" local out_dir="${3}" local manifest_dir="${4}" + local srcrpm_buildtime + local srcrpm_pkgid + local srcrpm_name + local srcrpm_version + local srcrpm_epoch + local srcrpm_release + local mimetype if [ -n "${RPM_DIR}" ]; then _debug "[$self] writing to $out_dir and $manifest_dir" - find "${RPM_DIR}" -type f -name '*src.rpm' | while read srcrpm ; do + find "${RPM_DIR}" -type f -name '*src.rpm' | while read -r srcrpm ; do cp "${srcrpm}" "${out_dir}" - srcrpm="$(basename ${srcrpm})" + srcrpm="$(basename "${srcrpm}")" _debug "[$self] --> ${srcrpm}" - local srcrpm_buildtime=$(rpm -qp --qf '%{buildtime}' ${out_dir}/${srcrpm} ) - local srcrpm_pkgid=$(rpm -qp --qf '%{pkgid}' ${out_dir}/${srcrpm} ) - local srcrpm_name=$(rpm -qp --qf '%{name}' ${out_dir}/${srcrpm} ) - local srcrpm_version=$(rpm -qp --qf '%{version}' ${out_dir}/${srcrpm} ) - local srcrpm_epoch=$(rpm -qp --qf '%{epoch}' ${out_dir}/${srcrpm} ) - local srcrpm_release=$(rpm -qp --qf '%{release}' ${out_dir}/${srcrpm} ) - local mimetype="$(file --brief --mime-type ${out_dir}/${srcrpm})" + srcrpm_buildtime=$(rpm -qp --qf '%{buildtime}' "${out_dir}"/"${srcrpm}" ) + srcrpm_pkgid=$(rpm -qp --qf '%{pkgid}' "${out_dir}"/"${srcrpm}" ) + srcrpm_name=$(rpm -qp --qf '%{name}' "${out_dir}"/"${srcrpm}" ) + srcrpm_version=$(rpm -qp --qf '%{version}' "${out_dir}"/"${srcrpm}" ) + srcrpm_epoch=$(rpm -qp --qf '%{epoch}' "${out_dir}"/"${srcrpm}" ) + srcrpm_release=$(rpm -qp --qf '%{release}' "${out_dir}"/"${srcrpm}" ) + mimetype="$(file --brief --mime-type "${out_dir}"/"${srcrpm}")" jq \ -n \ --arg filename "${srcrpm}" \ @@ -684,6 +731,7 @@ sourcedriver_rpm_dir() { --arg release "${srcrpm_release}" \ --arg buildtime "${srcrpm_buildtime}" \ --arg mimetype "${mimetype}" \ + --arg pkgid "${srcrpm_pkgid}" \ ' { "source.artifact.filename": $filename, @@ -692,6 +740,7 @@ sourcedriver_rpm_dir() { "source.artifact.epoch": $version, "source.artifact.release": $release, "source.artifact.mimetype": $mimetype, + "source.artifact.pkgid": $pkgid, "source.artifact.buildtime": $buildtime } ' \ @@ -715,15 +764,18 @@ sourcedriver_context_dir() { local rootfs="${2}" local out_dir="${3}" local manifest_dir="${4}" + local tarname + local mimetype + local source_info if [ -n "${CONTEXT_DIR}" ]; then _debug "$self: writing to $out_dir and $manifest_dir" - local tarname="context.tar" + tarname="context.tar" _tar -C "${CONTEXT_DIR}" \ --mtime=@0 --owner=0 --group=0 --mode='a+rw' --no-xattrs --no-selinux --no-acls \ -cf "${out_dir}/${tarname}" . - local mimetype="$(file --brief --mime-type ${out_dir}/${tarname})" - local source_info="${manifest_dir}/${tarname}.json" + mimetype="$(file --brief --mime-type "${out_dir}"/"${tarname}")" + source_info="${manifest_dir}/${tarname}.json" jq \ -n \ --arg name "${tarname}" \ @@ -753,15 +805,18 @@ sourcedriver_extra_src_dir() { local rootfs="${2}" local out_dir="${3}" local manifest_dir="${4}" + local tarname + local mimetype + local source_info if [ -n "${EXTRA_SRC_DIR}" ]; then _debug "$self: writing to $out_dir and $manifest_dir" - local tarname="extra-src.tar" + tarname="extra-src.tar" _tar -C "${EXTRA_SRC_DIR}" \ --mtime=@0 --owner=0 --group=0 --mode='a+rw' --no-xattrs --no-selinux --no-acls \ -cf "${out_dir}/${tarname}" . - local mimetype="$(file --brief --mime-type ${out_dir}/${tarname})" - local source_info="${manifest_dir}/${tarname}.json" + mimetype="$(file --brief --mime-type "${out_dir}"/"${tarname}")" + source_info="${manifest_dir}/${tarname}.json" jq \ -n \ --arg name "${tarname}" \ @@ -782,9 +837,29 @@ sourcedriver_extra_src_dir() { main() { - _init ${@} + local base_dir + local context_dir + local drivers + local extra_src_dir + local image_ref + local img_layout + local inspect_image_ref + local list_drivers + local output_dir + local push_image_ref + local ret + local rootfs + local rpm_dir + local src_dir + local src_img_dir + local src_img_tag + local src_name + local unpack_dir + local work_dir - local base_dir="$(pwd)/${ABV_NAME}" + _init "${@}" + + base_dir="$(pwd)/${ABV_NAME}" # using the bash builtin to parse while getopts ":hlDi:c:r:e:o:b:d:p:" opts; do case "${opts}" in @@ -792,31 +867,31 @@ main() { base_dir="${OPTARG}" ;; c) - local context_dir=${OPTARG} + context_dir=${OPTARG} ;; e) - local extra_src_dir=${OPTARG} + extra_src_dir=${OPTARG} ;; d) - local drivers=${OPTARG} + drivers=${OPTARG} ;; h) _usage ;; i) - local inspect_image_ref=${OPTARG} + inspect_image_ref=${OPTARG} ;; l) - local list_drivers=1 + list_drivers=1 ;; o) - local output_dir=${OPTARG} + output_dir=${OPTARG} ;; p) - local push_image_ref=${OPTARG} + push_image_ref=${OPTARG} ;; r) - local rpm_dir=${OPTARG} + rpm_dir=${OPTARG} ;; D) export DEBUG=1 @@ -839,7 +914,7 @@ main() { export EXTRA_SRC_DIR="${EXTRA_SRC_DIR:-$extra_src_dir}" export RPM_DIR="${RPM_DIR:-$rpm_dir}" - local output_dir="${OUTPUT_DIR:-$output_dir}" + output_dir="${OUTPUT_DIR:-$output_dir}" export TMPDIR="${base_dir}/tmp" if [ -d "${TMPDIR}" ] ; then @@ -848,42 +923,42 @@ main() { _mkdir_p "${TMPDIR}" # setup rootfs to be inspected (if any) - local rootfs="" - local image_ref="" - local src_dir="" - local work_dir="${base_dir}/work" + rootfs="" + image_ref="" + src_dir="" + work_dir="${base_dir}/work" if [ -n "${inspect_image_ref}" ] ; then _debug "Image Reference provided: ${inspect_image_ref}" - _debug "Image Reference base: $(parse_img_base ${inspect_image_ref})" - _debug "Image Reference tag: $(parse_img_tag ${inspect_image_ref})" + _debug "Image Reference base: $(parse_img_base "${inspect_image_ref}")" + _debug "Image Reference tag: $(parse_img_tag "${inspect_image_ref}")" - inspect_image_digest="$(parse_img_digest ${inspect_image_ref})" + inspect_image_digest="$(parse_img_digest "${inspect_image_ref}")" # determine missing digest before fetch, so that we fetch the precise image # including its digest. if [ -z "${inspect_image_digest}" ] ; then - inspect_image_digest="$(fetch_img_digest $(parse_img_base ${inspect_image_ref}):$(parse_img_tag ${inspect_image_ref}))" + inspect_image_digest="$(fetch_img_digest "$(parse_img_base "${inspect_image_ref}"):$(parse_img_tag "${inspect_image_ref}")")" fi _debug "inspect_image_digest: ${inspect_image_digest}" - local img_layout="" + img_layout="" # if inspect and fetch image, then to an OCI layout dir if [ ! -d "${work_dir}/layouts/${inspect_image_digest/:/\/}" ] ; then # we'll store the image to a path based on its digest, that it can be reused - img_layout="$(fetch_img $(parse_img_base ${inspect_image_ref}):$(parse_img_tag ${inspect_image_ref})@${inspect_image_digest} ${work_dir}/layouts/${inspect_image_digest/:/\/} )" + img_layout="$(fetch_img "$(parse_img_base "${inspect_image_ref}")":"$(parse_img_tag "${inspect_image_ref}")"@"${inspect_image_digest}" "${work_dir}"/layouts/"${inspect_image_digest/:/\/}" )" else - img_layout="${work_dir}/layouts/${inspect_image_digest/:/\/}:$(parse_img_tag ${inspect_image_ref})" + img_layout="${work_dir}/layouts/${inspect_image_digest/:/\/}:$(parse_img_tag "${inspect_image_ref}")" fi _debug "image layout: ${img_layout}" # unpack or reuse fetched image - local unpack_dir="${work_dir}/unpacked/${inspect_image_digest/:/\/}" + unpack_dir="${work_dir}/unpacked/${inspect_image_digest/:/\/}" if [ -d "${unpack_dir}" ] ; then _rm_rf "${unpack_dir}" fi - unpack_img ${img_layout} ${unpack_dir} + unpack_img "${img_layout}" "${unpack_dir}" rootfs="${unpack_dir}/rootfs" - image_ref="$(parse_img_base ${inspect_image_ref}):$(parse_img_tag ${inspect_image_ref})@${inspect_image_digest}" + image_ref="$(parse_img_base "${inspect_image_ref}"):$(parse_img_tag "${inspect_image_ref}")@${inspect_image_digest}" src_dir="${base_dir}/src/${inspect_image_digest/:/\/}" work_dir="${base_dir}/work/${inspect_image_digest/:/\/}" _info "inspecting image reference ${image_ref}" @@ -909,14 +984,14 @@ main() { if [ -n "${drivers}" ] ; then # clean up the args passed by the caller ... - drivers="$(echo ${drivers} | tr ',' ' '| tr '\n' ' ')" + drivers="$(echo "${drivers}" | tr ',' ' '| tr '\n' ' ')" else drivers="$(set | grep '^sourcedriver_.* () ' | tr -d ' ()' | tr '\n' ' ')" fi # Prep the OCI layout for the source image - local src_img_dir="$(_mktemp_d)" - local src_img_tag="latest-source" #XXX this tag needs to be a reference to the image built from + src_img_dir="$(_mktemp_d)" + src_img_tag="latest-source" # XXX this tag needs to be a reference to the image built from layout_new "${src_img_dir}" "${src_img_tag}" # iterate on the drivers @@ -930,14 +1005,14 @@ main() { "${rootfs}" \ "${src_dir}/${driver#sourcedriver_*}" \ "${work_dir}/driver/${driver#sourcedriver_*}" - local ret=$? + ret=$? if [ $ret -ne 0 ] ; then _error "$driver failed" fi # walk the driver output to determine layers to be added - find "${work_dir}/driver/${driver#sourcedriver_*}" -type f -name '*.json' | while read src_json ; do - local src_name=$(basename "${src_json}" .json) + find "${work_dir}/driver/${driver#sourcedriver_*}" -type f -name '*.json' | while read -r src_json ; do + src_name=$(basename "${src_json}" .json) layout_insert \ "${src_img_dir}" \ "${src_dir}/${driver#sourcedriver_*}/${src_name}" \ @@ -956,14 +1031,14 @@ main() { # TODO maybe look to a directory like /usr/libexec/BuildSourceImage/drivers/ for drop-ins to run - _info "succesfully packed 'oci:$src_img_dir:${src_img_tag}'" - _debug "$(skopeo inspect oci:$src_img_dir:${src_img_tag})" + _info "succesfully packed 'oci:${src_img_dir}:${src_img_tag}'" + _debug "$(skopeo inspect oci:"${src_img_dir}":"${src_img_tag}")" ## if an output directory is provided then save a copy to it if [ -n "${output_dir}" ] ; then _mkdir_p "${output_dir}" # XXX this $inspect_image_ref currently relies on the user passing in the `-i` flag - push_img "oci:$src_img_dir:${src_img_tag}" "oci:$output_dir:$(ref_src_img_tag $(parse_img_tag ${inspect_image_ref}))" + push_img "oci:$src_img_dir:${src_img_tag}" "oci:$output_dir:$(ref_src_img_tag "$(parse_img_tag "${inspect_image_ref}")")" fi if [ -n "${push_image_ref}" ] ; then @@ -974,6 +1049,6 @@ main() { } # only exec main if this is being called (this way we can source and test the functions) -_is_sourced || main ${@} +_is_sourced || main "${@}" # vim:set shiftwidth=4 softtabstop=4 expandtab: From 9f68658629f65e12b05e31f1614abfb46093f1cc Mon Sep 17 00:00:00 2001 From: Vincent Batts Date: Fri, 6 Sep 2019 16:42:59 -0400 Subject: [PATCH 11/17] BuildSourceImage: changes for VR's review Signed-off-by: Vincent Batts --- BuildSourceImage.sh | 39 ++++++++++++++++++++++++++++++++++----- 1 file changed, 34 insertions(+), 5 deletions(-) diff --git a/BuildSourceImage.sh b/BuildSourceImage.sh index b9f57b7..b63a6f8 100755 --- a/BuildSourceImage.sh +++ b/BuildSourceImage.sh @@ -13,12 +13,12 @@ _usage() { echo -e " -b \tbase path for source image builds" echo -e " -c \tbuild context for the container image. Can be provided via CONTEXT_DIR env variable" echo -e " -e \textra src for the container image. Can be provided via EXTRA_SRC_DIR env variable" - echo -e " -r \tdirectory of RPMS to add. Can be provided via RPM_DIR env variable" + echo -e " -r \tdirectory of SRPMS to add. Can be provided via RPM_DIR env variable" echo -e " -o \toutput the OCI image to path. Can be provided via OUTPUT_DIR env variable" echo -e " -d \tenumerate specific source drivers to run" echo -e " -l\t\tlist the source drivers available" - echo -e " -i \timage reference to fetch and inspect its rootfs" - echo -e " -p \tpush source image to reference after build" + echo -e " -i \timage reference to fetch and inspect its rootfs to derive sources" + echo -e " -p \tpush source image to specified reference after build" echo -e " -D\t\tdebuging output. Can be set via DEBUG env variable" exit 1 } @@ -270,6 +270,7 @@ fetch_img() { unpack_img() { local image_dir="${1}" local unpack_dir="${2}" + local ret if [ -d "${unpack_dir}" ] ; then _rm_rf "${unpack_dir}" @@ -279,9 +280,17 @@ unpack_img() { if [ -z "$(command -v umoci)" ] ; then # can be done as non-root (even in a non-root container) unpack_img_umoci "${image_dir}" "${unpack_dir}" + ret=$? + if [ ${ret} -ne 0 ] ; then + return ${ret} + fi else # can be done as non-root (even in a non-root container) unpack_img_bash "${image_dir}" "${unpack_dir}" + ret=$? + if [ ${ret} -ne 0 ] ; then + return ${ret} + fi fi } @@ -293,6 +302,9 @@ unpack_img_bash() { local unpack_dir="${2}" local mnfst_dgst local layer_dgsts + local ret + + _debug "unpacking with bash+jq" # for compat with umoci (which wants the image tag as well) if echo "${image_dir}" | grep -q ":" ; then @@ -300,16 +312,24 @@ unpack_img_bash() { fi mnfst_dgst="$(jq '.manifests[0].digest' "${image_dir}"/index.json | tr -d \")" + ret=$? + if [ ${ret} -ne 0 ] ; then + return ${ret} + fi # Since we're landing the reference as an OCI layout, this mediaType is fairly predictable # TODO don't always assume +gzip layer_dgsts="$(jq '.layers[] | select(.mediaType == "application/vnd.oci.image.layer.v1.tar+gzip") | .digest' "${image_dir}"/blobs/"${mnfst_dgst/:/\/}" | tr -d \")" + ret=$? + if [ ${ret} -ne 0 ] ; then + return ${ret} + fi _mkdir_p "${unpack_dir}/rootfs" for dgst in ${layer_dgsts} ; do path="${image_dir}/blobs/${dgst/:/\/}" tmp_file=$(_mktemp) - zcat "${path}" | _tar -t > "$tmp_file" # TODO cleanup these files + zcat "${path}" | _tar -t > "$tmp_file" # look for '.wh.' entries. They must be removed from the rootfs # _before_ extracting the archive, then the .wh. entries themselves @@ -328,6 +348,10 @@ unpack_img_bash() { _info "[unpacking] layer ${dgst}" # unpack layer to rootfs (without whiteouts) zcat "${path}" | _tar --restrict --no-xattr --no-acls --no-selinux --exclude='*.wh.*' -x -C "${unpack_dir}/rootfs" + ret=$? + if [ ${ret} -ne 0 ] ; then + return ${ret} + fi # some of the directories get unpacked as 0555, so removing them gives an EPERM find "${unpack_dir}" -type d -exec chmod 0755 "{}" \; @@ -341,7 +365,7 @@ unpack_img_umoci() { local image_dir="${1}" local unpack_dir="${2}" - _debug "unpackging with umoci" + _debug "unpacking with umoci" # always assume we're not root I reckon umoci unpack --rootless --image "${image_dir}" "${unpack_dir}" >&2 ret=$? @@ -355,6 +379,7 @@ push_img() { local src="${1}" local dst="${2}" + _debug "pushing image ${src} to ${dst}" ## TODO: check for authfile, creds, and whether it's an insecure registry skopeo copy --dest-tls-verify=false "$(ref_prefix "${src}")" "$(ref_prefix "${dst}")" # XXX for demo only #skopeo copy "$(ref_prefix "${src}")" "$(ref_prefix "${dst}")" @@ -956,6 +981,10 @@ main() { _rm_rf "${unpack_dir}" fi unpack_img "${img_layout}" "${unpack_dir}" + ret=$? + if [ ${ret} -ne 0 ] ; then + return ${ret} + fi rootfs="${unpack_dir}/rootfs" image_ref="$(parse_img_base "${inspect_image_ref}"):$(parse_img_tag "${inspect_image_ref}")@${inspect_image_digest}" From e6ca9482c8f4e88bec220b542942602cb723d454 Mon Sep 17 00:00:00 2001 From: Vincent Batts Date: Fri, 6 Sep 2019 17:04:04 -0400 Subject: [PATCH 12/17] BuildSourceImage: flipped condition! Signed-off-by: Vincent Batts --- BuildSourceImage.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/BuildSourceImage.sh b/BuildSourceImage.sh index b63a6f8..77efcd0 100755 --- a/BuildSourceImage.sh +++ b/BuildSourceImage.sh @@ -277,7 +277,7 @@ unpack_img() { fi # TODO perhaps if uid == 0 and podman is present then we can try it? - if [ -z "$(command -v umoci)" ] ; then + if [ -n "$(command -v umoci)" ] ; then # can be done as non-root (even in a non-root container) unpack_img_umoci "${image_dir}" "${unpack_dir}" ret=$? From e99be3b2f84b3f08c1b188260a64199fdbe72aa0 Mon Sep 17 00:00:00 2001 From: Vincent Batts Date: Fri, 6 Sep 2019 17:05:52 -0400 Subject: [PATCH 13/17] Dockerfile: plugins for `dnf download` Signed-off-by: Vincent Batts --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 6948a5b..1b1ab6f 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,6 +1,6 @@ FROM fedora -RUN dnf install -y jq skopeo findutils file +RUN dnf install -y jq skopeo findutils file 'dnf-command(download)' COPY . /usr/local/bin/ From 088e55ee767607f19b49a3fe0e94cbcdaed25644 Mon Sep 17 00:00:00 2001 From: Vincent Batts Date: Mon, 9 Sep 2019 14:32:13 -0400 Subject: [PATCH 14/17] *: more review updates from VR Signed-off-by: Vincent Batts --- BuildSourceImage.sh | 60 ++++++++++++++++++++++++++++----------------- Dockerfile | 2 +- 2 files changed, 39 insertions(+), 23 deletions(-) diff --git a/BuildSourceImage.sh b/BuildSourceImage.sh index 77efcd0..291ac51 100755 --- a/BuildSourceImage.sh +++ b/BuildSourceImage.sh @@ -13,7 +13,7 @@ _usage() { echo -e " -b \tbase path for source image builds" echo -e " -c \tbuild context for the container image. Can be provided via CONTEXT_DIR env variable" echo -e " -e \textra src for the container image. Can be provided via EXTRA_SRC_DIR env variable" - echo -e " -r \tdirectory of SRPMS to add. Can be provided via RPM_DIR env variable" + echo -e " -r \tdirectory of SRPMS to add. Can be provided via SRPM_DIR env variable" echo -e " -o \toutput the OCI image to path. Can be provided via OUTPUT_DIR env variable" echo -e " -d \tenumerate specific source drivers to run" echo -e " -l\t\tlist the source drivers available" @@ -23,9 +23,7 @@ _usage() { exit 1 } -# # sanity checks on startup -# _init() { set -o pipefail @@ -34,11 +32,12 @@ _init() { if [ -z "$(command -v ${cmd})" ] ; then # TODO: maybe this could be individual checks so it can report # where to find the tools - echo "ERROR: please install package to provide '${cmd}'" + _error "please install package to provide '${cmd}'" fi done } +# _is_sourced tests whether this script is being source, or executed directly _is_sourced() { # https://unix.stackexchange.com/a/215279 # thanks @tianon @@ -46,16 +45,14 @@ _is_sourced() { } # count $character $string -_count() { - #expr $(echo "${2}" | tr "${1}" '\n' | wc -l) - 1 +_count_char_in_string() { c="${2//[^${1}]}" echo -n ${#c} } -# size of file in bytes +# size of file/directory in bytes _size() { - local file="${1}" - stat -c "%s" "${file}" | tr -d '\n' + du -b "${1}" | awk '{ ORS=""; print $1 }' } # date timestamp in RFC 3339, to the nanosecond, but slightly golang style ... @@ -65,12 +62,18 @@ _date_ns() { # local `mktemp -d` _mktemp_d() { - mktemp -d "${TMPDIR:-/tmp}/${ABV_NAME}.XXXXXX" + local v + v=$(mktemp -d "${TMPDIR:-/tmp}/${ABV_NAME}.XXXXXX") + _debug "mktemp -d --> ${v}" + echo "${v}" } # local `mktemp` _mktemp() { - mktemp "${TMPDIR:-/tmp}/${ABV_NAME}.XXXXXX" + local v + v=$(mktemp "${TMPDIR:-/tmp}/${ABV_NAME}.XXXXXX") + _debug "mktemp --> ${v}" + echo "${v}" } # local rm -rf @@ -100,7 +103,7 @@ _tar() { # output things, only when $DEBUG is set _debug() { if [ -n "${DEBUG}" ] ; then - echo "[${ABV_NAME}][DEBUG] ${*}" + echo "[${ABV_NAME}][DEBUG] ${*}" >&2 fi } @@ -133,7 +136,7 @@ _error() { parse_img_digest() { local ref="${1}" local digest="" - if [ "$(_count '@' "${ref}")" -gt 0 ] ; then + if [ "$(_count_char_in_string '@' "${ref}")" -gt 0 ] ; then digest="${ref##*@}" # the digest after the "@" fi echo -n "${digest}" @@ -147,7 +150,7 @@ parse_img_base() { local base="${ref}" # default base is their reference local last_word="" # splitting up their reference to get the last word/chunk last_word="$(echo "${ref}" | tr '/' '\n' | tail -1 )" - if [ "$(_count ':' "${last_word}")" -gt 0 ] ; then + if [ "$(_count_char_in_string ':' "${last_word}")" -gt 0 ] ; then # which means everything before it is the base image name, **including # transport (which could have a port delineation), and even a URI like network ports. base="$(echo "${ref}" | rev | cut -d : -f 2 | rev )" @@ -169,7 +172,7 @@ parse_img_tag() { local last_word="" # splitting up their reference to get the last word/chunk last_word="$(echo "${ref}" | tr '/' '\n' | tail -1 )" - if [ "$(_count ':' "${last_word}")" -gt 0 ] ; then + if [ "$(_count_char_in_string ':' "${last_word}")" -gt 0 ] ; then # if there are colons in the last segment after '/', then get that tag name tag="${last_word#*:}" # this parameter expansion removes the prefix pattern before the ':' fi @@ -182,9 +185,14 @@ parse_img_tag() { ref_prefix() { local ref="${1}" local pfxs + local ret # get the supported prefixes of the current version of skopeo mapfile -t pfxs < <(skopeo copy --help | grep -A1 "Supported transports:" | grep -v "Supported transports" | sed 's/, /\n/g') + ret=$? + if [ ${ret} -ne 0 ] ; then + return ${ret} + fi for pfx in "${pfxs[@]}" ; do if echo "${ref}" | grep -q "^${pfx}:" ; then @@ -241,6 +249,7 @@ fetch_img() { local tag local dgst local from + local ret _mkdir_p "${dst}" @@ -261,6 +270,10 @@ fetch_img() { copy \ "${from}" \ "oci:${dst}:${tag}" >&2 + ret=$? + if [ ${ret} -ne 0 ] ; then + return ${ret} + fi echo -n "${dst}:${tag}" } @@ -276,7 +289,6 @@ unpack_img() { _rm_rf "${unpack_dir}" fi - # TODO perhaps if uid == 0 and podman is present then we can try it? if [ -n "$(command -v umoci)" ] ; then # can be done as non-root (even in a non-root container) unpack_img_umoci "${image_dir}" "${unpack_dir}" @@ -734,9 +746,9 @@ sourcedriver_rpm_dir() { local srcrpm_release local mimetype - if [ -n "${RPM_DIR}" ]; then + if [ -n "${SRPM_DIR}" ]; then _debug "[$self] writing to $out_dir and $manifest_dir" - find "${RPM_DIR}" -type f -name '*src.rpm' | while read -r srcrpm ; do + find "${SRPM_DIR}" -type f -name '*src.rpm' | while read -r srcrpm ; do cp "${srcrpm}" "${out_dir}" srcrpm="$(basename "${srcrpm}")" _debug "[$self] --> ${srcrpm}" @@ -874,7 +886,7 @@ main() { local push_image_ref local ret local rootfs - local rpm_dir + local srpm_dir local src_dir local src_img_dir local src_img_tag @@ -916,7 +928,7 @@ main() { push_image_ref=${OPTARG} ;; r) - rpm_dir=${OPTARG} + srpm_dir=${OPTARG} ;; D) export DEBUG=1 @@ -937,7 +949,7 @@ main() { # specific drivers will expect. export CONTEXT_DIR="${CONTEXT_DIR:-$context_dir}" export EXTRA_SRC_DIR="${EXTRA_SRC_DIR:-$extra_src_dir}" - export RPM_DIR="${RPM_DIR:-$rpm_dir}" + export SRPM_DIR="${SRPM_DIR:-$srpm_dir}" output_dir="${OUTPUT_DIR:-$output_dir}" @@ -970,6 +982,10 @@ main() { if [ ! -d "${work_dir}/layouts/${inspect_image_digest/:/\/}" ] ; then # we'll store the image to a path based on its digest, that it can be reused img_layout="$(fetch_img "$(parse_img_base "${inspect_image_ref}")":"$(parse_img_tag "${inspect_image_ref}")"@"${inspect_image_digest}" "${work_dir}"/layouts/"${inspect_image_digest/:/\/}" )" + ret=$? + if [ ${ret} -ne 0 ] ; then + _error "failed to copy image: $(parse_img_base "${inspect_image_ref}"):$(parse_img_tag "${inspect_image_ref}")@${inspect_image_digest}" + fi else img_layout="${work_dir}/layouts/${inspect_image_digest/:/\/}:$(parse_img_tag "${inspect_image_ref}")" fi @@ -1003,7 +1019,7 @@ main() { # setup rootfs, from that OCI layout local unpack_dir="${work_dir}/unpacked/${IMAGE_DIGEST/:/\/}" if [ ! -d "${unpack_dir}" ] ; then - unpack_img ${img_layout} ${unpack_dir} + unpack_img "${img_layout}" "${unpack_dir}" fi _debug "unpacked dir: ${unpack_dir}" _debug "rootfs dir: ${rootfs}" diff --git a/Dockerfile b/Dockerfile index 1b1ab6f..94cbed7 100644 --- a/Dockerfile +++ b/Dockerfile @@ -2,7 +2,7 @@ FROM fedora RUN dnf install -y jq skopeo findutils file 'dnf-command(download)' -COPY . /usr/local/bin/ +COPY ./BuildSourceImage.sh /usr/local/bin/BuildSourceImage.sh RUN mkdir -p /output ENV OUTPUT_DIR=/output From 433272896fc25b1746760536d21baee4c60890b3 Mon Sep 17 00:00:00 2001 From: Vincent Batts Date: Tue, 10 Sep 2019 09:10:56 -0400 Subject: [PATCH 15/17] BuildSourceImage: switch to -s for SRPMs Signed-off-by: Vincent Batts --- BuildSourceImage.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/BuildSourceImage.sh b/BuildSourceImage.sh index 291ac51..1f58520 100755 --- a/BuildSourceImage.sh +++ b/BuildSourceImage.sh @@ -13,7 +13,7 @@ _usage() { echo -e " -b \tbase path for source image builds" echo -e " -c \tbuild context for the container image. Can be provided via CONTEXT_DIR env variable" echo -e " -e \textra src for the container image. Can be provided via EXTRA_SRC_DIR env variable" - echo -e " -r \tdirectory of SRPMS to add. Can be provided via SRPM_DIR env variable" + echo -e " -s \tdirectory of SRPMS to add. Can be provided via SRPM_DIR env variable" echo -e " -o \toutput the OCI image to path. Can be provided via OUTPUT_DIR env variable" echo -e " -d \tenumerate specific source drivers to run" echo -e " -l\t\tlist the source drivers available" @@ -898,7 +898,7 @@ main() { base_dir="$(pwd)/${ABV_NAME}" # using the bash builtin to parse - while getopts ":hlDi:c:r:e:o:b:d:p:" opts; do + while getopts ":hlDi:c:s:e:o:b:d:p:" opts; do case "${opts}" in b) base_dir="${OPTARG}" @@ -927,7 +927,7 @@ main() { p) push_image_ref=${OPTARG} ;; - r) + s) srpm_dir=${OPTARG} ;; D) From 84b889e1ad7d60511d3632aeddec4b1dd56d720a Mon Sep 17 00:00:00 2001 From: Vincent Batts Date: Tue, 10 Sep 2019 09:17:24 -0400 Subject: [PATCH 16/17] BuildSourceImage: version flag Signed-off-by: Vincent Batts --- BuildSourceImage.sh | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/BuildSourceImage.sh b/BuildSourceImage.sh index 1f58520..48e76ac 100755 --- a/BuildSourceImage.sh +++ b/BuildSourceImage.sh @@ -6,8 +6,14 @@ export ABV_NAME="SrcImg" # TODO maybe a flag for this? export source_image_suffix="-source" +# output version string +_version() { + echo "$(basename "${0}") version 0.1" +} +# output the cli usage and exit _usage() { + _version echo "Usage: $(basename "$0") [-D] [-b ] [-c ] [-e ] [-r ] [-o ] [-i ] [-p ] [-l] [-d ]" echo "" echo -e " -b \tbase path for source image builds" @@ -20,6 +26,8 @@ _usage() { echo -e " -i \timage reference to fetch and inspect its rootfs to derive sources" echo -e " -p \tpush source image to specified reference after build" echo -e " -D\t\tdebuging output. Can be set via DEBUG env variable" + echo -e " -h\t\tthis usage information" + echo -e " -v\t\tversion" exit 1 } @@ -898,7 +906,7 @@ main() { base_dir="$(pwd)/${ABV_NAME}" # using the bash builtin to parse - while getopts ":hlDi:c:s:e:o:b:d:p:" opts; do + while getopts ":hlvDi:c:s:e:o:b:d:p:" opts; do case "${opts}" in b) base_dir="${OPTARG}" @@ -930,6 +938,10 @@ main() { s) srpm_dir=${OPTARG} ;; + v) + _version + exit 0 + ;; D) export DEBUG=1 ;; From 9fabac2890a69194f186e15399ae66df24ca47f7 Mon Sep 17 00:00:00 2001 From: Vincent Batts Date: Tue, 10 Sep 2019 09:35:00 -0400 Subject: [PATCH 17/17] BuildSourceImage: bail with information, if not inputs are provided Signed-off-by: Vincent Batts --- BuildSourceImage.sh | 51 +++++++++++++++++++++++++-------------------- 1 file changed, 28 insertions(+), 23 deletions(-) diff --git a/BuildSourceImage.sh b/BuildSourceImage.sh index 48e76ac..6992c64 100755 --- a/BuildSourceImage.sh +++ b/BuildSourceImage.sh @@ -883,18 +883,18 @@ sourcedriver_extra_src_dir() { main() { local base_dir - local context_dir + local input_context_dir + local input_extra_src_dir + local input_inspect_image_ref + local input_srpm_dir local drivers - local extra_src_dir local image_ref local img_layout - local inspect_image_ref local list_drivers local output_dir local push_image_ref local ret local rootfs - local srpm_dir local src_dir local src_img_dir local src_img_tag @@ -912,10 +912,10 @@ main() { base_dir="${OPTARG}" ;; c) - context_dir=${OPTARG} + input_context_dir=${OPTARG} ;; e) - extra_src_dir=${OPTARG} + input_extra_src_dir=${OPTARG} ;; d) drivers=${OPTARG} @@ -924,7 +924,7 @@ main() { _usage ;; i) - inspect_image_ref=${OPTARG} + input_inspect_image_ref=${OPTARG} ;; l) list_drivers=1 @@ -936,7 +936,7 @@ main() { push_image_ref=${OPTARG} ;; s) - srpm_dir=${OPTARG} + input_srpm_dir=${OPTARG} ;; v) _version @@ -957,11 +957,16 @@ main() { exit 0 fi + # "local" variables are not set in `env`, but are seen in `set` + if [ "$(set | grep -c '^input_')" -eq 0 ] ; then + _error "provide an input (example: $(basename "${0}") -i docker.io/centos -e ./my-sources/ )" + fi + # These three variables are slightly special, in that they're globals that # specific drivers will expect. - export CONTEXT_DIR="${CONTEXT_DIR:-$context_dir}" - export EXTRA_SRC_DIR="${EXTRA_SRC_DIR:-$extra_src_dir}" - export SRPM_DIR="${SRPM_DIR:-$srpm_dir}" + export CONTEXT_DIR="${CONTEXT_DIR:-$input_context_dir}" + export EXTRA_SRC_DIR="${EXTRA_SRC_DIR:-$input_extra_src_dir}" + export SRPM_DIR="${SRPM_DIR:-$input_srpm_dir}" output_dir="${OUTPUT_DIR:-$output_dir}" @@ -976,16 +981,16 @@ main() { image_ref="" src_dir="" work_dir="${base_dir}/work" - if [ -n "${inspect_image_ref}" ] ; then - _debug "Image Reference provided: ${inspect_image_ref}" - _debug "Image Reference base: $(parse_img_base "${inspect_image_ref}")" - _debug "Image Reference tag: $(parse_img_tag "${inspect_image_ref}")" + if [ -n "${input_inspect_image_ref}" ] ; then + _debug "Image Reference provided: ${input_inspect_image_ref}" + _debug "Image Reference base: $(parse_img_base "${input_inspect_image_ref}")" + _debug "Image Reference tag: $(parse_img_tag "${input_inspect_image_ref}")" - inspect_image_digest="$(parse_img_digest "${inspect_image_ref}")" + inspect_image_digest="$(parse_img_digest "${input_inspect_image_ref}")" # determine missing digest before fetch, so that we fetch the precise image # including its digest. if [ -z "${inspect_image_digest}" ] ; then - inspect_image_digest="$(fetch_img_digest "$(parse_img_base "${inspect_image_ref}"):$(parse_img_tag "${inspect_image_ref}")")" + inspect_image_digest="$(fetch_img_digest "$(parse_img_base "${input_inspect_image_ref}"):$(parse_img_tag "${input_inspect_image_ref}")")" fi _debug "inspect_image_digest: ${inspect_image_digest}" @@ -993,13 +998,13 @@ main() { # if inspect and fetch image, then to an OCI layout dir if [ ! -d "${work_dir}/layouts/${inspect_image_digest/:/\/}" ] ; then # we'll store the image to a path based on its digest, that it can be reused - img_layout="$(fetch_img "$(parse_img_base "${inspect_image_ref}")":"$(parse_img_tag "${inspect_image_ref}")"@"${inspect_image_digest}" "${work_dir}"/layouts/"${inspect_image_digest/:/\/}" )" + img_layout="$(fetch_img "$(parse_img_base "${input_inspect_image_ref}")":"$(parse_img_tag "${input_inspect_image_ref}")"@"${inspect_image_digest}" "${work_dir}"/layouts/"${inspect_image_digest/:/\/}" )" ret=$? if [ ${ret} -ne 0 ] ; then - _error "failed to copy image: $(parse_img_base "${inspect_image_ref}"):$(parse_img_tag "${inspect_image_ref}")@${inspect_image_digest}" + _error "failed to copy image: $(parse_img_base "${input_inspect_image_ref}"):$(parse_img_tag "${input_inspect_image_ref}")@${inspect_image_digest}" fi else - img_layout="${work_dir}/layouts/${inspect_image_digest/:/\/}:$(parse_img_tag "${inspect_image_ref}")" + img_layout="${work_dir}/layouts/${inspect_image_digest/:/\/}:$(parse_img_tag "${input_inspect_image_ref}")" fi _debug "image layout: ${img_layout}" @@ -1015,7 +1020,7 @@ main() { fi rootfs="${unpack_dir}/rootfs" - image_ref="$(parse_img_base "${inspect_image_ref}"):$(parse_img_tag "${inspect_image_ref}")@${inspect_image_digest}" + image_ref="$(parse_img_base "${input_inspect_image_ref}"):$(parse_img_tag "${input_inspect_image_ref}")@${inspect_image_digest}" src_dir="${base_dir}/src/${inspect_image_digest/:/\/}" work_dir="${base_dir}/work/${inspect_image_digest/:/\/}" _info "inspecting image reference ${image_ref}" @@ -1094,8 +1099,8 @@ main() { ## if an output directory is provided then save a copy to it if [ -n "${output_dir}" ] ; then _mkdir_p "${output_dir}" - # XXX this $inspect_image_ref currently relies on the user passing in the `-i` flag - push_img "oci:$src_img_dir:${src_img_tag}" "oci:$output_dir:$(ref_src_img_tag "$(parse_img_tag "${inspect_image_ref}")")" + # XXX this $input_inspect_image_ref currently relies on the user passing in the `-i` flag + push_img "oci:$src_img_dir:${src_img_tag}" "oci:$output_dir:$(ref_src_img_tag "$(parse_img_tag "${input_inspect_image_ref}")")" fi if [ -n "${push_image_ref}" ] ; then