From d5b5028cb93fbf51342e1e1620a05492864950d1 Mon Sep 17 00:00:00 2001 From: umohnani8 Date: Fri, 22 Sep 2017 11:10:15 -0400 Subject: [PATCH] Add secrets patch to crio Allows the user to define secret paths in /etc/containers/mounts.conf These are then volume mounted into the container Signed-off-by: umohnani8 --- cmd/crio/config.go | 3 + cmd/crio/main.go | 8 ++ libkpod/config.go | 8 ++ server/container_create.go | 25 +++++ server/secrets.go | 186 +++++++++++++++++++++++++++++++++++++ test/crio_secrets.bats | 44 +++++++++ test/helpers.bash | 9 +- 7 files changed, 282 insertions(+), 1 deletion(-) create mode 100644 server/secrets.go create mode 100644 test/crio_secrets.bats diff --git a/cmd/crio/config.go b/cmd/crio/config.go index 855bd466..af1af302 100644 --- a/cmd/crio/config.go +++ b/cmd/crio/config.go @@ -108,6 +108,9 @@ cgroup_manager = "{{ .CgroupManager }}" # hooks_dir_path is the oci hooks directory for automatically executed hooks hooks_dir_path = "{{ .HooksDirPath }}" +# default_mounts_path is the secrets mounts file path +default_mounts_path = "{{ .DefaultMountsPath }}" + # pids_limit is the number of processes allowed in a container pids_limit = {{ .PidsLimit }} diff --git a/cmd/crio/main.go b/cmd/crio/main.go index 93044e88..1059d60f 100644 --- a/cmd/crio/main.go +++ b/cmd/crio/main.go @@ -127,6 +127,9 @@ func mergeConfig(config *server.Config, ctx *cli.Context) error { if ctx.GlobalIsSet("hooks-dir-path") { config.HooksDirPath = ctx.GlobalString("hooks-dir-path") } + if ctx.GlobalIsSet("default-mounts-path") { + config.DefaultMountsPath = ctx.GlobalString("default-mounts-path") + } if ctx.GlobalIsSet("pids-limit") { config.PidsLimit = ctx.GlobalInt64("pids-limit") } @@ -322,6 +325,11 @@ func main() { Value: libkpod.DefaultHooksDirPath, Hidden: true, }, + cli.StringFlag{ + Name: "default-mounts-path", + Usage: "set the default mounts file path", + Hidden: true, + }, cli.BoolFlag{ Name: "profile", Usage: "enable pprof remote profiler on localhost:6060", diff --git a/libkpod/config.go b/libkpod/config.go index 123bece8..1fd86a3b 100644 --- a/libkpod/config.go +++ b/libkpod/config.go @@ -24,6 +24,10 @@ const ( cgroupManager = oci.CgroupfsCgroupsManager lockPath = "/run/crio.lock" containerExitsDir = oci.ContainerExitsDir + // DefaultMountsFile holds the default mount paths in the form "host:container" + DefaultMountsFile = "/usr/share/containers/mounts.conf" + // OverrideMountsFile holds the override mount paths in the form "host:container" + OverrideMountsFile = "/etc/containers/mounts.conf" ) // Config represents the entire set of configuration values that can be set for @@ -145,6 +149,9 @@ type RuntimeConfig struct { // HooksDirPath location of oci hooks config files HooksDirPath string `toml:"hooks_dir_path"` + // DefaultMountsPath location of the default mounts file + DefaultMountsPath string `toml:"default_mounts_path"` + // Hooks List of hooks to run with container Hooks map[string]HookParams @@ -288,6 +295,7 @@ func DefaultConfig() *Config { ContainerExitsDir: containerExitsDir, HooksDirPath: DefaultHooksDirPath, LogSizeMax: DefaultLogSizeMax, + DefaultMountsPath: DefaultMountsFile, }, ImageConfig: ImageConfig{ DefaultTransport: defaultTransport, diff --git a/server/container_create.go b/server/container_create.go index 6d93408c..1e036554 100644 --- a/server/container_create.go +++ b/server/container_create.go @@ -384,6 +384,27 @@ func ensureSaneLogPath(logPath string) error { return nil } +// addSecretsBindMounts mounts user defined secrets to the container +func addSecretsBindMounts(mountLabel, ctrRunDir, configDefaultMountsPath string, specgen generate.Generator) error { + mountPaths := []string{libkpod.OverrideMountsFile, libkpod.DefaultMountsFile} + // configDefaultMountsPath is used to override the mount file path for testing purposes only when set in the runtime config + if configDefaultMountsPath != "" { + mountPaths = []string{configDefaultMountsPath} + } + for _, path := range mountPaths { + containerMounts := specgen.Spec().Mounts + mounts, err := secretMounts(mountLabel, path, ctrRunDir, containerMounts) + if err != nil { + return err + } + for _, m := range mounts { + specgen.AddBindMount(m.Source, m.Destination, nil) + + } + } + return nil +} + // CreateContainer creates a new container in specified PodSandbox func (s *Server) CreateContainer(ctx context.Context, req *pb.CreateContainerRequest) (res *pb.CreateContainerResponse, err error) { logrus.Debugf("CreateContainerRequest %+v", req) @@ -911,6 +932,10 @@ func (s *Server) createSandboxContainer(ctx context.Context, containerID string, return nil, err } + if err = addSecretsBindMounts(mountLabel, containerInfo.RunDir, s.config.DefaultMountsPath, specgen); err != nil { + return nil, fmt.Errorf("failed to mount secrets: %v", err) + } + mountPoint, err := s.StorageRuntimeServer().StartContainer(containerID) if err != nil { return nil, fmt.Errorf("failed to mount container %s(%s): %v", containerName, containerID, err) diff --git a/server/secrets.go b/server/secrets.go new file mode 100644 index 00000000..34236c1f --- /dev/null +++ b/server/secrets.go @@ -0,0 +1,186 @@ +package server + +import ( + "bufio" + "fmt" + "io/ioutil" + "os" + "path/filepath" + "strings" + + rspec "github.com/opencontainers/runtime-spec/specs-go" + "github.com/opencontainers/selinux/go-selinux/label" + "github.com/pkg/errors" + "github.com/sirupsen/logrus" +) + +// SecretData info +type SecretData struct { + Name string + Data []byte +} + +// SaveTo saves secret data to given directory +func (s SecretData) SaveTo(dir string) error { + path := filepath.Join(dir, s.Name) + if err := os.MkdirAll(filepath.Dir(path), 0700); err != nil && !os.IsExist(err) { + return err + } + return ioutil.WriteFile(path, s.Data, 0700) +} + +// readMountFile returns a list of the host:container paths +func readMountFile(mountFilePath string) ([]string, error) { + var mountPaths []string + file, err := os.Open(mountFilePath) + if err != nil { + logrus.Warnf("file doesn't exist %q", mountFilePath) + return nil, nil + } + defer file.Close() + + scanner := bufio.NewScanner(file) + scanner.Split(bufio.ScanLines) + for scanner.Scan() { + mountPaths = append(mountPaths, scanner.Text()) + } + + return mountPaths, nil +} + +func readAll(root, prefix string) ([]SecretData, error) { + path := filepath.Join(root, prefix) + + data := []SecretData{} + + files, err := ioutil.ReadDir(path) + if err != nil { + if os.IsNotExist(err) { + return data, nil + } + + return nil, err + } + + for _, f := range files { + fileData, err := readFile(root, filepath.Join(prefix, f.Name())) + if err != nil { + // If the file did not exist, might be a dangling symlink + // Ignore the error + if os.IsNotExist(err) { + continue + } + return nil, err + } + data = append(data, fileData...) + } + + return data, nil +} + +func readFile(root, name string) ([]SecretData, error) { + path := filepath.Join(root, name) + + s, err := os.Stat(path) + if err != nil { + return nil, err + } + + if s.IsDir() { + dirData, err := readAll(root, name) + if err != nil { + return nil, err + } + return dirData, nil + } + bytes, err := ioutil.ReadFile(path) + if err != nil { + return nil, err + } + return []SecretData{{Name: name, Data: bytes}}, nil +} + +// getHostAndCtrDir separates the host:container paths +func getMountsMap(path string) (string, string, error) { + arr := strings.SplitN(path, ":", 2) + if len(arr) == 2 { + return arr[0], arr[1], nil + } + return "", "", errors.Errorf("unable to get host and container dir") +} + +func getHostSecretData(hostDir string) ([]SecretData, error) { + var allSecrets []SecretData + hostSecrets, err := readAll(hostDir, "") + if err != nil { + return nil, errors.Wrapf(err, "failed to read secrets from %q", hostDir) + } + return append(allSecrets, hostSecrets...), nil +} + +// secretMount copies the contents of host directory to container directory +// and returns a list of mounts +func secretMounts(mountLabel, mountFilePath, containerWorkingDir string, runtimeMounts []rspec.Mount) ([]rspec.Mount, error) { + var mounts []rspec.Mount + mountPaths, err := readMountFile(mountFilePath) + if err != nil { + return nil, err + } + for _, path := range mountPaths { + hostDir, ctrDir, err := getMountsMap(path) + if err != nil { + return nil, err + } + // skip if the hostDir path doesn't exist + if _, err := os.Stat(hostDir); os.IsNotExist(err) { + logrus.Warnf("%q doesn't exist, skipping", hostDir) + continue + } + + ctrDir = filepath.Join(containerWorkingDir, ctrDir) + // skip if ctrDir has already been mounted by caller + if isAlreadyMounted(runtimeMounts, ctrDir) { + logrus.Warnf("%q has already been mounted; cannot override mount", ctrDir) + continue + } + + if err := os.RemoveAll(ctrDir); err != nil { + return nil, fmt.Errorf("remove container directory failed: %v", err) + } + + if err := os.MkdirAll(ctrDir, 0755); err != nil { + return nil, fmt.Errorf("making container directory failed: %v", err) + } + + hostDir, err = resolveSymbolicLink(hostDir) + if err != nil { + return nil, err + } + + data, err := getHostSecretData(hostDir) + if err != nil { + return nil, errors.Wrapf(err, "getting host secret data failed") + } + for _, s := range data { + s.SaveTo(ctrDir) + } + label.Relabel(ctrDir, mountLabel, false) + + m := rspec.Mount{ + Source: hostDir, + Destination: ctrDir, + } + + mounts = append(mounts, m) + } + return mounts, nil +} + +func isAlreadyMounted(mounts []rspec.Mount, mountPath string) bool { + for _, mount := range mounts { + if mount.Destination == mountPath { + return true + } + } + return false +} diff --git a/test/crio_secrets.bats b/test/crio_secrets.bats new file mode 100644 index 00000000..83945aea --- /dev/null +++ b/test/crio_secrets.bats @@ -0,0 +1,44 @@ +#!/usr/bin/env bats + +load helpers + +IMAGE="redis:alpine" + +function teardown() { + cleanup_test +} + +function setup() { + MOUNT_PATH="$TESTDIR/secrets" + mkdir ${MOUNT_PATH} + MOUNT_FILE="${MOUNT_PATH}/test.txt" + touch ${MOUNT_FILE} + echo "Testing secrets mounts!" > ${MOUNT_FILE} + + echo "${MOUNT_PATH}:/container/path1" > ${DEFAULT_MOUNTS_FILE} +} + +@test "bind secrets mounts to container" { + start_crio + run crioctl pod run --config "$TESTDATA"/sandbox_config.json + echo "$output" + [ "$status" -eq 0 ] + pod_id="$output" + run crioctl image pull "$IMAGE" + [ "$status" -eq 0 ] + run crioctl ctr create --config "$TESTDATA"/container_redis.json --pod "$pod_id" + echo "$output" + [ "$status" -eq 0 ] + ctr_id="$output" + run crioctl ctr execsync --id "$ctr_id" mount + echo "$output" + [ "$status" -eq 0 ] + mount_info="$output" + grep $ctr_id/userdata/container/path1 <<< "$mount_info" + echo "$output" + [ "$status" -eq 0 ] + rm -rf MOUNT_PATH + cleanup_ctrs + cleanup_pods + stop_crio +} diff --git a/test/helpers.bash b/test/helpers.bash index ac30f22d..8d38f1fd 100644 --- a/test/helpers.bash +++ b/test/helpers.bash @@ -69,6 +69,13 @@ HOOKSDIR=$TESTDIR/hooks mkdir ${HOOKSDIR} HOOKS_OPTS="--hooks-dir-path=$HOOKSDIR" +# Setup default secrets mounts file +MOUNTS_DIR="$TESTDIR/containers" +mkdir ${MOUNTS_DIR} +DEFAULT_MOUNTS_FILE="${MOUNTS_DIR}/mounts.conf" +touch ${DEFAULT_MOUNTS_FILE} +DEFAULT_MOUNTS_OPTS="--default-mounts-path=$DEFAULT_MOUNTS_FILE" + # We may need to set some default storage options. case "$(stat -f -c %T ${TESTDIR})" in aufs) @@ -235,7 +242,7 @@ function start_crio() { "$COPYIMG_BINARY" --root "$TESTDIR/crio" $STORAGE_OPTS --runroot "$TESTDIR/crio-run" --image-name=mrunalp/image-volume-test --import-from=dir:"$ARTIFACTS_PATH"/image-volume-test-image --add-name=docker.io/library/mrunalp/image-volume-test --signature-policy="$INTEGRATION_ROOT"/policy.json "$COPYIMG_BINARY" --root "$TESTDIR/crio" $STORAGE_OPTS --runroot "$TESTDIR/crio-run" --image-name=busybox:latest --import-from=dir:"$ARTIFACTS_PATH"/busybox-image --add-name=docker.io/library/busybox:latest --signature-policy="$INTEGRATION_ROOT"/policy.json "$COPYIMG_BINARY" --root "$TESTDIR/crio" $STORAGE_OPTS --runroot "$TESTDIR/crio-run" --image-name=runcom/stderr-test:latest --import-from=dir:"$ARTIFACTS_PATH"/stderr-test --add-name=docker.io/runcom/stderr-test:latest --signature-policy="$INTEGRATION_ROOT"/policy.json - "$CRIO_BINARY" ${HOOKS_OPTS} --conmon "$CONMON_BINARY" --listen "$CRIO_SOCKET" --cgroup-manager "$CGROUP_MANAGER" --registry "docker.io" --runtime "$RUNTIME_BINARY" --root "$TESTDIR/crio" --runroot "$TESTDIR/crio-run" $STORAGE_OPTS --seccomp-profile "$seccomp" --apparmor-profile "$apparmor" --cni-config-dir "$CRIO_CNI_CONFIG" --cni-plugin-dir "$CRIO_CNI_PLUGIN" --signature-policy "$INTEGRATION_ROOT"/policy.json --image-volumes "$IMAGE_VOLUMES" --pids-limit "$PIDS_LIMIT" --log-size-max "$LOG_SIZE_MAX_LIMIT" --config /dev/null config >$CRIO_CONFIG + "$CRIO_BINARY" ${DEFAULT_MOUNTS_OPTS} ${HOOKS_OPTS} --conmon "$CONMON_BINARY" --listen "$CRIO_SOCKET" --cgroup-manager "$CGROUP_MANAGER" --registry "docker.io" --runtime "$RUNTIME_BINARY" --root "$TESTDIR/crio" --runroot "$TESTDIR/crio-run" $STORAGE_OPTS --seccomp-profile "$seccomp" --apparmor-profile "$apparmor" --cni-config-dir "$CRIO_CNI_CONFIG" --signature-policy "$INTEGRATION_ROOT"/policy.json --image-volumes "$IMAGE_VOLUMES" --pids-limit "$PIDS_LIMIT" --log-size-max "$LOG_SIZE_MAX_LIMIT" --config /dev/null config >$CRIO_CONFIG # Prepare the CNI configuration files, we're running with non host networking by default if [[ -n "$4" ]]; then