From dffe302f55b19fc312e2a8307df9808becb8c781 Mon Sep 17 00:00:00 2001 From: Matthew Heon Date: Sat, 18 Mar 2017 21:19:49 -0400 Subject: [PATCH 1/7] *WIP* Initial Implementation of kpod launch Signed-off-by: Matthew Heon --- cmd/kpod/launch.go | 544 ++++++++++++++++++++++++++++++++++++++++++++- cmd/kpod/main.go | 15 ++ 2 files changed, 556 insertions(+), 3 deletions(-) diff --git a/cmd/kpod/launch.go b/cmd/kpod/launch.go index 51cf0191..2be34b1f 100644 --- a/cmd/kpod/launch.go +++ b/cmd/kpod/launch.go @@ -1,16 +1,554 @@ package main import ( + "encoding/hex" "fmt" + "os" + "strings" + "github.com/Sirupsen/logrus" + "github.com/kubernetes-incubator/cri-o/server" + "github.com/opencontainers/runc/libcontainer/selinux" "github.com/urfave/cli" + "golang.org/x/net/context" + pb "k8s.io/kubernetes/pkg/kubelet/api/v1alpha1/runtime" ) -// TODO implement +// LAUNCH TODO +// Remove container (and sandbox if we created it) - high priority for usability & testing +// Terminal attach implementation (kpod attach command?) +// Logging (interaction with crio daemon?) +// Properly place created containers in cgroups +// Sanely populate metadata for sandbox +// Missing parsing in CLI handling - DNS, port forwards, mounts, devices, resource limits etc +// Labels and Annotations (pod & container) +// Security & confinement - SELinux, AppArmor, seccomp, capabilities, run as users +// Interface with crio daemon - locking to prevent trampling pod status +// Launch containers in existing sandboxes +// Integration tests +// Man pages + var launchCommand = cli.Command{ Name: "launch", Usage: "launch a pod", - Action: func(context *cli.Context) error { - return fmt.Errorf("this functionality is not yet implemented") + Flags: []cli.Flag{ + cli.StringFlag{ + Name: "image", + Value: "", + Usage: "image to launch", + }, + cli.BoolFlag{ + Name: "attach", + Usage: "attach to the container once it is created", + }, + cli.StringSliceFlag{ + Name: "env", + Usage: "specify environment variables to be set inside launched container, specified as KEY=VALUE", + }, + cli.StringFlag{ + Name: "labels, l", + Value: "", + Usage: "specify labels to be set on launched container", + }, + cli.StringFlag{ + Name: "limits", + Value: "", + Usage: "specify resource limits for launched container", + }, + cli.StringFlag{ + Name: "ports", + Value: "", + Usage: "specify ports to be forwarded to launched container", + }, + cli.BoolFlag{ + Name: "rm", + Usage: "remove launched container (and pod, if a new pod was created) after it exits", + }, + cli.BoolFlag{ + Name: "stdin, i", + Usage: "keep stdin open on launched container", + }, + cli.BoolFlag{ + Name: "tty, t", + Usage: "allocate a TTY for launched container", + }, + cli.StringSliceFlag{ + Name: "mount", + Usage: "attach mounts on the host to created container", + }, + cli.StringSliceFlag{ + Name: "device", + Usage: "make host devices available inside the container", + }, + cli.StringSliceFlag{ + Name: "dns", + Usage: "set DNS servers for container", + }, + cli.StringSliceFlag{ + Name: "dns-search", + Usage: "set DNS search domains for container", + }, + cli.StringFlag{ + Name: "pod", + Value: "", + Usage: "launch container inside an existing pod", + }, + cli.BoolFlag{ + Name: "privileged", + Usage: "launch a privileged container", + }, + cli.BoolFlag{ + Name: "read-only", + Usage: "mount root of created container as read only", + }, + cli.BoolFlag{ + Name: "host-network", + Usage: "don't join a network namespace, and use host's network namespace", + }, + cli.BoolFlag{ + Name: "host-ipc", + Usage: "don't join an IPC namespace, and use the host's IPC namespace", + }, + cli.BoolFlag{ + Name: "host-pid", + Usage: "don't join a PID namespace, and use the host's PID namespace", + }, + }, + Action: func(ctx *cli.Context) error { + if ctx.GlobalBool("debug") { + logrus.SetLevel(logrus.DebugLevel) + } + + // TODO enable selinux support + selinux.SetDisabled() + + // Parse CLI options + cliConfig, err := parseLaunchCLI(ctx) + if err != nil { + return fmt.Errorf("error parsing CLI arguments: %v", err) + } + + // TODO consider moving to a different configuration file syntax for kpod + // For now, use the existing cri-o one for convenience + config := new(server.Config) + err = config.FromFile(cliConfig.configPath) + if err != nil { + return fmt.Errorf("could not load configuration file %v: %v", cliConfig.configPath, err) + } + + if _, err := os.Stat(config.Runtime); os.IsNotExist(err) { + return fmt.Errorf("specified runtime %v does not exist", config.Runtime) + } + + server, err := server.New(config) + if err != nil { + return fmt.Errorf("error creating server: %v", err) + } + // Don't bother with a graceful shutdown for now - containers exist beyond the life of the kpod executable + // We don't want to shut them & related storage down when kpod itself exits + //defer server.Shutdown() + + logrus.Debugf("Going to create container named %v with image %v", cliConfig.containerName, cliConfig.image) + + var sandboxId string + var sandboxConfig *pb.PodSandboxConfig + + sandboxSecurityConfig, containerSecurityConfig, err := generateLinuxSecurityConfigs(cliConfig) + if err != nil { + return fmt.Errorf("error generating security configuration: %v", err) + } + + if cliConfig.pod != "" { + // We were passed a pod to join + // Don't create our own + sandboxId = cliConfig.pod + + logrus.Debugf("Joining existing sandbox with ID %v", sandboxId) + + // Get status of pod - verify it exists + sandboxStatusRequest := pb.PodSandboxStatusRequest{ + PodSandboxId: sandboxId, + } + sandboxStatusResp, err := server.PodSandboxStatus(context.Background(), &sandboxStatusRequest) + if err != nil { + return fmt.Errorf("error getting status of sandbox %v: %v", sandboxId, err) + } + + if sandboxStatusResp.Status.State != pb.PodSandboxState_SANDBOX_READY { + return fmt.Errorf("cannot attach to sandbox %v as it is not ready", sandboxId) + } + + // TODO - Before this can work, we need a way of getting the PodSandboxConfig from the server + // As we are not the executable that created the sandbox, we don't have the config to use + sandboxConfig = nil + + // TODO validity checks - once we get the sandbox's config, we should check given CLI options + // To check for conflicts between them and server config + } else { + logrus.Debugf("Creating new sandbox for container") + + sandboxConfig, err = makePodSandboxConfig(cliConfig, sandboxSecurityConfig) + if err != nil { + return fmt.Errorf("error creating PodSandboxConfig: %v", err) + } + + sandboxRequest := pb.RunPodSandboxRequest{ + Config: sandboxConfig, + } + sandboxResp, err := server.RunPodSandbox(context.Background(), &sandboxRequest) + if err != nil { + return fmt.Errorf("error creating sandbox: %v", err) + } + sandboxId = sandboxResp.PodSandboxId + + logrus.Infof("Successfully created sandbox with ID %v", sandboxId) + } + + createRequest, err := makeContainerCreateRequest(cliConfig, containerSecurityConfig, sandboxId, sandboxConfig) + if err != nil { + return fmt.Errorf("error creating ContainerCreateRequest: %v", err) + } + + containerResp, err := server.CreateContainer(context.Background(), createRequest) + if err != nil { + if cliConfig.pod != "" { + // If we did not join an existing pod, clean up the sandbox we made + defer func() { + removeRequest := pb.RemovePodSandboxRequest{ + PodSandboxId: sandboxId, + } + _, err = server.RemovePodSandbox(context.Background(), &removeRequest) + if err != nil { + logrus.Fatalf("Failed to remove sandbox %v: %v", sandboxId, err) + } + }() + } + + return fmt.Errorf("error creating container: %v", err) + } + + containerId := containerResp.ContainerId + + logrus.Debugf("Successfully created container with ID %v in sandbox %v", sandboxId, containerId) + + startRequest := pb.StartContainerRequest{ + ContainerId: containerId, + } + + // StartContainerResponse is an empty struct, just ignore it + _, err = server.StartContainer(context.Background(), &startRequest) + if err != nil { + return fmt.Errorf("error starting container: %v", err) + } + + logrus.Infof("Successfully started container with ID %v", containerId) + + return nil }, } + +// TODO add: +// - seccomp +// - apparmor +// - capabilities +// - selinux +// - annotations +// - logging configuration +// - cgroup config +// - Maybe working directory of container? +type launchConfig struct { + configPath string + containerName string + command string + args *[]string + image string + attach bool + env []*pb.KeyValue + labels *map[string]string // TODO - right now we set same labels for both pod and container. Worth splitting? + limits *pb.LinuxContainerResources + ports []*pb.PortMapping + remove bool + stdin bool + stdinOnce bool + tty bool + mount []*pb.Mount + devices []*pb.Device + dns *pb.DNSConfig + pod string + privileged bool + readOnlyRoot bool + hostNet bool + hostIpc bool + hostPid bool +} + +func parseLaunchCLI(ctx *cli.Context) (*launchConfig, error) { + config := new(launchConfig) + + // Parse required flags + + if !ctx.IsSet("image") { + return nil, fmt.Errorf("must provide image to launch") + } + config.image = ctx.String("image") + + // Parse required arguments + + if ctx.NArg() == 0 { + return nil, fmt.Errorf("must provide name of container") + } + config.containerName = ctx.Args().First() + + // TODO: currently it is very easy to confuse name and command + // Requiring a -- to separate the two (akin to kubectl run) could improve matters + + // Parse optional arguments + + if ctx.NArg() > 1 { + // First additional arg after name is command + config.command = ctx.Args().Get(1) + // Any others after that are ordered args + if ctx.NArg() > 2 { + args := make([]string, ctx.NArg()-2) + for i := 2; i < ctx.NArg(); i++ { + args[i-2] = ctx.Args().Get(i) + } + config.args = &args + } + } + + // Parse optional flags + + config.configPath = ctx.GlobalString("config") + + // TODO remove when terminal forwarding is finished + if ctx.IsSet("attach") { + return nil, fmt.Errorf("attach functionality is not yet implemented") + } + config.attach = ctx.Bool("attach") + + if ctx.IsSet("env") { + env := ctx.StringSlice("env") + config.env = make([]*pb.KeyValue, len(env)) + + for i := 0; i < len(env); i++ { + splitResult := strings.Split(env[i], "=") + if len(splitResult) != 2 { + return nil, fmt.Errorf("environment must be formatted KEY=VALUE, instead got %v", env[i]) + } + + keyValue := pb.KeyValue{ + Key: splitResult[0], + Value: splitResult[1], + } + config.env[i] = &keyValue + } + } + + if ctx.IsSet("labels") { + // TODO label parsing code + return nil, fmt.Errorf("label parsing is not yet implemented") + } else { + config.labels = new(map[string]string) + } + // TODO should set some default label values - indicate we were launched by kpod, maybe? + + if ctx.IsSet("limits") { + // TODO resource limit parsing code + // Should be relatively straightforward aside from units + return nil, fmt.Errorf("resource limit parsing is not yet implemented") + } + + if ctx.IsSet("ports") { + // TODO port mapping parsing + // Biggest difficulty: probably don't want to force users to specify host IP and protocol + // Protocol is easy - just create identical rules for TCP and UDP + // Host IP, maybe not so much, experimentation required to verify + return nil, fmt.Errorf("port mapping parsing is not yet implemented") + } + + // TODO remove when container remove is finished + if ctx.IsSet("rm") { + return nil, fmt.Errorf("remove functionality is not yet implemented") + } + config.remove = ctx.Bool("rm") + + if ctx.IsSet("stdin") { + config.stdin = true + config.stdinOnce = false + } + + config.tty = ctx.Bool("tty") + + if ctx.IsSet("mount") { + // TODO mount parsing + return nil, fmt.Errorf("mount parsing is not yet implemented") + } + + if ctx.IsSet("device") { + // TODO device parsing + return nil, fmt.Errorf("device parsing is not yet implemented") + } + + if ctx.IsSet("dns") || ctx.IsSet("dns-search") { + // TODO dns parsing + return nil, fmt.Errorf("DNS parsing is not yet implemented") + } + + config.privileged = ctx.Bool("privileged") + + config.readOnlyRoot = ctx.Bool("read-only") + + config.hostNet = ctx.Bool("host-net") + config.hostIpc = ctx.Bool("host-ipc") + config.hostPid = ctx.Bool("host-pid") + + if ctx.IsSet("pod") { + // TODO implement joining existing pods + // Needs modifications to server code to support + return nil, fmt.Errorf("joining an existing pod is not yet implemented") + } + + return config, nil +} + +func makePodSandboxConfig(cliConfig *launchConfig, securityConfig *pb.LinuxSandboxSecurityContext) (*pb.PodSandboxConfig, error) { + sandboxId, err := getRandomId() + if err != nil { + return nil, fmt.Errorf("error generating sandbox id: %v", err) + } + + metadata := pb.PodSandboxMetadata{ + Name: "kpod_launch_" + sandboxId, + Uid: "kpod_default", + Namespace: "kpod_default", + Attempt: 0, + } + + // TODO cgroup config + linuxSandboxConfig := pb.LinuxPodSandboxConfig{ + CgroupParent: "", + SecurityContext: securityConfig, + } + + config := pb.PodSandboxConfig{ + Metadata: &metadata, + Hostname: "kpod_launch_" + sandboxId, + // TODO Logging + LogDirectory: "", + DnsConfig: cliConfig.dns, + PortMappings: cliConfig.ports, + Labels: *cliConfig.labels, + // TODO appropriate annotations + Annotations: make(map[string]string), + // TODO populate cgroup and security configuration + Linux: &linuxSandboxConfig, + } + + return &config, nil +} + +func makeContainerCreateRequest(cliConfig *launchConfig, securityConfig *pb.LinuxContainerSecurityContext, sandboxId string, sandboxConfig *pb.PodSandboxConfig) (*pb.CreateContainerRequest, error) { + metadata := pb.ContainerMetadata{ + Name: cliConfig.containerName, + Attempt: 0, + } + + linuxConfig := pb.LinuxContainerConfig{ + Resources: cliConfig.limits, + SecurityContext: securityConfig, + } + + config := pb.ContainerConfig{ + Metadata: &metadata, + Image: &pb.ImageSpec{Image: cliConfig.image}, + Command: []string{cliConfig.command}, + Args: []string{}, + // TODO: allow passing this via CLI (if worthwhile) + WorkingDir: "", + Envs: cliConfig.env, + Mounts: cliConfig.mount, + Devices: cliConfig.devices, + Labels: make(map[string]string), + // TODO allow setting annotations + Annotations: nil, + // TODO Logging + LogPath: "", + Stdin: cliConfig.stdin, + StdinOnce: cliConfig.stdinOnce, + Tty: cliConfig.tty, + Linux: &linuxConfig, + } + + if cliConfig.args != nil { + config.Args = *cliConfig.args + } + + if cliConfig.labels != nil { + config.Labels = *cliConfig.labels + } + + req := pb.CreateContainerRequest{ + PodSandboxId: sandboxId, + Config: &config, + SandboxConfig: sandboxConfig, + } + + return &req, nil +} + +// TODO: Capabilities, SELinux, set non-root user, add additional groups +func generateLinuxSecurityConfigs(cliConfig *launchConfig) (*pb.LinuxSandboxSecurityContext, *pb.LinuxContainerSecurityContext, error) { + linuxNamespaceOption := pb.NamespaceOption{ + HostNetwork: cliConfig.hostNet, + HostPid: cliConfig.hostPid, + HostIpc: cliConfig.hostIpc, + } + + // Just run as root for now + runAsUser := pb.Int64Value{ + Value: 0, + } + + sandboxConfig := pb.LinuxSandboxSecurityContext{ + NamespaceOptions: &linuxNamespaceOption, + SelinuxOptions: nil, + RunAsUser: &runAsUser, // Probably not strictly necessary to set this here if we're doing it below + ReadonlyRootfs: cliConfig.readOnlyRoot, + SupplementalGroups: []int64{}, + Privileged: cliConfig.privileged, + } + + containerConfig := pb.LinuxContainerSecurityContext{ + Capabilities: &pb.Capability{}, + Privileged: cliConfig.privileged, + NamespaceOptions: &linuxNamespaceOption, + SelinuxOptions: nil, + RunAsUser: &runAsUser, + RunAsUsername: "", + ReadonlyRootfs: cliConfig.readOnlyRoot, + SupplementalGroups: []int64{}, + } + + return &sandboxConfig, &containerConfig, nil +} + +// Generate and hex-encode 128-bit random ID +func getRandomId() (string, error) { + urandom, err := os.Open("/dev/urandom") + if err != nil { + return "", fmt.Errorf("could not open urandom for reading: %v", err) + } + + defer urandom.Close() + + data := make([]byte, 16) + count, err := urandom.Read(data) + if err != nil { + return "", fmt.Errorf("error reading from urandom: %v", err) + } else if count != 16 { + return "", fmt.Errorf("read too few bytes from urandom") + } + + return hex.EncodeToString(data), nil +} diff --git a/cmd/kpod/main.go b/cmd/kpod/main.go index c1c34e4b..0e3c4f6b 100644 --- a/cmd/kpod/main.go +++ b/cmd/kpod/main.go @@ -7,12 +7,27 @@ import ( "github.com/urfave/cli" ) +// TODO should share this default with ocid - move inside server, maybe? +const ociConfigPath = "/etc/ocid/ocid.conf" + func main() { app := cli.NewApp() app.Name = "kpod" app.Usage = "manage pods and images" app.Version = "0.0.1" + app.Flags = []cli.Flag{ + cli.StringFlag{ + Name: "config", + Value: ociConfigPath, + Usage: "path to OCI config file", + }, + cli.BoolFlag{ + Name: "debug", + Usage: "enable debug level log output", + }, + } + app.Commands = []cli.Command{ launchCommand, } From 154e18b9b9e7f7775b49113778bdf7489cfbb8c7 Mon Sep 17 00:00:00 2001 From: Matthew Heon Date: Sun, 19 Mar 2017 14:02:18 -0400 Subject: [PATCH 2/7] Fix lint errors Signed-off-by: Matthew Heon --- cmd/kpod/launch.go | 45 ++++++++++++++++++++++----------------------- 1 file changed, 22 insertions(+), 23 deletions(-) diff --git a/cmd/kpod/launch.go b/cmd/kpod/launch.go index 2be34b1f..c594cba7 100644 --- a/cmd/kpod/launch.go +++ b/cmd/kpod/launch.go @@ -150,7 +150,7 @@ var launchCommand = cli.Command{ logrus.Debugf("Going to create container named %v with image %v", cliConfig.containerName, cliConfig.image) - var sandboxId string + var sandboxID string var sandboxConfig *pb.PodSandboxConfig sandboxSecurityConfig, containerSecurityConfig, err := generateLinuxSecurityConfigs(cliConfig) @@ -161,21 +161,21 @@ var launchCommand = cli.Command{ if cliConfig.pod != "" { // We were passed a pod to join // Don't create our own - sandboxId = cliConfig.pod + sandboxID = cliConfig.pod - logrus.Debugf("Joining existing sandbox with ID %v", sandboxId) + logrus.Debugf("Joining existing sandbox with ID %v", sandboxID) // Get status of pod - verify it exists sandboxStatusRequest := pb.PodSandboxStatusRequest{ - PodSandboxId: sandboxId, + PodSandboxId: sandboxID, } sandboxStatusResp, err := server.PodSandboxStatus(context.Background(), &sandboxStatusRequest) if err != nil { - return fmt.Errorf("error getting status of sandbox %v: %v", sandboxId, err) + return fmt.Errorf("error getting status of sandbox %v: %v", sandboxID, err) } if sandboxStatusResp.Status.State != pb.PodSandboxState_SANDBOX_READY { - return fmt.Errorf("cannot attach to sandbox %v as it is not ready", sandboxId) + return fmt.Errorf("cannot attach to sandbox %v as it is not ready", sandboxID) } // TODO - Before this can work, we need a way of getting the PodSandboxConfig from the server @@ -199,12 +199,12 @@ var launchCommand = cli.Command{ if err != nil { return fmt.Errorf("error creating sandbox: %v", err) } - sandboxId = sandboxResp.PodSandboxId + sandboxID = sandboxResp.PodSandboxId - logrus.Infof("Successfully created sandbox with ID %v", sandboxId) + logrus.Infof("Successfully created sandbox with ID %v", sandboxID) } - createRequest, err := makeContainerCreateRequest(cliConfig, containerSecurityConfig, sandboxId, sandboxConfig) + createRequest, err := makeContainerCreateRequest(cliConfig, containerSecurityConfig, sandboxID, sandboxConfig) if err != nil { return fmt.Errorf("error creating ContainerCreateRequest: %v", err) } @@ -215,11 +215,11 @@ var launchCommand = cli.Command{ // If we did not join an existing pod, clean up the sandbox we made defer func() { removeRequest := pb.RemovePodSandboxRequest{ - PodSandboxId: sandboxId, + PodSandboxId: sandboxID, } _, err = server.RemovePodSandbox(context.Background(), &removeRequest) if err != nil { - logrus.Fatalf("Failed to remove sandbox %v: %v", sandboxId, err) + logrus.Fatalf("Failed to remove sandbox %v: %v", sandboxID, err) } }() } @@ -227,12 +227,12 @@ var launchCommand = cli.Command{ return fmt.Errorf("error creating container: %v", err) } - containerId := containerResp.ContainerId + containerID := containerResp.ContainerId - logrus.Debugf("Successfully created container with ID %v in sandbox %v", sandboxId, containerId) + logrus.Debugf("Successfully created container with ID %v in sandbox %v", sandboxID, containerID) startRequest := pb.StartContainerRequest{ - ContainerId: containerId, + ContainerId: containerID, } // StartContainerResponse is an empty struct, just ignore it @@ -241,7 +241,7 @@ var launchCommand = cli.Command{ return fmt.Errorf("error starting container: %v", err) } - logrus.Infof("Successfully started container with ID %v", containerId) + logrus.Infof("Successfully started container with ID %v", containerID) return nil }, @@ -348,9 +348,8 @@ func parseLaunchCLI(ctx *cli.Context) (*launchConfig, error) { if ctx.IsSet("labels") { // TODO label parsing code return nil, fmt.Errorf("label parsing is not yet implemented") - } else { - config.labels = new(map[string]string) } + config.labels = new(map[string]string) // TODO should set some default label values - indicate we were launched by kpod, maybe? if ctx.IsSet("limits") { @@ -413,13 +412,13 @@ func parseLaunchCLI(ctx *cli.Context) (*launchConfig, error) { } func makePodSandboxConfig(cliConfig *launchConfig, securityConfig *pb.LinuxSandboxSecurityContext) (*pb.PodSandboxConfig, error) { - sandboxId, err := getRandomId() + sandboxID, err := getRandomID() if err != nil { return nil, fmt.Errorf("error generating sandbox id: %v", err) } metadata := pb.PodSandboxMetadata{ - Name: "kpod_launch_" + sandboxId, + Name: "kpod_launch_" + sandboxID, Uid: "kpod_default", Namespace: "kpod_default", Attempt: 0, @@ -433,7 +432,7 @@ func makePodSandboxConfig(cliConfig *launchConfig, securityConfig *pb.LinuxSandb config := pb.PodSandboxConfig{ Metadata: &metadata, - Hostname: "kpod_launch_" + sandboxId, + Hostname: "kpod_launch_" + sandboxID, // TODO Logging LogDirectory: "", DnsConfig: cliConfig.dns, @@ -448,7 +447,7 @@ func makePodSandboxConfig(cliConfig *launchConfig, securityConfig *pb.LinuxSandb return &config, nil } -func makeContainerCreateRequest(cliConfig *launchConfig, securityConfig *pb.LinuxContainerSecurityContext, sandboxId string, sandboxConfig *pb.PodSandboxConfig) (*pb.CreateContainerRequest, error) { +func makeContainerCreateRequest(cliConfig *launchConfig, securityConfig *pb.LinuxContainerSecurityContext, sandboxID string, sandboxConfig *pb.PodSandboxConfig) (*pb.CreateContainerRequest, error) { metadata := pb.ContainerMetadata{ Name: cliConfig.containerName, Attempt: 0, @@ -489,7 +488,7 @@ func makeContainerCreateRequest(cliConfig *launchConfig, securityConfig *pb.Linu } req := pb.CreateContainerRequest{ - PodSandboxId: sandboxId, + PodSandboxId: sandboxID, Config: &config, SandboxConfig: sandboxConfig, } @@ -534,7 +533,7 @@ func generateLinuxSecurityConfigs(cliConfig *launchConfig) (*pb.LinuxSandboxSecu } // Generate and hex-encode 128-bit random ID -func getRandomId() (string, error) { +func getRandomID() (string, error) { urandom, err := os.Open("/dev/urandom") if err != nil { return "", fmt.Errorf("could not open urandom for reading: %v", err) From cf67b84ce345faeadfc8cfc2f577caece1a3c29f Mon Sep 17 00:00:00 2001 From: Matthew Heon Date: Mon, 20 Mar 2017 11:29:56 -0400 Subject: [PATCH 3/7] kpod: Add CLI flag for additional groups Signed-off-by: Matthew Heon --- cmd/kpod/launch.go | 73 +++++++++++++++++++++++++++++----------------- 1 file changed, 47 insertions(+), 26 deletions(-) diff --git a/cmd/kpod/launch.go b/cmd/kpod/launch.go index c594cba7..6c797e8e 100644 --- a/cmd/kpod/launch.go +++ b/cmd/kpod/launch.go @@ -9,6 +9,7 @@ import ( "github.com/Sirupsen/logrus" "github.com/kubernetes-incubator/cri-o/server" "github.com/opencontainers/runc/libcontainer/selinux" + "github.com/opencontainers/runc/libcontainer/user" "github.com/urfave/cli" "golang.org/x/net/context" pb "k8s.io/kubernetes/pkg/kubelet/api/v1alpha1/runtime" @@ -113,6 +114,11 @@ var launchCommand = cli.Command{ Name: "host-pid", Usage: "don't join a PID namespace, and use the host's PID namespace", }, + cli.StringFlag{ + Name: "group-add", + Value: "", + Usage: "comma-separated list of additional groups to run as", + }, }, Action: func(ctx *cli.Context) error { if ctx.GlobalBool("debug") { @@ -257,29 +263,30 @@ var launchCommand = cli.Command{ // - cgroup config // - Maybe working directory of container? type launchConfig struct { - configPath string - containerName string - command string - args *[]string - image string - attach bool - env []*pb.KeyValue - labels *map[string]string // TODO - right now we set same labels for both pod and container. Worth splitting? - limits *pb.LinuxContainerResources - ports []*pb.PortMapping - remove bool - stdin bool - stdinOnce bool - tty bool - mount []*pb.Mount - devices []*pb.Device - dns *pb.DNSConfig - pod string - privileged bool - readOnlyRoot bool - hostNet bool - hostIpc bool - hostPid bool + configPath string + containerName string + command string + args *[]string // TODO - this and labels should not be pointers. Made sense before, but nothing else is. + image string + attach bool + env []*pb.KeyValue + labels *map[string]string // TODO - right now we set same labels for both pod and container. Worth splitting? + limits *pb.LinuxContainerResources + ports []*pb.PortMapping + remove bool + stdin bool + stdinOnce bool + tty bool + mount []*pb.Mount + devices []*pb.Device + dns *pb.DNSConfig + pod string + privileged bool + readOnlyRoot bool + hostNet bool + hostIpc bool + hostPid bool + additionalGroups []int64 } func parseLaunchCLI(ctx *cli.Context) (*launchConfig, error) { @@ -402,6 +409,20 @@ func parseLaunchCLI(ctx *cli.Context) (*launchConfig, error) { config.hostIpc = ctx.Bool("host-ipc") config.hostPid = ctx.Bool("host-pid") + if ctx.IsSet("group-add") { + groupsAsStrings := strings.Split(ctx.String("group-add"), ",") + + groupsAsInts, err := user.GetAdditionalGroupsPath(groupsAsStrings, "/etc/group") + if err != nil { + return nil, fmt.Errorf("error parsing groups from group-add: %v", err) + } + + config.additionalGroups = make([]int64, len(groupsAsInts)) + for i := 0; i < len(groupsAsInts); i++ { + config.additionalGroups[i] = int64(groupsAsInts[i]) + } + } + if ctx.IsSet("pod") { // TODO implement joining existing pods // Needs modifications to server code to support @@ -496,7 +517,7 @@ func makeContainerCreateRequest(cliConfig *launchConfig, securityConfig *pb.Linu return &req, nil } -// TODO: Capabilities, SELinux, set non-root user, add additional groups +// TODO: Capabilities, SELinux, set non-root user func generateLinuxSecurityConfigs(cliConfig *launchConfig) (*pb.LinuxSandboxSecurityContext, *pb.LinuxContainerSecurityContext, error) { linuxNamespaceOption := pb.NamespaceOption{ HostNetwork: cliConfig.hostNet, @@ -514,7 +535,7 @@ func generateLinuxSecurityConfigs(cliConfig *launchConfig) (*pb.LinuxSandboxSecu SelinuxOptions: nil, RunAsUser: &runAsUser, // Probably not strictly necessary to set this here if we're doing it below ReadonlyRootfs: cliConfig.readOnlyRoot, - SupplementalGroups: []int64{}, + SupplementalGroups: cliConfig.additionalGroups, Privileged: cliConfig.privileged, } @@ -526,7 +547,7 @@ func generateLinuxSecurityConfigs(cliConfig *launchConfig) (*pb.LinuxSandboxSecu RunAsUser: &runAsUser, RunAsUsername: "", ReadonlyRootfs: cliConfig.readOnlyRoot, - SupplementalGroups: []int64{}, + SupplementalGroups: cliConfig.additionalGroups, } return &sandboxConfig, &containerConfig, nil From f96f4c85f6a4485ddae17460f761ae9915f2c69e Mon Sep 17 00:00:00 2001 From: Matthew Heon Date: Mon, 20 Mar 2017 13:27:37 -0400 Subject: [PATCH 4/7] kpod: Add flag to specify user to run as Signed-off-by: Matthew Heon --- cmd/kpod/launch.go | 27 +++++++++++++++++++++++---- 1 file changed, 23 insertions(+), 4 deletions(-) diff --git a/cmd/kpod/launch.go b/cmd/kpod/launch.go index 6c797e8e..9bbe6f14 100644 --- a/cmd/kpod/launch.go +++ b/cmd/kpod/launch.go @@ -23,7 +23,7 @@ import ( // Sanely populate metadata for sandbox // Missing parsing in CLI handling - DNS, port forwards, mounts, devices, resource limits etc // Labels and Annotations (pod & container) -// Security & confinement - SELinux, AppArmor, seccomp, capabilities, run as users +// Security & confinement - SELinux, AppArmor, seccomp, capabilities // Interface with crio daemon - locking to prevent trampling pod status // Launch containers in existing sandboxes // Integration tests @@ -119,6 +119,11 @@ var launchCommand = cli.Command{ Value: "", Usage: "comma-separated list of additional groups to run as", }, + cli.StringFlag{ + Name: "user", + Value: "", + Usage: "specify user to run container as", + }, }, Action: func(ctx *cli.Context) error { if ctx.GlobalBool("debug") { @@ -287,6 +292,7 @@ type launchConfig struct { hostIpc bool hostPid bool additionalGroups []int64 + runAsUser int64 } func parseLaunchCLI(ctx *cli.Context) (*launchConfig, error) { @@ -423,6 +429,18 @@ func parseLaunchCLI(ctx *cli.Context) (*launchConfig, error) { } } + if ctx.IsSet("user") { + user, err := user.GetExecUserPath(ctx.String("user"), nil, "/etc/passwd", "/etc/group") + if err != nil { + return nil, fmt.Errorf("error setting container user: %v", err) + } + + config.runAsUser = int64(user.Uid) + } else { + // Default to root + config.runAsUser = 0 + } + if ctx.IsSet("pod") { // TODO implement joining existing pods // Needs modifications to server code to support @@ -517,7 +535,7 @@ func makeContainerCreateRequest(cliConfig *launchConfig, securityConfig *pb.Linu return &req, nil } -// TODO: Capabilities, SELinux, set non-root user +// TODO: Capabilities, SELinux func generateLinuxSecurityConfigs(cliConfig *launchConfig) (*pb.LinuxSandboxSecurityContext, *pb.LinuxContainerSecurityContext, error) { linuxNamespaceOption := pb.NamespaceOption{ HostNetwork: cliConfig.hostNet, @@ -525,9 +543,8 @@ func generateLinuxSecurityConfigs(cliConfig *launchConfig) (*pb.LinuxSandboxSecu HostIpc: cliConfig.hostIpc, } - // Just run as root for now runAsUser := pb.Int64Value{ - Value: 0, + Value: cliConfig.runAsUser, } sandboxConfig := pb.LinuxSandboxSecurityContext{ @@ -539,6 +556,8 @@ func generateLinuxSecurityConfigs(cliConfig *launchConfig) (*pb.LinuxSandboxSecu Privileged: cliConfig.privileged, } + // TODO: We can't get group independently of user. SupplementalGroups helps alleviate this, but it's desirable. + // Adding this would require changes to the server, so it's not really worth it for now containerConfig := pb.LinuxContainerSecurityContext{ Capabilities: &pb.Capability{}, Privileged: cliConfig.privileged, From ddeff0fcefff2183fa46c0617dc2e8995847fecd Mon Sep 17 00:00:00 2001 From: Matthew Heon Date: Tue, 21 Mar 2017 10:27:56 -0400 Subject: [PATCH 5/7] kpod: Partially address comments by Dan Signed-off-by: Matthew Heon --- cmd/kpod/launch.go | 45 ++++++++++++++++++++++----------------------- 1 file changed, 22 insertions(+), 23 deletions(-) diff --git a/cmd/kpod/launch.go b/cmd/kpod/launch.go index 9bbe6f14..426fe665 100644 --- a/cmd/kpod/launch.go +++ b/cmd/kpod/launch.go @@ -20,7 +20,6 @@ import ( // Terminal attach implementation (kpod attach command?) // Logging (interaction with crio daemon?) // Properly place created containers in cgroups -// Sanely populate metadata for sandbox // Missing parsing in CLI handling - DNS, port forwards, mounts, devices, resource limits etc // Labels and Annotations (pod & container) // Security & confinement - SELinux, AppArmor, seccomp, capabilities @@ -31,76 +30,76 @@ import ( var launchCommand = cli.Command{ Name: "launch", - Usage: "launch a pod", + Usage: "launch a pod or insert a container into an existing pod", Flags: []cli.Flag{ cli.StringFlag{ Name: "image", Value: "", - Usage: "image to launch", + Usage: "`image` to launch", }, cli.BoolFlag{ Name: "attach", - Usage: "attach to the container once it is created", + Usage: "`attach` to the primary container once it is created", }, cli.StringSliceFlag{ Name: "env", - Usage: "specify environment variables to be set inside launched container, specified as KEY=VALUE", + Usage: "specify environment `variable`s to be set inside the primary container, specified as `VARIABLE=VALUE`", }, cli.StringFlag{ Name: "labels, l", Value: "", - Usage: "specify labels to be set on launched container", + Usage: "specify `label`s to be set on launched pod", }, cli.StringFlag{ Name: "limits", Value: "", - Usage: "specify resource limits for launched container", + Usage: "specify resource `limit`s for the primary container", }, cli.StringFlag{ Name: "ports", Value: "", - Usage: "specify ports to be forwarded to launched container", + Usage: "specify `port`s to be forwarded to the launched pod", }, cli.BoolFlag{ Name: "rm", - Usage: "remove launched container (and pod, if a new pod was created) after it exits", + Usage: "`remove` launched container (and pod, if a new pod was created) after it exits", }, cli.BoolFlag{ Name: "stdin, i", - Usage: "keep stdin open on launched container", + Usage: "keep `stdin` open on primary container", }, cli.BoolFlag{ Name: "tty, t", - Usage: "allocate a TTY for launched container", + Usage: "allocate a `TTY` for primary container", }, cli.StringSliceFlag{ Name: "mount", - Usage: "attach mounts on the host to created container", + Usage: "attach `mount`s on the host to primary container", }, cli.StringSliceFlag{ Name: "device", - Usage: "make host devices available inside the container", + Usage: "make host `device`s available inside the primary container", }, cli.StringSliceFlag{ Name: "dns", - Usage: "set DNS servers for container", + Usage: "set `DNS server`s for pod", }, cli.StringSliceFlag{ Name: "dns-search", - Usage: "set DNS search domains for container", + Usage: "set `DNS search domain`s for pod", }, cli.StringFlag{ Name: "pod", Value: "", - Usage: "launch container inside an existing pod", + Usage: "launch container inside an `existing pod`", }, cli.BoolFlag{ Name: "privileged", - Usage: "launch a privileged container", + Usage: "launch a `privileged` pod", }, cli.BoolFlag{ Name: "read-only", - Usage: "mount root of created container as read only", + Usage: "mount root of primary container as `read only`", }, cli.BoolFlag{ Name: "host-network", @@ -117,12 +116,12 @@ var launchCommand = cli.Command{ cli.StringFlag{ Name: "group-add", Value: "", - Usage: "comma-separated list of additional groups to run as", + Usage: "comma-separated list of additional `group`s to run as", }, cli.StringFlag{ Name: "user", Value: "", - Usage: "specify user to run container as", + Usage: "specify `user` to run primary container as", }, }, Action: func(ctx *cli.Context) error { @@ -576,7 +575,7 @@ func generateLinuxSecurityConfigs(cliConfig *launchConfig) (*pb.LinuxSandboxSecu func getRandomID() (string, error) { urandom, err := os.Open("/dev/urandom") if err != nil { - return "", fmt.Errorf("could not open urandom for reading: %v", err) + return "", fmt.Errorf("could not open /dev/urandom for reading: %v", err) } defer urandom.Close() @@ -584,9 +583,9 @@ func getRandomID() (string, error) { data := make([]byte, 16) count, err := urandom.Read(data) if err != nil { - return "", fmt.Errorf("error reading from urandom: %v", err) + return "", fmt.Errorf("error reading from /dev/urandom: %v", err) } else if count != 16 { - return "", fmt.Errorf("read too few bytes from urandom") + return "", fmt.Errorf("read too few bytes from /dev/urandom") } return hex.EncodeToString(data), nil From 5f74cffd1217ab36db419508561dc523251e70d7 Mon Sep 17 00:00:00 2001 From: Matthew Heon Date: Tue, 21 Mar 2017 10:30:31 -0400 Subject: [PATCH 6/7] kpod: Use build tags in makefile Signed-off-by: Matthew Heon --- Makefile | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/Makefile b/Makefile index f565d65c..33436174 100644 --- a/Makefile +++ b/Makefile @@ -66,7 +66,9 @@ ocic: check-gopath $(GO) build -o $@ $(PROJECT)/cmd/ocic kpod: check-gopath - $(GO) build -o $@ $(PROJECT)/cmd/kpod + $(GO) build -o $@ \ + -tags "$(BUILDTAGS)" \ + $(PROJECT)/cmd/kpod ocid.conf: ocid ./ocid --config="" config --default > ocid.conf From 12f2dbe2a0d2b7509e147400e0f13683e0916a76 Mon Sep 17 00:00:00 2001 From: Matthew Heon Date: Tue, 21 Mar 2017 12:59:57 -0400 Subject: [PATCH 7/7] Replace ID generation function with vendored equivalent Signed-off-by: Matthew Heon --- cmd/kpod/launch.go | 27 ++------------------------- 1 file changed, 2 insertions(+), 25 deletions(-) diff --git a/cmd/kpod/launch.go b/cmd/kpod/launch.go index 426fe665..d0aae5fa 100644 --- a/cmd/kpod/launch.go +++ b/cmd/kpod/launch.go @@ -1,12 +1,12 @@ package main import ( - "encoding/hex" "fmt" "os" "strings" "github.com/Sirupsen/logrus" + "github.com/docker/docker/pkg/stringid" "github.com/kubernetes-incubator/cri-o/server" "github.com/opencontainers/runc/libcontainer/selinux" "github.com/opencontainers/runc/libcontainer/user" @@ -450,10 +450,7 @@ func parseLaunchCLI(ctx *cli.Context) (*launchConfig, error) { } func makePodSandboxConfig(cliConfig *launchConfig, securityConfig *pb.LinuxSandboxSecurityContext) (*pb.PodSandboxConfig, error) { - sandboxID, err := getRandomID() - if err != nil { - return nil, fmt.Errorf("error generating sandbox id: %v", err) - } + sandboxID := stringid.GenerateNonCryptoID() metadata := pb.PodSandboxMetadata{ Name: "kpod_launch_" + sandboxID, @@ -570,23 +567,3 @@ func generateLinuxSecurityConfigs(cliConfig *launchConfig) (*pb.LinuxSandboxSecu return &sandboxConfig, &containerConfig, nil } - -// Generate and hex-encode 128-bit random ID -func getRandomID() (string, error) { - urandom, err := os.Open("/dev/urandom") - if err != nil { - return "", fmt.Errorf("could not open /dev/urandom for reading: %v", err) - } - - defer urandom.Close() - - data := make([]byte, 16) - count, err := urandom.Read(data) - if err != nil { - return "", fmt.Errorf("error reading from /dev/urandom: %v", err) - } else if count != 16 { - return "", fmt.Errorf("read too few bytes from /dev/urandom") - } - - return hex.EncodeToString(data), nil -}