From c2ee13d187e34029b93de5d011ebd450a2fd9c77 Mon Sep 17 00:00:00 2001 From: Haiyan Meng Date: Mon, 1 Aug 2016 13:39:42 -0400 Subject: [PATCH] Implement CreateContainer Signed-off-by: Haiyan Meng --- cmd/client/main.go | 81 +++++- oci/oci.go | 51 ++++ server/runtime.go | 249 +++++++++++++++++- server/server.go | 29 +- testdata/README.md | 15 ++ testdata/container_config.json | 95 +++++++ .../sandbox_config.json | 0 utils/utils.go | 59 +++++ 8 files changed, 567 insertions(+), 12 deletions(-) create mode 100644 testdata/README.md create mode 100644 testdata/container_config.json rename cmd/client/podsandboxconfig.json => testdata/sandbox_config.json (100%) diff --git a/cmd/client/main.go b/cmd/client/main.go index ca6a4a77..f684a3c7 100644 --- a/cmd/client/main.go +++ b/cmd/client/main.go @@ -31,14 +31,22 @@ func getClientConnection() (*grpc.ClientConn, error) { return conn, nil } -func loadPodSandboxConfig(path string) (*pb.PodSandboxConfig, error) { +func openFile(path string) (*os.File, error) { f, err := os.Open(path) if err != nil { if os.IsNotExist(err) { - return nil, fmt.Errorf("pod sandbox config at %s not found", path) + return nil, fmt.Errorf("config at %s not found", path) } return nil, err } + return f, nil +} + +func loadPodSandboxConfig(path string) (*pb.PodSandboxConfig, error) { + f, err := openFile(path) + if err != nil { + return nil, err + } defer f.Close() var config pb.PodSandboxConfig @@ -48,6 +56,20 @@ func loadPodSandboxConfig(path string) (*pb.PodSandboxConfig, error) { return &config, nil } +func loadContainerConfig(path string) (*pb.ContainerConfig, error) { + f, err := openFile(path) + if err != nil { + return nil, err + } + defer f.Close() + + var config pb.ContainerConfig + if err := json.NewDecoder(f).Decode(&config); err != nil { + return nil, err + } + return &config, nil +} + // CreatePodSandbox sends a CreatePodSandboxRequest to the server, and parses // the returned CreatePodSandboxResponse. func CreatePodSandbox(client pb.RuntimeServiceClient, path string) error { @@ -64,6 +86,25 @@ func CreatePodSandbox(client pb.RuntimeServiceClient, path string) error { return nil } +// CreateContainer sends a CreateContainerRequest to the server, and parses +// the returned CreateContainerResponse. +func CreateContainer(client pb.RuntimeServiceClient, sandbox string, path string) error { + config, err := loadContainerConfig(path) + if err != nil { + return err + } + + r, err := client.CreateContainer(context.Background(), &pb.CreateContainerRequest{ + PodSandboxId: &sandbox, + Config: config, + }) + if err != nil { + return err + } + fmt.Println(r) + return nil +} + // Version sends a VersionRequest to the server, and parses the returned VersionResponse. func Version(client pb.RuntimeServiceClient, version string) error { r, err := client.Version(context.Background(), &pb.VersionRequest{Version: &version}) @@ -82,6 +123,7 @@ func main() { app.Commands = []cli.Command{ runtimeVersionCommand, createPodSandboxCommand, + createContainerCommand, pullImageCommand, } @@ -168,3 +210,38 @@ var createPodSandboxCommand = cli.Command{ return nil }, } + +var createContainerCommand = cli.Command{ + Name: "createcontainer", + Usage: "create a container", + Flags: []cli.Flag{ + cli.StringFlag{ + Name: "sandbox", + Usage: "the id of the pod sandbox to which the container belongs", + }, + cli.StringFlag{ + Name: "config", + Value: "config.json", + Usage: "the path of a container config file", + }, + }, + Action: func(context *cli.Context) error { + // Set up a connection to the server. + conn, err := getClientConnection() + if err != nil { + return fmt.Errorf("Failed to connect: %v", err) + } + defer conn.Close() + client := pb.NewRuntimeServiceClient(conn) + + if !context.IsSet("sandbox") { + return fmt.Errorf("Please specify the id of the pod sandbox to which the container belongs via the --sandbox option") + } + // Test RuntimeServiceClient.CreateContainer + err = CreateContainer(client, context.String("sandbox"), context.String("config")) + if err != nil { + return fmt.Errorf("Creating the pod sandbox failed: %v", err) + } + return nil + }, +} diff --git a/oci/oci.go b/oci/oci.go index 9d9638e4..2ae7ce5a 100644 --- a/oci/oci.go +++ b/oci/oci.go @@ -1,6 +1,7 @@ package oci import ( + "os" "path/filepath" "strings" @@ -58,3 +59,53 @@ func getOCIVersion(name string, args ...string) (string, error) { v := firstLine[strings.LastIndex(firstLine, " ")+1:] return v, nil } + +// CreateContainer creates a container. +func (r *Runtime) CreateContainer(c *Container) error { + return utils.ExecCmdWithStdStreams(os.Stdin, os.Stdout, os.Stderr, r.path, "create", "--bundle", c.bundlePath, c.name) +} + +// Container respresents a runtime container. +type Container struct { + name string + bundlePath string + logPath string + labels map[string]string + sandbox string +} + +func NewContainer(name string, bundlePath string, logPath string, labels map[string]string, sandbox string) (*Container, error) { + c := &Container{ + name: name, + bundlePath: bundlePath, + logPath: logPath, + labels: labels, + sandbox: sandbox, + } + return c, nil +} + +// Name returns the name of the container. +func (c *Container) Name() string { + return c.name +} + +// BundlePath returns the bundlePath of the container. +func (c *Container) BundlePath() string { + return c.bundlePath +} + +// LogPath returns the log path of the container. +func (c *Container) LogPath() string { + return c.logPath +} + +// Labels returns the labels of the container. +func (c *Container) Labels() map[string]string { + return c.labels +} + +// Sandbox returns the sandbox name of the container. +func (c *Container) Sandbox() string { + return c.sandbox +} diff --git a/server/runtime.go b/server/runtime.go index 3c352a99..8a402d24 100644 --- a/server/runtime.go +++ b/server/runtime.go @@ -6,6 +6,8 @@ import ( "path/filepath" pb "github.com/kubernetes/kubernetes/pkg/kubelet/api/v1alpha1/runtime" + "github.com/mrunalp/ocid/oci" + "github.com/mrunalp/ocid/utils" "github.com/opencontainers/ocitools/generate" "golang.org/x/net/context" ) @@ -87,9 +89,10 @@ func (s *Server) CreatePodSandbox(ctx context.Context, req *pb.CreatePodSandboxR labels := req.GetConfig().GetLabels() s.addSandbox(&sandbox{ - name: name, - logDir: logDir, - labels: labels, + name: name, + logDir: logDir, + labels: labels, + containers: []*oci.Container{}, }) annotations := req.GetConfig().GetAnnotations() @@ -156,8 +159,244 @@ func (s *Server) ListPodSandbox(context.Context, *pb.ListPodSandboxRequest) (*pb } // CreateContainer creates a new container in specified PodSandbox -func (s *Server) CreateContainer(context.Context, *pb.CreateContainerRequest) (*pb.CreateContainerResponse, error) { - return nil, nil +func (s *Server) CreateContainer(ctx context.Context, req *pb.CreateContainerRequest) (*pb.CreateContainerResponse, error) { + // The id of the PodSandbox + podSandboxId := req.GetPodSandboxId() + if !s.hasSandbox(podSandboxId) { + return nil, fmt.Errorf("the pod sandbox (%s) does not exist", podSandboxId) + } + + // The config of the container + containerConfig := req.GetConfig() + if containerConfig == nil { + return nil, fmt.Errorf("CreateContainerRequest.ContainerConfig is nil") + } + + name := containerConfig.GetName() + if name == "" { + return nil, fmt.Errorf("CreateContainerRequest.ContainerConfig.Name is empty") + } + + // containerDir is the dir for the container bundle. + containerDir := filepath.Join(s.runtime.ContainerDir(), name) + + if _, err := os.Stat(containerDir); err == nil { + return nil, fmt.Errorf("container (%s) already exists", containerDir) + } + + if err := os.MkdirAll(containerDir, 0755); err != nil { + return nil, err + } + + imageSpec := containerConfig.GetImage() + if imageSpec == nil { + return nil, fmt.Errorf("CreateContainerRequest.ContainerConfig.Image is nil") + } + + image := imageSpec.GetImage() + if image == "" { + return nil, fmt.Errorf("CreateContainerRequest.ContainerConfig.Image.Image is empty") + } + + // creates a spec Generator with the default spec. + specgen := generate.New() + + // by default, the root path is an empty string. + // here set it to be "rootfs". + specgen.SetRootPath("rootfs") + + args := containerConfig.GetArgs() + if args == nil { + args = []string{"/bin/sh"} + } + specgen.SetProcessArgs(args) + + cwd := containerConfig.GetWorkingDir() + if cwd == "" { + cwd = "/" + } + specgen.SetProcessCwd(cwd) + + envs := containerConfig.GetEnvs() + if envs != nil { + for _, item := range envs { + key := item.GetKey() + value := item.GetValue() + if key == "" { + continue + } + env := fmt.Sprintf("%s=%s", key, value) + specgen.AddProcessEnv(env) + } + } + + mounts := containerConfig.GetMounts() + for _, mount := range mounts { + dest := mount.GetContainerPath() + if dest == "" { + return nil, fmt.Errorf("Mount.ContainerPath is empty") + } + + src := mount.GetHostPath() + if src == "" { + return nil, fmt.Errorf("Mount.HostPath is empty") + } + + options := "rw" + if mount.GetReadonly() { + options = "ro" + } + + //TODO(hmeng): how to use this info? Do we need to handle relabel a FS with Selinux? + selinuxRelabel := mount.GetSelinuxRelabel() + fmt.Printf("selinuxRelabel: %v\n", selinuxRelabel) + + specgen.AddBindMount(src, dest, options) + + } + + labels := containerConfig.GetLabels() + + annotations := containerConfig.GetAnnotations() + if annotations != nil { + for k, v := range annotations { + specgen.AddAnnotation(k, v) + } + } + + if containerConfig.GetPrivileged() { + specgen.SetupPrivileged(true) + } + + if containerConfig.GetReadonlyRootfs() { + specgen.SetRootReadonly(true) + } + + logPath := containerConfig.GetLogPath() + + if containerConfig.GetTty() { + specgen.SetProcessTerminal(true) + } + + linux := containerConfig.GetLinux() + if linux != nil { + resources := linux.GetResources() + if resources != nil { + cpuPeriod := resources.GetCpuPeriod() + if cpuPeriod != 0 { + specgen.SetLinuxResourcesCPUPeriod(uint64(cpuPeriod)) + } + + cpuQuota := resources.GetCpuQuota() + if cpuQuota != 0 { + specgen.SetLinuxResourcesCPUQuota(uint64(cpuQuota)) + } + + cpuShares := resources.GetCpuShares() + if cpuShares != 0 { + specgen.SetLinuxResourcesCPUShares(uint64(cpuShares)) + } + + memoryLimit := resources.GetMemoryLimitInBytes() + if memoryLimit != 0 { + specgen.SetLinuxResourcesMemoryLimit(uint64(memoryLimit)) + } + + oomScoreAdj := resources.GetOomScoreAdj() + specgen.SetLinuxResourcesOOMScoreAdj(int(oomScoreAdj)) + } + + capabilities := linux.GetCapabilities() + if capabilities != nil { + addCaps := capabilities.GetAddCapabilities() + if addCaps != nil { + for _, cap := range addCaps { + if err := specgen.AddProcessCapability(cap); err != nil { + return nil, err + } + } + } + + dropCaps := capabilities.GetDropCapabilities() + if dropCaps != nil { + for _, cap := range dropCaps { + if err := specgen.DropProcessCapability(cap); err != nil { + return nil, err + } + } + } + } + + selinuxOptions := linux.GetSelinuxOptions() + if selinuxOptions != nil { + user := selinuxOptions.GetUser() + if user == "" { + return nil, fmt.Errorf("SELinuxOption.User is empty") + } + + role := selinuxOptions.GetRole() + if role == "" { + return nil, fmt.Errorf("SELinuxOption.Role is empty") + } + + t := selinuxOptions.GetType() + if t == "" { + return nil, fmt.Errorf("SELinuxOption.Type is empty") + } + + level := selinuxOptions.GetLevel() + if level == "" { + return nil, fmt.Errorf("SELinuxOption.Level is empty") + } + + specgen.SetProcessSelinuxLabel(fmt.Sprintf("%s:%s:%s:%s", user, role, t, level)) + } + + user := linux.GetUser() + if user != nil { + uid := user.GetUid() + specgen.SetProcessUID(uint32(uid)) + + gid := user.GetGid() + specgen.SetProcessGID(uint32(gid)) + + groups := user.GetAdditionalGids() + if groups != nil { + for _, group := range groups { + specgen.AddProcessAdditionalGid(uint32(group)) + } + } + } + } + + // The config of the PodSandbox + sandboxConfig := req.GetSandboxConfig() + fmt.Printf("sandboxConfig: %v\n", sandboxConfig) + + if err := specgen.SaveToFile(filepath.Join(containerDir, "config.json")); err != nil { + return nil, err + } + + // TODO: copy the rootfs into the bundle. + // Currently, utils.CreateFakeRootfs is used to populate the rootfs. + if err := utils.CreateFakeRootfs(containerDir, image); err != nil { + return nil, err + } + + container, err := oci.NewContainer(name, containerDir, logPath, labels, podSandboxId) + if err != nil { + return nil, err + } + + if err := s.runtime.CreateContainer(container); err != nil { + return nil, err + } + + s.addContainer(container) + + return &pb.CreateContainerResponse{ + ContainerId: &name, + }, nil } // StartContainer starts the container. diff --git a/server/server.go b/server/server.go index 91ee80da..af87668c 100644 --- a/server/server.go +++ b/server/server.go @@ -37,25 +37,44 @@ func New(runtimePath, sandboxDir, containerDir string) (*Server, error) { return nil, err } sandboxes := make(map[string]*sandbox) + containers := make(map[string]*oci.Container) return &Server{ runtime: r, sandboxDir: sandboxDir, state: &serverState{ - sandboxes: sandboxes, + sandboxes: sandboxes, + containers: containers, }, }, nil } type serverState struct { - sandboxes map[string]*sandbox + sandboxes map[string]*sandbox + containers map[string]*oci.Container } type sandbox struct { - name string - logDir string - labels map[string]string + name string + logDir string + labels map[string]string + containers []*oci.Container } func (s *Server) addSandbox(sb *sandbox) { s.state.sandboxes[sb.name] = sb } + +func (s *Server) hasSandbox(name string) bool { + _, ok := s.state.sandboxes[name] + return ok +} + +func (s *sandbox) addContainer(c *oci.Container) { + s.containers = append(s.containers, c) +} + +func (s *Server) addContainer(c *oci.Container) { + sandbox := s.state.sandboxes[c.Sandbox()] + sandbox.addContainer(c) + s.state.containers[c.Name()] = c +} diff --git a/testdata/README.md b/testdata/README.md new file mode 100644 index 00000000..b97243e5 --- /dev/null +++ b/testdata/README.md @@ -0,0 +1,15 @@ +In terminal 1: +``` +sudo ./ocid +``` + +In terminal 2: +``` +sudo ./ocic runtimeversion + +sudo rm -rf /var/lib/ocid/sandboxes/podsandbox1 +sudo ./ocic createpodsandbox --config testdata/sandbox_config.json + +sudo rm -rf /var/lib/ocid/containers/container1 +sudo ./ocic createcontainer --sandbox podsandbox1 --config testdata/container_config.json +``` diff --git a/testdata/container_config.json b/testdata/container_config.json new file mode 100644 index 00000000..b882ef58 --- /dev/null +++ b/testdata/container_config.json @@ -0,0 +1,95 @@ +{ + "name": "container1", + "image": { + "image": "docker://redis:latest" + }, + "command": [ + "/bin/bash" + ], + "args": [ + "/bin/ls" + ], + "working_dir": "/", + "envs": [ + { + "key": "PATH", + "value": "/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin" + }, + { + "key": "TERM", + "value": "xterm" + }, + { + "key": "TESTDIR", + "value": "test/dir1" + }, + { + "key": "TESTFILE", + "value": "test/file1" + } + ], + "mounts": [ + { + "name": "mount1", + "container_path": "/dir1", + "host_path": "/tmp/dir1", + "readonly": false, + "selinux_relabel": true + }, + { + "name": "mount2", + "container_path": "/dir2", + "host_path": "/tmp/dir2", + "readonly": true, + "selinux_relabel": true + } + ], + "labels": { + "type": "small", + "batch": "no" + }, + "annotations": { + "owner": "dragon", + "daemon": "ocid" + }, + "privileged": true, + "readonly_rootfs": true, + "log_path": "container.log", + "stdin": false, + "stdin_once": false, + "tty": false, + "linux": { + "resources": { + "cpu_period": 10000, + "cpu_quota": 20000, + "cpu_shares": 512, + "memory_limit_in_bytes": 88000000, + "oom_score_adj": 30 + }, + "capabilities": { + "add_capabilities": [ + "setuid", + "setgid" + ], + "drop_capabilities": [ + "audit_write", + "audit_read" + ] + }, + "selinux_options": { + "user": "selinux_test", + "role": "control", + "type": "dogfood", + "level": "s1" + }, + "user": { + "uid": 5, + "gid": 300, + "additional_gids": [ + 400, + 401, + 402 + ] + } + } +} diff --git a/cmd/client/podsandboxconfig.json b/testdata/sandbox_config.json similarity index 100% rename from cmd/client/podsandboxconfig.json rename to testdata/sandbox_config.json diff --git a/utils/utils.go b/utils/utils.go index 6cbfa00d..ae93cdbe 100644 --- a/utils/utils.go +++ b/utils/utils.go @@ -3,7 +3,9 @@ package utils import ( "bytes" "fmt" + "os" "os/exec" + "path/filepath" "strings" "syscall" ) @@ -25,6 +27,21 @@ func ExecCmd(name string, args ...string) (string, error) { return stdout.String(), nil } +// ExecCmdWithStdStreams execute a command with the specified standard streams. +func ExecCmdWithStdStreams(stdin, stdout, stderr *os.File, name string, args ...string) error { + cmd := exec.Command(name, args...) + cmd.Stdin = stdin + cmd.Stdout = stdout + cmd.Stderr = stderr + + err := cmd.Run() + if err != nil { + return fmt.Errorf("`%v %v` failed: %v", name, strings.Join(args, " "), err) + } + + return nil +} + // SetSubreaper sets the value i as the subreaper setting for the calling process func SetSubreaper(i int) error { return Prctl(PR_SET_CHILD_SUBREAPER, uintptr(i), 0, 0, 0) @@ -38,3 +55,45 @@ func Prctl(option int, arg2, arg3, arg4, arg5 uintptr) (err error) { } return } + +// CreateFakeRootfs creates a fake rootfs for test. +func CreateFakeRootfs(dir string, image string) error { + if len(image) <= 9 || image[:9] != "docker://" { + return fmt.Errorf("CreateFakeRootfs only support docker images currently") + } + + rootfs := filepath.Join(dir, "rootfs") + if err := os.MkdirAll(rootfs, 0755); err != nil { + return err + } + + // docker export $(docker create image[9:]) | tar -C rootfs -xf - + return dockerExport(image[9:], rootfs) +} + +func dockerExport(image string, rootfs string) error { + out, err := ExecCmd("docker", "create", image) + if err != nil { + return err + } + + container := out[:strings.Index(out, "\n")] + + cmd := fmt.Sprintf("docker export %s | tar -C %s -xf -", container, rootfs) + if _, err := ExecCmd("/bin/bash", "-c", cmd); err != nil { + err1 := dockerRemove(container) + if err1 == nil { + return err + } + return fmt.Errorf("%v; %v", err, err1) + } + + return dockerRemove(container) +} + +func dockerRemove(container string) error { + if _, err := ExecCmd("docker", "rm", container); err != nil { + return err + } + return nil +}