package server import ( "encoding/json" "fmt" "io/ioutil" "os" "path/filepath" "github.com/Sirupsen/logrus" "github.com/kubernetes-incubator/ocid/oci" "github.com/kubernetes-incubator/ocid/utils" pb "github.com/kubernetes/kubernetes/pkg/kubelet/api/v1alpha1/runtime" "github.com/opencontainers/ocitools/generate" "golang.org/x/net/context" ) // Version returns the runtime name, runtime version and runtime API version func (s *Server) Version(ctx context.Context, req *pb.VersionRequest) (*pb.VersionResponse, error) { version, err := getGPRCVersion() if err != nil { return nil, err } runtimeVersion, err := s.runtime.Version() if err != nil { return nil, err } // taking const address rav := runtimeAPIVersion runtimeName := s.runtime.Name() return &pb.VersionResponse{ Version: &version, RuntimeName: &runtimeName, RuntimeVersion: &runtimeVersion, RuntimeApiVersion: &rav, }, nil } // CreatePodSandbox creates a pod-level sandbox. // The definition of PodSandbox is at https://github.com/kubernetes/kubernetes/pull/25899 func (s *Server) CreatePodSandbox(ctx context.Context, req *pb.CreatePodSandboxRequest) (*pb.CreatePodSandboxResponse, error) { var err error if err := os.MkdirAll(s.sandboxDir, 0755); err != nil { return nil, err } // process req.Name name := req.GetConfig().GetMetadata().GetName() if name == "" { return nil, fmt.Errorf("PodSandboxConfig.Name should not be empty") } podSandboxDir := filepath.Join(s.sandboxDir, name) if _, err := os.Stat(podSandboxDir); err == nil { return nil, fmt.Errorf("pod sandbox (%s) already exists", podSandboxDir) } if err := os.MkdirAll(podSandboxDir, 0755); err != nil { return nil, err } defer func() { if err != nil { os.RemoveAll(podSandboxDir) } }() // creates a spec Generator with the default spec. g := generate.New() // setup defaults for the pod sandbox g.SetRootPath("/var/lib/ocid/graph/vfs/pause") g.SetRootReadonly(true) g.SetProcessArgs([]string{"/pause"}) // process req.Hostname hostname := req.GetConfig().GetHostname() if hostname != "" { g.SetHostname(hostname) } // process req.LogDirectory logDir := req.GetConfig().GetLogDirectory() if logDir == "" { logDir = fmt.Sprintf("/var/log/ocid/pods/%s", name) } dnsServers := req.GetConfig().GetDnsOptions().GetServers() dnsSearches := req.GetConfig().GetDnsOptions().GetSearches() resolvPath := fmt.Sprintf("%s/resolv.conf", podSandboxDir) err = parseDNSOptions(dnsServers, dnsSearches, resolvPath) if err != nil { err1 := removeFile(resolvPath) if err1 != nil { err = err1 return nil, fmt.Errorf("%v; failed to remove %s: %v", err, resolvPath, err1) } return nil, err } g.AddBindMount(resolvPath, "/etc/resolv.conf", "ro") labels := req.GetConfig().GetLabels() s.addSandbox(&sandbox{ name: name, logDir: logDir, labels: labels, containers: oci.NewMemoryStore(), }) annotations := req.GetConfig().GetAnnotations() for k, v := range annotations { g.AddAnnotation(k, v) } cgroupParent := req.GetConfig().GetLinux().GetCgroupParent() if cgroupParent != "" { g.SetLinuxCgroupsPath(cgroupParent) } // set up namespaces if req.GetConfig().GetLinux().GetNamespaceOptions().GetHostNetwork() { err = g.RemoveLinuxNamespace("network") if err != nil { return nil, err } } if req.GetConfig().GetLinux().GetNamespaceOptions().GetHostPid() { err = g.RemoveLinuxNamespace("pid") if err != nil { return nil, err } } if req.GetConfig().GetLinux().GetNamespaceOptions().GetHostIpc() { err = g.RemoveLinuxNamespace("ipc") if err != nil { return nil, err } } err = g.SaveToFile(filepath.Join(podSandboxDir, "config.json")) if err != nil { return nil, err } containerName := name + "-infra" container, err := oci.NewContainer(containerName, podSandboxDir, podSandboxDir, labels, name, false) if err != nil { return nil, err } if err = s.runtime.CreateContainer(container); err != nil { return nil, err } if err = s.runtime.UpdateStatus(container); err != nil { return nil, err } // Setup the network podNamespace := "" netnsPath, err := container.NetNsPath() if err != nil { return nil, err } if err = s.netPlugin.SetUpPod(netnsPath, podNamespace, name, containerName); err != nil { return nil, fmt.Errorf("failed to create network for container %s in sandbox %s: %v", containerName, name, err) } if err = s.runtime.StartContainer(container); err != nil { return nil, err } s.addContainer(container) meta := &metadata{ LogDir: logDir, ContainerName: containerName, Labels: labels, } b, err := json.Marshal(meta) if err != nil { return nil, err } // TODO: eventually we would track all containers in this pod so on server start // we can repopulate the structs in memory properly... // e.g. each container can write itself in podSandboxDir if err := ioutil.WriteFile(filepath.Join(podSandboxDir, "metadata.json"), b, 0644); err != nil { return nil, err } if err = s.runtime.UpdateStatus(container); err != nil { return nil, err } return &pb.CreatePodSandboxResponse{PodSandboxId: &name}, nil } // StopPodSandbox stops the sandbox. If there are any running containers in the // sandbox, they should be force terminated. func (s *Server) StopPodSandbox(ctx context.Context, req *pb.StopPodSandboxRequest) (*pb.StopPodSandboxResponse, error) { sbName := req.PodSandboxId if *sbName == "" { return nil, fmt.Errorf("PodSandboxId should not be empty") } sb := s.getSandbox(*sbName) if sb == nil { return nil, fmt.Errorf("specified sandbox not found: %s", *sbName) } podInfraContainer := *sbName + "-infra" containersList := sb.containers.List() for _, c := range containersList { if podInfraContainer == c.Name() { podNamespace := "" netnsPath, err := c.NetNsPath() if err != nil { return nil, err } if err := s.netPlugin.TearDownPod(netnsPath, podNamespace, *sbName, podInfraContainer); err != nil { return nil, fmt.Errorf("failed to destroy network for container %s in sandbox %s: %v", c.Name(), *sbName, err) } } if err := s.runtime.StopContainer(c); err != nil { return nil, fmt.Errorf("failed to stop container %s in sandbox %s: %v", c.Name(), *sbName, err) } } return &pb.StopPodSandboxResponse{}, nil } // RemovePodSandbox deletes the sandbox. If there are any running containers in the // sandbox, they should be force deleted. func (s *Server) RemovePodSandbox(ctx context.Context, req *pb.RemovePodSandboxRequest) (*pb.RemovePodSandboxResponse, error) { sbName := req.PodSandboxId if *sbName == "" { return nil, fmt.Errorf("PodSandboxId should not be empty") } sb := s.getSandbox(*sbName) if sb == nil { return nil, fmt.Errorf("specified sandbox not found: %s", *sbName) } podInfraContainer := *sbName + "-infra" // Delete all the containers in the sandbox containersList := sb.containers.List() for _, c := range containersList { if err := s.runtime.DeleteContainer(c); err != nil { return nil, fmt.Errorf("failed to delete container %s in sandbox %s: %v", c.Name(), *sbName, err) } if podInfraContainer == c.Name() { continue } containerDir := filepath.Join(s.runtime.ContainerDir(), c.Name()) if err := os.RemoveAll(containerDir); err != nil { return nil, fmt.Errorf("failed to remove container %s directory: %v", c.Name(), err) } } // Remove the files related to the sandbox podSandboxDir := filepath.Join(s.sandboxDir, *sbName) if err := os.RemoveAll(podSandboxDir); err != nil { return nil, fmt.Errorf("failed to remove sandbox %s directory: %v", *sbName, err) } return &pb.RemovePodSandboxResponse{}, nil } func int64Ptr(i int64) *int64 { return &i } func int32Ptr(i int32) *int32 { return &i } func sPtr(s string) *string { return &s } // PodSandboxStatus returns the Status of the PodSandbox. func (s *Server) PodSandboxStatus(ctx context.Context, req *pb.PodSandboxStatusRequest) (*pb.PodSandboxStatusResponse, error) { sbName := req.PodSandboxId if *sbName == "" { return nil, fmt.Errorf("PodSandboxId should not be empty") } sb := s.getSandbox(*sbName) if sb == nil { return nil, fmt.Errorf("specified sandbox not found: %s", *sbName) } podInfraContainerName := *sbName + "-infra" podInfraContainer := sb.getContainer(podInfraContainerName) cState := s.runtime.ContainerStatus(podInfraContainer) created := cState.Created.Unix() netNsPath, err := podInfraContainer.NetNsPath() if err != nil { return nil, err } podNamespace := "" ip, err := s.netPlugin.GetContainerNetworkStatus(netNsPath, podNamespace, *sbName, podInfraContainerName) if err != nil { // ignore the error on network status ip = "" } return &pb.PodSandboxStatusResponse{ Status: &pb.PodSandboxStatus{ Id: sbName, CreatedAt: int64Ptr(created), Linux: &pb.LinuxPodSandboxStatus{ Namespaces: &pb.Namespace{ Network: sPtr(netNsPath), }, }, Network: &pb.PodSandboxNetworkStatus{Ip: &ip}, }, }, nil } // ListPodSandbox returns a list of SandBox. func (s *Server) ListPodSandbox(context.Context, *pb.ListPodSandboxRequest) (*pb.ListPodSandboxResponse, error) { return nil, nil } // CreateContainer creates a new container in specified PodSandbox func (s *Server) CreateContainer(ctx context.Context, req *pb.CreateContainerRequest) (*pb.CreateContainerResponse, error) { // The id of the PodSandbox podSandboxId := req.GetPodSandboxId() if !s.hasSandbox(podSandboxId) { return nil, fmt.Errorf("the pod sandbox (%s) does not exist", podSandboxId) } // The config of the container containerConfig := req.GetConfig() if containerConfig == nil { return nil, fmt.Errorf("CreateContainerRequest.ContainerConfig is nil") } name := containerConfig.GetMetadata().GetName() if name == "" { return nil, fmt.Errorf("CreateContainerRequest.ContainerConfig.Name is empty") } // containerDir is the dir for the container bundle. containerDir := filepath.Join(s.runtime.ContainerDir(), name) if _, err := os.Stat(containerDir); err == nil { return nil, fmt.Errorf("container (%s) already exists", containerDir) } if err := os.MkdirAll(containerDir, 0755); err != nil { return nil, err } imageSpec := containerConfig.GetImage() if imageSpec == nil { return nil, fmt.Errorf("CreateContainerRequest.ContainerConfig.Image is nil") } image := imageSpec.GetImage() if image == "" { return nil, fmt.Errorf("CreateContainerRequest.ContainerConfig.Image.Image is empty") } // creates a spec Generator with the default spec. specgen := generate.New() // by default, the root path is an empty string. // here set it to be "rootfs". specgen.SetRootPath("rootfs") args := containerConfig.GetArgs() if args == nil { args = []string{"/bin/sh"} } specgen.SetProcessArgs(args) cwd := containerConfig.GetWorkingDir() if cwd == "" { cwd = "/" } specgen.SetProcessCwd(cwd) envs := containerConfig.GetEnvs() if envs != nil { for _, item := range envs { key := item.GetKey() value := item.GetValue() if key == "" { continue } env := fmt.Sprintf("%s=%s", key, value) specgen.AddProcessEnv(env) } } mounts := containerConfig.GetMounts() for _, mount := range mounts { dest := mount.GetContainerPath() if dest == "" { return nil, fmt.Errorf("Mount.ContainerPath is empty") } src := mount.GetHostPath() if src == "" { return nil, fmt.Errorf("Mount.HostPath is empty") } options := "rw" if mount.GetReadonly() { options = "ro" } //TODO(hmeng): how to use this info? Do we need to handle relabel a FS with Selinux? selinuxRelabel := mount.GetSelinuxRelabel() fmt.Printf("selinuxRelabel: %v\n", selinuxRelabel) specgen.AddBindMount(src, dest, options) } labels := containerConfig.GetLabels() annotations := containerConfig.GetAnnotations() if annotations != nil { for k, v := range annotations { specgen.AddAnnotation(k, v) } } if containerConfig.GetPrivileged() { specgen.SetupPrivileged(true) } if containerConfig.GetReadonlyRootfs() { specgen.SetRootReadonly(true) } logPath := containerConfig.GetLogPath() if containerConfig.GetTty() { specgen.SetProcessTerminal(true) } linux := containerConfig.GetLinux() if linux != nil { resources := linux.GetResources() if resources != nil { cpuPeriod := resources.GetCpuPeriod() if cpuPeriod != 0 { specgen.SetLinuxResourcesCPUPeriod(uint64(cpuPeriod)) } cpuQuota := resources.GetCpuQuota() if cpuQuota != 0 { specgen.SetLinuxResourcesCPUQuota(uint64(cpuQuota)) } cpuShares := resources.GetCpuShares() if cpuShares != 0 { specgen.SetLinuxResourcesCPUShares(uint64(cpuShares)) } memoryLimit := resources.GetMemoryLimitInBytes() if memoryLimit != 0 { specgen.SetLinuxResourcesMemoryLimit(uint64(memoryLimit)) } oomScoreAdj := resources.GetOomScoreAdj() specgen.SetLinuxResourcesOOMScoreAdj(int(oomScoreAdj)) } capabilities := linux.GetCapabilities() if capabilities != nil { addCaps := capabilities.GetAddCapabilities() if addCaps != nil { for _, cap := range addCaps { if err := specgen.AddProcessCapability(cap); err != nil { return nil, err } } } dropCaps := capabilities.GetDropCapabilities() if dropCaps != nil { for _, cap := range dropCaps { if err := specgen.DropProcessCapability(cap); err != nil { return nil, err } } } } selinuxOptions := linux.GetSelinuxOptions() if selinuxOptions != nil { user := selinuxOptions.GetUser() if user == "" { return nil, fmt.Errorf("SELinuxOption.User is empty") } role := selinuxOptions.GetRole() if role == "" { return nil, fmt.Errorf("SELinuxOption.Role is empty") } t := selinuxOptions.GetType() if t == "" { return nil, fmt.Errorf("SELinuxOption.Type is empty") } level := selinuxOptions.GetLevel() if level == "" { return nil, fmt.Errorf("SELinuxOption.Level is empty") } specgen.SetProcessSelinuxLabel(fmt.Sprintf("%s:%s:%s:%s", user, role, t, level)) } user := linux.GetUser() if user != nil { uid := user.GetUid() specgen.SetProcessUID(uint32(uid)) gid := user.GetGid() specgen.SetProcessGID(uint32(gid)) groups := user.GetAdditionalGids() if groups != nil { for _, group := range groups { specgen.AddProcessAdditionalGid(uint32(group)) } } } } // The config of the PodSandbox sandboxConfig := req.GetSandboxConfig() fmt.Printf("sandboxConfig: %v\n", sandboxConfig) // Join the namespace paths for the pod sandbox container. podContainerName := podSandboxId + "-infra" podInfraContainer := s.state.containers.Get(podContainerName) podInfraState := s.runtime.ContainerStatus(podInfraContainer) logrus.Infof("pod container state %v", podInfraState) for nsType, nsFile := range map[string]string{ "ipc": "ipc", "uts": "uts", "network": "net", } { nsPath := fmt.Sprintf("/proc/%d/ns/%s", podInfraState.Pid, nsFile) specgen.AddOrReplaceLinuxNamespace(nsType, nsPath) } if err := specgen.SaveToFile(filepath.Join(containerDir, "config.json")); err != nil { return nil, err } // TODO: copy the rootfs into the bundle. // Currently, utils.CreateFakeRootfs is used to populate the rootfs. if err := utils.CreateFakeRootfs(containerDir, image); err != nil { return nil, err } container, err := oci.NewContainer(name, containerDir, logPath, labels, podSandboxId, containerConfig.GetTty()) if err != nil { return nil, err } if err := s.runtime.CreateContainer(container); err != nil { return nil, err } if err := s.runtime.UpdateStatus(container); err != nil { return nil, err } s.addContainer(container) return &pb.CreateContainerResponse{ ContainerId: &name, }, nil } // StartContainer starts the container. func (s *Server) StartContainer(ctx context.Context, req *pb.StartContainerRequest) (*pb.StartContainerResponse, error) { containerName := req.ContainerId if *containerName == "" { return nil, fmt.Errorf("container ID should not be empty") } c := s.state.containers.Get(*containerName) if c == nil { return nil, fmt.Errorf("specified container not found: %s", *containerName) } if err := s.runtime.StartContainer(c); err != nil { return nil, fmt.Errorf("failed to start container %s in sandbox %s: %v", c.Name(), *containerName, err) } return &pb.StartContainerResponse{}, nil } // StopContainer stops a running container with a grace period (i.e., timeout). func (s *Server) StopContainer(ctx context.Context, req *pb.StopContainerRequest) (*pb.StopContainerResponse, error) { containerName := req.ContainerId if *containerName == "" { return nil, fmt.Errorf("container ID should not be empty") } c := s.state.containers.Get(*containerName) if c == nil { return nil, fmt.Errorf("specified container not found: %s", *containerName) } if err := s.runtime.StopContainer(c); err != nil { return nil, fmt.Errorf("failed to stop container %s: %v", *containerName, err) } return &pb.StopContainerResponse{}, nil } // RemoveContainer removes the container. If the container is running, the container // should be force removed. func (s *Server) RemoveContainer(ctx context.Context, req *pb.RemoveContainerRequest) (*pb.RemoveContainerResponse, error) { containerName := req.ContainerId if *containerName == "" { return nil, fmt.Errorf("container ID should not be empty") } c := s.state.containers.Get(*containerName) if c == nil { return nil, fmt.Errorf("specified container not found: %s", *containerName) } if err := s.runtime.DeleteContainer(c); err != nil { return nil, fmt.Errorf("failed to delete container %s: %v", *containerName, err) } containerDir := filepath.Join(s.runtime.ContainerDir(), *containerName) if err := os.RemoveAll(containerDir); err != nil { return nil, fmt.Errorf("failed to remove container %s directory: %v", *containerName, err) } s.removeContainer(c) return &pb.RemoveContainerResponse{}, nil } // ListContainers lists all containers by filters. func (s *Server) ListContainers(context.Context, *pb.ListContainersRequest) (*pb.ListContainersResponse, error) { return nil, nil } // ContainerStatus returns status of the container. func (s *Server) ContainerStatus(ctx context.Context, req *pb.ContainerStatusRequest) (*pb.ContainerStatusResponse, error) { containerName := req.ContainerId if *containerName == "" { return nil, fmt.Errorf("container ID should not be empty") } c := s.state.containers.Get(*containerName) if c == nil { return nil, fmt.Errorf("specified container not found: %s", *containerName) } if err := s.runtime.UpdateStatus(c); err != nil { return nil, err } csr := &pb.ContainerStatusResponse{ Status: &pb.ContainerStatus{ Id: containerName, }, } cState := s.runtime.ContainerStatus(c) rStatus := pb.ContainerState_UNKNOWN switch cState.Status { case "created": rStatus = pb.ContainerState_CREATED created := cState.Created.Unix() csr.Status.CreatedAt = int64Ptr(created) case "running": rStatus = pb.ContainerState_RUNNING created := cState.Created.Unix() csr.Status.CreatedAt = int64Ptr(created) started := cState.Started.Unix() csr.Status.StartedAt = int64Ptr(started) case "stopped": rStatus = pb.ContainerState_EXITED created := cState.Created.Unix() csr.Status.CreatedAt = int64Ptr(created) started := cState.Started.Unix() csr.Status.StartedAt = int64Ptr(started) finished := cState.Finished.Unix() csr.Status.FinishedAt = int64Ptr(finished) csr.Status.ExitCode = int32Ptr(cState.ExitCode) } csr.Status.State = &rStatus return csr, nil } // Exec executes the command in the container. func (s *Server) Exec(pb.RuntimeService_ExecServer) error { return nil }