diff --git a/pkg/annotations/annotations.go b/pkg/annotations/annotations.go index 26ead571..85f959dc 100644 --- a/pkg/annotations/annotations.go +++ b/pkg/annotations/annotations.go @@ -25,6 +25,9 @@ const ( // KubeName is the kubernetes name annotation KubeName = "io.kubernetes.cri-o.KubeName" + // Namespace is the kubernetes pod namespace annotation + Namespace = "io.kubernetes.cri-o.Namespace" + // Labels are the kubernetes labels annotation Labels = "io.kubernetes.cri-o.Labels" @@ -63,6 +66,9 @@ const ( // StdinOnce is the stdin_once annotation StdinOnce = "io.kubernetes.cri-o.StdinOnce" + + // PortMappings is the annotation for sandbox port mappings + PortMappings = "io.kubernetes.cri-o.PortMappings" ) // ContainerType values diff --git a/server/container.go b/server/container.go index 1a10cfa4..03962217 100644 --- a/server/container.go +++ b/server/container.go @@ -11,14 +11,10 @@ func (s *Server) getContainerFromRequest(cid string) (*oci.Container, error) { return nil, fmt.Errorf("container ID should not be empty") } - containerID, err := s.ctrIDIndex.Get(cid) + c, err := s.state.LookupContainerByID(cid) if err != nil { - return nil, fmt.Errorf("container with ID starting with %s not found: %v", cid, err) + return nil, fmt.Errorf("container with ID starting with %s could not be retrieved: %v", cid, err) } - c := s.state.containers.Get(containerID) - if c == nil { - return nil, fmt.Errorf("specified container not found: %s", containerID) - } return c, nil } diff --git a/server/container_attach.go b/server/container_attach.go index e7823e1e..8d6dd9b5 100644 --- a/server/container_attach.go +++ b/server/container_attach.go @@ -38,10 +38,9 @@ func (s *Server) Attach(ctx context.Context, req *pb.AttachRequest) (*pb.AttachR // Attach endpoint for streaming.Runtime func (ss streamService) Attach(containerID string, inputStream io.Reader, outputStream, errorStream io.WriteCloser, tty bool, resize <-chan term.Size) error { - c := ss.runtimeServer.GetContainer(containerID) - - if c == nil { - return fmt.Errorf("could not find container %q", containerID) + c, err := ss.runtimeServer.GetContainer(containerID) + if err != nil { + return err } if err := ss.runtimeServer.runtime.UpdateStatus(c); err != nil { diff --git a/server/container_create.go b/server/container_create.go index ed7dd126..4a476d25 100644 --- a/server/container_create.go +++ b/server/container_create.go @@ -17,6 +17,7 @@ import ( "github.com/kubernetes-incubator/cri-o/oci" "github.com/kubernetes-incubator/cri-o/pkg/annotations" "github.com/kubernetes-incubator/cri-o/server/apparmor" + "github.com/kubernetes-incubator/cri-o/server/sandbox" "github.com/kubernetes-incubator/cri-o/server/seccomp" "github.com/opencontainers/image-spec/specs-go/v1" "github.com/opencontainers/runc/libcontainer/devices" @@ -34,7 +35,7 @@ const ( seccompLocalhostPrefix = "localhost/" ) -func addOCIBindMounts(sb *sandbox, containerConfig *pb.ContainerConfig, specgen *generate.Generator) error { +func addOCIBindMounts(sb *sandbox.Sandbox, containerConfig *pb.ContainerConfig, specgen *generate.Generator) error { mounts := containerConfig.GetMounts() for _, mount := range mounts { dest := mount.ContainerPath @@ -60,7 +61,7 @@ func addOCIBindMounts(sb *sandbox, containerConfig *pb.ContainerConfig, specgen if mount.SelinuxRelabel { // Need a way in kubernetes to determine if the volume is shared or private - if err := label.Relabel(src, sb.mountLabel, true); err != nil && err != syscall.ENOTSUP { + if err := label.Relabel(src, sb.MountLabel(), true); err != nil && err != syscall.ENOTSUP { return fmt.Errorf("relabel failed %s: %v", src, err) } } @@ -71,7 +72,7 @@ func addOCIBindMounts(sb *sandbox, containerConfig *pb.ContainerConfig, specgen return nil } -func addDevices(sb *sandbox, containerConfig *pb.ContainerConfig, specgen *generate.Generator) error { +func addDevices(sb *sandbox.Sandbox, containerConfig *pb.ContainerConfig, specgen *generate.Generator) error { sp := specgen.Spec() for _, device := range containerConfig.GetDevices() { dev, err := devices.DeviceFromPath(device.HostPath, device.Permissions) @@ -236,14 +237,9 @@ func (s *Server) CreateContainer(ctx context.Context, req *pb.CreateContainerReq return nil, fmt.Errorf("PodSandboxId should not be empty") } - sandboxID, err := s.podIDIndex.Get(sbID) + sb, err := s.state.LookupSandboxByID(sbID) if err != nil { - return nil, fmt.Errorf("PodSandbox with ID starting with %s not found: %v", sbID, err) - } - - sb := s.getSandbox(sandboxID) - if sb == nil { - return nil, fmt.Errorf("specified sandbox not found: %s", sandboxID) + return nil, err } // The config of the container @@ -257,17 +253,11 @@ func (s *Server) CreateContainer(ctx context.Context, req *pb.CreateContainerReq return nil, fmt.Errorf("CreateContainerRequest.ContainerConfig.Name is empty") } - containerID, containerName, err := s.generateContainerIDandName(sb.metadata, containerConfig) + containerID, containerName, err := s.generateContainerIDandName(sb.Metadata(), containerConfig) if err != nil { return nil, err } - defer func() { - if err != nil { - s.releaseContainerName(containerName) - } - }() - container, err := s.createSandboxContainer(ctx, containerID, containerName, sb, req.GetSandboxConfig(), containerConfig) if err != nil { return nil, err @@ -281,7 +271,7 @@ func (s *Server) CreateContainer(ctx context.Context, req *pb.CreateContainerReq } }() - if err = s.runtime.CreateContainer(container, sb.cgroupParent); err != nil { + if err = s.runtime.CreateContainer(container, sb.CgroupParent()); err != nil { return nil, err } @@ -289,10 +279,7 @@ func (s *Server) CreateContainer(ctx context.Context, req *pb.CreateContainerReq return nil, err } - s.addContainer(container) - - if err = s.ctrIDIndex.Add(containerID); err != nil { - s.removeContainer(container) + if err := s.addContainer(container); err != nil { return nil, err } @@ -306,7 +293,7 @@ func (s *Server) CreateContainer(ctx context.Context, req *pb.CreateContainerReq return resp, nil } -func (s *Server) createSandboxContainer(ctx context.Context, containerID string, containerName string, sb *sandbox, SandboxConfig *pb.PodSandboxConfig, containerConfig *pb.ContainerConfig) (*oci.Container, error) { +func (s *Server) createSandboxContainer(ctx context.Context, containerID string, containerName string, sb *sandbox.Sandbox, SandboxConfig *pb.PodSandboxConfig, containerConfig *pb.ContainerConfig) (*oci.Container, error) { if sb == nil { return nil, errors.New("createSandboxContainer needs a sandbox") } @@ -340,7 +327,7 @@ func (s *Server) createSandboxContainer(ctx context.Context, containerID string, // set this container's apparmor profile if it is set by sandbox if s.appArmorEnabled { - appArmorProfileName := s.getAppArmorProfileName(sb.annotations, metadata.Name) + appArmorProfileName := s.getAppArmorProfileName(sb.Annotations(), metadata.Name) if appArmorProfileName != "" { // reload default apparmor profile if it is unloaded. if s.appArmorProfile == apparmor.DefaultApparmorProfile { @@ -367,12 +354,12 @@ func (s *Server) createSandboxContainer(ctx context.Context, containerID string, logPath := containerConfig.LogPath if logPath == "" { // TODO: Should we use sandboxConfig.GetLogDirectory() here? - logPath = filepath.Join(sb.logDir, containerID+".log") + logPath = filepath.Join(sb.LogDir(), containerID+".log") } if !filepath.IsAbs(logPath) { // XXX: It's not really clear what this should be versus the sbox logDirectory. logrus.Warnf("requested logPath for ctr id %s is a relative path: %s", containerID, logPath) - logPath = filepath.Join(sb.logDir, logPath) + logPath = filepath.Join(sb.LogDir(), logPath) } // Handle https://issues.k8s.io/44043 @@ -381,7 +368,7 @@ func (s *Server) createSandboxContainer(ctx context.Context, containerID string, } logrus.WithFields(logrus.Fields{ - "sbox.logdir": sb.logDir, + "sbox.logdir": sb.LogDir(), "ctr.logfile": containerConfig.LogPath, "log_path": logPath, }).Debugf("setting container's log_path") @@ -416,12 +403,12 @@ func (s *Server) createSandboxContainer(ctx context.Context, containerID string, specgen.SetLinuxResourcesOOMScoreAdj(int(oomScoreAdj)) } - if sb.cgroupParent != "" { + if sb.CgroupParent() != "" { if s.config.CgroupManager == "systemd" { - cgPath := sb.cgroupParent + ":" + "crio" + ":" + containerID + cgPath := sb.CgroupParent() + ":" + "crio" + ":" + containerID specgen.SetLinuxCgroupsPath(cgPath) } else { - specgen.SetLinuxCgroupsPath(sb.cgroupParent + "/" + containerID) + specgen.SetLinuxCgroupsPath(sb.CgroupParent() + "/" + containerID) } } @@ -452,8 +439,8 @@ func (s *Server) createSandboxContainer(ctx context.Context, containerID string, } } - specgen.SetProcessSelinuxLabel(sb.processLabel) - specgen.SetLinuxMountLabel(sb.mountLabel) + specgen.SetProcessSelinuxLabel(sb.ProcessLabel()) + specgen.SetLinuxMountLabel(sb.MountLabel()) if containerConfig.GetLinux().GetSecurityContext() != nil && !containerConfig.GetLinux().GetSecurityContext().Privileged { @@ -481,7 +468,7 @@ func (s *Server) createSandboxContainer(ctx context.Context, containerID string, } } // Join the namespace paths for the pod sandbox container. - podInfraState := s.runtime.ContainerStatus(sb.infraContainer) + podInfraState := s.runtime.ContainerStatus(sb.InfraContainer()) logrus.Debugf("pod container state %+v", podInfraState) @@ -490,7 +477,7 @@ func (s *Server) createSandboxContainer(ctx context.Context, containerID string, return nil, err } - netNsPath := sb.netNsPath() + netNsPath := sb.NetNsPath() if netNsPath == "" { // The sandbox does not have a permanent namespace, // it's on the host one. @@ -512,15 +499,15 @@ func (s *Server) createSandboxContainer(ctx context.Context, containerID string, } // bind mount the pod shm - specgen.AddBindMount(sb.shmPath, "/dev/shm", []string{"rw"}) + specgen.AddBindMount(sb.ShmPath(), "/dev/shm", []string{"rw"}) options := []string{"rw"} if readOnlyRootfs { options = []string{"ro"} } - if sb.resolvPath != "" { + if sb.ResolvPath() != "" { // bind mount the pod resolver file - specgen.AddBindMount(sb.resolvPath, "/etc/resolv.conf", options) + specgen.AddBindMount(sb.ResolvPath(), "/etc/resolv.conf", options) } // Bind mount /etc/hosts for host networking containers @@ -528,14 +515,14 @@ func (s *Server) createSandboxContainer(ctx context.Context, containerID string, specgen.AddBindMount("/etc/hosts", "/etc/hosts", options) } - if sb.hostname != "" { - specgen.SetHostname(sb.hostname) + if sb.Hostname() != "" { + specgen.SetHostname(sb.Hostname()) } specgen.AddAnnotation(annotations.Name, containerName) specgen.AddAnnotation(annotations.ContainerID, containerID) - specgen.AddAnnotation(annotations.SandboxID, sb.id) - specgen.AddAnnotation(annotations.SandboxName, sb.infraContainer.Name()) + specgen.AddAnnotation(annotations.SandboxID, sb.ID()) + specgen.AddAnnotation(annotations.SandboxName, sb.InfraContainer().Name()) specgen.AddAnnotation(annotations.ContainerType, annotations.ContainerTypeContainer) specgen.AddAnnotation(annotations.LogPath, logPath) specgen.AddAnnotation(annotations.TTY, fmt.Sprintf("%v", containerConfig.Tty)) @@ -564,19 +551,19 @@ func (s *Server) createSandboxContainer(ctx context.Context, containerID string, } specgen.AddAnnotation(annotations.Annotations, string(kubeAnnotationsJSON)) - if err = s.setupSeccomp(&specgen, containerName, sb.annotations); err != nil { + if err = s.setupSeccomp(&specgen, containerName, sb.Annotations()); err != nil { return nil, err } metaname := metadata.Name attempt := metadata.Attempt containerInfo, err := s.storageRuntimeServer.CreateContainer(s.imageContext, - sb.name, sb.id, + sb.Name(), sb.ID(), image, image, containerName, containerID, metaname, attempt, - sb.mountLabel, + sb.MountLabel(), nil) if err != nil { return nil, err @@ -684,7 +671,7 @@ func (s *Server) createSandboxContainer(ctx context.Context, containerID string, return nil, err } - container, err := oci.NewContainer(containerID, containerName, containerInfo.RunDir, logPath, sb.netNs(), labels, kubeAnnotations, imageSpec, metadata, sb.id, containerConfig.Tty, containerConfig.Stdin, containerConfig.StdinOnce, sb.privileged, sb.trusted, containerInfo.Dir, created, containerImageConfig.Config.StopSignal) + container, err := oci.NewContainer(containerID, containerName, containerInfo.RunDir, logPath, sb.NetNs(), labels, kubeAnnotations, imageSpec, metadata, sb.ID(), containerConfig.Tty, containerConfig.Stdin, containerConfig.StdinOnce, sb.Privileged(), sb.Trusted(), containerInfo.Dir, created, containerImageConfig.Config.StopSignal) if err != nil { return nil, err } diff --git a/server/container_exec.go b/server/container_exec.go index 0711acd0..7340cb2f 100644 --- a/server/container_exec.go +++ b/server/container_exec.go @@ -30,10 +30,14 @@ func (s *Server) Exec(ctx context.Context, req *pb.ExecRequest) (*pb.ExecRespons // Exec endpoint for streaming.Runtime func (ss streamService) Exec(containerID string, cmd []string, stdin io.Reader, stdout, stderr io.WriteCloser, tty bool, resize <-chan term.Size) error { - c := ss.runtimeServer.GetContainer(containerID) + sbID, err := ss.runtimeServer.state.GetContainerSandbox(containerID) + if err != nil { + return err + } - if c == nil { - return fmt.Errorf("could not find container %q", containerID) + c, err := ss.runtimeServer.state.GetContainer(containerID, sbID) + if err != nil { + return err } if err := ss.runtimeServer.runtime.UpdateStatus(c); err != nil { diff --git a/server/container_list.go b/server/container_list.go index 9e372a5f..5d76af55 100644 --- a/server/container_list.go +++ b/server/container_list.go @@ -3,6 +3,7 @@ package server import ( "github.com/Sirupsen/logrus" "github.com/kubernetes-incubator/cri-o/oci" + "github.com/kubernetes-incubator/cri-o/server/state" "golang.org/x/net/context" "k8s.io/apimachinery/pkg/fields" pb "k8s.io/kubernetes/pkg/kubelet/api/v1alpha1/runtime" @@ -31,17 +32,19 @@ func (s *Server) ListContainers(ctx context.Context, req *pb.ListContainersReque logrus.Debugf("ListContainersRequest %+v", req) var ctrs []*pb.Container filter := req.Filter - ctrList := s.state.containers.List() + ctrList, _ := s.state.GetAllContainers() // Filter using container id and pod id first. if filter != nil { if filter.Id != "" { - id, err := s.ctrIDIndex.Get(filter.Id) + c, err := s.state.LookupContainerByID(filter.Id) if err != nil { - return nil, err - } - c := s.state.containers.Get(id) - if c != nil { + if !state.IsCtrNotExist(err) { + return nil, err + } + + ctrList = []*oci.Container{} + } else { if filter.PodSandboxId != "" { if c.Sandbox() == filter.PodSandboxId { ctrList = []*oci.Container{c} @@ -55,11 +58,15 @@ func (s *Server) ListContainers(ctx context.Context, req *pb.ListContainersReque } } else { if filter.PodSandboxId != "" { - pod := s.state.sandboxes[filter.PodSandboxId] - if pod == nil { + pod, err := s.state.GetSandbox(filter.PodSandboxId) + if err != nil { + if !state.IsSandboxNotExist(err) { + return nil, err + } + ctrList = []*oci.Container{} } else { - ctrList = pod.containers.List() + ctrList = pod.Containers() } } } diff --git a/server/container_portforward.go b/server/container_portforward.go index bc222d7d..9edbed06 100644 --- a/server/container_portforward.go +++ b/server/container_portforward.go @@ -28,10 +28,9 @@ func (s *Server) PortForward(ctx context.Context, req *pb.PortForwardRequest) (* } func (ss streamService) PortForward(podSandboxID string, port int32, stream io.ReadWriteCloser) error { - c := ss.runtimeServer.GetSandboxContainer(podSandboxID) - - if c == nil { - return fmt.Errorf("could not find container for sandbox %q", podSandboxID) + c, err := ss.runtimeServer.GetSandboxContainer(podSandboxID) + if err != nil { + return err } if err := ss.runtimeServer.runtime.UpdateStatus(c); err != nil { diff --git a/server/container_remove.go b/server/container_remove.go index 1d1d67a2..89fec1f5 100644 --- a/server/container_remove.go +++ b/server/container_remove.go @@ -36,18 +36,14 @@ func (s *Server) RemoveContainer(ctx context.Context, req *pb.RemoveContainerReq return nil, fmt.Errorf("failed to delete container %s: %v", c.ID(), err) } - s.removeContainer(c) + if err := s.removeContainer(c); err != nil { + return nil, fmt.Errorf("failed to remove container %s: %v", c.ID(), err) + } if err := s.storageRuntimeServer.DeleteContainer(c.ID()); err != nil { return nil, fmt.Errorf("failed to delete storage for container %s: %v", c.ID(), err) } - s.releaseContainerName(c.Name()) - - if err := s.ctrIDIndex.Delete(c.ID()); err != nil { - return nil, err - } - resp := &pb.RemoveContainerResponse{} logrus.Debugf("RemoveContainerResponse: %+v", resp) return resp, nil diff --git a/server/naming.go b/server/naming.go index 0175158b..b783e959 100644 --- a/server/naming.go +++ b/server/naming.go @@ -48,39 +48,30 @@ func makeContainerName(sandboxMetadata *pb.PodSandboxMetadata, containerConfig * func (s *Server) generatePodIDandName(sandboxConfig *pb.PodSandboxConfig) (string, string, error) { var ( - err error - id = stringid.GenerateNonCryptoID() + err error + id = stringid.GenerateNonCryptoID() + name = makeSandboxName(sandboxConfig) ) if sandboxConfig.Metadata.Namespace == "" { return "", "", fmt.Errorf("cannot generate pod ID without namespace") } - name, err := s.reservePodName(id, makeSandboxName(sandboxConfig)) - if err != nil { - return "", "", err - } return id, name, err } func (s *Server) generateContainerIDandNameForSandbox(sandboxConfig *pb.PodSandboxConfig) (string, string, error) { var ( - err error - id = stringid.GenerateNonCryptoID() + err error + id = stringid.GenerateNonCryptoID() + name = makeSandboxContainerName(sandboxConfig) ) - name, err := s.reserveContainerName(id, makeSandboxContainerName(sandboxConfig)) - if err != nil { - return "", "", err - } return id, name, err } func (s *Server) generateContainerIDandName(sandboxMetadata *pb.PodSandboxMetadata, containerConfig *pb.ContainerConfig) (string, string, error) { var ( - err error - id = stringid.GenerateNonCryptoID() + err error + id = stringid.GenerateNonCryptoID() + name = makeContainerName(sandboxMetadata, containerConfig) ) - name, err := s.reserveContainerName(id, makeContainerName(sandboxMetadata, containerConfig)) - if err != nil { - return "", "", err - } return id, name, err } diff --git a/server/sandbox.go b/server/sandbox.go deleted file mode 100644 index 0823fad1..00000000 --- a/server/sandbox.go +++ /dev/null @@ -1,283 +0,0 @@ -package server - -import ( - "crypto/rand" - "errors" - "fmt" - "os" - "path/filepath" - "sync" - - "github.com/Sirupsen/logrus" - "github.com/containernetworking/cni/pkg/ns" - "github.com/docker/docker/pkg/mount" - "github.com/docker/docker/pkg/symlink" - "github.com/kubernetes-incubator/cri-o/oci" - "golang.org/x/sys/unix" - "k8s.io/apimachinery/pkg/fields" - pb "k8s.io/kubernetes/pkg/kubelet/api/v1alpha1/runtime" - "k8s.io/kubernetes/pkg/kubelet/network/hostport" -) - -type sandboxNetNs struct { - sync.Mutex - ns ns.NetNS - symlink *os.File - closed bool - restored bool -} - -func (ns *sandboxNetNs) symlinkCreate(name string) error { - b := make([]byte, 4) - _, randErr := rand.Reader.Read(b) - if randErr != nil { - return randErr - } - - nsName := fmt.Sprintf("%s-%x", name, b) - symlinkPath := filepath.Join(nsRunDir, nsName) - - if err := os.Symlink(ns.ns.Path(), symlinkPath); err != nil { - return err - } - - fd, err := os.Open(symlinkPath) - if err != nil { - if removeErr := os.RemoveAll(symlinkPath); removeErr != nil { - return removeErr - } - - return err - } - - ns.symlink = fd - - return nil -} - -func (ns *sandboxNetNs) symlinkRemove() error { - if err := ns.symlink.Close(); err != nil { - return err - } - - return os.RemoveAll(ns.symlink.Name()) -} - -func isSymbolicLink(path string) (bool, error) { - fi, err := os.Lstat(path) - if err != nil { - return false, err - } - - return fi.Mode()&os.ModeSymlink == os.ModeSymlink, nil -} - -func netNsGet(nspath, name string) (*sandboxNetNs, error) { - if err := ns.IsNSorErr(nspath); err != nil { - return nil, errSandboxClosedNetNS - } - - symlink, symlinkErr := isSymbolicLink(nspath) - if symlinkErr != nil { - return nil, symlinkErr - } - - var resolvedNsPath string - if symlink { - path, err := os.Readlink(nspath) - if err != nil { - return nil, err - } - resolvedNsPath = path - } else { - resolvedNsPath = nspath - } - - netNS, err := ns.GetNS(resolvedNsPath) - if err != nil { - return nil, err - } - - netNs := &sandboxNetNs{ns: netNS, closed: false, restored: true} - - if symlink { - fd, err := os.Open(nspath) - if err != nil { - return nil, err - } - - netNs.symlink = fd - } else { - if err := netNs.symlinkCreate(name); err != nil { - return nil, err - } - } - - return netNs, nil -} - -func hostNetNsPath() (string, error) { - netNS, err := ns.GetCurrentNS() - if err != nil { - return "", err - } - - defer netNS.Close() - return netNS.Path(), nil -} - -type sandbox struct { - id string - namespace string - // OCI pod name (eg "--") - name string - // Kubernetes pod name (eg, "") - kubeName string - logDir string - labels fields.Set - annotations map[string]string - infraContainer *oci.Container - containers oci.ContainerStorer - processLabel string - mountLabel string - netns *sandboxNetNs - metadata *pb.PodSandboxMetadata - shmPath string - cgroupParent string - privileged bool - trusted bool - resolvPath string - hostname string - portMappings []*hostport.PortMapping -} - -const ( - defaultShmSize = 64 * 1024 * 1024 - nsRunDir = "/var/run/netns" - podInfraCommand = "/pause" -) - -var ( - errSandboxIDEmpty = errors.New("PodSandboxId should not be empty") - errSandboxClosedNetNS = errors.New("PodSandbox networking namespace is closed") -) - -func (s *sandbox) addContainer(c *oci.Container) { - s.containers.Add(c.Name(), c) -} - -func (s *sandbox) getContainer(name string) *oci.Container { - return s.containers.Get(name) -} - -func (s *sandbox) removeContainer(c *oci.Container) { - s.containers.Delete(c.Name()) -} - -func (s *sandbox) netNs() ns.NetNS { - if s.netns == nil { - return nil - } - - return s.netns.ns -} - -func (s *sandbox) netNsPath() string { - if s.netns == nil { - return "" - } - - return s.netns.symlink.Name() -} - -func (s *sandbox) netNsCreate() error { - if s.netns != nil { - return fmt.Errorf("net NS already created") - } - - netNS, err := ns.NewNS() - if err != nil { - return err - } - - s.netns = &sandboxNetNs{ - ns: netNS, - closed: false, - } - - if err := s.netns.symlinkCreate(s.name); err != nil { - logrus.Warnf("Could not create nentns symlink %v", err) - - if err1 := s.netns.ns.Close(); err1 != nil { - return err1 - } - - return err - } - - return nil -} - -func (s *sandbox) netNsRemove() error { - if s.netns == nil { - logrus.Warn("no networking namespace") - return nil - } - - s.netns.Lock() - defer s.netns.Unlock() - - if s.netns.closed { - // netNsRemove() can be called multiple - // times without returning an error. - return nil - } - - if err := s.netns.symlinkRemove(); err != nil { - return err - } - - if err := s.netns.ns.Close(); err != nil { - return err - } - - if s.netns.restored { - // we got namespaces in the form of - // /var/run/netns/cni-0d08effa-06eb-a963-f51a-e2b0eceffc5d - // but /var/run on most system is symlinked to /run so we first resolve - // the symlink and then try and see if it's mounted - fp, err := symlink.FollowSymlinkInScope(s.netns.ns.Path(), "/") - if err != nil { - return err - } - if mounted, err := mount.Mounted(fp); err == nil && mounted { - if err := unix.Unmount(fp, unix.MNT_DETACH); err != nil { - return err - } - } - - if err := os.RemoveAll(s.netns.ns.Path()); err != nil { - return err - } - } - - s.netns.closed = true - return nil -} - -func (s *Server) getPodSandboxFromRequest(podSandboxID string) (*sandbox, error) { - if podSandboxID == "" { - return nil, errSandboxIDEmpty - } - - sandboxID, err := s.podIDIndex.Get(podSandboxID) - if err != nil { - return nil, fmt.Errorf("PodSandbox with ID starting with %s not found: %v", podSandboxID, err) - } - - sb := s.getSandbox(sandboxID) - if sb == nil { - return nil, fmt.Errorf("specified pod sandbox not found: %s", sandboxID) - } - return sb, nil -} diff --git a/server/sandbox/sandbox.go b/server/sandbox/sandbox.go new file mode 100644 index 00000000..a9f37469 --- /dev/null +++ b/server/sandbox/sandbox.go @@ -0,0 +1,438 @@ +package sandbox + +import ( + "crypto/rand" + "errors" + "fmt" + "os" + "path/filepath" + "sync" + + "github.com/Sirupsen/logrus" + "github.com/containernetworking/cni/pkg/ns" + "github.com/docker/docker/pkg/mount" + "github.com/docker/docker/pkg/symlink" + "github.com/kubernetes-incubator/cri-o/oci" + "golang.org/x/sys/unix" + "k8s.io/apimachinery/pkg/fields" + pb "k8s.io/kubernetes/pkg/kubelet/api/v1alpha1/runtime" + "k8s.io/kubernetes/pkg/kubelet/network/hostport" +) + +type sandboxNetNs struct { + sync.Mutex + ns ns.NetNS + symlink *os.File + closed bool + restored bool +} + +func (ns *sandboxNetNs) symlinkCreate(name string) error { + b := make([]byte, 4) + _, randErr := rand.Reader.Read(b) + if randErr != nil { + return randErr + } + + nsName := fmt.Sprintf("%s-%x", name, b) + symlinkPath := filepath.Join(nsRunDir, nsName) + + if err := os.Symlink(ns.ns.Path(), symlinkPath); err != nil { + return err + } + + fd, err := os.Open(symlinkPath) + if err != nil { + if removeErr := os.RemoveAll(symlinkPath); removeErr != nil { + return removeErr + } + + return err + } + + ns.symlink = fd + + return nil +} + +func (ns *sandboxNetNs) symlinkRemove() error { + if err := ns.symlink.Close(); err != nil { + return err + } + + return os.RemoveAll(ns.symlink.Name()) +} + +func isSymbolicLink(path string) (bool, error) { + fi, err := os.Lstat(path) + if err != nil { + return false, err + } + + return fi.Mode()&os.ModeSymlink == os.ModeSymlink, nil +} + +func netNsGet(nspath, name string) (*sandboxNetNs, error) { + if err := ns.IsNSorErr(nspath); err != nil { + return nil, ErrSandboxClosedNetNS + } + + symlink, symlinkErr := isSymbolicLink(nspath) + if symlinkErr != nil { + return nil, symlinkErr + } + + var resolvedNsPath string + if symlink { + path, err := os.Readlink(nspath) + if err != nil { + return nil, err + } + resolvedNsPath = path + } else { + resolvedNsPath = nspath + } + + netNS, err := ns.GetNS(resolvedNsPath) + if err != nil { + return nil, err + } + + netNs := &sandboxNetNs{ns: netNS, closed: false, restored: true} + + if symlink { + fd, err := os.Open(nspath) + if err != nil { + return nil, err + } + + netNs.symlink = fd + } else { + if err := netNs.symlinkCreate(name); err != nil { + return nil, err + } + } + + return netNs, nil +} + +// HostNetNsPath returns the path of the host's network namespace +func HostNetNsPath() (string, error) { + netNS, err := ns.GetCurrentNS() + if err != nil { + return "", err + } + + defer netNS.Close() + + return netNS.Path(), nil +} + +// Sandbox represents a single pod sandbox +type Sandbox struct { + id string + namespace string + // OCI pod name (eg "--") + name string + // Kubernetes pod name (eg, "") + kubeName string + logDir string + labels fields.Set + annotations map[string]string + infraContainer *oci.Container + containers oci.ContainerStorer + processLabel string + mountLabel string + netns *sandboxNetNs + metadata *pb.PodSandboxMetadata + shmPath string + cgroupParent string + privileged bool + trusted bool + resolvPath string + hostname string + portMappings []*hostport.PortMapping +} + +const ( + // DefaultShmSize is the default size of the SHM device for sandboxs + DefaultShmSize = 64 * 1024 * 1024 + // PodInfraCommand is the default pause command for pods + PodInfraCommand = "/pause" + nsRunDir = "/var/run/netns" +) + +var ( + // ErrSandboxIDEmpty is the error returned when an operation passes "" instead of a sandbox ID + ErrSandboxIDEmpty = errors.New("PodSandboxId should not be empty") + // ErrSandboxClosedNetNS is the error returned when a network namespace is closed and cannot be joined + ErrSandboxClosedNetNS = errors.New("PodSandbox networking namespace is closed") +) + +// New creates and populates a new sandbox +// New sandboxes have no containers, no infra container, and no network namespace associated with them. +// An infra container must be attached before the sandbox is added to the state +func New(id, namespace, name, kubeName, logDir string, labels, annotations map[string]string, processLabel, mountLabel string, metadata *pb.PodSandboxMetadata, shmPath, cgroupParent string, privileged bool, trusted bool, resolvPath, hostname string, portMappings []*hostport.PortMapping) (*Sandbox, error) { + sb := new(Sandbox) + sb.id = id + sb.namespace = namespace + sb.name = name + sb.kubeName = kubeName + sb.logDir = logDir + sb.labels = labels + sb.annotations = annotations + sb.containers = oci.NewMemoryStore() + sb.processLabel = processLabel + sb.mountLabel = mountLabel + sb.metadata = metadata + sb.shmPath = shmPath + sb.cgroupParent = cgroupParent + sb.privileged = privileged + sb.trusted = trusted + sb.resolvPath = resolvPath + sb.hostname = hostname + sb.portMappings = portMappings + + return sb, nil +} + +// ID returns the sandbox's ID +func (s *Sandbox) ID() string { + return s.id +} + +// Namespace returns the sandbox's namespace +func (s *Sandbox) Namespace() string { + return s.namespace +} + +// Name returns the sandbox's name +func (s *Sandbox) Name() string { + return s.name +} + +// KubeName returns the name the sandbox was given by Kubernetes +// This is not a fully qualified, globally unique name and cannot be used to look up the sandbox +func (s *Sandbox) KubeName() string { + return s.kubeName +} + +// LogDir returns the directory the sandbox logs to +func (s *Sandbox) LogDir() string { + return s.logDir +} + +// Labels returns the sandbox's labels +func (s *Sandbox) Labels() map[string]string { + return s.labels +} + +// Annotations returns the sandbox's annotations +func (s *Sandbox) Annotations() map[string]string { + return s.annotations +} + +// InfraContainer returns the sandbox's infrastructure container +func (s *Sandbox) InfraContainer() *oci.Container { + return s.infraContainer +} + +// Containers returns an array of all the containers in the sandbox +func (s *Sandbox) Containers() []*oci.Container { + return s.containers.List() +} + +// ProcessLabel returns the SELinux process label of the sandbox +func (s *Sandbox) ProcessLabel() string { + return s.processLabel +} + +// MountLabel returns the SELinux mount label of the sandbox +func (s *Sandbox) MountLabel() string { + return s.mountLabel +} + +// Metadata returns Kubernetes metadata associated with the sandbox +func (s *Sandbox) Metadata() *pb.PodSandboxMetadata { + return s.metadata +} + +// ShmPath returns the path to the sandbox's shared memory device +func (s *Sandbox) ShmPath() string { + return s.shmPath +} + +// CgroupParent returns the sandbox's CGroup parent +func (s *Sandbox) CgroupParent() string { + return s.cgroupParent +} + +// Privileged returns whether the sandbox can support privileged containers +func (s *Sandbox) Privileged() bool { + return s.privileged +} + +// Trusted returns whether the sandbox is a trusted workload +func (s *Sandbox) Trusted() bool { + return s.trusted +} + +// ResolvPath returns the path to the sandbox's DNS resolver configuration +func (s *Sandbox) ResolvPath() string { + return s.resolvPath +} + +// Hostname returns the sandbox's hostname +func (s *Sandbox) Hostname() string { + return s.hostname +} + +// PortMappings gets the sandbox's host port mappings +func (s *Sandbox) PortMappings() []*hostport.PortMapping { + return s.portMappings +} + +// AddContainer adds a container to the sandbox +func (s *Sandbox) AddContainer(c *oci.Container) { + s.containers.Add(c.ID(), c) +} + +// GetContainer retrieves the container with given ID from the sandbox +// Returns nil if no such container exists +func (s *Sandbox) GetContainer(id string) *oci.Container { + return s.containers.Get(id) +} + +// RemoveContainer removes the container with given ID from the sandbox +// If no container with that ID exists in the sandbox, no action is taken +func (s *Sandbox) RemoveContainer(id string) { + s.containers.Delete(id) +} + +// SetInfraContainer sets the infrastructure container of a sandbox +// Attempts to set the infrastructure container after one is already present will throw an error +func (s *Sandbox) SetInfraContainer(infraCtr *oci.Container) error { + if s.infraContainer != nil { + return fmt.Errorf("sandbox already has an infra container") + } else if infraCtr == nil { + return fmt.Errorf("must provide non-nil infra container") + } + + s.infraContainer = infraCtr + + return nil +} + +// NetNs retrieves the network namespace of the sandbox +// If the sandbox uses the host namespace, nil is returned +func (s *Sandbox) NetNs() ns.NetNS { + if s.netns == nil { + return nil + } + + return s.netns.ns +} + +// NetNsPath returns the path to the network namespace +// If the sandbox uses the host namespace, "" is returned +func (s *Sandbox) NetNsPath() string { + if s.netns == nil { + return "" + } + + return s.netns.symlink.Name() +} + +// NetNsCreate creates a new network namespace for the sandbox +func (s *Sandbox) NetNsCreate() error { + if s.netns != nil { + return fmt.Errorf("net NS already created") + } + + netNS, err := ns.NewNS() + if err != nil { + return err + } + + s.netns = &sandboxNetNs{ + ns: netNS, + closed: false, + } + + if err := s.netns.symlinkCreate(s.name); err != nil { + logrus.Warnf("Could not create nentns symlink %v", err) + + if err1 := s.netns.ns.Close(); err1 != nil { + return err1 + } + + return err + } + + return nil +} + +// NetNsJoin attempts to join the sandbox to an existing network namespace +// This will fail if the sandbox is already part of a network namespace +func (s *Sandbox) NetNsJoin(nspath, name string) error { + if s.netns != nil { + return fmt.Errorf("sandbox already has a network namespace, cannot join another") + } + + netNS, err := netNsGet(nspath, name) + if err != nil { + return err + } + + s.netns = netNS + + return nil +} + +// NetNsRemove removes the network namespace associated with the sandbox +func (s *Sandbox) NetNsRemove() error { + if s.netns == nil { + logrus.Warn("no networking namespace") + return nil + } + + s.netns.Lock() + defer s.netns.Unlock() + + if s.netns.closed { + // netNsRemove() can be called multiple + // times without returning an error. + return nil + } + + if err := s.netns.symlinkRemove(); err != nil { + return err + } + + if err := s.netns.ns.Close(); err != nil { + return err + } + + if s.netns.restored { + // we got namespaces in the form of + // /var/run/netns/cni-0d08effa-06eb-a963-f51a-e2b0eceffc5d + // but /var/run on most system is symlinked to /run so we first resolve + // the symlink and then try and see if it's mounted + fp, err := symlink.FollowSymlinkInScope(s.netns.ns.Path(), "/") + if err != nil { + return err + } + if mounted, err := mount.Mounted(fp); err == nil && mounted { + if err := unix.Unmount(fp, unix.MNT_DETACH); err != nil { + return err + } + } + + if err := os.RemoveAll(s.netns.ns.Path()); err != nil { + return err + } + } + + s.netns.closed = true + return nil +} diff --git a/server/sandbox_list.go b/server/sandbox_list.go index 155c3d59..c01ac808 100644 --- a/server/sandbox_list.go +++ b/server/sandbox_list.go @@ -1,8 +1,12 @@ package server import ( + "fmt" + "github.com/Sirupsen/logrus" "github.com/kubernetes-incubator/cri-o/oci" + "github.com/kubernetes-incubator/cri-o/server/sandbox" + "github.com/kubernetes-incubator/cri-o/server/state" "golang.org/x/net/context" "k8s.io/apimachinery/pkg/fields" pb "k8s.io/kubernetes/pkg/kubelet/api/v1alpha1/runtime" @@ -30,30 +34,34 @@ func filterSandbox(p *pb.PodSandbox, filter *pb.PodSandboxFilter) bool { func (s *Server) ListPodSandbox(ctx context.Context, req *pb.ListPodSandboxRequest) (*pb.ListPodSandboxResponse, error) { logrus.Debugf("ListPodSandboxRequest %+v", req) var pods []*pb.PodSandbox - var podList []*sandbox - for _, sb := range s.state.sandboxes { - podList = append(podList, sb) + var podList []*sandbox.Sandbox + + sandboxes, err := s.state.GetAllSandboxes() + if err != nil { + return nil, fmt.Errorf("error retrieving sandboxes: %v", err) } + podList = append(podList, sandboxes...) + filter := req.Filter // Filter by pod id first. if filter != nil { if filter.Id != "" { - id, err := s.podIDIndex.Get(filter.Id) + sb, err := s.state.LookupSandboxByID(filter.Id) if err != nil { - return nil, err - } - sb := s.getSandbox(id) - if sb == nil { - podList = []*sandbox{} + if !state.IsCtrNotExist(err) { + return nil, err + } + + podList = []*sandbox.Sandbox{} } else { - podList = []*sandbox{sb} + podList = []*sandbox.Sandbox{sb} } } } for _, sb := range podList { - podInfraContainer := sb.infraContainer + podInfraContainer := sb.InfraContainer() if podInfraContainer == nil { // this can't really happen, but if it does because of a bug // it's better not to panic @@ -71,12 +79,12 @@ func (s *Server) ListPodSandbox(ctx context.Context, req *pb.ListPodSandboxReque } pod := &pb.PodSandbox{ - Id: sb.id, + Id: sb.ID(), CreatedAt: created, State: rStatus, - Labels: sb.labels, - Annotations: sb.annotations, - Metadata: sb.metadata, + Labels: sb.Labels(), + Annotations: sb.Annotations(), + Metadata: sb.Metadata(), } // Filter by other criteria such as state and labels. diff --git a/server/sandbox_remove.go b/server/sandbox_remove.go index cd4955f8..58f5ef9d 100644 --- a/server/sandbox_remove.go +++ b/server/sandbox_remove.go @@ -7,6 +7,7 @@ import ( "github.com/containers/storage" "github.com/kubernetes-incubator/cri-o/oci" pkgstorage "github.com/kubernetes-incubator/cri-o/pkg/storage" + "github.com/kubernetes-incubator/cri-o/server/sandbox" "golang.org/x/net/context" pb "k8s.io/kubernetes/pkg/kubelet/api/v1alpha1/runtime" ) @@ -17,7 +18,7 @@ func (s *Server) RemovePodSandbox(ctx context.Context, req *pb.RemovePodSandboxR logrus.Debugf("RemovePodSandboxRequest %+v", req) sb, err := s.getPodSandboxFromRequest(req.PodSandboxId) if err != nil { - if err == errSandboxIDEmpty { + if err == sandbox.ErrSandboxIDEmpty { return nil, err } @@ -30,8 +31,8 @@ func (s *Server) RemovePodSandbox(ctx context.Context, req *pb.RemovePodSandboxR return resp, nil } - podInfraContainer := sb.infraContainer - containers := sb.containers.List() + podInfraContainer := sb.InfraContainer() + containers := sb.Containers() containers = append(containers, podInfraContainer) // Delete all the containers in the sandbox @@ -49,7 +50,7 @@ func (s *Server) RemovePodSandbox(ctx context.Context, req *pb.RemovePodSandboxR } if err := s.runtime.DeleteContainer(c); err != nil { - return nil, fmt.Errorf("failed to delete container %s in pod sandbox %s: %v", c.Name(), sb.id, err) + return nil, fmt.Errorf("failed to delete container %s in pod sandbox %s: %v", c.Name(), sb.ID(), err) } if c.ID() == podInfraContainer.ID() { @@ -58,38 +59,27 @@ func (s *Server) RemovePodSandbox(ctx context.Context, req *pb.RemovePodSandboxR if err := s.storageRuntimeServer.StopContainer(c.ID()); err != nil && err != storage.ErrContainerUnknown { // assume container already umounted - logrus.Warnf("failed to stop container %s in pod sandbox %s: %v", c.Name(), sb.id, err) + logrus.Warnf("failed to stop container %s in pod sandbox %s: %v", c.Name(), sb.ID(), err) } if err := s.storageRuntimeServer.DeleteContainer(c.ID()); err != nil && err != storage.ErrContainerUnknown { - return nil, fmt.Errorf("failed to delete container %s in pod sandbox %s: %v", c.Name(), sb.id, err) + return nil, fmt.Errorf("failed to delete container %s in pod sandbox %s: %v", c.Name(), sb.ID(), err) } - s.releaseContainerName(c.Name()) - s.removeContainer(c) - if err := s.ctrIDIndex.Delete(c.ID()); err != nil { - return nil, fmt.Errorf("failed to delete container %s in pod sandbox %s from index: %v", c.Name(), sb.id, err) + if err := s.removeContainer(c); err != nil { + return nil, err } } - s.removeContainer(podInfraContainer) + if err := s.removeSandbox(sb.ID()); err != nil { + return nil, fmt.Errorf("error removing sandbox %s: %v", sb.ID(), err) + } // Remove the files related to the sandbox - if err := s.storageRuntimeServer.StopContainer(sb.id); err != nil && err != storage.ErrContainerUnknown { - logrus.Warnf("failed to stop sandbox container in pod sandbox %s: %v", sb.id, err) + if err := s.storageRuntimeServer.StopContainer(sb.ID()); err != nil && err != storage.ErrContainerUnknown { + logrus.Warnf("failed to stop sandbox container in pod sandbox %s: %v", sb.ID(), err) } - if err := s.storageRuntimeServer.RemovePodSandbox(sb.id); err != nil && err != pkgstorage.ErrInvalidSandboxID { - return nil, fmt.Errorf("failed to remove pod sandbox %s: %v", sb.id, err) - } - - s.releaseContainerName(podInfraContainer.Name()) - if err := s.ctrIDIndex.Delete(podInfraContainer.ID()); err != nil { - return nil, fmt.Errorf("failed to delete infra container %s in pod sandbox %s from index: %v", podInfraContainer.ID(), sb.id, err) - } - - s.releasePodName(sb.name) - s.removeSandbox(sb.id) - if err := s.podIDIndex.Delete(sb.id); err != nil { - return nil, fmt.Errorf("failed to delete pod sandbox %s from index: %v", sb.id, err) + if err := s.storageRuntimeServer.RemovePodSandbox(sb.ID()); err != nil && err != pkgstorage.ErrInvalidSandboxID { + return nil, fmt.Errorf("failed to remove pod sandbox %s: %v", sb.ID(), err) } resp := &pb.RemovePodSandboxResponse{} diff --git a/server/sandbox_run.go b/server/sandbox_run.go index 2f7ea045..37cd8e7e 100644 --- a/server/sandbox_run.go +++ b/server/sandbox_run.go @@ -16,6 +16,7 @@ import ( "github.com/containers/storage" "github.com/kubernetes-incubator/cri-o/oci" "github.com/kubernetes-incubator/cri-o/pkg/annotations" + "github.com/kubernetes-incubator/cri-o/server/sandbox" "github.com/opencontainers/runc/libcontainer/cgroups/systemd" "github.com/opencontainers/runtime-tools/generate" "github.com/opencontainers/selinux/go-selinux/label" @@ -135,23 +136,11 @@ func (s *Server) RunPodSandbox(ctx context.Context, req *pb.RunPodSandboxRequest } } - defer func() { - if err != nil { - s.releasePodName(name) - } - }() - _, containerName, err := s.generateContainerIDandNameForSandbox(req.GetConfig()) if err != nil { return nil, err } - defer func() { - if err != nil { - s.releaseContainerName(containerName) - } - }() - podContainer, err := s.storageRuntimeServer.CreatePodSandbox(s.imageContext, name, id, s.config.PauseImage, "", @@ -186,7 +175,7 @@ func (s *Server) RunPodSandbox(ctx context.Context, req *pb.RunPodSandboxRequest if podContainer.Config != nil { g.SetProcessArgs(podContainer.Config.Config.Cmd) } else { - g.SetProcessArgs([]string{podInfraCommand}) + g.SetProcessArgs([]string{sandbox.PodInfraCommand}) } } else { g.SetProcessArgs([]string{s.config.PauseCommand}) @@ -283,18 +272,6 @@ func (s *Server) RunPodSandbox(ctx context.Context, req *pb.RunPodSandboxRequest return nil, err } - if err = s.ctrIDIndex.Add(id); err != nil { - return nil, err - } - - defer func() { - if err != nil { - if err2 := s.ctrIDIndex.Delete(id); err2 != nil { - logrus.Warnf("couldn't delete ctr id %s from idIndex", id) - } - } - }() - // set log path inside log directory logPath := filepath.Join(logDir, id+".log") @@ -320,6 +297,7 @@ func (s *Server) RunPodSandbox(ctx context.Context, req *pb.RunPodSandboxRequest g.AddAnnotation(annotations.ResolvPath, resolvPath) g.AddAnnotation(annotations.HostName, hostname) g.AddAnnotation(annotations.KubeName, kubeName) + g.AddAnnotation(annotations.Namespace, namespace) if podContainer.Config.Config.StopSignal != "" { // this key is defined in image-spec conversion document at https://github.com/opencontainers/image-spec/pull/492/files#diff-8aafbe2c3690162540381b8cdb157112R57 g.AddAnnotation("org.opencontainers.image.stopSignal", podContainer.Config.Config.StopSignal) @@ -329,45 +307,11 @@ func (s *Server) RunPodSandbox(ctx context.Context, req *pb.RunPodSandboxRequest g.AddAnnotation(annotations.Created, created.Format(time.RFC3339Nano)) portMappings := convertPortMappings(req.GetConfig().GetPortMappings()) - - sb := &sandbox{ - id: id, - namespace: namespace, - name: name, - kubeName: kubeName, - logDir: logDir, - labels: labels, - annotations: kubeAnnotations, - containers: oci.NewMemoryStore(), - processLabel: processLabel, - mountLabel: mountLabel, - metadata: metadata, - shmPath: shmPath, - privileged: privileged, - trusted: trusted, - resolvPath: resolvPath, - hostname: hostname, - portMappings: portMappings, - } - - s.addSandbox(sb) - defer func() { - if err != nil { - s.removeSandbox(id) - } - }() - - if err = s.podIDIndex.Add(id); err != nil { + mappingsJSON, err := json.Marshal(portMappings) + if err != nil { return nil, err } - - defer func() { - if err != nil { - if err := s.podIDIndex.Delete(id); err != nil { - logrus.Warnf("couldn't delete pod id %s from idIndex", id) - } - } - }() + g.AddAnnotation(annotations.PortMappings, string(mappingsJSON)) for k, v := range kubeAnnotations { g.AddAnnotation(k, v) @@ -394,10 +338,9 @@ func (s *Server) RunPodSandbox(ctx context.Context, req *pb.RunPodSandboxRequest return nil, err } g.SetLinuxCgroupsPath(cgPath + ":" + "crio" + ":" + id) - sb.cgroupParent = cgPath + cgroupParent = cgPath } else { g.SetLinuxCgroupsPath(cgroupParent + "/" + id) - sb.cgroupParent = cgroupParent } } @@ -405,6 +348,11 @@ func (s *Server) RunPodSandbox(ctx context.Context, req *pb.RunPodSandboxRequest // so it doesn't get killed. g.SetLinuxResourcesOOMScoreAdj(PodInfraOOMAdj) + sb, err := sandbox.New(id, namespace, name, kubeName, logDir, labels, kubeAnnotations, processLabel, mountLabel, metadata, shmPath, cgroupParent, privileged, trusted, resolvPath, hostname, portMappings) + if err != nil { + return nil, err + } + hostNetwork := req.GetConfig().GetLinux().GetSecurityContext().GetNamespaceOptions().HostNetwork // set up namespaces @@ -414,13 +362,13 @@ func (s *Server) RunPodSandbox(ctx context.Context, req *pb.RunPodSandboxRequest return nil, err } - netNsPath, err = hostNetNsPath() + netNsPath, err = sandbox.HostNetNsPath() if err != nil { return nil, err } } else { // Create the sandbox network namespace - if err = sb.netNsCreate(); err != nil { + if err = sb.NetNsCreate(); err != nil { return nil, err } @@ -429,18 +377,18 @@ func (s *Server) RunPodSandbox(ctx context.Context, req *pb.RunPodSandboxRequest return } - if netnsErr := sb.netNsRemove(); netnsErr != nil { + if netnsErr := sb.NetNsRemove(); netnsErr != nil { logrus.Warnf("Failed to remove networking namespace: %v", netnsErr) } }() // Pass the created namespace path to the runtime - err = g.AddOrReplaceLinuxNamespace("network", sb.netNsPath()) + err = g.AddOrReplaceLinuxNamespace("network", sb.NetNsPath()) if err != nil { return nil, err } - netNsPath = sb.netNsPath() + netNsPath = sb.NetNsPath() } if req.GetConfig().GetLinux().GetSecurityContext().GetNamespaceOptions().HostPid { @@ -464,23 +412,28 @@ func (s *Server) RunPodSandbox(ctx context.Context, req *pb.RunPodSandboxRequest saveOptions := generate.ExportOptions{} mountPoint, err := s.storageRuntimeServer.StartContainer(id) if err != nil { - return nil, fmt.Errorf("failed to mount container %s in pod sandbox %s(%s): %v", containerName, sb.name, id, err) + return nil, fmt.Errorf("failed to mount container %s in pod sandbox %s(%s): %v", containerName, name, id, err) } g.SetRootPath(mountPoint) err = g.SaveToFile(filepath.Join(podContainer.Dir, "config.json"), saveOptions) if err != nil { - return nil, fmt.Errorf("failed to save template configuration for pod sandbox %s(%s): %v", sb.name, id, err) + return nil, fmt.Errorf("failed to save template configuration for pod sandbox %s(%s): %v", name, id, err) } if err = g.SaveToFile(filepath.Join(podContainer.RunDir, "config.json"), saveOptions); err != nil { - return nil, fmt.Errorf("failed to write runtime configuration for pod sandbox %s(%s): %v", sb.name, id, err) + return nil, fmt.Errorf("failed to write runtime configuration for pod sandbox %s(%s): %v", name, id, err) } - container, err := oci.NewContainer(id, containerName, podContainer.RunDir, logPath, sb.netNs(), labels, kubeAnnotations, nil, nil, id, false, false, false, sb.privileged, sb.trusted, podContainer.Dir, created, podContainer.Config.Config.StopSignal) + container, err := oci.NewContainer(id, containerName, podContainer.RunDir, logPath, sb.NetNs(), labels, kubeAnnotations, nil, nil, id, false, false, false, sb.Privileged(), sb.Trusted(), podContainer.Dir, created, podContainer.Config.Config.StopSignal) if err != nil { return nil, err } + if err := sb.SetInfraContainer(container); err != nil { + return nil, err + } - sb.infraContainer = container + if err = s.addSandbox(sb); err != nil { + return nil, err + } // setup the network if !hostNetwork { @@ -511,7 +464,7 @@ func (s *Server) RunPodSandbox(ctx context.Context, req *pb.RunPodSandboxRequest } } - if err = s.runContainer(container, sb.cgroupParent); err != nil { + if err = s.runContainer(container, sb.CgroupParent()); err != nil { return nil, err } @@ -579,7 +532,7 @@ func setupShm(podSandboxRunDir, mountLabel string) (shmPath string, err error) { if err = os.Mkdir(shmPath, 0700); err != nil { return "", err } - shmOptions := "mode=1777,size=" + strconv.Itoa(defaultShmSize) + shmOptions := "mode=1777,size=" + strconv.Itoa(sandbox.DefaultShmSize) if err = syscall.Mount("shm", shmPath, "tmpfs", uintptr(syscall.MS_NOEXEC|syscall.MS_NOSUID|syscall.MS_NODEV), label.FormatMountLabel(shmOptions, mountLabel)); err != nil { return "", fmt.Errorf("failed to mount shm tmpfs for pod: %v", err) diff --git a/server/sandbox_status.go b/server/sandbox_status.go index 2ec89795..974e01c7 100644 --- a/server/sandbox_status.go +++ b/server/sandbox_status.go @@ -15,7 +15,7 @@ func (s *Server) PodSandboxStatus(ctx context.Context, req *pb.PodSandboxStatusR return nil, err } - podInfraContainer := sb.infraContainer + podInfraContainer := sb.InfraContainer() if err = s.runtime.UpdateStatus(podInfraContainer); err != nil { return nil, err } @@ -27,7 +27,7 @@ func (s *Server) PodSandboxStatus(ctx context.Context, req *pb.PodSandboxStatusR if err != nil { return nil, err } - ip, err := s.netPlugin.GetContainerNetworkStatus(netNsPath, sb.namespace, sb.kubeName, sb.id) + ip, err := s.netPlugin.GetContainerNetworkStatus(netNsPath, sb.Namespace(), sb.KubeName(), sb.ID()) if err != nil { // ignore the error on network status ip = "" @@ -38,7 +38,7 @@ func (s *Server) PodSandboxStatus(ctx context.Context, req *pb.PodSandboxStatusR rStatus = pb.PodSandboxState_SANDBOX_READY } - sandboxID := sb.id + sandboxID := sb.ID() resp := &pb.PodSandboxStatusResponse{ Status: &pb.PodSandboxStatus{ Id: sandboxID, @@ -50,9 +50,9 @@ func (s *Server) PodSandboxStatus(ctx context.Context, req *pb.PodSandboxStatusR }, Network: &pb.PodSandboxNetworkStatus{Ip: ip}, State: rStatus, - Labels: sb.labels, - Annotations: sb.annotations, - Metadata: sb.metadata, + Labels: sb.Labels(), + Annotations: sb.Annotations(), + Metadata: sb.Metadata(), }, } diff --git a/server/sandbox_stop.go b/server/sandbox_stop.go index a5422274..6b9bf3d6 100644 --- a/server/sandbox_stop.go +++ b/server/sandbox_stop.go @@ -9,6 +9,7 @@ import ( "github.com/docker/docker/pkg/mount" "github.com/docker/docker/pkg/symlink" "github.com/kubernetes-incubator/cri-o/oci" + "github.com/kubernetes-incubator/cri-o/server/sandbox" "github.com/opencontainers/selinux/go-selinux/label" "golang.org/x/net/context" "golang.org/x/sys/unix" @@ -22,7 +23,7 @@ func (s *Server) StopPodSandbox(ctx context.Context, req *pb.StopPodSandboxReque logrus.Debugf("StopPodSandboxRequest %+v", req) sb, err := s.getPodSandboxFromRequest(req.PodSandboxId) if err != nil { - if err == errSandboxIDEmpty { + if err == sandbox.ErrSandboxIDEmpty { return nil, err } @@ -35,36 +36,36 @@ func (s *Server) StopPodSandbox(ctx context.Context, req *pb.StopPodSandboxReque return resp, nil } - podInfraContainer := sb.infraContainer + podInfraContainer := sb.InfraContainer() netnsPath, err := podInfraContainer.NetNsPath() if err != nil { return nil, err } if _, err := os.Stat(netnsPath); err == nil { - if err2 := s.hostportManager.Remove(sb.id, &hostport.PodPortMapping{ - Name: sb.name, - PortMappings: sb.portMappings, + if err2 := s.hostportManager.Remove(sb.ID(), &hostport.PodPortMapping{ + Name: sb.Name(), + PortMappings: sb.PortMappings(), HostNetwork: false, }); err2 != nil { logrus.Warnf("failed to remove hostport for container %s in sandbox %s: %v", - podInfraContainer.Name(), sb.id, err2) + podInfraContainer.Name(), sb.ID(), err2) } - if err2 := s.netPlugin.TearDownPod(netnsPath, sb.namespace, sb.kubeName, sb.id); err2 != nil { + if err2 := s.netPlugin.TearDownPod(netnsPath, sb.Namespace(), sb.KubeName(), sb.ID()); err2 != nil { logrus.Warnf("failed to destroy network for container %s in sandbox %s: %v", - podInfraContainer.Name(), sb.id, err2) + podInfraContainer.Name(), sb.ID(), err2) } } else if !os.IsNotExist(err) { // it's ok for netnsPath to *not* exist return nil, fmt.Errorf("failed to stat netns path for container %s in sandbox %s before tearing down the network: %v", - sb.name, sb.id, err) + podInfraContainer.Name(), sb.ID(), err) } // Close the sandbox networking namespace. - if err := sb.netNsRemove(); err != nil { + if err := sb.NetNsRemove(); err != nil { return nil, err } - containers := sb.containers.List() + containers := sb.Containers() containers = append(containers, podInfraContainer) for _, c := range containers { @@ -74,30 +75,30 @@ func (s *Server) StopPodSandbox(ctx context.Context, req *pb.StopPodSandboxReque cStatus := s.runtime.ContainerStatus(c) if cStatus.Status != oci.ContainerStateStopped { if err := s.runtime.StopContainer(c, -1); err != nil { - return nil, fmt.Errorf("failed to stop container %s in pod sandbox %s: %v", c.Name(), sb.id, err) + return nil, fmt.Errorf("failed to stop container %s in pod sandbox %s: %v", c.Name(), sb.ID(), err) } if c.ID() == podInfraContainer.ID() { continue } if err := s.storageRuntimeServer.StopContainer(c.ID()); err != nil && err != storage.ErrContainerUnknown { // assume container already umounted - logrus.Warnf("failed to stop container %s in pod sandbox %s: %v", c.Name(), sb.id, err) + logrus.Warnf("failed to stop container %s in pod sandbox %s: %v", c.Name(), sb.ID(), err) } } s.containerStateToDisk(c) } - if err := label.ReleaseLabel(sb.processLabel); err != nil { + if err := label.ReleaseLabel(sb.ProcessLabel()); err != nil { return nil, err } // unmount the shm for the pod - if sb.shmPath != "/dev/shm" { + if sb.ShmPath() != "/dev/shm" { // we got namespaces in the form of // /var/run/containers/storage/overlay-containers/CID/userdata/shm // but /var/run on most system is symlinked to /run so we first resolve // the symlink and then try and see if it's mounted - fp, err := symlink.FollowSymlinkInScope(sb.shmPath, "/") + fp, err := symlink.FollowSymlinkInScope(sb.ShmPath(), "/") if err != nil { return nil, err } @@ -107,8 +108,8 @@ func (s *Server) StopPodSandbox(ctx context.Context, req *pb.StopPodSandboxReque } } } - if err := s.storageRuntimeServer.StopContainer(sb.id); err != nil && err != storage.ErrContainerUnknown { - logrus.Warnf("failed to stop sandbox container in pod sandbox %s: %v", sb.id, err) + if err := s.storageRuntimeServer.StopContainer(sb.ID()); err != nil && err != storage.ErrContainerUnknown { + logrus.Warnf("failed to stop sandbox container in pod sandbox %s: %v", sb.ID(), err) } resp := &pb.StopPodSandboxResponse{} @@ -119,12 +120,17 @@ func (s *Server) StopPodSandbox(ctx context.Context, req *pb.StopPodSandboxReque // StopAllPodSandboxes removes all pod sandboxes func (s *Server) StopAllPodSandboxes() { logrus.Debugf("StopAllPodSandboxes") - for _, sb := range s.state.sandboxes { + sandboxes, err := s.state.GetAllSandboxes() + if err != nil { + logrus.Errorf("error retrieving sandboxes: %v", err) + return + } + for _, sb := range sandboxes { pod := &pb.StopPodSandboxRequest{ - PodSandboxId: sb.id, + PodSandboxId: sb.ID(), } if _, err := s.StopPodSandbox(nil, pod); err != nil { - logrus.Warnf("could not StopPodSandbox %s: %v", sb.id, err) + logrus.Warnf("could not StopPodSandbox %s: %v", sb.ID(), err) } } } diff --git a/server/server.go b/server/server.go index 17ca8364..c97a248c 100644 --- a/server/server.go +++ b/server/server.go @@ -14,14 +14,14 @@ import ( "github.com/containers/image/types" sstorage "github.com/containers/storage" "github.com/docker/docker/pkg/ioutils" - "github.com/docker/docker/pkg/registrar" - "github.com/docker/docker/pkg/truncindex" "github.com/kubernetes-incubator/cri-o/oci" "github.com/kubernetes-incubator/cri-o/pkg/annotations" "github.com/kubernetes-incubator/cri-o/pkg/ocicni" "github.com/kubernetes-incubator/cri-o/pkg/storage" "github.com/kubernetes-incubator/cri-o/server/apparmor" + "github.com/kubernetes-incubator/cri-o/server/sandbox" "github.com/kubernetes-incubator/cri-o/server/seccomp" + "github.com/kubernetes-incubator/cri-o/server/state" rspec "github.com/opencontainers/runtime-spec/specs-go" "github.com/opencontainers/selinux/go-selinux/label" knet "k8s.io/apimachinery/pkg/util/net" @@ -57,15 +57,10 @@ type Server struct { store sstorage.Store storageImageServer storage.ImageServer storageRuntimeServer storage.RuntimeServer - stateLock sync.Mutex updateLock sync.RWMutex - state *serverState + state state.RuntimeStateStorer netPlugin ocicni.CNIPlugin hostportManager hostport.HostPortManager - podNameIndex *registrar.Registrar - podIDIndex *truncindex.TruncIndex - ctrNameIndex *registrar.Registrar - ctrIDIndex *truncindex.TruncIndex imageContext *types.SystemContext seccompEnabled bool @@ -106,23 +101,13 @@ func (s *Server) loadContainer(id string) error { return err } name := m.Annotations[annotations.Name] - name, err = s.reserveContainerName(id, name) - if err != nil { - return err - } - - defer func() { - if err != nil { - s.releaseContainerName(name) - } - }() var metadata pb.ContainerMetadata if err = json.Unmarshal([]byte(m.Annotations[annotations.Metadata]), &metadata); err != nil { return err } - sb := s.getSandbox(m.Annotations[annotations.SandboxID]) - if sb == nil { + sb, err := s.getSandbox(m.Annotations[annotations.SandboxID]) + if err != nil { return fmt.Errorf("could not get sandbox with id %s, skipping", m.Annotations[annotations.SandboxID]) } @@ -158,15 +143,14 @@ func (s *Server) loadContainer(id string) error { return err } - ctr, err := oci.NewContainer(id, name, containerPath, m.Annotations[annotations.LogPath], sb.netNs(), labels, kubeAnnotations, img, &metadata, sb.id, tty, stdin, stdinOnce, sb.privileged, sb.trusted, containerDir, created, m.Annotations["org.opencontainers.image.stopSignal"]) + ctr, err := oci.NewContainer(id, name, containerPath, m.Annotations[annotations.LogPath], sb.NetNs(), labels, kubeAnnotations, img, &metadata, sb.ID(), tty, stdin, stdinOnce, sb.Privileged(), sb.Trusted(), containerDir, created, m.Annotations["org.opencontainers.image.stopSignal"]) if err != nil { return err } s.containerStateFromDisk(ctr) - s.addContainer(ctr) - return s.ctrIDIndex.Add(id) + return s.addContainer(ctr) } func (s *Server) containerStateFromDisk(c *oci.Container) error { @@ -224,15 +208,6 @@ func (s *Server) loadSandbox(id string) error { return err } name := m.Annotations[annotations.Name] - name, err = s.reservePodName(id, name) - if err != nil { - return err - } - defer func() { - if err != nil { - s.releasePodName(name) - } - }() var metadata pb.PodSandboxMetadata if err = json.Unmarshal([]byte(m.Annotations[annotations.Metadata]), &metadata); err != nil { return err @@ -248,49 +223,32 @@ func (s *Server) loadSandbox(id string) error { return err } + portMappings := []*hostport.PortMapping{} + if err = json.Unmarshal([]byte(m.Annotations[annotations.PortMappings]), &portMappings); err != nil { + return err + } + privileged := isTrue(m.Annotations[annotations.PrivilegedRuntime]) trusted := isTrue(m.Annotations[annotations.TrustedSandbox]) - sb := &sandbox{ - id: id, - name: name, - kubeName: m.Annotations[annotations.KubeName], - logDir: filepath.Dir(m.Annotations[annotations.LogPath]), - labels: labels, - containers: oci.NewMemoryStore(), - processLabel: processLabel, - mountLabel: mountLabel, - annotations: kubeAnnotations, - metadata: &metadata, - shmPath: m.Annotations[annotations.ShmPath], - privileged: privileged, - trusted: trusted, - resolvPath: m.Annotations[annotations.ResolvPath], + sb, err := sandbox.New(id, m.Annotations[annotations.Namespace], name, m.Annotations[annotations.KubeName], filepath.Dir(m.Annotations[annotations.LogPath]), labels, kubeAnnotations, processLabel, mountLabel, &metadata, m.Annotations[annotations.ShmPath], m.Linux.CgroupsPath, privileged, trusted, m.Annotations[annotations.ResolvPath], m.Annotations[annotations.HostName], portMappings) + if err != nil { + return err } // We add a netNS only if we can load a permanent one. // Otherwise, the sandbox will live in the host namespace. netNsPath, err := configNetNsPath(m) if err == nil { - netNS, nsErr := netNsGet(netNsPath, sb.name) // If we can't load the networking namespace - // because it's closed, we just set the sb netns - // pointer to nil. Otherwise we return an error. - if nsErr != nil && nsErr != errSandboxClosedNetNS { - return nsErr + // because it's closed, just leave the sandbox's netns pointer as nil + if nsErr := sb.NetNsJoin(netNsPath, sb.Name()); err != nil { + if nsErr != sandbox.ErrSandboxClosedNetNS { + return nsErr + } } - - sb.netns = netNS } - s.addSandbox(sb) - - defer func() { - if err != nil { - s.removeSandbox(sb.id) - } - }() - sandboxPath, err := s.store.ContainerRunDirectory(id) if err != nil { return err @@ -301,22 +259,12 @@ func (s *Server) loadSandbox(id string) error { return err } - cname, err := s.reserveContainerName(m.Annotations[annotations.ContainerID], m.Annotations[annotations.ContainerName]) - if err != nil { - return err - } - defer func() { - if err != nil { - s.releaseContainerName(cname) - } - }() - created, err := time.Parse(time.RFC3339Nano, m.Annotations[annotations.Created]) if err != nil { return err } - scontainer, err := oci.NewContainer(m.Annotations[annotations.ContainerID], cname, sandboxPath, m.Annotations[annotations.LogPath], sb.netNs(), labels, kubeAnnotations, nil, nil, id, false, false, false, privileged, trusted, sandboxDir, created, m.Annotations["org.opencontainers.image.stopSignal"]) + scontainer, err := oci.NewContainer(m.Annotations[annotations.ContainerID], m.Annotations[annotations.ContainerName], sandboxPath, m.Annotations[annotations.LogPath], sb.NetNs(), labels, kubeAnnotations, nil, nil, id, false, false, false, privileged, trusted, sandboxDir, created, m.Annotations["org.opencontainers.image.stopSignal"]) if err != nil { return err } @@ -326,14 +274,11 @@ func (s *Server) loadSandbox(id string) error { if err = label.ReserveLabel(processLabel); err != nil { return err } - sb.infraContainer = scontainer - if err = s.ctrIDIndex.Add(scontainer.ID()); err != nil { + if err = sb.SetInfraContainer(scontainer); err != nil { return err } - if err = s.podIDIndex.Add(id); err != nil { - return err - } - return nil + + return s.addSandbox(sb) } func (s *Server) restore() { @@ -399,7 +344,7 @@ func (s *Server) update() error { oldPodContainers[container.ID] = container.ID continue } - if s.getContainer(container.ID) != nil { + if _, err := s.getContainer(container.ID); err == nil { // FIXME: do we need to reload/update any info about the container? oldPodContainers[container.ID] = container.ID continue @@ -416,52 +361,54 @@ func (s *Server) update() error { newPodContainers[container.ID] = &metadata } } - s.ctrIDIndex.Iterate(func(id string) { - if _, ok := oldPodContainers[id]; !ok { + + // TODO this will not check pod infra containers - should we care about this? + stateContainers, err := s.state.GetAllContainers() + if err != nil { + return fmt.Errorf("error retrieving containers list: %v", err) + } + for _, ctr := range stateContainers { + if _, ok := oldPodContainers[ctr.ID()]; !ok { // this container's ID wasn't in the updated list -> removed - removedPodContainers[id] = id + removedPodContainers[ctr.ID()] = ctr.ID() } - }) + } + for removedPodContainer := range removedPodContainers { // forget this container - c := s.getContainer(removedPodContainer) - if c == nil { + c, err := s.getContainer(removedPodContainer) + if err != nil { logrus.Warnf("bad state when getting container removed %+v", removedPodContainer) continue } - s.releaseContainerName(c.Name()) - s.removeContainer(c) - if err = s.ctrIDIndex.Delete(c.ID()); err != nil { - return err + if err := s.removeContainer(c); err != nil { + return fmt.Errorf("error forgetting removed pod container %s: %v", c.ID(), err) } logrus.Debugf("forgetting removed pod container %s", c.ID()) } - s.podIDIndex.Iterate(func(id string) { - if _, ok := oldPods[id]; !ok { + + pods, err := s.state.GetAllSandboxes() + if err != nil { + return fmt.Errorf("error retrieving pods list: %v", err) + } + for _, pod := range pods { + if _, ok := oldPods[pod.ID()]; !ok { // this pod's ID wasn't in the updated list -> removed - removedPods[id] = id + removedPods[pod.ID()] = pod.ID() } - }) + } + for removedPod := range removedPods { // forget this pod - sb := s.getSandbox(removedPod) - if sb == nil { + sb, err := s.getSandbox(removedPod) + if err != nil { logrus.Warnf("bad state when getting pod to remove %+v", removedPod) continue } - podInfraContainer := sb.infraContainer - s.releaseContainerName(podInfraContainer.Name()) - s.removeContainer(podInfraContainer) - if err = s.ctrIDIndex.Delete(podInfraContainer.ID()); err != nil { - return err + if err := s.removeSandbox(sb.ID()); err != nil { + return fmt.Errorf("error removing sandbox %s: %v", sb.ID(), err) } - sb.infraContainer = nil - s.releasePodName(sb.name) - s.removeSandbox(sb.id) - if err = s.podIDIndex.Delete(sb.id); err != nil { - return err - } - logrus.Debugf("forgetting removed pod %s", sb.id) + logrus.Debugf("forgetting removed pod %s", sb.ID()) } for sandboxID := range newPods { // load this pod @@ -482,44 +429,6 @@ func (s *Server) update() error { return nil } -func (s *Server) reservePodName(id, name string) (string, error) { - if err := s.podNameIndex.Reserve(name, id); err != nil { - if err == registrar.ErrNameReserved { - id, err := s.podNameIndex.Get(name) - if err != nil { - logrus.Warnf("conflict, pod name %q already reserved", name) - return "", err - } - return "", fmt.Errorf("conflict, name %q already reserved for pod %q", name, id) - } - return "", fmt.Errorf("error reserving pod name %q", name) - } - return name, nil -} - -func (s *Server) releasePodName(name string) { - s.podNameIndex.Release(name) -} - -func (s *Server) reserveContainerName(id, name string) (string, error) { - if err := s.ctrNameIndex.Reserve(name, id); err != nil { - if err == registrar.ErrNameReserved { - id, err := s.ctrNameIndex.Get(name) - if err != nil { - logrus.Warnf("conflict, ctr name %q already reserved", name) - return "", err - } - return "", fmt.Errorf("conflict, name %q already reserved for ctr %q", name, id) - } - return "", fmt.Errorf("error reserving ctr name %s", name) - } - return name, nil -} - -func (s *Server) releaseContainerName(name string) { - s.ctrNameIndex.Release(name) -} - // cleanupSandboxesOnShutdown Remove all running Sandboxes on system shutdown func (s *Server) cleanupSandboxesOnShutdown() { _, err := os.Stat(shutdownFile) @@ -575,8 +484,6 @@ func New(config *Config) (*Server, error) { if err != nil { return nil, err } - sandboxes := make(map[string]*sandbox) - containers := oci.NewMemoryStore() netPlugin, err := ocicni.InitCNI(config.NetworkDir, config.PluginDir) if err != nil { return nil, err @@ -592,13 +499,10 @@ func New(config *Config) (*Server, error) { netPlugin: netPlugin, hostportManager: hostportManager, config: *config, - state: &serverState{ - sandboxes: sandboxes, - containers: containers, - }, - seccompEnabled: seccomp.IsEnabled(), - appArmorEnabled: apparmor.IsEnabled(), - appArmorProfile: config.ApparmorProfile, + state: state.NewInMemoryState(), + seccompEnabled: seccomp.IsEnabled(), + appArmorEnabled: apparmor.IsEnabled(), + appArmorProfile: config.ApparmorProfile, } if s.seccompEnabled { seccompProfile, fileErr := ioutil.ReadFile(config.SeccompProfile) @@ -618,10 +522,6 @@ func New(config *Config) (*Server, error) { } } - s.podIDIndex = truncindex.NewTruncIndex([]string{}) - s.podNameIndex = registrar.NewRegistrar() - s.ctrIDIndex = truncindex.NewTruncIndex([]string{}) - s.ctrNameIndex = registrar.NewRegistrar() s.imageContext = &types.SystemContext{ SignaturePolicyPath: config.ImageConfig.SignaturePolicyPath, } @@ -656,73 +556,66 @@ func New(config *Config) (*Server, error) { s.stream.streamServer.Start(true) }() - logrus.Debugf("sandboxes: %v", s.state.sandboxes) - logrus.Debugf("containers: %v", s.state.containers) return s, nil } -type serverState struct { - sandboxes map[string]*sandbox - containers oci.ContainerStorer +func (s *Server) addSandbox(sb *sandbox.Sandbox) error { + return s.state.AddSandbox(sb) } -func (s *Server) addSandbox(sb *sandbox) { - s.stateLock.Lock() - s.state.sandboxes[sb.id] = sb - s.stateLock.Unlock() -} - -func (s *Server) getSandbox(id string) *sandbox { - s.stateLock.Lock() - sb := s.state.sandboxes[id] - s.stateLock.Unlock() - return sb +func (s *Server) getSandbox(id string) (*sandbox.Sandbox, error) { + return s.state.GetSandbox(id) } func (s *Server) hasSandbox(id string) bool { - s.stateLock.Lock() - _, ok := s.state.sandboxes[id] - s.stateLock.Unlock() - return ok + return s.state.HasSandbox(id) } -func (s *Server) removeSandbox(id string) { - s.stateLock.Lock() - delete(s.state.sandboxes, id) - s.stateLock.Unlock() +func (s *Server) removeSandbox(id string) error { + return s.state.DeleteSandbox(id) } -func (s *Server) addContainer(c *oci.Container) { - s.stateLock.Lock() - sandbox := s.state.sandboxes[c.Sandbox()] - // TODO(runcom): handle !ok above!!! otherwise it panics! - sandbox.addContainer(c) - s.state.containers.Add(c.ID(), c) - s.stateLock.Unlock() +func (s *Server) addContainer(c *oci.Container) error { + return s.state.AddContainer(c) } -func (s *Server) getContainer(id string) *oci.Container { - s.stateLock.Lock() - c := s.state.containers.Get(id) - s.stateLock.Unlock() - return c +func (s *Server) getContainer(id string) (*oci.Container, error) { + sbID, err := s.state.GetContainerSandbox(id) + if err != nil { + return nil, err + } + + return s.state.GetContainer(id, sbID) } // GetSandboxContainer returns the infra container for a given sandbox -func (s *Server) GetSandboxContainer(id string) *oci.Container { - sb := s.getSandbox(id) - return sb.infraContainer +func (s *Server) GetSandboxContainer(id string) (*oci.Container, error) { + sb, err := s.getSandbox(id) + if err != nil { + return nil, err + } + + return sb.InfraContainer(), nil } // GetContainer returns a container by its ID -func (s *Server) GetContainer(id string) *oci.Container { +func (s *Server) GetContainer(id string) (*oci.Container, error) { return s.getContainer(id) } -func (s *Server) removeContainer(c *oci.Container) { - s.stateLock.Lock() - sandbox := s.state.sandboxes[c.Sandbox()] - sandbox.removeContainer(c) - s.state.containers.Delete(c.ID()) - s.stateLock.Unlock() +func (s *Server) removeContainer(c *oci.Container) error { + return s.state.DeleteContainer(c.ID(), c.Sandbox()) +} + +func (s *Server) getPodSandboxFromRequest(podSandboxID string) (*sandbox.Sandbox, error) { + if podSandboxID == "" { + return nil, sandbox.ErrSandboxIDEmpty + } + + sb, err := s.state.LookupSandboxByID(podSandboxID) + if err != nil { + return nil, err + } + + return sb, nil } diff --git a/server/state/error.go b/server/state/error.go new file mode 100644 index 00000000..ffbf04f1 --- /dev/null +++ b/server/state/error.go @@ -0,0 +1,98 @@ +package state + +import ( + "fmt" +) + +// NoSuchSandboxError is an error occurring when requested sandbox does not exist +type NoSuchSandboxError struct { + id string + name string + inner error +} + +// Error produces an human-readable error message +func (e NoSuchSandboxError) Error() string { + if e.id != "" { + if e.inner == nil { + return fmt.Sprintf("no sandbox found with ID %s", e.id) + } + + return fmt.Sprintf("no sandbox found with ID %s: %v", e.id, e.inner) + } else if e.name != "" { + if e.inner == nil { + return fmt.Sprintf("no sandbox found with name %s", e.name) + } + + return fmt.Sprintf("no sandbox found with name %s: %v", e.name, e.inner) + } else if e.inner != nil { + return fmt.Sprintf("no such sandbox: %v", e.inner) + } + + return "no such sandbox" +} + +// NoSuchCtrError is an error occurring when requested container does not exist +type NoSuchCtrError struct { + id string + name string + sandbox string + inner error +} + +// Error produces a human-readable error message +func (e NoSuchCtrError) Error() string { + if e.id != "" { + if e.sandbox != "" { + if e.inner == nil { + return fmt.Sprintf("no container found with ID %s in sandbox %s", e.id, e.sandbox) + } + + return fmt.Sprintf("no container found with ID %s in sandbox %s: %v", e.id, e.sandbox, e.inner) + } + if e.inner == nil { + return fmt.Sprintf("no container found with ID %s", e.id) + } + + return fmt.Sprintf("no container found with ID %s: %v", e.id, e.inner) + } else if e.name != "" { + if e.sandbox != "" { + if e.inner == nil { + return fmt.Sprintf("no container found with name %s in sandbox %s", e.name, e.sandbox) + } + + return fmt.Sprintf("no container found with name %s in sandbox %s: %v", e.name, e.sandbox, e.inner) + } + if e.inner == nil { + return fmt.Sprintf("no container found with name %s", e.name) + } + + return fmt.Sprintf("no container found with name %s: %v", e.name, e.inner) + } else if e.inner != nil { + return fmt.Sprintf("no such container: %v", e.inner) + } + + return "no such container" +} + +// Functions for verifying errors + +// IsSandboxNotExist checks if an error indicated that given sandbox does not exist +func IsSandboxNotExist(err error) bool { + switch err.(type) { + case *NoSuchSandboxError: + return true + default: + return false + } +} + +// IsCtrNotExist checks if an error indicates that given container does not exist +func IsCtrNotExist(err error) bool { + switch err.(type) { + case *NoSuchCtrError: + return true + default: + return false + } +} diff --git a/server/state/in_memory_state.go b/server/state/in_memory_state.go new file mode 100644 index 00000000..2259673d --- /dev/null +++ b/server/state/in_memory_state.go @@ -0,0 +1,469 @@ +package state + +import ( + "fmt" + "sync" + + "github.com/Sirupsen/logrus" + "github.com/docker/docker/pkg/registrar" + "github.com/docker/docker/pkg/truncindex" + "github.com/kubernetes-incubator/cri-o/oci" + "github.com/kubernetes-incubator/cri-o/server/sandbox" +) + +// TODO: make operations atomic to greatest extent possible + +// InMemoryState is an in-memory state store, suitable for use when no other +// programs are expected to interact with the server +type InMemoryState struct { + lock sync.Mutex + sandboxes map[string]*sandbox.Sandbox + containers map[string]*oci.Container + podNameIndex *registrar.Registrar + podIDIndex *truncindex.TruncIndex + ctrNameIndex *registrar.Registrar + ctrIDIndex *truncindex.TruncIndex +} + +// NewInMemoryState creates a new, empty server state +func NewInMemoryState() RuntimeStateStorer { + state := new(InMemoryState) + state.sandboxes = make(map[string]*sandbox.Sandbox) + state.containers = make(map[string]*oci.Container) + state.podNameIndex = registrar.NewRegistrar() + state.podIDIndex = truncindex.NewTruncIndex([]string{}) + state.ctrNameIndex = registrar.NewRegistrar() + state.ctrIDIndex = truncindex.NewTruncIndex([]string{}) + + return state +} + +// AddSandbox adds a sandbox and any containers in it to the state +func (s *InMemoryState) AddSandbox(sandbox *sandbox.Sandbox) (err error) { + s.lock.Lock() + defer s.lock.Unlock() + + if sandbox == nil { + return fmt.Errorf("nil passed as sandbox to AddSandbox") + } + + // We shouldn't share ID with any containers + // Our pod infra container will share our ID and we don't want it to conflict with anything + // An equivalent check for sandbox IDs is done in addSandboxMappings() + if _, exist := s.containers[sandbox.ID()]; exist { + return fmt.Errorf("requested sandbox ID %v conflicts with existing container ID", sandbox.ID()) + } + + if err := s.addSandboxMappings(sandbox); err != nil { + return err + } + defer func() { + if err != nil { + if err2 := s.deleteSandboxMappings(sandbox); err2 != nil { + logrus.Errorf("Error removing mappings for incompletely-added sandbox %s: %v", sandbox.ID(), err2) + } + } + }() + + // If there are containers in the sandbox add them to the mapping + for _, ctr := range sandbox.Containers() { + if err := s.addContainerMappings(ctr, true); err != nil { + return fmt.Errorf("error adding container %v mappings in sandbox %v", ctr.ID(), sandbox.ID()) + } + + defer func(c *oci.Container) { + if err != nil { + if err2 := s.deleteContainerMappings(c, true); err2 != nil { + logrus.Errorf("Error removing container %s mappings: %v", c.ID(), err2) + } + } + }(ctr) + } + + // Add the pod infrastructure container to mappings + // TODO: Right now, we don't add it to the all containers listing. We may want to change this. + if err := s.addContainerMappings(sandbox.InfraContainer(), false); err != nil { + return fmt.Errorf("error adding infrastructure container %v to mappings: %v", sandbox.InfraContainer().ID(), err) + } + + return nil +} + +// Add sandbox name, ID to appropriate mappings +func (s *InMemoryState) addSandboxMappings(sb *sandbox.Sandbox) error { + if _, exist := s.sandboxes[sb.ID()]; exist { + return fmt.Errorf("sandbox with ID %s already exists in sandboxes map", sb.ID()) + } + + if err := s.podNameIndex.Reserve(sb.Name(), sb.ID()); err != nil { + return fmt.Errorf("error registering sandbox name: %v", err) + } + if err := s.podIDIndex.Add(sb.ID()); err != nil { + s.podNameIndex.Release(sb.Name()) + return fmt.Errorf("error registering sandbox ID: %v", err) + } + s.sandboxes[sb.ID()] = sb + + return nil +} + +// HasSandbox determines if a given sandbox exists in the state +func (s *InMemoryState) HasSandbox(id string) bool { + s.lock.Lock() + defer s.lock.Unlock() + + _, exist := s.sandboxes[id] + + return exist +} + +// DeleteSandbox removes a sandbox from the state +func (s *InMemoryState) DeleteSandbox(id string) (err error) { + s.lock.Lock() + defer s.lock.Unlock() + + if _, exist := s.sandboxes[id]; !exist { + return &NoSuchSandboxError{id: id} + } + + sb := s.sandboxes[id] + + if err := s.deleteSandboxMappings(sb); err != nil { + return err + } + defer func() { + if err != nil { + if err2 := s.addSandboxMappings(sb); err2 != nil { + logrus.Errorf("Error re-adding sandbox mappings: %v", err2) + } + } + }() + + // If there are containers left in the sandbox delete them from the mappings + for _, ctr := range sb.Containers() { + if err := s.deleteContainerMappings(ctr, true); err != nil { + return fmt.Errorf("error removing container %v mappings: %v", ctr.ID(), err) + } + + defer func(c *oci.Container) { + if err != nil { + if err2 := s.addContainerMappings(c, true); err2 != nil { + logrus.Errorf("Error re-adding mappings for container %s: %v", c.ID(), err2) + } + } + }(ctr) + } + + // Delete infra container from mappings + if err := s.deleteContainerMappings(sb.InfraContainer(), false); err != nil { + return fmt.Errorf("error removing infra container %v from mappings: %v", sb.InfraContainer().ID(), err) + } + + return nil +} + +// Remove sandbox name, ID to appropriate mappings +func (s *InMemoryState) deleteSandboxMappings(sb *sandbox.Sandbox) error { + if _, exist := s.sandboxes[sb.ID()]; !exist { + return fmt.Errorf("sandbox with ID %s does not exist in sandboxes map", sb.ID()) + } + + if err := s.podIDIndex.Delete(sb.ID()); err != nil { + return fmt.Errorf("error unregistering sandbox %s: %v", sb.ID(), err) + } + delete(s.sandboxes, sb.ID()) + s.podNameIndex.Release(sb.Name()) + + return nil +} + +// GetSandbox returns a sandbox given its full ID +func (s *InMemoryState) GetSandbox(id string) (*sandbox.Sandbox, error) { + s.lock.Lock() + defer s.lock.Unlock() + + sandbox, ok := s.sandboxes[id] + if !ok { + return nil, &NoSuchSandboxError{id: id} + } + + return sandbox, nil +} + +// LookupSandboxByName returns a sandbox given its full or partial name +func (s *InMemoryState) LookupSandboxByName(name string) (*sandbox.Sandbox, error) { + s.lock.Lock() + defer s.lock.Unlock() + + id, err := s.podNameIndex.Get(name) + if err != nil { + return nil, &NoSuchSandboxError{ + name: name, + inner: err, + } + } + + sandbox, ok := s.sandboxes[id] + if !ok { + // This should never happen - our internal state must be desynced + return nil, fmt.Errorf("cannot find sandbox %v in sandboxes map", id) + } + + return sandbox, nil +} + +// LookupSandboxByID returns a sandbox given its full or partial ID +// An error will be returned if the partial ID given is not unique +func (s *InMemoryState) LookupSandboxByID(id string) (*sandbox.Sandbox, error) { + s.lock.Lock() + defer s.lock.Unlock() + + fullID, err := s.podIDIndex.Get(id) + if err != nil { + return nil, &NoSuchSandboxError{ + id: id, + inner: err, + } + } + + sandbox, ok := s.sandboxes[fullID] + if !ok { + // This should never happen, internal state must be desynced + return nil, fmt.Errorf("cannot find sandbox %v in sandboxes map", fullID) + } + + return sandbox, nil +} + +// GetAllSandboxes returns all sandboxes in the state +func (s *InMemoryState) GetAllSandboxes() ([]*sandbox.Sandbox, error) { + s.lock.Lock() + defer s.lock.Unlock() + + sandboxes := make([]*sandbox.Sandbox, 0, len(s.sandboxes)) + for _, sb := range s.sandboxes { + sandboxes = append(sandboxes, sb) + } + + return sandboxes, nil +} + +// AddContainer adds a single container to a given sandbox in the state +func (s *InMemoryState) AddContainer(c *oci.Container) error { + s.lock.Lock() + defer s.lock.Unlock() + + if c == nil { + return fmt.Errorf("nil passed as container to AddContainer") + } + + sandbox, ok := s.sandboxes[c.Sandbox()] + if !ok { + return &NoSuchSandboxError{id: c.Sandbox()} + } + + if ctr := sandbox.GetContainer(c.ID()); ctr != nil { + return fmt.Errorf("container with ID %v already exists in sandbox %v", c.ID(), c.Sandbox()) + } + + if sandbox.InfraContainer().ID() == c.ID() { + return fmt.Errorf("container is infra container of sandbox %s, refusing to add to containers list", c.Sandbox()) + } + + if err := s.addContainerMappings(c, true); err != nil { + return err + } + + sandbox.AddContainer(c) + + return nil +} + +// Add container ID, Name and Sandbox mappings +func (s *InMemoryState) addContainerMappings(c *oci.Container, addToContainers bool) error { + if addToContainers { + if _, exist := s.containers[c.ID()]; exist { + return fmt.Errorf("container with ID %v already exists in containers store", c.ID()) + } + } + + // TODO: if not a pod infra container, check if it conflicts with existing sandbox ID? + // Does this matter? + + if err := s.ctrNameIndex.Reserve(c.Name(), c.ID()); err != nil { + return fmt.Errorf("error registering name for container %s: %v", c.ID(), err) + } + + if err := s.ctrIDIndex.Add(c.ID()); err != nil { + s.ctrNameIndex.Release(c.ID()) + return fmt.Errorf("error registering ID for container %s: %v", c.ID(), err) + } + + if addToContainers { + s.containers[c.ID()] = c + } + + return nil +} + +// HasContainer checks if a container with the given ID exists in a given sandbox +func (s *InMemoryState) HasContainer(id, sandboxID string) bool { + s.lock.Lock() + defer s.lock.Unlock() + + sandbox, ok := s.sandboxes[sandboxID] + if !ok { + return false + } + + ctr := sandbox.GetContainer(id) + + return ctr != nil +} + +// DeleteContainer removes the container with given ID from the given sandbox +func (s *InMemoryState) DeleteContainer(id, sandboxID string) error { + s.lock.Lock() + defer s.lock.Unlock() + + sandbox, ok := s.sandboxes[sandboxID] + if !ok { + return &NoSuchSandboxError{id: sandboxID} + } + + ctr := sandbox.GetContainer(id) + if ctr == nil { + return &NoSuchCtrError{ + id: id, + sandbox: id, + } + } + + if err := s.deleteContainerMappings(ctr, true); err != nil { + return nil + } + + sandbox.RemoveContainer(id) + + return nil +} + +// Deletes container from the ID and Name mappings and optionally from the global containers list +func (s *InMemoryState) deleteContainerMappings(ctr *oci.Container, deleteFromContainers bool) error { + if deleteFromContainers { + if _, exist := s.containers[ctr.ID()]; !exist { + return fmt.Errorf("container ID %v does not exist in containers store", ctr.ID()) + } + } + + if err := s.ctrIDIndex.Delete(ctr.ID()); err != nil { + return fmt.Errorf("error unregistering container ID for %s: %v", ctr.ID(), err) + } + + s.ctrNameIndex.Release(ctr.Name()) + + if deleteFromContainers { + delete(s.containers, ctr.ID()) + } + + return nil +} + +// GetContainer returns the container with given ID in the given sandbox +func (s *InMemoryState) GetContainer(id, sandboxID string) (*oci.Container, error) { + s.lock.Lock() + defer s.lock.Unlock() + + return s.getContainerFromSandbox(id, sandboxID) +} + +// GetContainerSandbox returns the ID of a container's sandbox from the full container ID +// May not find the ID of pod infrastructure containers +func (s *InMemoryState) GetContainerSandbox(id string) (string, error) { + s.lock.Lock() + defer s.lock.Unlock() + + ctr, exist := s.containers[id] + if !exist { + return "", &NoSuchCtrError{id: id} + } + + return ctr.Sandbox(), nil +} + +// LookupContainerByName returns the full ID of a container given its full or partial name +func (s *InMemoryState) LookupContainerByName(name string) (*oci.Container, error) { + s.lock.Lock() + defer s.lock.Unlock() + + fullID, err := s.ctrNameIndex.Get(name) + if err != nil { + return nil, &NoSuchCtrError{ + name: name, + inner: err, + } + } + + return s.getContainer(fullID) +} + +// LookupContainerByID returns the full ID of a container given a full or partial ID +// If the given ID is not unique, an error is returned +func (s *InMemoryState) LookupContainerByID(id string) (*oci.Container, error) { + s.lock.Lock() + defer s.lock.Unlock() + + fullID, err := s.ctrIDIndex.Get(id) + if err != nil { + return nil, &NoSuchCtrError{ + id: id, + inner: err, + } + } + + return s.getContainer(fullID) +} + +// GetAllContainers returns all containers in the state, regardless of which sandbox they belong to +// Pod Infra containers are not included +func (s *InMemoryState) GetAllContainers() ([]*oci.Container, error) { + s.lock.Lock() + defer s.lock.Unlock() + + containers := make([]*oci.Container, 0, len(s.containers)) + for _, ctr := range s.containers { + containers = append(containers, ctr) + } + + return containers, nil +} + +// Returns a single container from any sandbox based on full ID +// TODO: is it worth making this public as an alternative to GetContainer +func (s *InMemoryState) getContainer(id string) (*oci.Container, error) { + ctr, exist := s.containers[id] + if !exist { + return nil, &NoSuchCtrError{id: id} + } + + return s.getContainerFromSandbox(id, ctr.Sandbox()) +} + +// Returns a single container from a sandbox based on its full ID +// Internal implementation of GetContainer() but does not lock so it can be used in other functions +func (s *InMemoryState) getContainerFromSandbox(id, sandboxID string) (*oci.Container, error) { + sandbox, ok := s.sandboxes[sandboxID] + if !ok { + return nil, &NoSuchSandboxError{id: sandboxID} + } + + ctr := sandbox.GetContainer(id) + if ctr == nil { + return nil, &NoSuchCtrError{ + id: id, + sandbox: sandboxID, + } + } + + return ctr, nil +} diff --git a/server/state/state_store.go b/server/state/state_store.go new file mode 100644 index 00000000..5a7f7683 --- /dev/null +++ b/server/state/state_store.go @@ -0,0 +1,29 @@ +package state + +import ( + "github.com/kubernetes-incubator/cri-o/oci" + "github.com/kubernetes-incubator/cri-o/server/sandbox" +) + +// RuntimeStateStorer stores the state of the CRI-O server, including active pods and containers +type RuntimeStateStorer interface { + AddSandbox(s *sandbox.Sandbox) error + HasSandbox(id string) bool + DeleteSandbox(id string) error + // These should modify the associated sandbox without prompting + AddContainer(c *oci.Container) error + HasContainer(id, sandboxID string) bool + DeleteContainer(id, sandboxID string) error + // These two require full, explicit ID + GetSandbox(id string) (*sandbox.Sandbox, error) + GetContainer(id, sandboxID string) (*oci.Container, error) + // Get ID of sandbox container belongs to + GetContainerSandbox(id string) (string, error) + // Following 4 should accept partial names as long as they are globally unique + LookupSandboxByName(name string) (*sandbox.Sandbox, error) + LookupSandboxByID(id string) (*sandbox.Sandbox, error) + LookupContainerByName(name string) (*oci.Container, error) + LookupContainerByID(id string) (*oci.Container, error) + GetAllSandboxes() ([]*sandbox.Sandbox, error) + GetAllContainers() ([]*oci.Container, error) +} diff --git a/test/ctr.bats b/test/ctr.bats index cf14c3ce..33d7dc2b 100644 --- a/test/ctr.bats +++ b/test/ctr.bats @@ -11,7 +11,7 @@ function teardown() { run crioctl ctr status --id randomid echo "$output" [ "$status" -eq 1 ] - [[ "$output" =~ "container with ID starting with randomid not found" ]] + [[ "$output" =~ "container with ID starting with randomid could not be retrieved: no container found with ID randomid: ID does not exist" ]] stop_crio } diff --git a/test/restore.bats b/test/restore.bats index 264096ed..d3ef5a41 100644 --- a/test/restore.bats +++ b/test/restore.bats @@ -166,7 +166,7 @@ function teardown() { run crioctl ctr stop --id "$ctr_id" echo "$output" [ "$status" -eq 1 ] - [[ "${output}" =~ "not found" ]] + [[ "${output}" =~ "no container found" ]] cleanup_ctrs cleanup_pods