diff --git a/server/container_create.go b/server/container_create.go index 89bab2c8..f95e73ab 100644 --- a/server/container_create.go +++ b/server/container_create.go @@ -16,6 +16,7 @@ import ( "github.com/docker/docker/pkg/symlink" "github.com/kubernetes-incubator/cri-o/oci" "github.com/kubernetes-incubator/cri-o/server/apparmor" + "github.com/kubernetes-incubator/cri-o/server/sandbox" "github.com/kubernetes-incubator/cri-o/server/seccomp" "github.com/opencontainers/image-spec/specs-go/v1" "github.com/opencontainers/runc/libcontainer/user" @@ -31,7 +32,7 @@ const ( seccompLocalhostPrefix = "localhost/" ) -func addOciBindMounts(sb *sandbox, containerConfig *pb.ContainerConfig, specgen *generate.Generator) error { +func addOciBindMounts(sb *sandbox.Sandbox, containerConfig *pb.ContainerConfig, specgen *generate.Generator) error { mounts := containerConfig.GetMounts() for _, mount := range mounts { dest := mount.ContainerPath @@ -51,7 +52,7 @@ func addOciBindMounts(sb *sandbox, containerConfig *pb.ContainerConfig, specgen if mount.SelinuxRelabel { // Need a way in kubernetes to determine if the volume is shared or private - if err := label.Relabel(src, sb.mountLabel, true); err != nil && err != syscall.ENOTSUP { + if err := label.Relabel(src, sb.MountLabel(), true); err != nil && err != syscall.ENOTSUP { return fmt.Errorf("relabel failed %s: %v", src, err) } } @@ -228,7 +229,7 @@ func (s *Server) CreateContainer(ctx context.Context, req *pb.CreateContainerReq } attempt := containerConfig.GetMetadata().Attempt - containerID, containerName, err := s.generateContainerIDandName(sb.name, name, attempt) + containerID, containerName, err := s.generateContainerIDandName(sb.Name(), name, attempt) if err != nil { return nil, err } @@ -246,7 +247,7 @@ func (s *Server) CreateContainer(ctx context.Context, req *pb.CreateContainerReq } }() - if err = s.runtime.CreateContainer(container, sb.cgroupParent); err != nil { + if err = s.runtime.CreateContainer(container, sb.CgroupParent()); err != nil { return nil, err } @@ -266,7 +267,7 @@ func (s *Server) CreateContainer(ctx context.Context, req *pb.CreateContainerReq return resp, nil } -func (s *Server) createSandboxContainer(ctx context.Context, containerID string, containerName string, sb *sandbox, SandboxConfig *pb.PodSandboxConfig, containerConfig *pb.ContainerConfig) (*oci.Container, error) { +func (s *Server) createSandboxContainer(ctx context.Context, containerID string, containerName string, sb *sandbox.Sandbox, SandboxConfig *pb.PodSandboxConfig, containerConfig *pb.ContainerConfig) (*oci.Container, error) { if sb == nil { return nil, errors.New("createSandboxContainer needs a sandbox") } @@ -294,7 +295,7 @@ func (s *Server) createSandboxContainer(ctx context.Context, containerID string, // set this container's apparmor profile if it is set by sandbox if s.appArmorEnabled { - appArmorProfileName := s.getAppArmorProfileName(sb.annotations, metadata.Name) + appArmorProfileName := s.getAppArmorProfileName(sb.Annotations(), metadata.Name) if appArmorProfileName != "" { // reload default apparmor profile if it is unloaded. if s.appArmorProfile == apparmor.DefaultApparmorProfile { @@ -319,12 +320,12 @@ func (s *Server) createSandboxContainer(ctx context.Context, containerID string, logPath := containerConfig.LogPath if logPath == "" { // TODO: Should we use sandboxConfig.GetLogDirectory() here? - logPath = filepath.Join(sb.logDir, containerID+".log") + logPath = filepath.Join(sb.LogDir(), containerID+".log") } if !filepath.IsAbs(logPath) { // XXX: It's not really clear what this should be versus the sbox logDirectory. logrus.Warnf("requested logPath for ctr id %s is a relative path: %s", containerID, logPath) - logPath = filepath.Join(sb.logDir, logPath) + logPath = filepath.Join(sb.LogDir(), logPath) } // Handle https://issues.k8s.io/44043 @@ -333,7 +334,7 @@ func (s *Server) createSandboxContainer(ctx context.Context, containerID string, } logrus.WithFields(logrus.Fields{ - "sbox.logdir": sb.logDir, + "sbox.logdir": sb.LogDir(), "ctr.logfile": containerConfig.LogPath, "log_path": logPath, }).Debugf("setting container's log_path") @@ -368,12 +369,12 @@ func (s *Server) createSandboxContainer(ctx context.Context, containerID string, specgen.SetLinuxResourcesOOMScoreAdj(int(oomScoreAdj)) } - if sb.cgroupParent != "" { + if sb.CgroupParent() != "" { if s.config.CgroupManager == "systemd" { - cgPath := sb.cgroupParent + ":" + "ocid" + ":" + containerID + cgPath := sb.CgroupParent() + ":" + "ocid" + ":" + containerID specgen.SetLinuxCgroupsPath(cgPath) } else { - specgen.SetLinuxCgroupsPath(sb.cgroupParent + "/" + containerID) + specgen.SetLinuxCgroupsPath(sb.CgroupParent() + "/" + containerID) } } @@ -398,12 +399,12 @@ func (s *Server) createSandboxContainer(ctx context.Context, containerID string, } } - specgen.SetProcessSelinuxLabel(sb.processLabel) - specgen.SetLinuxMountLabel(sb.mountLabel) + specgen.SetProcessSelinuxLabel(sb.ProcessLabel()) + specgen.SetLinuxMountLabel(sb.MountLabel()) } // Join the namespace paths for the pod sandbox container. - podInfraState := s.runtime.ContainerStatus(sb.infraContainer) + podInfraState := s.runtime.ContainerStatus(sb.InfraContainer()) logrus.Debugf("pod container state %+v", podInfraState) @@ -412,7 +413,7 @@ func (s *Server) createSandboxContainer(ctx context.Context, containerID string, return nil, err } - netNsPath := sb.netNsPath() + netNsPath := sb.NetNsPath() if netNsPath == "" { // The sandbox does not have a permanent namespace, // it's on the host one. @@ -434,20 +435,20 @@ func (s *Server) createSandboxContainer(ctx context.Context, containerID string, } // bind mount the pod shm - specgen.AddBindMount(sb.shmPath, "/dev/shm", []string{"rw"}) + specgen.AddBindMount(sb.ShmPath(), "/dev/shm", []string{"rw"}) - if sb.resolvPath != "" { + if sb.ResolvPath() != "" { // bind mount the pod resolver file - specgen.AddBindMount(sb.resolvPath, "/etc/resolv.conf", []string{"ro"}) + specgen.AddBindMount(sb.ResolvPath(), "/etc/resolv.conf", []string{"ro"}) } - if sb.hostname != "" { - specgen.SetHostname(sb.hostname) + if sb.Hostname() != "" { + specgen.SetHostname(sb.Hostname()) } specgen.AddAnnotation("ocid/name", containerName) - specgen.AddAnnotation("ocid/sandbox_id", sb.id) - specgen.AddAnnotation("ocid/sandbox_name", sb.infraContainer.Name()) + specgen.AddAnnotation("ocid/sandbox_id", sb.ID()) + specgen.AddAnnotation("ocid/sandbox_name", sb.InfraContainer().Name()) specgen.AddAnnotation("ocid/container_type", containerTypeContainer) specgen.AddAnnotation("ocid/log_path", logPath) specgen.AddAnnotation("ocid/tty", fmt.Sprintf("%v", containerConfig.Tty)) @@ -471,19 +472,19 @@ func (s *Server) createSandboxContainer(ctx context.Context, containerID string, } specgen.AddAnnotation("ocid/annotations", string(annotationsJSON)) - if err = s.setupSeccomp(&specgen, containerName, sb.annotations); err != nil { + if err = s.setupSeccomp(&specgen, containerName, sb.Annotations()); err != nil { return nil, err } metaname := metadata.Name attempt := metadata.Attempt containerInfo, err := s.storage.CreateContainer(s.imageContext, - sb.name, sb.id, + sb.Name(), sb.ID(), image, image, containerName, containerID, metaname, attempt, - sb.mountLabel, + sb.MountLabel(), nil) if err != nil { return nil, err @@ -561,7 +562,7 @@ func (s *Server) createSandboxContainer(ctx context.Context, containerID string, return nil, err } - container, err := oci.NewContainer(containerID, containerName, containerInfo.RunDir, logPath, sb.netNs(), labels, annotations, imageSpec, metadata, sb.id, containerConfig.Tty, sb.privileged) + container, err := oci.NewContainer(containerID, containerName, containerInfo.RunDir, logPath, sb.NetNs(), labels, annotations, imageSpec, metadata, sb.ID(), containerConfig.Tty, sb.Privileged()) if err != nil { return nil, err } diff --git a/server/container_list.go b/server/container_list.go index 4c097345..3722f6c5 100644 --- a/server/container_list.go +++ b/server/container_list.go @@ -58,7 +58,7 @@ func (s *Server) ListContainers(ctx context.Context, req *pb.ListContainersReque if err != nil { ctrList = []*oci.Container{} } else { - ctrList = pod.containers.List() + ctrList = pod.Containers() } } } diff --git a/server/sandbox.go b/server/sandbox.go deleted file mode 100644 index 7a4a853e..00000000 --- a/server/sandbox.go +++ /dev/null @@ -1,274 +0,0 @@ -package server - -import ( - "crypto/rand" - "errors" - "fmt" - "os" - "path/filepath" - "sync" - - "github.com/Sirupsen/logrus" - "github.com/containernetworking/cni/pkg/ns" - "github.com/docker/docker/pkg/stringid" - "github.com/kubernetes-incubator/cri-o/oci" - "golang.org/x/sys/unix" - "k8s.io/apimachinery/pkg/fields" - pb "k8s.io/kubernetes/pkg/kubelet/api/v1alpha1/runtime" -) - -type sandboxNetNs struct { - sync.Mutex - ns ns.NetNS - symlink *os.File - closed bool - restored bool -} - -func (ns *sandboxNetNs) symlinkCreate(name string) error { - b := make([]byte, 4) - _, randErr := rand.Reader.Read(b) - if randErr != nil { - return randErr - } - - nsName := fmt.Sprintf("%s-%x", name, b) - symlinkPath := filepath.Join(nsRunDir, nsName) - - if err := os.Symlink(ns.ns.Path(), symlinkPath); err != nil { - return err - } - - fd, err := os.Open(symlinkPath) - if err != nil { - if removeErr := os.RemoveAll(symlinkPath); removeErr != nil { - return removeErr - } - - return err - } - - ns.symlink = fd - - return nil -} - -func (ns *sandboxNetNs) symlinkRemove() error { - if err := ns.symlink.Close(); err != nil { - return err - } - - return os.RemoveAll(ns.symlink.Name()) -} - -func isSymbolicLink(path string) (bool, error) { - fi, err := os.Lstat(path) - if err != nil { - return false, err - } - - return fi.Mode()&os.ModeSymlink == os.ModeSymlink, nil -} - -func netNsGet(nspath, name string) (*sandboxNetNs, error) { - if err := ns.IsNSorErr(nspath); err != nil { - return nil, errSandboxClosedNetNS - } - - symlink, symlinkErr := isSymbolicLink(nspath) - if symlinkErr != nil { - return nil, symlinkErr - } - - var resolvedNsPath string - if symlink { - path, err := os.Readlink(nspath) - if err != nil { - return nil, err - } - resolvedNsPath = path - } else { - resolvedNsPath = nspath - } - - netNS, err := ns.GetNS(resolvedNsPath) - if err != nil { - return nil, err - } - - netNs := &sandboxNetNs{ns: netNS, closed: false, restored: true} - - if symlink { - fd, err := os.Open(nspath) - if err != nil { - return nil, err - } - - netNs.symlink = fd - } else { - if err := netNs.symlinkCreate(name); err != nil { - return nil, err - } - } - - return netNs, nil -} - -func hostNetNsPath() (string, error) { - netNS, err := ns.GetCurrentNS() - if err != nil { - return "", err - } - - defer netNS.Close() - return netNS.Path(), nil -} - -type sandbox struct { - id string - name string - logDir string - labels fields.Set - annotations map[string]string - infraContainer *oci.Container - containers oci.Store - processLabel string - mountLabel string - netns *sandboxNetNs - metadata *pb.PodSandboxMetadata - shmPath string - cgroupParent string - privileged bool - resolvPath string - hostname string -} - -const ( - podDefaultNamespace = "default" - defaultShmSize = 64 * 1024 * 1024 - nsRunDir = "/var/run/netns" - podInfraCommand = "/pause" -) - -var ( - errSandboxIDEmpty = errors.New("PodSandboxId should not be empty") - errSandboxClosedNetNS = errors.New("PodSandbox networking namespace is closed") -) - -func (s *sandbox) addContainer(c *oci.Container) { - s.containers.Add(c.Name(), c) -} - -func (s *sandbox) getContainer(name string) *oci.Container { - return s.containers.Get(name) -} - -func (s *sandbox) removeContainer(c *oci.Container) { - s.containers.Delete(c.Name()) -} - -func (s *sandbox) netNs() ns.NetNS { - if s.netns == nil { - return nil - } - - return s.netns.ns -} - -func (s *sandbox) netNsPath() string { - if s.netns == nil { - return "" - } - - return s.netns.symlink.Name() -} - -func (s *sandbox) netNsCreate() error { - if s.netns != nil { - return fmt.Errorf("net NS already created") - } - - netNS, err := ns.NewNS() - if err != nil { - return err - } - - s.netns = &sandboxNetNs{ - ns: netNS, - closed: false, - } - - if err := s.netns.symlinkCreate(s.name); err != nil { - logrus.Warnf("Could not create nentns symlink %v", err) - - if err1 := s.netns.ns.Close(); err1 != nil { - return err1 - } - - return err - } - - return nil -} - -func (s *sandbox) netNsRemove() error { - if s.netns == nil { - logrus.Warn("no networking namespace") - return nil - } - - s.netns.Lock() - defer s.netns.Unlock() - - if s.netns.closed { - // netNsRemove() can be called multiple - // times without returning an error. - return nil - } - - if err := s.netns.symlinkRemove(); err != nil { - return err - } - - if err := s.netns.ns.Close(); err != nil { - return err - } - - if s.netns.restored { - if err := unix.Unmount(s.netns.ns.Path(), unix.MNT_DETACH); err != nil { - return err - } - - if err := os.RemoveAll(s.netns.ns.Path()); err != nil { - return err - } - } - - s.netns.closed = true - return nil -} - -func (s *Server) generatePodIDandName(name string, namespace string, attempt uint32) (string, string, error) { - var ( - err error - id = stringid.GenerateNonCryptoID() - ) - if namespace == "" { - namespace = podDefaultNamespace - } - - return id, fmt.Sprintf("%s-%s-%v", namespace, name, attempt), err -} - -func (s *Server) getPodSandboxFromRequest(podSandboxID string) (*sandbox, error) { - if podSandboxID == "" { - return nil, errSandboxIDEmpty - } - - sb, err := s.state.LookupSandboxByID(podSandboxID) - if err != nil { - return nil, fmt.Errorf("could not retrieve pod sandbox with ID starting with %v: %v", podSandboxID, err) - } - - return sb, nil -} diff --git a/server/sandbox/sandbox.go b/server/sandbox/sandbox.go new file mode 100644 index 00000000..ad934d86 --- /dev/null +++ b/server/sandbox/sandbox.go @@ -0,0 +1,396 @@ +package sandbox + +import ( + "crypto/rand" + "errors" + "fmt" + "os" + "path/filepath" + "sync" + + "github.com/Sirupsen/logrus" + "github.com/containernetworking/cni/pkg/ns" + "github.com/kubernetes-incubator/cri-o/oci" + "golang.org/x/sys/unix" + "k8s.io/apimachinery/pkg/fields" + pb "k8s.io/kubernetes/pkg/kubelet/api/v1alpha1/runtime" +) + +type sandboxNetNs struct { + sync.Mutex + ns ns.NetNS + symlink *os.File + closed bool + restored bool +} + +func (ns *sandboxNetNs) symlinkCreate(name string) error { + b := make([]byte, 4) + _, randErr := rand.Reader.Read(b) + if randErr != nil { + return randErr + } + + nsName := fmt.Sprintf("%s-%x", name, b) + symlinkPath := filepath.Join(nsRunDir, nsName) + + if err := os.Symlink(ns.ns.Path(), symlinkPath); err != nil { + return err + } + + fd, err := os.Open(symlinkPath) + if err != nil { + if removeErr := os.RemoveAll(symlinkPath); removeErr != nil { + return removeErr + } + + return err + } + + ns.symlink = fd + + return nil +} + +func (ns *sandboxNetNs) symlinkRemove() error { + if err := ns.symlink.Close(); err != nil { + return err + } + + return os.RemoveAll(ns.symlink.Name()) +} + +func isSymbolicLink(path string) (bool, error) { + fi, err := os.Lstat(path) + if err != nil { + return false, err + } + + return fi.Mode()&os.ModeSymlink == os.ModeSymlink, nil +} + +func netNsGet(nspath, name string) (*sandboxNetNs, error) { + if err := ns.IsNSorErr(nspath); err != nil { + return nil, ErrSandboxClosedNetNS + } + + symlink, symlinkErr := isSymbolicLink(nspath) + if symlinkErr != nil { + return nil, symlinkErr + } + + var resolvedNsPath string + if symlink { + path, err := os.Readlink(nspath) + if err != nil { + return nil, err + } + resolvedNsPath = path + } else { + resolvedNsPath = nspath + } + + netNS, err := ns.GetNS(resolvedNsPath) + if err != nil { + return nil, err + } + + netNs := &sandboxNetNs{ns: netNS, closed: false, restored: true} + + if symlink { + fd, err := os.Open(nspath) + if err != nil { + return nil, err + } + + netNs.symlink = fd + } else { + if err := netNs.symlinkCreate(name); err != nil { + return nil, err + } + } + + return netNs, nil +} + +// HostNetNsPath returns the path of the host's network namespace +func HostNetNsPath() (string, error) { + netNS, err := ns.GetCurrentNS() + if err != nil { + return "", err + } + + defer netNS.Close() + + return netNS.Path(), nil +} + +// Sandbox represents a single pod sandbox +type Sandbox struct { + id string + name string + logDir string + labels fields.Set + annotations map[string]string + infraContainer *oci.Container + containers oci.Store + processLabel string + mountLabel string + netns *sandboxNetNs + metadata *pb.PodSandboxMetadata + shmPath string + cgroupParent string + privileged bool + resolvPath string + hostname string +} + +const ( + // PodDefaultNamespace is the default namespace name for pods + PodDefaultNamespace = "default" + // DefaultShmSize is the default size of the SHM device for sandboxs + DefaultShmSize = 64 * 1024 * 1024 + // PodInfraCommand is the default pause command for pods + PodInfraCommand = "/pause" + nsRunDir = "/var/run/netns" +) + +var ( + // ErrSandboxIDEmpty is the error returned when an operation passes "" instead of a sandbox ID + ErrSandboxIDEmpty = errors.New("PodSandboxId should not be empty") + // ErrSandboxClosedNetNS is the error returned when a network namespace is closed and cannot be joined + ErrSandboxClosedNetNS = errors.New("PodSandbox networking namespace is closed") +) + +// New creates and populates a new sandbox +// New sandboxes have no containers, no infra container, and no network namespace associated with them. +// An infra container must be attached before the sandbox is added to the state +func New(id, name, logDir string, labels, annotations map[string]string, processLabel, mountLabel string, metadata *pb.PodSandboxMetadata, shmPath, cgroupParent string, privileged bool, resolvPath, hostname string) (*Sandbox, error) { + sb := new(Sandbox) + sb.id = id + sb.name = name + sb.logDir = logDir + sb.labels = labels + sb.annotations = annotations + sb.containers = oci.NewMemoryStore() + sb.processLabel = processLabel + sb.mountLabel = mountLabel + sb.metadata = metadata + sb.shmPath = shmPath + sb.cgroupParent = cgroupParent + sb.privileged = privileged + sb.resolvPath = resolvPath + sb.hostname = hostname + + return sb, nil +} + +// ID returns the sandbox's ID +func (s *Sandbox) ID() string { + return s.id +} + +// Name returns the sandbox's name +func (s *Sandbox) Name() string { + return s.name +} + +// LogDir returns the directory the sandbox logs to +func (s *Sandbox) LogDir() string { + return s.logDir +} + +// Labels returns the sandbox's labels +func (s *Sandbox) Labels() map[string]string { + return s.labels +} + +// Annotations returns the sandbox's annotations +func (s *Sandbox) Annotations() map[string]string { + return s.annotations +} + +// InfraContainer returns the sandbox's infrastructure container +func (s *Sandbox) InfraContainer() *oci.Container { + return s.infraContainer +} + +// Containers returns an array of all the containers in the sandbox +func (s *Sandbox) Containers() []*oci.Container { + return s.containers.List() +} + +// ProcessLabel returns the SELinux process label of the sandbox +func (s *Sandbox) ProcessLabel() string { + return s.processLabel +} + +// MountLabel returns the SELinux mount label of the sandbox +func (s *Sandbox) MountLabel() string { + return s.mountLabel +} + +// Metadata returns Kubernetes metadata associated with the sandbox +func (s *Sandbox) Metadata() *pb.PodSandboxMetadata { + return s.metadata +} + +// ShmPath returns the path to the sandbox's shared memory device +func (s *Sandbox) ShmPath() string { + return s.shmPath +} + +// CgroupParent returns the sandbox's CGroup parent +func (s *Sandbox) CgroupParent() string { + return s.cgroupParent +} + +// Privileged returns whether the sandbox can support privileged containers +func (s *Sandbox) Privileged() bool { + return s.privileged +} + +// ResolvPath returns the path to the sandbox's DNS resolver configuration +func (s *Sandbox) ResolvPath() string { + return s.resolvPath +} + +// Hostname returns the sandbox's hostname +func (s *Sandbox) Hostname() string { + return s.hostname +} + +// AddContainer adds a container to the sandbox +func (s *Sandbox) AddContainer(c *oci.Container) { + s.containers.Add(c.ID(), c) +} + +// GetContainer retrieves the container with given ID from the sandbox +// Returns nil if no such container exists +func (s *Sandbox) GetContainer(id string) *oci.Container { + return s.containers.Get(id) +} + +// RemoveContainer removes the container with given ID from the sandbox +// If no container with that ID exists in the sandbox, no action is taken +func (s *Sandbox) RemoveContainer(id string) { + s.containers.Delete(id) +} + +// SetInfraContainer sets the infrastructure container of a sandbox +// Attempts to set the infrastructure container after one is already present will throw an error +func (s *Sandbox) SetInfraContainer(infraCtr *oci.Container) error { + if s.infraContainer != nil { + return fmt.Errorf("sandbox already has an infra container") + } else if infraCtr == nil { + return fmt.Errorf("must provide non-nil infra container") + } + + s.infraContainer = infraCtr + + return nil +} + +// NetNs retrieves the network namespace of the sandbox +// If the sandbox uses the host namespace, nil is returned +func (s *Sandbox) NetNs() ns.NetNS { + if s.netns == nil { + return nil + } + + return s.netns.ns +} + +// NetNsPath returns the path to the network namespace +// If the sandbox uses the host namespace, "" is returned +func (s *Sandbox) NetNsPath() string { + if s.netns == nil { + return "" + } + + return s.netns.symlink.Name() +} + +// NetNsCreate creates a new network namespace for the sandbox +func (s *Sandbox) NetNsCreate() error { + if s.netns != nil { + return fmt.Errorf("net NS already created") + } + + netNS, err := ns.NewNS() + if err != nil { + return err + } + + s.netns = &sandboxNetNs{ + ns: netNS, + closed: false, + } + + if err := s.netns.symlinkCreate(s.name); err != nil { + logrus.Warnf("Could not create nentns symlink %v", err) + + if err1 := s.netns.ns.Close(); err1 != nil { + return err1 + } + + return err + } + + return nil +} + +// NetNsJoin attempts to join the sandbox to an existing network namespace +// This will fail if the sandbox is already part of a network namespace +func (s *Sandbox) NetNsJoin(nspath, name string) error { + if s.netns != nil { + return fmt.Errorf("sandbox already has a network namespace, cannot join another") + } + + netNS, err := netNsGet(nspath, name) + if err != nil { + return err + } + + s.netns = netNS + + return nil +} + +// NetNsRemove removes the network namespace associated with the sandbox +func (s *Sandbox) NetNsRemove() error { + if s.netns == nil { + logrus.Warn("no networking namespace") + return nil + } + + s.netns.Lock() + defer s.netns.Unlock() + + if s.netns.closed { + // netNsRemove() can be called multiple + // times without returning an error. + return nil + } + + if err := s.netns.symlinkRemove(); err != nil { + return err + } + + if err := s.netns.ns.Close(); err != nil { + return err + } + + if s.netns.restored { + if err := unix.Unmount(s.netns.ns.Path(), unix.MNT_DETACH); err != nil { + return err + } + + if err := os.RemoveAll(s.netns.ns.Path()); err != nil { + return err + } + } + + s.netns.closed = true + return nil +} diff --git a/server/sandbox_list.go b/server/sandbox_list.go index 090637e4..e2b96015 100644 --- a/server/sandbox_list.go +++ b/server/sandbox_list.go @@ -5,6 +5,7 @@ import ( "github.com/Sirupsen/logrus" "github.com/kubernetes-incubator/cri-o/oci" + "github.com/kubernetes-incubator/cri-o/server/sandbox" "golang.org/x/net/context" "k8s.io/apimachinery/pkg/fields" pb "k8s.io/kubernetes/pkg/kubelet/api/v1alpha1/runtime" @@ -33,16 +34,14 @@ func (s *Server) ListPodSandbox(ctx context.Context, req *pb.ListPodSandboxReque logrus.Debugf("ListPodSandboxRequest %+v", req) s.Update() var pods []*pb.PodSandbox - var podList []*sandbox + var podList []*sandbox.Sandbox sandboxes, err := s.state.GetAllSandboxes() if err != nil { return nil, fmt.Errorf("error retrieving sandboxes: %v", err) } - for _, sb := range sandboxes { - podList = append(podList, sb) - } + podList = append(podList, sandboxes...) filter := req.Filter // Filter by pod id first. @@ -51,15 +50,15 @@ func (s *Server) ListPodSandbox(ctx context.Context, req *pb.ListPodSandboxReque sb, err := s.state.LookupSandboxByID(filter.Id) // TODO if we return something other than a No Such Sandbox should we throw an error instead? if err != nil { - podList = []*sandbox{} + podList = []*sandbox.Sandbox{} } else { - podList = []*sandbox{sb} + podList = []*sandbox.Sandbox{sb} } } } for _, sb := range podList { - podInfraContainer := sb.infraContainer + podInfraContainer := sb.InfraContainer() if podInfraContainer == nil { // this can't really happen, but if it does because of a bug // it's better not to panic @@ -76,12 +75,12 @@ func (s *Server) ListPodSandbox(ctx context.Context, req *pb.ListPodSandboxReque } pod := &pb.PodSandbox{ - Id: sb.id, + Id: sb.ID(), CreatedAt: created, State: rStatus, - Labels: sb.labels, - Annotations: sb.annotations, - Metadata: sb.metadata, + Labels: sb.Labels(), + Annotations: sb.Annotations(), + Metadata: sb.Metadata(), } // Filter by other criteria such as state and labels. diff --git a/server/sandbox_remove.go b/server/sandbox_remove.go index 0410f3ce..aaeee9b5 100644 --- a/server/sandbox_remove.go +++ b/server/sandbox_remove.go @@ -6,6 +6,7 @@ import ( "github.com/Sirupsen/logrus" "github.com/kubernetes-incubator/cri-o/oci" + "github.com/kubernetes-incubator/cri-o/server/sandbox" "github.com/opencontainers/selinux/go-selinux/label" "golang.org/x/net/context" pb "k8s.io/kubernetes/pkg/kubelet/api/v1alpha1/runtime" @@ -18,7 +19,7 @@ func (s *Server) RemovePodSandbox(ctx context.Context, req *pb.RemovePodSandboxR s.Update() sb, err := s.getPodSandboxFromRequest(req.PodSandboxId) if err != nil { - if err == errSandboxIDEmpty { + if err == sandbox.ErrSandboxIDEmpty { return nil, err } @@ -27,8 +28,8 @@ func (s *Server) RemovePodSandbox(ctx context.Context, req *pb.RemovePodSandboxR return resp, nil } - podInfraContainer := sb.infraContainer - containers := sb.containers.List() + podInfraContainer := sb.InfraContainer() + containers := sb.Containers() containers = append(containers, podInfraContainer) // Delete all the containers in the sandbox @@ -45,7 +46,7 @@ func (s *Server) RemovePodSandbox(ctx context.Context, req *pb.RemovePodSandboxR } if err := s.runtime.DeleteContainer(c); err != nil { - return nil, fmt.Errorf("failed to delete container %s in pod sandbox %s: %v", c.Name(), sb.id, err) + return nil, fmt.Errorf("failed to delete container %s in pod sandbox %s: %v", c.Name(), sb.ID(), err) } if c == podInfraContainer { @@ -53,45 +54,42 @@ func (s *Server) RemovePodSandbox(ctx context.Context, req *pb.RemovePodSandboxR } if err := s.storage.StopContainer(c.ID()); err != nil { - return nil, fmt.Errorf("failed to delete container %s in pod sandbox %s: %v", c.Name(), sb.id, err) + return nil, fmt.Errorf("failed to delete container %s in pod sandbox %s: %v", c.Name(), sb.ID(), err) } if err := s.storage.DeleteContainer(c.ID()); err != nil { - return nil, fmt.Errorf("failed to delete container %s in pod sandbox %s: %v", c.Name(), sb.id, err) + return nil, fmt.Errorf("failed to delete container %s in pod sandbox %s: %v", c.Name(), sb.ID(), err) } if err := s.removeContainer(c); err != nil { - return nil, fmt.Errorf("failed to delete container %s in pod sandbox %s: %v", c.Name(), sb.id, err) + return nil, fmt.Errorf("failed to delete container %s in pod sandbox %s: %v", c.Name(), sb.ID(), err) } } - if err := label.ReleaseLabel(sb.processLabel); err != nil { + if err := label.ReleaseLabel(sb.ProcessLabel()); err != nil { return nil, err } // unmount the shm for the pod - if sb.shmPath != "/dev/shm" { - if err := syscall.Unmount(sb.shmPath, syscall.MNT_DETACH); err != nil { + if sb.ShmPath() != "/dev/shm" { + if err := syscall.Unmount(sb.ShmPath(), syscall.MNT_DETACH); err != nil { return nil, err } } - if err := sb.netNsRemove(); err != nil { - return nil, fmt.Errorf("failed to remove networking namespace for sandbox %s: %v", sb.id, err) + if err := sb.NetNsRemove(); err != nil { + return nil, fmt.Errorf("failed to remove networking namespace for sandbox %s: %v", sb.ID(), err) } - // Must happen before we set infraContainer to nil, so the infra container is properly removed - if err := s.removeSandbox(sb.id); err != nil { - return nil, fmt.Errorf("error removing sandbox %s: %v", sb.id, err) + if err := s.removeSandbox(sb.ID()); err != nil { + return nil, fmt.Errorf("error removing sandbox %s: %v", sb.ID(), err) } - sb.infraContainer = nil - // Remove the files related to the sandbox - if err := s.storage.StopContainer(sb.id); err != nil { - return nil, fmt.Errorf("failed to delete sandbox container in pod sandbox %s: %v", sb.id, err) + if err := s.storage.StopContainer(sb.ID()); err != nil { + return nil, fmt.Errorf("failed to delete sandbox container in pod sandbox %s: %v", sb.ID(), err) } - if err := s.storage.RemovePodSandbox(sb.id); err != nil { - return nil, fmt.Errorf("failed to remove pod sandbox %s: %v", sb.id, err) + if err := s.storage.RemovePodSandbox(sb.ID()); err != nil { + return nil, fmt.Errorf("failed to remove pod sandbox %s: %v", sb.ID(), err) } resp := &pb.RemovePodSandboxResponse{} diff --git a/server/sandbox_run.go b/server/sandbox_run.go index 7f6f0afb..eeb48ffa 100644 --- a/server/sandbox_run.go +++ b/server/sandbox_run.go @@ -11,6 +11,7 @@ import ( "github.com/Sirupsen/logrus" "github.com/containers/storage/storage" "github.com/kubernetes-incubator/cri-o/oci" + "github.com/kubernetes-incubator/cri-o/server/sandbox" "github.com/opencontainers/runtime-tools/generate" "github.com/opencontainers/selinux/go-selinux/label" "golang.org/x/net/context" @@ -122,7 +123,7 @@ func (s *Server) RunPodSandbox(ctx context.Context, req *pb.RunPodSandboxRequest if podContainer.Config != nil { g.SetProcessArgs(podContainer.Config.Config.Cmd) } else { - g.SetProcessArgs([]string{podInfraCommand}) + g.SetProcessArgs([]string{sandbox.PodInfraCommand}) } } else { g.SetProcessArgs([]string{s.config.PauseCommand}) @@ -242,22 +243,6 @@ func (s *Server) RunPodSandbox(ctx context.Context, req *pb.RunPodSandboxRequest g.AddAnnotation("ocid/resolv_path", resolvPath) g.AddAnnotation("ocid/hostname", hostname) - sb := &sandbox{ - id: id, - name: name, - logDir: logDir, - labels: labels, - annotations: annotations, - containers: oci.NewMemoryStore(), - processLabel: processLabel, - mountLabel: mountLabel, - metadata: metadata, - shmPath: shmPath, - privileged: privileged, - resolvPath: resolvPath, - hostname: hostname, - } - for k, v := range annotations { g.AddAnnotation(k, v) } @@ -283,9 +268,12 @@ func (s *Server) RunPodSandbox(ctx context.Context, req *pb.RunPodSandboxRequest } else { g.SetLinuxCgroupsPath(cgroupParent + "/" + id) - } - sb.cgroupParent = cgroupParent + } + + sb, err := sandbox.New(id, name, logDir, labels, annotations, processLabel, mountLabel, metadata, shmPath, cgroupParent, privileged, resolvPath, hostname) + if err != nil { + return nil, err } hostNetwork := req.GetConfig().GetLinux().GetSecurityContext().GetNamespaceOptions().HostNetwork @@ -297,13 +285,13 @@ func (s *Server) RunPodSandbox(ctx context.Context, req *pb.RunPodSandboxRequest return nil, err } - netNsPath, err = hostNetNsPath() + netNsPath, err = sandbox.HostNetNsPath() if err != nil { return nil, err } } else { // Create the sandbox network namespace - if err = sb.netNsCreate(); err != nil { + if err = sb.NetNsCreate(); err != nil { return nil, err } @@ -312,18 +300,18 @@ func (s *Server) RunPodSandbox(ctx context.Context, req *pb.RunPodSandboxRequest return } - if netnsErr := sb.netNsRemove(); netnsErr != nil { + if netnsErr := sb.NetNsRemove(); netnsErr != nil { logrus.Warnf("Failed to remove networking namespace: %v", netnsErr) } }() // Pass the created namespace path to the runtime - err = g.AddOrReplaceLinuxNamespace("network", sb.netNsPath()) + err = g.AddOrReplaceLinuxNamespace("network", sb.NetNsPath()) if err != nil { return nil, err } - netNsPath = sb.netNsPath() + netNsPath = sb.NetNsPath() } if req.GetConfig().GetLinux().GetSecurityContext().GetNamespaceOptions().HostPid { @@ -347,25 +335,25 @@ func (s *Server) RunPodSandbox(ctx context.Context, req *pb.RunPodSandboxRequest saveOptions := generate.ExportOptions{} mountPoint, err := s.storage.StartContainer(id) if err != nil { - return nil, fmt.Errorf("failed to mount container %s in pod sandbox %s(%s): %v", containerName, sb.name, id, err) + return nil, fmt.Errorf("failed to mount container %s in pod sandbox %s(%s): %v", containerName, name, id, err) } g.SetRootPath(mountPoint) err = g.SaveToFile(filepath.Join(podContainer.Dir, "config.json"), saveOptions) if err != nil { - return nil, fmt.Errorf("failed to save template configuration for pod sandbox %s(%s): %v", sb.name, id, err) + return nil, fmt.Errorf("failed to save template configuration for pod sandbox %s(%s): %v", name, id, err) } if err = g.SaveToFile(filepath.Join(podContainer.RunDir, "config.json"), saveOptions); err != nil { - return nil, fmt.Errorf("failed to write runtime configuration for pod sandbox %s(%s): %v", sb.name, id, err) + return nil, fmt.Errorf("failed to write runtime configuration for pod sandbox %s(%s): %v", name, id, err) } - container, err := oci.NewContainer(id, containerName, podContainer.RunDir, logPath, sb.netNs(), labels, annotations, nil, nil, id, false, sb.privileged) + container, err := oci.NewContainer(id, containerName, podContainer.RunDir, logPath, sb.NetNs(), labels, annotations, nil, nil, id, false, sb.Privileged()) if err != nil { return nil, err } + if err := sb.SetInfraContainer(container); err != nil { + return nil, err + } - sb.infraContainer = container - - // Only register the sandbox after infra container has been added if err = s.addSandbox(sb); err != nil { return nil, err } @@ -378,7 +366,7 @@ func (s *Server) RunPodSandbox(ctx context.Context, req *pb.RunPodSandboxRequest } } - if err = s.runContainer(container, sb.cgroupParent); err != nil { + if err = s.runContainer(container, sb.CgroupParent()); err != nil { return nil, err } @@ -428,7 +416,7 @@ func setupShm(podSandboxRunDir, mountLabel string) (shmPath string, err error) { if err = os.Mkdir(shmPath, 0700); err != nil { return "", err } - shmOptions := "mode=1777,size=" + strconv.Itoa(defaultShmSize) + shmOptions := "mode=1777,size=" + strconv.Itoa(sandbox.DefaultShmSize) if err = syscall.Mount("shm", shmPath, "tmpfs", uintptr(syscall.MS_NOEXEC|syscall.MS_NOSUID|syscall.MS_NODEV), label.FormatMountLabel(shmOptions, mountLabel)); err != nil { return "", fmt.Errorf("failed to mount shm tmpfs for pod: %v", err) diff --git a/server/sandbox_status.go b/server/sandbox_status.go index ea16802d..9e041de0 100644 --- a/server/sandbox_status.go +++ b/server/sandbox_status.go @@ -16,7 +16,7 @@ func (s *Server) PodSandboxStatus(ctx context.Context, req *pb.PodSandboxStatusR return nil, err } - podInfraContainer := sb.infraContainer + podInfraContainer := sb.InfraContainer() if err = s.runtime.UpdateStatus(podInfraContainer); err != nil { return nil, err } @@ -29,7 +29,7 @@ func (s *Server) PodSandboxStatus(ctx context.Context, req *pb.PodSandboxStatusR return nil, err } podNamespace := "" - ip, err := s.netPlugin.GetContainerNetworkStatus(netNsPath, podNamespace, sb.id, podInfraContainer.Name()) + ip, err := s.netPlugin.GetContainerNetworkStatus(netNsPath, podNamespace, sb.ID(), podInfraContainer.Name()) if err != nil { // ignore the error on network status ip = "" @@ -40,7 +40,7 @@ func (s *Server) PodSandboxStatus(ctx context.Context, req *pb.PodSandboxStatusR rStatus = pb.PodSandboxState_SANDBOX_READY } - sandboxID := sb.id + sandboxID := sb.ID() resp := &pb.PodSandboxStatusResponse{ Status: &pb.PodSandboxStatus{ Id: sandboxID, @@ -52,9 +52,9 @@ func (s *Server) PodSandboxStatus(ctx context.Context, req *pb.PodSandboxStatusR }, Network: &pb.PodSandboxNetworkStatus{Ip: ip}, State: rStatus, - Labels: sb.labels, - Annotations: sb.annotations, - Metadata: sb.metadata, + Labels: sb.Labels(), + Annotations: sb.Annotations(), + Metadata: sb.Metadata(), }, } diff --git a/server/sandbox_stop.go b/server/sandbox_stop.go index e8dd61c6..2d306bad 100644 --- a/server/sandbox_stop.go +++ b/server/sandbox_stop.go @@ -21,27 +21,27 @@ func (s *Server) StopPodSandbox(ctx context.Context, req *pb.StopPodSandboxReque } podNamespace := "" - podInfraContainer := sb.infraContainer + podInfraContainer := sb.InfraContainer() netnsPath, err := podInfraContainer.NetNsPath() if err != nil { return nil, err } if _, err := os.Stat(netnsPath); err == nil { - if err2 := s.netPlugin.TearDownPod(netnsPath, podNamespace, sb.id, podInfraContainer.Name()); err2 != nil { + if err2 := s.netPlugin.TearDownPod(netnsPath, podNamespace, sb.ID(), podInfraContainer.Name()); err2 != nil { return nil, fmt.Errorf("failed to destroy network for container %s in sandbox %s: %v", - podInfraContainer.Name(), sb.id, err2) + podInfraContainer.Name(), sb.ID(), err2) } } else if !os.IsNotExist(err) { // it's ok for netnsPath to *not* exist return nil, fmt.Errorf("failed to stat netns path for container %s in sandbox %s before tearing down the network: %v", - podInfraContainer.Name(), sb.id, err) + podInfraContainer.Name(), sb.ID(), err) } // Close the sandbox networking namespace. - if err := sb.netNsRemove(); err != nil { + if err := sb.NetNsRemove(); err != nil { return nil, err } - containers := sb.containers.List() + containers := sb.Containers() containers = append(containers, podInfraContainer) for _, c := range containers { @@ -51,7 +51,7 @@ func (s *Server) StopPodSandbox(ctx context.Context, req *pb.StopPodSandboxReque cStatus := s.runtime.ContainerStatus(c) if cStatus.Status != oci.ContainerStateStopped { if err := s.runtime.StopContainer(c); err != nil { - return nil, fmt.Errorf("failed to stop container %s in pod sandbox %s: %v", c.Name(), sb.id, err) + return nil, fmt.Errorf("failed to stop container %s in pod sandbox %s: %v", c.Name(), sb.ID(), err) } } } diff --git a/server/server.go b/server/server.go index 7768a402..583747be 100644 --- a/server/server.go +++ b/server/server.go @@ -11,11 +11,14 @@ import ( "github.com/Sirupsen/logrus" "github.com/containers/image/types" sstorage "github.com/containers/storage/storage" + "github.com/docker/docker/pkg/stringid" "github.com/kubernetes-incubator/cri-o/oci" "github.com/kubernetes-incubator/cri-o/pkg/ocicni" "github.com/kubernetes-incubator/cri-o/pkg/storage" "github.com/kubernetes-incubator/cri-o/server/apparmor" + "github.com/kubernetes-incubator/cri-o/server/sandbox" "github.com/kubernetes-incubator/cri-o/server/seccomp" + "github.com/kubernetes-incubator/cri-o/server/state" rspec "github.com/opencontainers/runtime-spec/specs-go" "github.com/opencontainers/selinux/go-selinux/label" pb "k8s.io/kubernetes/pkg/kubelet/api/v1alpha1/runtime" @@ -33,7 +36,7 @@ type Server struct { images storage.ImageServer storage storage.RuntimeServer updateLock sync.RWMutex - state StateStore + state state.Store netPlugin ocicni.CNIPlugin imageContext *types.SystemContext @@ -90,7 +93,7 @@ func (s *Server) loadContainer(id string) error { return err } - ctr, err := oci.NewContainer(id, name, containerPath, m.Annotations["ocid/log_path"], sb.netNs(), labels, annotations, img, &metadata, sb.id, tty, sb.privileged) + ctr, err := oci.NewContainer(id, name, containerPath, m.Annotations["ocid/log_path"], sb.NetNs(), labels, annotations, img, &metadata, sb.ID(), tty, sb.Privileged()) if err != nil { return err } @@ -147,34 +150,22 @@ func (s *Server) loadSandbox(id string) error { privileged := m.Annotations["ocid/privileged_runtime"] == "true" - sb := &sandbox{ - id: id, - name: name, - logDir: filepath.Dir(m.Annotations["ocid/log_path"]), - labels: labels, - containers: oci.NewMemoryStore(), - processLabel: processLabel, - mountLabel: mountLabel, - annotations: annotations, - metadata: &metadata, - shmPath: m.Annotations["ocid/shm_path"], - privileged: privileged, - resolvPath: m.Annotations["ocid/resolv_path"], + sb, err := sandbox.New(id, name, filepath.Dir(m.Annotations["ocid/log_path"]), labels, annotations, processLabel, mountLabel, &metadata, m.Annotations["ocid/shm_path"], *m.Linux.CgroupsPath, privileged, m.Annotations["ocid/resolv_path"], m.Annotations["ocid/hostname"]) + if err != nil { + return err } // We add a netNS only if we can load a permanent one. // Otherwise, the sandbox will live in the host namespace. netNsPath, err := configNetNsPath(m) if err == nil { - netNS, nsErr := netNsGet(netNsPath, sb.name) // If we can't load the networking namespace - // because it's closed, we just set the sb netns - // pointer to nil. Otherwise we return an error. - if nsErr != nil && nsErr != errSandboxClosedNetNS { - return nsErr + // because it's closed, just leave the sandbox's netns pointer as nil + if nsErr := sb.NetNsJoin(netNsPath, sb.Name()); err != nil { + if nsErr != sandbox.ErrSandboxClosedNetNS { + return nsErr + } } - - sb.netns = netNS } sandboxPath, err := s.store.GetContainerRunDirectory(id) @@ -182,7 +173,7 @@ func (s *Server) loadSandbox(id string) error { return err } - scontainer, err := oci.NewContainer(m.Annotations["ocid/container_id"], m.Annotations["ocid/container_name"], sandboxPath, m.Annotations["ocid/log_path"], sb.netNs(), labels, annotations, nil, nil, id, false, privileged) + scontainer, err := oci.NewContainer(m.Annotations["ocid/container_id"], m.Annotations["ocid/container_name"], sandboxPath, m.Annotations["ocid/log_path"], sb.NetNs(), labels, annotations, nil, nil, id, false, privileged) if err != nil { return err } @@ -192,7 +183,9 @@ func (s *Server) loadSandbox(id string) error { if err = label.ReserveLabel(processLabel); err != nil { return err } - sb.infraContainer = scontainer + if err = sb.SetInfraContainer(scontainer); err != nil { + return err + } return s.addSandbox(sb) } @@ -308,9 +301,9 @@ func (s *Server) update() error { return fmt.Errorf("error retrieving pods list: %v", err) } for _, pod := range pods { - if _, ok := oldPods[pod.id]; !ok { + if _, ok := oldPods[pod.ID()]; !ok { // this pod's ID wasn't in the updated list -> removed - removedPods[pod.id] = pod.id + removedPods[pod.ID()] = pod.ID() } } @@ -321,11 +314,10 @@ func (s *Server) update() error { logrus.Warnf("bad state when getting pod to remove %+v", removedPod) continue } - if err := s.removeSandbox(sb.id); err != nil { - return fmt.Errorf("error removing sandbox %s: %v", sb.id, err) + if err := s.removeSandbox(sb.ID()); err != nil { + return fmt.Errorf("error removing sandbox %s: %v", sb.ID(), err) } - sb.infraContainer = nil - logrus.Debugf("forgetting removed pod %s", sb.id) + logrus.Debugf("forgetting removed pod %s", sb.ID()) } for sandboxID := range newPods { // load this pod @@ -389,7 +381,7 @@ func New(config *Config) (*Server, error) { storage: storageRuntimeService, netPlugin: netPlugin, config: *config, - state: NewInMemoryState(), + state: state.NewInMemoryState(), seccompEnabled: seccomp.IsEnabled(), appArmorEnabled: apparmor.IsEnabled(), appArmorProfile: config.ApparmorProfile, @@ -421,11 +413,11 @@ func New(config *Config) (*Server, error) { return s, nil } -func (s *Server) addSandbox(sb *sandbox) error { +func (s *Server) addSandbox(sb *sandbox.Sandbox) error { return s.state.AddSandbox(sb) } -func (s *Server) getSandbox(id string) (*sandbox, error) { +func (s *Server) getSandbox(id string) (*sandbox.Sandbox, error) { return s.state.GetSandbox(id) } @@ -453,3 +445,28 @@ func (s *Server) getContainer(id string) (*oci.Container, error) { func (s *Server) removeContainer(c *oci.Container) error { return s.state.DeleteContainer(c.ID(), c.Sandbox()) } + +func (s *Server) generatePodIDandName(name string, namespace string, attempt uint32) (string, string, error) { + var ( + err error + id = stringid.GenerateNonCryptoID() + ) + if namespace == "" { + namespace = sandbox.PodDefaultNamespace + } + + return id, fmt.Sprintf("%s-%s-%v", namespace, name, attempt), err +} + +func (s *Server) getPodSandboxFromRequest(podSandboxID string) (*sandbox.Sandbox, error) { + if podSandboxID == "" { + return nil, sandbox.ErrSandboxIDEmpty + } + + sb, err := s.state.LookupSandboxByID(podSandboxID) + if err != nil { + return nil, fmt.Errorf("could not retrieve pod sandbox with ID starting with %v: %v", podSandboxID, err) + } + + return sb, nil +} diff --git a/server/file_state.go b/server/state/file_state.go similarity index 88% rename from server/file_state.go rename to server/state/file_state.go index 0798f799..c94dd92b 100644 --- a/server/file_state.go +++ b/server/state/file_state.go @@ -1,4 +1,4 @@ -package server +package state import ( "encoding/json" @@ -12,6 +12,7 @@ import ( "github.com/containernetworking/cni/pkg/ns" "github.com/containers/storage/storage" "github.com/kubernetes-incubator/cri-o/oci" + "github.com/kubernetes-incubator/cri-o/server/sandbox" "k8s.io/apimachinery/pkg/fields" pb "k8s.io/kubernetes/pkg/kubelet/api/v1alpha1/runtime" ) @@ -28,7 +29,7 @@ import ( type FileState struct { rootPath string lockfile storage.Locker - memoryState StateStore + memoryState Store } // Net namespace is taken from enclosing sandbox @@ -65,6 +66,8 @@ type sandboxFile struct { ShmPath string `json:"shmPath"` CgroupParent string `json:"cgroupParent"` Privileged bool `json:"privileged"` + ResolvPath string `json:"resolvPath"` + Hostname string `json:"hostname"` } // Sync the in-memory state and the state on disk @@ -111,96 +114,61 @@ func (s *FileState) syncWithDisk() error { } // Convert a sandbox to on-disk format -func sandboxToSandboxFile(sb *sandbox) *sandboxFile { +func sandboxToSandboxFile(sb *sandbox.Sandbox) *sandboxFile { sbFile := sandboxFile{ - ID: sb.id, - Name: sb.name, - LogDir: sb.logDir, - Labels: sb.labels, - Annotations: sb.annotations, - Containers: make([]string, 0, len(sb.containers.List())), - ProcessLabel: sb.processLabel, - MountLabel: sb.mountLabel, - Metadata: sb.metadata, - ShmPath: sb.shmPath, - CgroupParent: sb.cgroupParent, - Privileged: sb.privileged, + ID: sb.ID(), + Name: sb.Name(), + LogDir: sb.LogDir(), + Labels: sb.Labels(), + Annotations: sb.Annotations(), + Containers: make([]string, 0, len(sb.Containers())), + ProcessLabel: sb.ProcessLabel(), + MountLabel: sb.MountLabel(), + Metadata: sb.Metadata(), + ShmPath: sb.ShmPath(), + CgroupParent: sb.CgroupParent(), + Privileged: sb.Privileged(), + ResolvPath: sb.ResolvPath(), + Hostname: sb.Hostname(), } - if sb.netns != nil { - sbFile.NetNsPath = sb.netNsPath() - sbFile.NetNsClosed = sb.netns.closed - sbFile.NetNsRestored = sb.netns.restored - - if sb.netns.symlink != nil { - sbFile.NetNsSymlinkPath = sb.netns.symlink.Name() - } - } - - for _, ctr := range sb.containers.List() { + for _, ctr := range sb.Containers() { sbFile.Containers = append(sbFile.Containers, ctr.ID()) } - sbFile.InfraContainer = sb.infraContainer.ID() + sbFile.InfraContainer = sb.InfraContainer().ID() return &sbFile } // Convert a sandbox from on-disk format to normal format -func (s *FileState) sandboxFileToSandbox(sbFile *sandboxFile) (*sandbox, error) { - sb := sandbox{ - id: sbFile.ID, - name: sbFile.Name, - logDir: sbFile.LogDir, - labels: sbFile.Labels, - annotations: sbFile.Annotations, - containers: oci.NewMemoryStore(), - processLabel: sbFile.ProcessLabel, - mountLabel: sbFile.MountLabel, - metadata: sbFile.Metadata, - shmPath: sbFile.ShmPath, - cgroupParent: sbFile.CgroupParent, - privileged: sbFile.Privileged, - } - - ns, err := ns.GetNS(sbFile.NetNsPath) +func (s *FileState) sandboxFileToSandbox(sbFile *sandboxFile) (*sandbox.Sandbox, error) { + sb, err := sandbox.New(sbFile.ID, sbFile.Name, sbFile.LogDir, sbFile.Labels, sbFile.Annotations, sbFile.ProcessLabel, sbFile.MountLabel, sbFile.Metadata, sbFile.ShmPath, sbFile.CgroupParent, sbFile.Privileged, sbFile.ResolvPath, sbFile.Hostname) if err != nil { - return nil, fmt.Errorf("error retrieving network namespace %v: %v", sbFile.NetNsPath, err) + return nil, fmt.Errorf("error creating sandbox with ID %v: %v", sbFile.ID, err) } - var symlink *os.File - if sbFile.NetNsSymlinkPath != "" { - symlink, err = os.Open(sbFile.NetNsSymlinkPath) - if err != nil { - return nil, fmt.Errorf("error retrieving network namespace symlink %v: %v", sbFile.NetNsSymlinkPath, err) - } - } - netns := sandboxNetNs{ - ns: ns, - symlink: symlink, - closed: sbFile.NetNsClosed, - restored: sbFile.NetNsRestored, - } - sb.netns = &netns - infraCtr, err := s.getContainerFromDisk(sbFile.InfraContainer, sbFile.ID, &netns) + infraCtr, err := s.getContainerFromDisk(sbFile.InfraContainer, sbFile.ID, nil) if err != nil { return nil, fmt.Errorf("error retrieving infra container for pod %v: %v", sbFile.ID, err) } - sb.infraContainer = infraCtr + if err := sb.SetInfraContainer(infraCtr); err != nil { + return nil, fmt.Errorf("error setting infra container for pod %v: %v", sbFile.ID, err) + } for _, id := range sbFile.Containers { - ctr, err := s.getContainerFromDisk(id, sbFile.ID, &netns) + ctr, err := s.getContainerFromDisk(id, sbFile.ID, nil) if err != nil { return nil, fmt.Errorf("error retrieving container ID %v in pod ID %v: %v", id, sbFile.ID, err) } - sb.containers.Add(ctr.ID(), ctr) + sb.AddContainer(ctr) } - return &sb, nil + return sb, nil } // Retrieve a sandbox and all associated containers from disk -func (s *FileState) getSandboxFromDisk(id string) (*sandbox, error) { +func (s *FileState) getSandboxFromDisk(id string) (*sandbox.Sandbox, error) { sbFile, err := s.getSandboxFileFromDisk(id) if err != nil { return nil, err @@ -230,7 +198,7 @@ func (s *FileState) getSandboxFileFromDisk(id string) (*sandboxFile, error) { // Save a sandbox to disk // Will save all associated containers, including infra container, as well -func (s *FileState) putSandboxToDisk(sb *sandbox) error { +func (s *FileState) putSandboxToDisk(sb *sandbox.Sandbox) error { sbFile := sandboxToSandboxFile(sb) if err := s.putSandboxFileToDisk(sbFile); err != nil { @@ -238,13 +206,13 @@ func (s *FileState) putSandboxToDisk(sb *sandbox) error { } // Need to put infra container and any additional containers to disk as well - if err := s.putContainerToDisk(sb.infraContainer, false); err != nil { - return fmt.Errorf("error storing sandbox %v infra container: %v", sb.id, err) + if err := s.putContainerToDisk(sb.InfraContainer(), false); err != nil { + return fmt.Errorf("error storing sandbox %v infra container: %v", sb.ID(), err) } - for _, ctr := range sb.containers.List() { + for _, ctr := range sb.Containers() { if err := s.putContainerToDisk(ctr, false); err != nil { - return fmt.Errorf("error storing container %v in sandbox %v: %v", ctr.ID(), sb.id, err) + return fmt.Errorf("error storing container %v in sandbox %v: %v", ctr.ID(), sb.ID(), err) } } @@ -388,12 +356,12 @@ func getContainerFileFromContainer(ctr *oci.Container) *containerFile { } // Convert on-disk container format to normal oci.Container -func getContainerFromContainerFile(ctrFile *containerFile, netNs *sandboxNetNs) (*oci.Container, error) { - return oci.NewContainer(ctrFile.ID, ctrFile.Name, ctrFile.BundlePath, ctrFile.LogPath, netNs.ns, ctrFile.Labels, ctrFile.Annotations, ctrFile.Image, ctrFile.Metadata, ctrFile.Sandbox, ctrFile.Terminal, ctrFile.Privileged) +func getContainerFromContainerFile(ctrFile *containerFile, ns ns.NetNS) (*oci.Container, error) { + return oci.NewContainer(ctrFile.ID, ctrFile.Name, ctrFile.BundlePath, ctrFile.LogPath, ns, ctrFile.Labels, ctrFile.Annotations, ctrFile.Image, ctrFile.Metadata, ctrFile.Sandbox, ctrFile.Terminal, ctrFile.Privileged) } // Get a container from disk -func (s *FileState) getContainerFromDisk(id, sandboxID string, netNs *sandboxNetNs) (*oci.Container, error) { +func (s *FileState) getContainerFromDisk(id, sandboxID string, netNs ns.NetNS) (*oci.Container, error) { ctrFile, err := s.getContainerFileFromDisk(id, sandboxID) if err != nil { return nil, err @@ -578,7 +546,7 @@ func decodeFromFile(fileName string, decodeInto interface{}) error { // NewFileState makes a new file-based state store at the given directory // TODO: Should we attempt to populate the state based on the directory that exists, // or should we let server's sync() handle that? -func NewFileState(statePath string) (StateStore, error) { +func NewFileState(statePath string) (Store, error) { state := new(FileState) state.rootPath = statePath state.memoryState = NewInMemoryState() @@ -632,7 +600,7 @@ func NewFileState(statePath string) (StateStore, error) { } // AddSandbox adds a sandbox and any containers in it to the state -func (s *FileState) AddSandbox(sb *sandbox) error { +func (s *FileState) AddSandbox(sb *sandbox.Sandbox) error { s.lockfile.Lock() defer s.lockfile.Unlock() @@ -640,8 +608,8 @@ func (s *FileState) AddSandbox(sb *sandbox) error { return fmt.Errorf("error syncing with on-disk state: %v", err) } - if s.memoryState.HasSandbox(sb.id) { - return fmt.Errorf("sandbox with ID %v already exists", sb.id) + if s.memoryState.HasSandbox(sb.ID()) { + return fmt.Errorf("sandbox with ID %v already exists", sb.ID()) } if err := s.putSandboxToDisk(sb); err != nil { @@ -649,7 +617,7 @@ func (s *FileState) AddSandbox(sb *sandbox) error { } if err := s.memoryState.AddSandbox(sb); err != nil { - return fmt.Errorf("error adding sandbox %v to in-memory state: %v", sb.id, err) + return fmt.Errorf("error adding sandbox %v to in-memory state: %v", sb.ID(), err) } return nil @@ -693,7 +661,7 @@ func (s *FileState) DeleteSandbox(id string) error { } // GetSandbox retrieves the given sandbox from the state -func (s *FileState) GetSandbox(id string) (*sandbox, error) { +func (s *FileState) GetSandbox(id string) (*sandbox.Sandbox, error) { s.lockfile.Lock() defer s.lockfile.Unlock() @@ -705,7 +673,7 @@ func (s *FileState) GetSandbox(id string) (*sandbox, error) { } // LookupSandboxByName returns a sandbox given its full or partial name -func (s *FileState) LookupSandboxByName(name string) (*sandbox, error) { +func (s *FileState) LookupSandboxByName(name string) (*sandbox.Sandbox, error) { s.lockfile.Lock() defer s.lockfile.Unlock() @@ -718,7 +686,7 @@ func (s *FileState) LookupSandboxByName(name string) (*sandbox, error) { // LookupSandboxByID returns a sandbox given its full or partial ID // An error will be returned if the partial ID given is not unique -func (s *FileState) LookupSandboxByID(id string) (*sandbox, error) { +func (s *FileState) LookupSandboxByID(id string) (*sandbox.Sandbox, error) { s.lockfile.Lock() defer s.lockfile.Unlock() @@ -730,7 +698,7 @@ func (s *FileState) LookupSandboxByID(id string) (*sandbox, error) { } // GetAllSandboxes returns all sandboxes in the state -func (s *FileState) GetAllSandboxes() ([]*sandbox, error) { +func (s *FileState) GetAllSandboxes() ([]*sandbox.Sandbox, error) { s.lockfile.Lock() defer s.lockfile.Unlock() diff --git a/server/in_memory_state.go b/server/state/in_memory_state.go similarity index 86% rename from server/in_memory_state.go rename to server/state/in_memory_state.go index 45541c59..b3fbf2ae 100644 --- a/server/in_memory_state.go +++ b/server/state/in_memory_state.go @@ -1,4 +1,4 @@ -package server +package state import ( "fmt" @@ -7,6 +7,7 @@ import ( "github.com/docker/docker/pkg/registrar" "github.com/docker/docker/pkg/truncindex" "github.com/kubernetes-incubator/cri-o/oci" + "github.com/kubernetes-incubator/cri-o/server/sandbox" ) // TODO: make operations atomic to greatest extent possible @@ -15,7 +16,7 @@ import ( // programs are expected to interact with the server type InMemoryState struct { lock sync.Mutex - sandboxes map[string]*sandbox + sandboxes map[string]*sandbox.Sandbox containers oci.Store podNameIndex *registrar.Registrar podIDIndex *truncindex.TruncIndex @@ -24,9 +25,9 @@ type InMemoryState struct { } // NewInMemoryState creates a new, empty server state -func NewInMemoryState() StateStore { +func NewInMemoryState() Store { state := new(InMemoryState) - state.sandboxes = make(map[string]*sandbox) + state.sandboxes = make(map[string]*sandbox.Sandbox) state.containers = oci.NewMemoryStore() state.podNameIndex = registrar.NewRegistrar() state.podIDIndex = truncindex.NewTruncIndex([]string{}) @@ -37,39 +38,40 @@ func NewInMemoryState() StateStore { } // AddSandbox adds a sandbox and any containers in it to the state -func (s *InMemoryState) AddSandbox(sandbox *sandbox) error { +func (s *InMemoryState) AddSandbox(sandbox *sandbox.Sandbox) error { s.lock.Lock() defer s.lock.Unlock() - if _, exist := s.sandboxes[sandbox.id]; exist { - return fmt.Errorf("sandbox with ID %v already exists", sandbox.id) + if _, exist := s.sandboxes[sandbox.ID()]; exist { + return fmt.Errorf("sandbox with ID %v already exists", sandbox.ID()) } // We shouldn't share ID with any containers, either - if ctrCheck := s.containers.Get(sandbox.id); ctrCheck != nil { - return fmt.Errorf("requested sandbox ID %v conflicts with existing container ID", sandbox.id) + // Our pod infra container will share our ID and we don't want it to conflict with anything + if ctrCheck := s.containers.Get(sandbox.ID()); ctrCheck != nil { + return fmt.Errorf("requested sandbox ID %v conflicts with existing container ID", sandbox.ID()) } - s.sandboxes[sandbox.id] = sandbox - if err := s.podNameIndex.Reserve(sandbox.name, sandbox.id); err != nil { + s.sandboxes[sandbox.ID()] = sandbox + if err := s.podNameIndex.Reserve(sandbox.Name(), sandbox.ID()); err != nil { return fmt.Errorf("error registering sandbox name: %v", err) } - if err := s.podIDIndex.Add(sandbox.id); err != nil { + if err := s.podIDIndex.Add(sandbox.ID()); err != nil { return fmt.Errorf("error registering sandbox ID: %v", err) } // If there are containers in the sandbox add them to the mapping - containers := sandbox.containers.List() + containers := sandbox.Containers() for _, ctr := range containers { if err := s.addContainerMappings(ctr, true); err != nil { - return fmt.Errorf("error adding container %v mappings in sandbox %v", ctr.ID(), sandbox.id) + return fmt.Errorf("error adding container %v mappings in sandbox %v", ctr.ID(), sandbox.ID()) } } // Add the pod infrastructure container to mappings // TODO: Right now, we don't add it to the all containers listing. We may want to change this. - if err := s.addContainerMappings(sandbox.infraContainer, false); err != nil { - return fmt.Errorf("error adding infrastructure container %v to mappings: %v", sandbox.infraContainer.ID(), err) + if err := s.addContainerMappings(sandbox.InfraContainer(), false); err != nil { + return fmt.Errorf("error adding infrastructure container %v to mappings: %v", sandbox.InfraContainer().ID(), err) } return nil @@ -94,9 +96,9 @@ func (s *InMemoryState) DeleteSandbox(id string) error { return fmt.Errorf("no sandbox with ID %v exists, cannot delete", id) } - name := s.sandboxes[id].name - containers := s.sandboxes[id].containers.List() - infraContainer := s.sandboxes[id].infraContainer + name := s.sandboxes[id].Name() + containers := s.sandboxes[id].Containers() + infraContainer := s.sandboxes[id].InfraContainer() delete(s.sandboxes, id) s.podNameIndex.Release(name) @@ -120,7 +122,7 @@ func (s *InMemoryState) DeleteSandbox(id string) error { } // GetSandbox returns a sandbox given its full ID -func (s *InMemoryState) GetSandbox(id string) (*sandbox, error) { +func (s *InMemoryState) GetSandbox(id string) (*sandbox.Sandbox, error) { s.lock.Lock() defer s.lock.Unlock() @@ -133,7 +135,7 @@ func (s *InMemoryState) GetSandbox(id string) (*sandbox, error) { } // LookupSandboxByName returns a sandbox given its full or partial name -func (s *InMemoryState) LookupSandboxByName(name string) (*sandbox, error) { +func (s *InMemoryState) LookupSandboxByName(name string) (*sandbox.Sandbox, error) { s.lock.Lock() defer s.lock.Unlock() @@ -153,7 +155,7 @@ func (s *InMemoryState) LookupSandboxByName(name string) (*sandbox, error) { // LookupSandboxByID returns a sandbox given its full or partial ID // An error will be returned if the partial ID given is not unique -func (s *InMemoryState) LookupSandboxByID(id string) (*sandbox, error) { +func (s *InMemoryState) LookupSandboxByID(id string) (*sandbox.Sandbox, error) { s.lock.Lock() defer s.lock.Unlock() @@ -172,11 +174,11 @@ func (s *InMemoryState) LookupSandboxByID(id string) (*sandbox, error) { } // GetAllSandboxes returns all sandboxes in the state -func (s *InMemoryState) GetAllSandboxes() ([]*sandbox, error) { +func (s *InMemoryState) GetAllSandboxes() ([]*sandbox.Sandbox, error) { s.lock.Lock() defer s.lock.Unlock() - sandboxes := make([]*sandbox, 0, len(s.sandboxes)) + sandboxes := make([]*sandbox.Sandbox, 0, len(s.sandboxes)) for _, sb := range s.sandboxes { sandboxes = append(sandboxes, sb) } @@ -198,11 +200,11 @@ func (s *InMemoryState) AddContainer(c *oci.Container, sandboxID string) error { return fmt.Errorf("sandbox with ID %v does not exist, cannot add container", sandboxID) } - if ctr := sandbox.containers.Get(c.ID()); ctr != nil { + if ctr := sandbox.GetContainer(c.ID()); ctr != nil { return fmt.Errorf("container with ID %v already exists in sandbox %v", c.ID(), sandboxID) } - sandbox.containers.Add(c.ID(), c) + sandbox.AddContainer(c) return s.addContainerMappings(c, true) } @@ -242,7 +244,7 @@ func (s *InMemoryState) HasContainer(id, sandboxID string) bool { return false } - ctr := sandbox.containers.Get(id) + ctr := sandbox.GetContainer(id) return ctr != nil } @@ -257,12 +259,12 @@ func (s *InMemoryState) DeleteContainer(id, sandboxID string) error { return fmt.Errorf("sandbox with ID %v does not exist", sandboxID) } - ctr := sandbox.containers.Get(id) + ctr := sandbox.GetContainer(id) if ctr == nil { return fmt.Errorf("sandbox %v has no container with ID %v", sandboxID, id) } - sandbox.containers.Delete(id) + sandbox.RemoveContainer(id) return s.deleteContainerMappings(ctr, true) } @@ -358,7 +360,7 @@ func (s *InMemoryState) getContainerFromSandbox(id, sandboxID string) (*oci.Cont return nil, fmt.Errorf("sandbox with ID %v does not exist", sandboxID) } - ctr := sandbox.containers.Get(id) + ctr := sandbox.GetContainer(id) if ctr == nil { return nil, fmt.Errorf("cannot find container %v in sandbox %v", id, sandboxID) } diff --git a/server/state_store.go b/server/state/state_store.go similarity index 64% rename from server/state_store.go rename to server/state/state_store.go index c3a4deb7..d7662285 100644 --- a/server/state_store.go +++ b/server/state/state_store.go @@ -1,13 +1,13 @@ -package server +package state import ( "github.com/kubernetes-incubator/cri-o/oci" + "github.com/kubernetes-incubator/cri-o/server/sandbox" ) -// StateStore stores the state of the CRI-O server, including active pods and -// containers -type StateStore interface { - AddSandbox(s *sandbox) error +// Store stores the state of the CRI-O server, including active pods and containers +type Store interface { + AddSandbox(s *sandbox.Sandbox) error HasSandbox(id string) bool DeleteSandbox(id string) error // These should modify the associated sandbox without prompting @@ -15,15 +15,15 @@ type StateStore interface { HasContainer(id, sandboxID string) bool DeleteContainer(id, sandboxID string) error // These two require full, explicit ID - GetSandbox(id string) (*sandbox, error) + GetSandbox(id string) (*sandbox.Sandbox, error) GetContainer(id, sandboxID string) (*oci.Container, error) // Get ID of sandbox container belongs to GetContainerSandbox(id string) (string, error) // Following 4 should accept partial names as long as they are globally unique - LookupSandboxByName(name string) (*sandbox, error) - LookupSandboxByID(id string) (*sandbox, error) + LookupSandboxByName(name string) (*sandbox.Sandbox, error) + LookupSandboxByID(id string) (*sandbox.Sandbox, error) LookupContainerByName(name string) (*oci.Container, error) LookupContainerByID(id string) (*oci.Container, error) - GetAllSandboxes() ([]*sandbox, error) + GetAllSandboxes() ([]*sandbox.Sandbox, error) GetAllContainers() ([]*oci.Container, error) }