diff --git a/server/sandbox.go b/libkpod/sandbox/sandbox.go similarity index 83% rename from server/sandbox.go rename to libkpod/sandbox/sandbox.go index 0e2ecd44..e3b94237 100644 --- a/server/sandbox.go +++ b/libkpod/sandbox/sandbox.go @@ -1,4 +1,4 @@ -package server +package sandbox import ( "crypto/rand" @@ -19,7 +19,8 @@ import ( "k8s.io/kubernetes/pkg/kubelet/network/hostport" ) -type sandboxNetNs struct { +// NetNs handles data pertaining a network namespace +type NetNs struct { sync.Mutex ns ns.NetNS symlink *os.File @@ -27,7 +28,7 @@ type sandboxNetNs struct { restored bool } -func (ns *sandboxNetNs) symlinkCreate(name string) error { +func (ns *NetNs) symlinkCreate(name string) error { b := make([]byte, 4) _, randErr := rand.Reader.Read(b) if randErr != nil { @@ -35,7 +36,7 @@ func (ns *sandboxNetNs) symlinkCreate(name string) error { } nsName := fmt.Sprintf("%s-%x", name, b) - symlinkPath := filepath.Join(nsRunDir, nsName) + symlinkPath := filepath.Join(NsRunDir, nsName) if err := os.Symlink(ns.ns.Path(), symlinkPath); err != nil { return err @@ -55,7 +56,7 @@ func (ns *sandboxNetNs) symlinkCreate(name string) error { return nil } -func (ns *sandboxNetNs) symlinkRemove() error { +func (ns *NetNs) symlinkRemove() error { if err := ns.symlink.Close(); err != nil { return err } @@ -72,9 +73,10 @@ func isSymbolicLink(path string) (bool, error) { return fi.Mode()&os.ModeSymlink == os.ModeSymlink, nil } -func netNsGet(nspath, name string) (*sandboxNetNs, error) { +// NetNsGet returns the NetNs associated with the given nspath and name +func NetNsGet(nspath, name string) (*NetNs, error) { if err := ns.IsNSorErr(nspath); err != nil { - return nil, errSandboxClosedNetNS + return nil, ErrClosedNetNS } symlink, symlinkErr := isSymbolicLink(nspath) @@ -98,7 +100,7 @@ func netNsGet(nspath, name string) (*sandboxNetNs, error) { return nil, err } - netNs := &sandboxNetNs{ns: netNS, closed: false, restored: true} + netNs := &NetNs{ns: netNS, closed: false, restored: true} if symlink { fd, err := os.Open(nspath) @@ -116,7 +118,8 @@ func netNsGet(nspath, name string) (*sandboxNetNs, error) { return netNs, nil } -func hostNetNsPath() (string, error) { +// HostNetNsPath returns the current network namespace for the host +func HostNetNsPath() (string, error) { netNS, err := ns.GetCurrentNS() if err != nil { return "", err @@ -141,7 +144,7 @@ type Sandbox struct { containers oci.ContainerStorer processLabel string mountLabel string - netns *sandboxNetNs + netns *NetNs metadata *pb.PodSandboxMetadata shmPath string cgroupParent string @@ -153,20 +156,28 @@ type Sandbox struct { } const ( - defaultShmSize = 64 * 1024 * 1024 - nsRunDir = "/var/run/netns" - podInfraCommand = "/pause" + // DefaultShmSize is the default shm size + DefaultShmSize = 64 * 1024 * 1024 + // NsRunDir is the default directory in which running network namespaces + // are stored + NsRunDir = "/var/run/netns" + // PodInfraCommand is the default command when starting a pod infrastructure + // container + PodInfraCommand = "/pause" ) var ( - errSandboxIDEmpty = errors.New("PodSandboxId should not be empty") - errSandboxClosedNetNS = errors.New("PodSandbox networking namespace is closed") + // ErrIDEmpty is the erro returned when the id of the sandbox is empty + ErrIDEmpty = errors.New("PodSandboxId should not be empty") + // ErrClosedNetNS is the error returned when the network namespace of the + // sandbox is closed + ErrClosedNetNS = errors.New("PodSandbox networking namespace is closed") ) -// NewSandbox creates and populates a new pod sandbox +// New creates and populates a new pod sandbox // New sandboxes have no containers, no infra container, and no network namespaces associated with them // An infra container must be attached before the sandbox is added to the state -func NewSandbox(id, namespace, name, kubeName, logDir string, labels, annotations map[string]string, processLabel, mountLabel string, metadata *pb.PodSandboxMetadata, shmPath, cgroupParent string, privileged, trusted bool, resolvPath, hostname string, portMappings []*hostport.PortMapping) (*Sandbox, error) { +func New(id, namespace, name, kubeName, logDir string, labels, annotations map[string]string, processLabel, mountLabel string, metadata *pb.PodSandboxMetadata, shmPath, cgroupParent string, privileged, trusted bool, resolvPath, hostname string, portMappings []*hostport.PortMapping) (*Sandbox, error) { sb := new(Sandbox) sb.id = id sb.namespace = namespace @@ -316,6 +327,11 @@ func (s *Sandbox) SetInfraContainer(infraCtr *oci.Container) error { return nil } +// RemoveInfraContainer removes the infrastructure container of a sandbox +func (s *Sandbox) RemoveInfraContainer() { + s.infraContainer = nil +} + // NetNs retrieves the network namespace of the sandbox // If the sandbox uses the host namespace, nil is returned func (s *Sandbox) NetNs() ns.NetNS { @@ -347,7 +363,7 @@ func (s *Sandbox) NetNsCreate() error { return err } - s.netns = &sandboxNetNs{ + s.netns = &NetNs{ ns: netNS, closed: false, } @@ -372,7 +388,7 @@ func (s *Sandbox) NetNsJoin(nspath, name string) error { return fmt.Errorf("sandbox already has a network namespace, cannot join another") } - netNS, err := netNsGet(nspath, name) + netNS, err := NetNsGet(nspath, name) if err != nil { return err } @@ -429,20 +445,3 @@ func (s *Sandbox) NetNsRemove() error { s.netns.closed = true return nil } - -func (s *Server) getPodSandboxFromRequest(podSandboxID string) (*Sandbox, error) { - if podSandboxID == "" { - return nil, errSandboxIDEmpty - } - - sandboxID, err := s.podIDIndex.Get(podSandboxID) - if err != nil { - return nil, fmt.Errorf("PodSandbox with ID starting with %s not found: %v", podSandboxID, err) - } - - sb := s.getSandbox(sandboxID) - if sb == nil { - return nil, fmt.Errorf("specified pod sandbox not found: %s", sandboxID) - } - return sb, nil -} diff --git a/server/container_create.go b/server/container_create.go index 0ee0aa9e..ebcd607f 100644 --- a/server/container_create.go +++ b/server/container_create.go @@ -14,6 +14,7 @@ import ( "github.com/Sirupsen/logrus" "github.com/docker/docker/pkg/stringid" "github.com/docker/docker/pkg/symlink" + "github.com/kubernetes-incubator/cri-o/libkpod/sandbox" "github.com/kubernetes-incubator/cri-o/oci" "github.com/kubernetes-incubator/cri-o/pkg/annotations" "github.com/kubernetes-incubator/cri-o/pkg/storage" @@ -37,7 +38,7 @@ const ( seccompLocalhostPrefix = "localhost/" ) -func addOCIBindMounts(sb *Sandbox, containerConfig *pb.ContainerConfig, specgen *generate.Generator) error { +func addOCIBindMounts(sb *sandbox.Sandbox, containerConfig *pb.ContainerConfig, specgen *generate.Generator) error { mounts := containerConfig.GetMounts() for _, mount := range mounts { dest := mount.ContainerPath @@ -63,7 +64,7 @@ func addOCIBindMounts(sb *Sandbox, containerConfig *pb.ContainerConfig, specgen if mount.SelinuxRelabel { // Need a way in kubernetes to determine if the volume is shared or private - if err := label.Relabel(src, sb.mountLabel, true); err != nil && err != unix.ENOTSUP { + if err := label.Relabel(src, sb.MountLabel(), true); err != nil && err != unix.ENOTSUP { return fmt.Errorf("relabel failed %s: %v", src, err) } } @@ -109,7 +110,7 @@ func addImageVolumes(rootfs string, s *Server, containerInfo *storage.ContainerI return nil } -func addDevices(sb *Sandbox, containerConfig *pb.ContainerConfig, specgen *generate.Generator) error { +func addDevices(sb *sandbox.Sandbox, containerConfig *pb.ContainerConfig, specgen *generate.Generator) error { sp := specgen.Spec() for _, device := range containerConfig.GetDevices() { dev, err := devices.DeviceFromPath(device.HostPath, device.Permissions) @@ -295,7 +296,7 @@ func (s *Server) CreateContainer(ctx context.Context, req *pb.CreateContainerReq return nil, fmt.Errorf("CreateContainerRequest.ContainerConfig.Name is empty") } - containerID, containerName, err := s.generateContainerIDandName(sb.metadata, containerConfig) + containerID, containerName, err := s.generateContainerIDandName(sb.Metadata(), containerConfig) if err != nil { return nil, err } @@ -319,7 +320,7 @@ func (s *Server) CreateContainer(ctx context.Context, req *pb.CreateContainerReq } }() - if err = s.Runtime().CreateContainer(container, sb.cgroupParent); err != nil { + if err = s.Runtime().CreateContainer(container, sb.CgroupParent()); err != nil { return nil, err } @@ -344,7 +345,7 @@ func (s *Server) CreateContainer(ctx context.Context, req *pb.CreateContainerReq return resp, nil } -func (s *Server) createSandboxContainer(ctx context.Context, containerID string, containerName string, sb *Sandbox, SandboxConfig *pb.PodSandboxConfig, containerConfig *pb.ContainerConfig) (*oci.Container, error) { +func (s *Server) createSandboxContainer(ctx context.Context, containerID string, containerName string, sb *sandbox.Sandbox, SandboxConfig *pb.PodSandboxConfig, containerConfig *pb.ContainerConfig) (*oci.Container, error) { if sb == nil { return nil, errors.New("createSandboxContainer needs a sandbox") } @@ -381,7 +382,7 @@ func (s *Server) createSandboxContainer(ctx context.Context, containerID string, // set this container's apparmor profile if it is set by sandbox if s.appArmorEnabled { - appArmorProfileName := s.getAppArmorProfileName(sb.annotations, metadata.Name) + appArmorProfileName := s.getAppArmorProfileName(sb.Annotations(), metadata.Name) if appArmorProfileName != "" { // reload default apparmor profile if it is unloaded. if s.appArmorProfile == apparmor.DefaultApparmorProfile { @@ -408,12 +409,12 @@ func (s *Server) createSandboxContainer(ctx context.Context, containerID string, logPath := containerConfig.LogPath if logPath == "" { // TODO: Should we use sandboxConfig.GetLogDirectory() here? - logPath = filepath.Join(sb.logDir, containerID+".log") + logPath = filepath.Join(sb.LogDir(), containerID+".log") } if !filepath.IsAbs(logPath) { // XXX: It's not really clear what this should be versus the sbox logDirectory. logrus.Warnf("requested logPath for ctr id %s is a relative path: %s", containerID, logPath) - logPath = filepath.Join(sb.logDir, logPath) + logPath = filepath.Join(sb.LogDir(), logPath) } // Handle https://issues.k8s.io/44043 @@ -422,7 +423,7 @@ func (s *Server) createSandboxContainer(ctx context.Context, containerID string, } logrus.WithFields(logrus.Fields{ - "sbox.logdir": sb.logDir, + "sbox.logdir": sb.LogDir(), "ctr.logfile": containerConfig.LogPath, "log_path": logPath, }).Debugf("setting container's log_path") @@ -457,12 +458,12 @@ func (s *Server) createSandboxContainer(ctx context.Context, containerID string, specgen.SetProcessOOMScoreAdj(int(oomScoreAdj)) } - if sb.cgroupParent != "" { + if sb.CgroupParent() != "" { if s.config.CgroupManager == "systemd" { - cgPath := sb.cgroupParent + ":" + "crio" + ":" + containerID + cgPath := sb.CgroupParent() + ":" + "crio" + ":" + containerID specgen.SetLinuxCgroupsPath(cgPath) } else { - specgen.SetLinuxCgroupsPath(sb.cgroupParent + "/" + containerID) + specgen.SetLinuxCgroupsPath(sb.CgroupParent() + "/" + containerID) } } @@ -493,8 +494,8 @@ func (s *Server) createSandboxContainer(ctx context.Context, containerID string, } } - specgen.SetProcessSelinuxLabel(sb.processLabel) - specgen.SetLinuxMountLabel(sb.mountLabel) + specgen.SetProcessSelinuxLabel(sb.ProcessLabel()) + specgen.SetLinuxMountLabel(sb.MountLabel()) if containerConfig.GetLinux().GetSecurityContext() != nil && !containerConfig.GetLinux().GetSecurityContext().Privileged { @@ -522,7 +523,7 @@ func (s *Server) createSandboxContainer(ctx context.Context, containerID string, } } // Join the namespace paths for the pod sandbox container. - podInfraState := s.Runtime().ContainerStatus(sb.infraContainer) + podInfraState := s.Runtime().ContainerStatus(sb.InfraContainer()) logrus.Debugf("pod container state %+v", podInfraState) @@ -553,15 +554,15 @@ func (s *Server) createSandboxContainer(ctx context.Context, containerID string, } // bind mount the pod shm - specgen.AddBindMount(sb.shmPath, "/dev/shm", []string{"rw"}) + specgen.AddBindMount(sb.ShmPath(), "/dev/shm", []string{"rw"}) options := []string{"rw"} if readOnlyRootfs { options = []string{"ro"} } - if sb.resolvPath != "" { + if sb.ResolvPath() != "" { // bind mount the pod resolver file - specgen.AddBindMount(sb.resolvPath, "/etc/resolv.conf", options) + specgen.AddBindMount(sb.ResolvPath(), "/etc/resolv.conf", options) } // Bind mount /etc/hosts for host networking containers @@ -569,14 +570,14 @@ func (s *Server) createSandboxContainer(ctx context.Context, containerID string, specgen.AddBindMount("/etc/hosts", "/etc/hosts", options) } - if sb.hostname != "" { - specgen.SetHostname(sb.hostname) + if sb.Hostname() != "" { + specgen.SetHostname(sb.Hostname()) } specgen.AddAnnotation(annotations.Name, containerName) specgen.AddAnnotation(annotations.ContainerID, containerID) - specgen.AddAnnotation(annotations.SandboxID, sb.id) - specgen.AddAnnotation(annotations.SandboxName, sb.infraContainer.Name()) + specgen.AddAnnotation(annotations.SandboxID, sb.ID()) + specgen.AddAnnotation(annotations.SandboxName, sb.InfraContainer().Name()) specgen.AddAnnotation(annotations.ContainerType, annotations.ContainerTypeContainer) specgen.AddAnnotation(annotations.LogPath, logPath) specgen.AddAnnotation(annotations.TTY, fmt.Sprintf("%v", containerConfig.Tty)) @@ -605,19 +606,19 @@ func (s *Server) createSandboxContainer(ctx context.Context, containerID string, } specgen.AddAnnotation(annotations.Annotations, string(kubeAnnotationsJSON)) - if err = s.setupSeccomp(&specgen, containerName, sb.annotations); err != nil { + if err = s.setupSeccomp(&specgen, containerName, sb.Annotations()); err != nil { return nil, err } metaname := metadata.Name attempt := metadata.Attempt containerInfo, err := s.storageRuntimeServer.CreateContainer(s.ImageContext(), - sb.name, sb.id, + sb.Name(), sb.ID(), image, image, containerName, containerID, metaname, attempt, - sb.mountLabel, + sb.MountLabel(), nil) if err != nil { return nil, err @@ -639,7 +640,7 @@ func (s *Server) createSandboxContainer(ctx context.Context, containerID string, } // Add image volumes - if err := addImageVolumes(mountPoint, s, &containerInfo, &specgen, sb.mountLabel); err != nil { + if err := addImageVolumes(mountPoint, s, &containerInfo, &specgen, sb.MountLabel()); err != nil { return nil, err } @@ -714,7 +715,7 @@ func (s *Server) createSandboxContainer(ctx context.Context, containerID string, return nil, err } - container, err := oci.NewContainer(containerID, containerName, containerInfo.RunDir, logPath, sb.NetNs(), labels, kubeAnnotations, image, metadata, sb.id, containerConfig.Tty, containerConfig.Stdin, containerConfig.StdinOnce, sb.privileged, sb.trusted, containerInfo.Dir, created, containerImageConfig.Config.StopSignal) + container, err := oci.NewContainer(containerID, containerName, containerInfo.RunDir, logPath, sb.NetNs(), labels, kubeAnnotations, image, metadata, sb.ID(), containerConfig.Tty, containerConfig.Stdin, containerConfig.StdinOnce, sb.Privileged(), sb.Trusted(), containerInfo.Dir, created, containerImageConfig.Config.StopSignal) if err != nil { return nil, err } diff --git a/server/container_list.go b/server/container_list.go index 10b93338..80871d59 100644 --- a/server/container_list.go +++ b/server/container_list.go @@ -59,7 +59,7 @@ func (s *Server) ListContainers(ctx context.Context, req *pb.ListContainersReque if pod == nil { ctrList = []*oci.Container{} } else { - ctrList = pod.containers.List() + ctrList = pod.Containers().List() } } } diff --git a/server/sandbox_list.go b/server/sandbox_list.go index 36ff860e..f1ccbe2c 100644 --- a/server/sandbox_list.go +++ b/server/sandbox_list.go @@ -2,6 +2,7 @@ package server import ( "github.com/Sirupsen/logrus" + "github.com/kubernetes-incubator/cri-o/libkpod/sandbox" "github.com/kubernetes-incubator/cri-o/oci" "golang.org/x/net/context" "k8s.io/apimachinery/pkg/fields" @@ -30,7 +31,7 @@ func filterSandbox(p *pb.PodSandbox, filter *pb.PodSandboxFilter) bool { func (s *Server) ListPodSandbox(ctx context.Context, req *pb.ListPodSandboxRequest) (*pb.ListPodSandboxResponse, error) { logrus.Debugf("ListPodSandboxRequest %+v", req) var pods []*pb.PodSandbox - var podList []*Sandbox + var podList []*sandbox.Sandbox for _, sb := range s.state.sandboxes { podList = append(podList, sb) } @@ -45,15 +46,15 @@ func (s *Server) ListPodSandbox(ctx context.Context, req *pb.ListPodSandboxReque } sb := s.getSandbox(id) if sb == nil { - podList = []*Sandbox{} + podList = []*sandbox.Sandbox{} } else { - podList = []*Sandbox{sb} + podList = []*sandbox.Sandbox{sb} } } } for _, sb := range podList { - podInfraContainer := sb.infraContainer + podInfraContainer := sb.InfraContainer() if podInfraContainer == nil { // this can't really happen, but if it does because of a bug // it's better not to panic @@ -71,12 +72,12 @@ func (s *Server) ListPodSandbox(ctx context.Context, req *pb.ListPodSandboxReque } pod := &pb.PodSandbox{ - Id: sb.id, + Id: sb.ID(), CreatedAt: created, State: rStatus, - Labels: sb.labels, - Annotations: sb.annotations, - Metadata: sb.metadata, + Labels: sb.Labels(), + Annotations: sb.Annotations(), + Metadata: sb.Metadata(), } // Filter by other criteria such as state and labels. diff --git a/server/sandbox_remove.go b/server/sandbox_remove.go index 383f1b57..d9efe7e6 100644 --- a/server/sandbox_remove.go +++ b/server/sandbox_remove.go @@ -5,6 +5,7 @@ import ( "github.com/Sirupsen/logrus" "github.com/containers/storage" + "github.com/kubernetes-incubator/cri-o/libkpod/sandbox" "github.com/kubernetes-incubator/cri-o/oci" pkgstorage "github.com/kubernetes-incubator/cri-o/pkg/storage" "golang.org/x/net/context" @@ -17,7 +18,7 @@ func (s *Server) RemovePodSandbox(ctx context.Context, req *pb.RemovePodSandboxR logrus.Debugf("RemovePodSandboxRequest %+v", req) sb, err := s.getPodSandboxFromRequest(req.PodSandboxId) if err != nil { - if err == errSandboxIDEmpty { + if err == sandbox.ErrIDEmpty { return nil, err } @@ -30,8 +31,8 @@ func (s *Server) RemovePodSandbox(ctx context.Context, req *pb.RemovePodSandboxR return resp, nil } - podInfraContainer := sb.infraContainer - containers := sb.containers.List() + podInfraContainer := sb.InfraContainer() + containers := sb.Containers().List() containers = append(containers, podInfraContainer) // Delete all the containers in the sandbox @@ -49,7 +50,7 @@ func (s *Server) RemovePodSandbox(ctx context.Context, req *pb.RemovePodSandboxR } if err := s.Runtime().DeleteContainer(c); err != nil { - return nil, fmt.Errorf("failed to delete container %s in pod sandbox %s: %v", c.Name(), sb.id, err) + return nil, fmt.Errorf("failed to delete container %s in pod sandbox %s: %v", c.Name(), sb.ID(), err) } if c.ID() == podInfraContainer.ID() { @@ -58,38 +59,38 @@ func (s *Server) RemovePodSandbox(ctx context.Context, req *pb.RemovePodSandboxR if err := s.storageRuntimeServer.StopContainer(c.ID()); err != nil && err != storage.ErrContainerUnknown { // assume container already umounted - logrus.Warnf("failed to stop container %s in pod sandbox %s: %v", c.Name(), sb.id, err) + logrus.Warnf("failed to stop container %s in pod sandbox %s: %v", c.Name(), sb.ID(), err) } if err := s.storageRuntimeServer.DeleteContainer(c.ID()); err != nil && err != storage.ErrContainerUnknown { - return nil, fmt.Errorf("failed to delete container %s in pod sandbox %s: %v", c.Name(), sb.id, err) + return nil, fmt.Errorf("failed to delete container %s in pod sandbox %s: %v", c.Name(), sb.ID(), err) } s.releaseContainerName(c.Name()) s.removeContainer(c) if err := s.CtrIDIndex().Delete(c.ID()); err != nil { - return nil, fmt.Errorf("failed to delete container %s in pod sandbox %s from index: %v", c.Name(), sb.id, err) + return nil, fmt.Errorf("failed to delete container %s in pod sandbox %s from index: %v", c.Name(), sb.ID(), err) } } s.removeContainer(podInfraContainer) // Remove the files related to the sandbox - if err := s.storageRuntimeServer.StopContainer(sb.id); err != nil && err != storage.ErrContainerUnknown { - logrus.Warnf("failed to stop sandbox container in pod sandbox %s: %v", sb.id, err) + if err := s.storageRuntimeServer.StopContainer(sb.ID()); err != nil && err != storage.ErrContainerUnknown { + logrus.Warnf("failed to stop sandbox container in pod sandbox %s: %v", sb.ID(), err) } - if err := s.storageRuntimeServer.RemovePodSandbox(sb.id); err != nil && err != pkgstorage.ErrInvalidSandboxID { - return nil, fmt.Errorf("failed to remove pod sandbox %s: %v", sb.id, err) + if err := s.storageRuntimeServer.RemovePodSandbox(sb.ID()); err != nil && err != pkgstorage.ErrInvalidSandboxID { + return nil, fmt.Errorf("failed to remove pod sandbox %s: %v", sb.ID(), err) } s.releaseContainerName(podInfraContainer.Name()) if err := s.CtrIDIndex().Delete(podInfraContainer.ID()); err != nil { - return nil, fmt.Errorf("failed to delete infra container %s in pod sandbox %s from index: %v", podInfraContainer.ID(), sb.id, err) + return nil, fmt.Errorf("failed to delete infra container %s in pod sandbox %s from index: %v", podInfraContainer.ID(), sb.ID(), err) } - s.releasePodName(sb.name) - s.removeSandbox(sb.id) - if err := s.podIDIndex.Delete(sb.id); err != nil { - return nil, fmt.Errorf("failed to delete pod sandbox %s from index: %v", sb.id, err) + s.releasePodName(sb.Name()) + s.removeSandbox(sb.ID()) + if err := s.podIDIndex.Delete(sb.ID()); err != nil { + return nil, fmt.Errorf("failed to delete pod sandbox %s from index: %v", sb.ID(), err) } resp := &pb.RemovePodSandboxResponse{} diff --git a/server/sandbox_run.go b/server/sandbox_run.go index 250b94ae..bfff47d2 100644 --- a/server/sandbox_run.go +++ b/server/sandbox_run.go @@ -13,6 +13,7 @@ import ( "github.com/Sirupsen/logrus" "github.com/containers/storage" + "github.com/kubernetes-incubator/cri-o/libkpod/sandbox" "github.com/kubernetes-incubator/cri-o/oci" "github.com/kubernetes-incubator/cri-o/pkg/annotations" "github.com/opencontainers/runc/libcontainer/cgroups/systemd" @@ -186,7 +187,7 @@ func (s *Server) RunPodSandbox(ctx context.Context, req *pb.RunPodSandboxRequest if podContainer.Config != nil { g.SetProcessArgs(podContainer.Config.Config.Cmd) } else { - g.SetProcessArgs([]string{podInfraCommand}) + g.SetProcessArgs([]string{sandbox.PodInfraCommand}) } } else { g.SetProcessArgs([]string{s.config.PauseCommand}) @@ -330,24 +331,24 @@ func (s *Server) RunPodSandbox(ctx context.Context, req *pb.RunPodSandboxRequest portMappings := convertPortMappings(req.GetConfig().GetPortMappings()) - sb := &Sandbox{ - id: id, - namespace: namespace, - name: name, - kubeName: kubeName, - logDir: logDir, - labels: labels, - annotations: kubeAnnotations, - containers: oci.NewMemoryStore(), - processLabel: processLabel, - mountLabel: mountLabel, - metadata: metadata, - shmPath: shmPath, - privileged: privileged, - trusted: trusted, - resolvPath: resolvPath, - hostname: hostname, - portMappings: portMappings, + // setup cgroup settings + cgroupParent := req.GetConfig().GetLinux().CgroupParent + if cgroupParent != "" { + if s.config.CgroupManager == "systemd" { + cgPath, err := convertCgroupNameToSystemd(cgroupParent, false) + if err != nil { + return nil, err + } + g.SetLinuxCgroupsPath(cgPath + ":" + "crio" + ":" + id) + cgroupParent = cgPath + } else { + g.SetLinuxCgroupsPath(cgroupParent + "/" + id) + } + } + + sb, err := sandbox.New(id, namespace, name, kubeName, logDir, labels, kubeAnnotations, processLabel, mountLabel, metadata, shmPath, cgroupParent, privileged, trusted, resolvPath, hostname, portMappings) + if err != nil { + return nil, err } s.addSandbox(sb) @@ -385,22 +386,6 @@ func (s *Server) RunPodSandbox(ctx context.Context, req *pb.RunPodSandboxRequest g.AddLinuxSysctl(sysctl.Name, sysctl.Value) } - // setup cgroup settings - cgroupParent := req.GetConfig().GetLinux().CgroupParent - if cgroupParent != "" { - if s.config.CgroupManager == "systemd" { - cgPath, err := convertCgroupNameToSystemd(cgroupParent, false) - if err != nil { - return nil, err - } - g.SetLinuxCgroupsPath(cgPath + ":" + "crio" + ":" + id) - sb.cgroupParent = cgPath - } else { - g.SetLinuxCgroupsPath(cgroupParent + "/" + id) - sb.cgroupParent = cgroupParent - } - } - // Set OOM score adjust of the infra container to be very low // so it doesn't get killed. g.SetProcessOOMScoreAdj(PodInfraOOMAdj) @@ -414,7 +399,7 @@ func (s *Server) RunPodSandbox(ctx context.Context, req *pb.RunPodSandboxRequest return nil, err } - netNsPath, err = hostNetNsPath() + netNsPath, err = sandbox.HostNetNsPath() if err != nil { return nil, err } @@ -464,23 +449,23 @@ func (s *Server) RunPodSandbox(ctx context.Context, req *pb.RunPodSandboxRequest saveOptions := generate.ExportOptions{} mountPoint, err := s.storageRuntimeServer.StartContainer(id) if err != nil { - return nil, fmt.Errorf("failed to mount container %s in pod sandbox %s(%s): %v", containerName, sb.name, id, err) + return nil, fmt.Errorf("failed to mount container %s in pod sandbox %s(%s): %v", containerName, sb.Name(), id, err) } g.SetRootPath(mountPoint) err = g.SaveToFile(filepath.Join(podContainer.Dir, "config.json"), saveOptions) if err != nil { - return nil, fmt.Errorf("failed to save template configuration for pod sandbox %s(%s): %v", sb.name, id, err) + return nil, fmt.Errorf("failed to save template configuration for pod sandbox %s(%s): %v", sb.Name(), id, err) } if err = g.SaveToFile(filepath.Join(podContainer.RunDir, "config.json"), saveOptions); err != nil { - return nil, fmt.Errorf("failed to write runtime configuration for pod sandbox %s(%s): %v", sb.name, id, err) + return nil, fmt.Errorf("failed to write runtime configuration for pod sandbox %s(%s): %v", sb.Name(), id, err) } - container, err := oci.NewContainer(id, containerName, podContainer.RunDir, logPath, sb.NetNs(), labels, kubeAnnotations, "", nil, id, false, false, false, sb.privileged, sb.trusted, podContainer.Dir, created, podContainer.Config.Config.StopSignal) + container, err := oci.NewContainer(id, containerName, podContainer.RunDir, logPath, sb.NetNs(), labels, kubeAnnotations, "", nil, id, false, false, false, sb.Privileged(), sb.Trusted(), podContainer.Dir, created, podContainer.Config.Config.StopSignal) if err != nil { return nil, err } - sb.infraContainer = container + sb.SetInfraContainer(container) // setup the network if !hostNetwork { @@ -511,7 +496,7 @@ func (s *Server) RunPodSandbox(ctx context.Context, req *pb.RunPodSandboxRequest } } - if err = s.runContainer(container, sb.cgroupParent); err != nil { + if err = s.runContainer(container, sb.CgroupParent()); err != nil { return nil, err } @@ -579,7 +564,7 @@ func setupShm(podSandboxRunDir, mountLabel string) (shmPath string, err error) { if err = os.Mkdir(shmPath, 0700); err != nil { return "", err } - shmOptions := "mode=1777,size=" + strconv.Itoa(defaultShmSize) + shmOptions := "mode=1777,size=" + strconv.Itoa(sandbox.DefaultShmSize) if err = unix.Mount("shm", shmPath, "tmpfs", unix.MS_NOEXEC|unix.MS_NOSUID|unix.MS_NODEV, label.FormatMountLabel(shmOptions, mountLabel)); err != nil { return "", fmt.Errorf("failed to mount shm tmpfs for pod: %v", err) diff --git a/server/sandbox_status.go b/server/sandbox_status.go index 7aa4d5de..ba00563d 100644 --- a/server/sandbox_status.go +++ b/server/sandbox_status.go @@ -15,7 +15,7 @@ func (s *Server) PodSandboxStatus(ctx context.Context, req *pb.PodSandboxStatusR return nil, err } - podInfraContainer := sb.infraContainer + podInfraContainer := sb.InfraContainer() if err = s.Runtime().UpdateStatus(podInfraContainer); err != nil { return nil, err } @@ -27,7 +27,7 @@ func (s *Server) PodSandboxStatus(ctx context.Context, req *pb.PodSandboxStatusR if err != nil { return nil, err } - ip, err := s.netPlugin.GetContainerNetworkStatus(netNsPath, sb.namespace, sb.kubeName, sb.id) + ip, err := s.netPlugin.GetContainerNetworkStatus(netNsPath, sb.Namespace(), sb.KubeName(), sb.ID()) if err != nil { // ignore the error on network status ip = "" @@ -38,7 +38,7 @@ func (s *Server) PodSandboxStatus(ctx context.Context, req *pb.PodSandboxStatusR rStatus = pb.PodSandboxState_SANDBOX_READY } - sandboxID := sb.id + sandboxID := sb.ID() resp := &pb.PodSandboxStatusResponse{ Status: &pb.PodSandboxStatus{ Id: sandboxID, @@ -50,9 +50,9 @@ func (s *Server) PodSandboxStatus(ctx context.Context, req *pb.PodSandboxStatusR }, Network: &pb.PodSandboxNetworkStatus{Ip: ip}, State: rStatus, - Labels: sb.labels, - Annotations: sb.annotations, - Metadata: sb.metadata, + Labels: sb.Labels(), + Annotations: sb.Annotations(), + Metadata: sb.Metadata(), }, } diff --git a/server/sandbox_stop.go b/server/sandbox_stop.go index e5c571c6..1f863fc2 100644 --- a/server/sandbox_stop.go +++ b/server/sandbox_stop.go @@ -8,6 +8,7 @@ import ( "github.com/containers/storage" "github.com/docker/docker/pkg/mount" "github.com/docker/docker/pkg/symlink" + "github.com/kubernetes-incubator/cri-o/libkpod/sandbox" "github.com/kubernetes-incubator/cri-o/oci" "github.com/opencontainers/selinux/go-selinux/label" "golang.org/x/net/context" @@ -22,7 +23,7 @@ func (s *Server) StopPodSandbox(ctx context.Context, req *pb.StopPodSandboxReque logrus.Debugf("StopPodSandboxRequest %+v", req) sb, err := s.getPodSandboxFromRequest(req.PodSandboxId) if err != nil { - if err == errSandboxIDEmpty { + if err == sandbox.ErrIDEmpty { return nil, err } @@ -35,28 +36,28 @@ func (s *Server) StopPodSandbox(ctx context.Context, req *pb.StopPodSandboxReque return resp, nil } - podInfraContainer := sb.infraContainer + podInfraContainer := sb.InfraContainer() netnsPath, err := podInfraContainer.NetNsPath() if err != nil { return nil, err } if _, err := os.Stat(netnsPath); err == nil { - if err2 := s.hostportManager.Remove(sb.id, &hostport.PodPortMapping{ - Name: sb.name, - PortMappings: sb.portMappings, + if err2 := s.hostportManager.Remove(sb.ID(), &hostport.PodPortMapping{ + Name: sb.Name(), + PortMappings: sb.PortMappings(), HostNetwork: false, }); err2 != nil { logrus.Warnf("failed to remove hostport for container %s in sandbox %s: %v", - podInfraContainer.Name(), sb.id, err2) + podInfraContainer.Name(), sb.ID(), err2) } - if err2 := s.netPlugin.TearDownPod(netnsPath, sb.namespace, sb.kubeName, sb.id); err2 != nil { + if err2 := s.netPlugin.TearDownPod(netnsPath, sb.Namespace(), sb.KubeName(), sb.ID()); err2 != nil { logrus.Warnf("failed to destroy network for container %s in sandbox %s: %v", - podInfraContainer.Name(), sb.id, err2) + podInfraContainer.Name(), sb.ID(), err2) } } else if !os.IsNotExist(err) { // it's ok for netnsPath to *not* exist return nil, fmt.Errorf("failed to stat netns path for container %s in sandbox %s before tearing down the network: %v", - sb.name, sb.id, err) + sb.Name(), sb.ID(), err) } // Close the sandbox networking namespace. @@ -64,7 +65,7 @@ func (s *Server) StopPodSandbox(ctx context.Context, req *pb.StopPodSandboxReque return nil, err } - containers := sb.containers.List() + containers := sb.Containers().List() containers = append(containers, podInfraContainer) for _, c := range containers { @@ -74,30 +75,30 @@ func (s *Server) StopPodSandbox(ctx context.Context, req *pb.StopPodSandboxReque cStatus := s.Runtime().ContainerStatus(c) if cStatus.Status != oci.ContainerStateStopped { if err := s.Runtime().StopContainer(c, -1); err != nil { - return nil, fmt.Errorf("failed to stop container %s in pod sandbox %s: %v", c.Name(), sb.id, err) + return nil, fmt.Errorf("failed to stop container %s in pod sandbox %s: %v", c.Name(), sb.ID(), err) } if c.ID() == podInfraContainer.ID() { continue } if err := s.storageRuntimeServer.StopContainer(c.ID()); err != nil && err != storage.ErrContainerUnknown { // assume container already umounted - logrus.Warnf("failed to stop container %s in pod sandbox %s: %v", c.Name(), sb.id, err) + logrus.Warnf("failed to stop container %s in pod sandbox %s: %v", c.Name(), sb.ID(), err) } } s.containerStateToDisk(c) } - if err := label.ReleaseLabel(sb.processLabel); err != nil { + if err := label.ReleaseLabel(sb.ProcessLabel()); err != nil { return nil, err } // unmount the shm for the pod - if sb.shmPath != "/dev/shm" { + if sb.ShmPath() != "/dev/shm" { // we got namespaces in the form of // /var/run/containers/storage/overlay-containers/CID/userdata/shm // but /var/run on most system is symlinked to /run so we first resolve // the symlink and then try and see if it's mounted - fp, err := symlink.FollowSymlinkInScope(sb.shmPath, "/") + fp, err := symlink.FollowSymlinkInScope(sb.ShmPath(), "/") if err != nil { return nil, err } @@ -107,8 +108,8 @@ func (s *Server) StopPodSandbox(ctx context.Context, req *pb.StopPodSandboxReque } } } - if err := s.storageRuntimeServer.StopContainer(sb.id); err != nil && err != storage.ErrContainerUnknown { - logrus.Warnf("failed to stop sandbox container in pod sandbox %s: %v", sb.id, err) + if err := s.storageRuntimeServer.StopContainer(sb.ID()); err != nil && err != storage.ErrContainerUnknown { + logrus.Warnf("failed to stop sandbox container in pod sandbox %s: %v", sb.ID(), err) } resp := &pb.StopPodSandboxResponse{} @@ -121,10 +122,10 @@ func (s *Server) StopAllPodSandboxes() { logrus.Debugf("StopAllPodSandboxes") for _, sb := range s.state.sandboxes { pod := &pb.StopPodSandboxRequest{ - PodSandboxId: sb.id, + PodSandboxId: sb.ID(), } if _, err := s.StopPodSandbox(nil, pod); err != nil { - logrus.Warnf("could not StopPodSandbox %s: %v", sb.id, err) + logrus.Warnf("could not StopPodSandbox %s: %v", sb.ID(), err) } } } diff --git a/server/server.go b/server/server.go index c18bff5d..71cb960b 100644 --- a/server/server.go +++ b/server/server.go @@ -17,6 +17,7 @@ import ( "github.com/docker/docker/pkg/registrar" "github.com/docker/docker/pkg/truncindex" "github.com/kubernetes-incubator/cri-o/libkpod" + "github.com/kubernetes-incubator/cri-o/libkpod/sandbox" "github.com/kubernetes-incubator/cri-o/oci" "github.com/kubernetes-incubator/cri-o/pkg/annotations" "github.com/kubernetes-incubator/cri-o/pkg/ocicni" @@ -152,7 +153,7 @@ func (s *Server) loadContainer(id string) error { return err } - ctr, err := oci.NewContainer(id, name, containerPath, m.Annotations[annotations.LogPath], sb.NetNs(), labels, kubeAnnotations, img, &metadata, sb.id, tty, stdin, stdinOnce, sb.privileged, sb.trusted, containerDir, created, m.Annotations["org.opencontainers.image.stopSignal"]) + ctr, err := oci.NewContainer(id, name, containerPath, m.Annotations[annotations.LogPath], sb.NetNs(), labels, kubeAnnotations, img, &metadata, sb.ID(), tty, stdin, stdinOnce, sb.Privileged(), sb.Trusted(), containerDir, created, m.Annotations["org.opencontainers.image.stopSignal"]) if err != nil { return err } @@ -245,43 +246,29 @@ func (s *Server) loadSandbox(id string) error { privileged := isTrue(m.Annotations[annotations.PrivilegedRuntime]) trusted := isTrue(m.Annotations[annotations.TrustedSandbox]) - sb := &Sandbox{ - id: id, - name: name, - kubeName: m.Annotations[annotations.KubeName], - logDir: filepath.Dir(m.Annotations[annotations.LogPath]), - labels: labels, - containers: oci.NewMemoryStore(), - processLabel: processLabel, - mountLabel: mountLabel, - annotations: kubeAnnotations, - metadata: &metadata, - shmPath: m.Annotations[annotations.ShmPath], - privileged: privileged, - trusted: trusted, - resolvPath: m.Annotations[annotations.ResolvPath], + sb, err := sandbox.New(id, name, m.Annotations[annotations.KubeName], filepath.Dir(m.Annotations[annotations.LogPath]), "", labels, kubeAnnotations, processLabel, mountLabel, &metadata, m.Annotations[annotations.ShmPath], "", privileged, trusted, m.Annotations[annotations.ResolvPath], "", nil) + if err != nil { + return err } // We add a netNS only if we can load a permanent one. // Otherwise, the sandbox will live in the host namespace. netNsPath, err := configNetNsPath(m) if err == nil { - netNS, nsErr := netNsGet(netNsPath, sb.name) + nsErr := sb.NetNsJoin(netNsPath, sb.Name()) // If we can't load the networking namespace // because it's closed, we just set the sb netns // pointer to nil. Otherwise we return an error. - if nsErr != nil && nsErr != errSandboxClosedNetNS { + if nsErr != nil && nsErr != sandbox.ErrClosedNetNS { return nsErr } - - sb.netns = netNS } s.addSandbox(sb) defer func() { if err != nil { - s.removeSandbox(sb.id) + s.removeSandbox(sb.ID()) } }() @@ -320,7 +307,7 @@ func (s *Server) loadSandbox(id string) error { if err = label.ReserveLabel(processLabel); err != nil { return err } - sb.infraContainer = scontainer + sb.SetInfraContainer(scontainer) if err = s.CtrIDIndex().Add(scontainer.ID()); err != nil { return err } @@ -443,19 +430,19 @@ func (s *Server) update() error { logrus.Warnf("bad state when getting pod to remove %+v", removedPod) continue } - podInfraContainer := sb.infraContainer + podInfraContainer := sb.InfraContainer() s.releaseContainerName(podInfraContainer.Name()) s.removeContainer(podInfraContainer) if err = s.CtrIDIndex().Delete(podInfraContainer.ID()); err != nil { return err } - sb.infraContainer = nil - s.releasePodName(sb.name) - s.removeSandbox(sb.id) - if err = s.podIDIndex.Delete(sb.id); err != nil { + sb.RemoveInfraContainer() + s.releasePodName(sb.Name()) + s.removeSandbox(sb.ID()) + if err = s.podIDIndex.Delete(sb.ID()); err != nil { return err } - logrus.Debugf("forgetting removed pod %s", sb.id) + logrus.Debugf("forgetting removed pod %s", sb.ID()) } for sandboxID := range newPods { // load this pod @@ -570,7 +557,7 @@ func New(config *Config) (*Server, error) { return nil, err } - sandboxes := make(map[string]*Sandbox) + sandboxes := make(map[string]*sandbox.Sandbox) netPlugin, err := ocicni.InitCNI(config.NetworkDir, config.PluginDir) if err != nil { return nil, err @@ -651,16 +638,16 @@ func New(config *Config) (*Server, error) { } type serverState struct { - sandboxes map[string]*Sandbox + sandboxes map[string]*sandbox.Sandbox } -func (s *Server) addSandbox(sb *Sandbox) { +func (s *Server) addSandbox(sb *sandbox.Sandbox) { s.stateLock.Lock() - s.state.sandboxes[sb.id] = sb + s.state.sandboxes[sb.ID()] = sb s.stateLock.Unlock() } -func (s *Server) getSandbox(id string) *Sandbox { +func (s *Server) getSandbox(id string) *sandbox.Sandbox { s.stateLock.Lock() sb := s.state.sandboxes[id] s.stateLock.Unlock() @@ -699,7 +686,7 @@ func (s *Server) getContainer(id string) *oci.Container { // GetSandboxContainer returns the infra container for a given sandbox func (s *Server) GetSandboxContainer(id string) *oci.Container { sb := s.getSandbox(id) - return sb.infraContainer + return sb.InfraContainer() } // GetContainer returns a container by its ID @@ -714,3 +701,20 @@ func (s *Server) removeContainer(c *oci.Container) { s.ContainerServer.RemoveContainer(c) s.stateLock.Unlock() } + +func (s *Server) getPodSandboxFromRequest(podSandboxID string) (*sandbox.Sandbox, error) { + if podSandboxID == "" { + return nil, sandbox.ErrIDEmpty + } + + sandboxID, err := s.podIDIndex.Get(podSandboxID) + if err != nil { + return nil, fmt.Errorf("PodSandbox with ID starting with %s not found: %v", podSandboxID, err) + } + + sb := s.getSandbox(sandboxID) + if sb == nil { + return nil, fmt.Errorf("specified pod sandbox not found: %s", sandboxID) + } + return sb, nil +}