vbatts/cri-o
1
0
Fork 0
Code Issues Pull requests Releases Wiki Activity

Move Sandbox and State into their own packages

Browse source 
This allows us to make sandboxes read-only to the greatest extent possible,
making syncronizing them to disk much easier. It also removes the state package
from server, cleaning up the interface a bit.

Signed-off-by: Matthew Heon <mheon@redhat.com>
This commit is contained in:
Matthew Heon 2017-03-30 15:36:46 -04:00
parent d530d36b4b
commit 05a2bc8f73
13 changed files with 635 additions and 469 deletions
Download patch file Download diff file Expand all files Collapse all files

6
pkg/annotations/annotations.go
View file

@ -25,6 +25,9 @@ const (
// KubeName is the kubernetes name annotation
KubeName = "io.kubernetes.cri-o.KubeName"
// Namespace is the kubernetes pod namespace annotation
Namespace = "io.kubernetes.cri-o.Namespace"
// Labels are the kubernetes labels annotation
Labels = "io.kubernetes.cri-o.Labels"
@ -63,6 +66,9 @@ const (
// StdinOnce is the stdin_once annotation
StdinOnce = "io.kubernetes.cri-o.StdinOnce"
// PortMappings is the annotation for sandbox port mappings
PortMappings = "io.kubernetes.cri-o.PortMappings"
)
// ContainerType values

57
server/container_create.go
View file

@ -17,6 +17,7 @@ import (
"github.com/kubernetes-incubator/cri-o/oci"
"github.com/kubernetes-incubator/cri-o/pkg/annotations"
"github.com/kubernetes-incubator/cri-o/server/apparmor"
"github.com/kubernetes-incubator/cri-o/server/sandbox"
"github.com/kubernetes-incubator/cri-o/server/seccomp"
"github.com/opencontainers/image-spec/specs-go/v1"
"github.com/opencontainers/runc/libcontainer/devices"
@ -34,7 +35,7 @@ const (
seccompLocalhostPrefix = "localhost/"
)
func addOCIBindMounts(sb *sandbox, containerConfig *pb.ContainerConfig, specgen *generate.Generator) error {
func addOCIBindMounts(sb *sandbox.Sandbox, containerConfig *pb.ContainerConfig, specgen *generate.Generator) error {
mounts := containerConfig.GetMounts()
for _, mount := range mounts {
dest := mount.ContainerPath
@ -60,7 +61,7 @@ func addOCIBindMounts(sb *sandbox, containerConfig *pb.ContainerConfig, specgen
if mount.SelinuxRelabel {
// Need a way in kubernetes to determine if the volume is shared or private
if err := label.Relabel(src, sb.mountLabel, true); err != nil && err != syscall.ENOTSUP {
if err := label.Relabel(src, sb.MountLabel(), true); err != nil && err != syscall.ENOTSUP {
return fmt.Errorf("relabel failed %s: %v", src, err)
}
}
@ -71,7 +72,7 @@ func addOCIBindMounts(sb *sandbox, containerConfig *pb.ContainerConfig, specgen
return nil
}
func addDevices(sb *sandbox, containerConfig *pb.ContainerConfig, specgen *generate.Generator) error {
func addDevices(sb *sandbox.Sandbox, containerConfig *pb.ContainerConfig, specgen *generate.Generator) error {
sp := specgen.Spec()
for _, device := range containerConfig.GetDevices() {
dev, err := devices.DeviceFromPath(device.HostPath, device.Permissions)
@ -252,7 +253,7 @@ func (s *Server) CreateContainer(ctx context.Context, req *pb.CreateContainerReq
return nil, fmt.Errorf("CreateContainerRequest.ContainerConfig.Name is empty")
}
containerID, containerName, err := s.generateContainerIDandName(sb.metadata, containerConfig)
containerID, containerName, err := s.generateContainerIDandName(sb.Metadata(), containerConfig)
if err != nil {
return nil, err
}
@ -270,7 +271,7 @@ func (s *Server) CreateContainer(ctx context.Context, req *pb.CreateContainerReq
}
}()
if err = s.runtime.CreateContainer(container, sb.cgroupParent); err != nil {
if err = s.runtime.CreateContainer(container, sb.CgroupParent()); err != nil {
return nil, err
}
@ -292,7 +293,7 @@ func (s *Server) CreateContainer(ctx context.Context, req *pb.CreateContainerReq
return resp, nil
}
func (s *Server) createSandboxContainer(ctx context.Context, containerID string, containerName string, sb *sandbox, SandboxConfig *pb.PodSandboxConfig, containerConfig *pb.ContainerConfig) (*oci.Container, error) {
func (s *Server) createSandboxContainer(ctx context.Context, containerID string, containerName string, sb *sandbox.Sandbox, SandboxConfig *pb.PodSandboxConfig, containerConfig *pb.ContainerConfig) (*oci.Container, error) {
if sb == nil {
return nil, errors.New("createSandboxContainer needs a sandbox")
}
@ -325,7 +326,7 @@ func (s *Server) createSandboxContainer(ctx context.Context, containerID string,
// set this container's apparmor profile if it is set by sandbox
if s.appArmorEnabled {
appArmorProfileName := s.getAppArmorProfileName(sb.annotations, metadata.Name)
appArmorProfileName := s.getAppArmorProfileName(sb.Annotations(), metadata.Name)
if appArmorProfileName != "" {
// reload default apparmor profile if it is unloaded.
if s.appArmorProfile == apparmor.DefaultApparmorProfile {
@ -352,12 +353,12 @@ func (s *Server) createSandboxContainer(ctx context.Context, containerID string,
logPath := containerConfig.LogPath
if logPath == "" {
// TODO: Should we use sandboxConfig.GetLogDirectory() here?
logPath = filepath.Join(sb.logDir, containerID+".log")
logPath = filepath.Join(sb.LogDir(), containerID+".log")
}
if !filepath.IsAbs(logPath) {
// XXX: It's not really clear what this should be versus the sbox logDirectory.
logrus.Warnf("requested logPath for ctr id %s is a relative path: %s", containerID, logPath)
logPath = filepath.Join(sb.logDir, logPath)
logPath = filepath.Join(sb.LogDir(), logPath)
}
// Handle https://issues.k8s.io/44043
@ -366,7 +367,7 @@ func (s *Server) createSandboxContainer(ctx context.Context, containerID string,
}
logrus.WithFields(logrus.Fields{
"sbox.logdir": sb.logDir,
"sbox.logdir": sb.LogDir(),
"ctr.logfile": containerConfig.LogPath,
"log_path": logPath,
}).Debugf("setting container's log_path")
@ -401,12 +402,12 @@ func (s *Server) createSandboxContainer(ctx context.Context, containerID string,
specgen.SetLinuxResourcesOOMScoreAdj(int(oomScoreAdj))
}
if sb.cgroupParent != "" {
if sb.CgroupParent() != "" {
if s.config.CgroupManager == "systemd" {
cgPath := sb.cgroupParent + ":" + "crio" + ":" + containerID
cgPath := sb.CgroupParent() + ":" + "crio" + ":" + containerID
specgen.SetLinuxCgroupsPath(cgPath)
} else {
specgen.SetLinuxCgroupsPath(sb.cgroupParent + "/" + containerID)
specgen.SetLinuxCgroupsPath(sb.CgroupParent() + "/" + containerID)
}
}
@ -437,8 +438,8 @@ func (s *Server) createSandboxContainer(ctx context.Context, containerID string,
}
}
specgen.SetProcessSelinuxLabel(sb.processLabel)
specgen.SetLinuxMountLabel(sb.mountLabel)
specgen.SetProcessSelinuxLabel(sb.ProcessLabel())
specgen.SetLinuxMountLabel(sb.MountLabel())
if containerConfig.GetLinux().GetSecurityContext() != nil &&
!containerConfig.GetLinux().GetSecurityContext().Privileged {
@ -466,7 +467,7 @@ func (s *Server) createSandboxContainer(ctx context.Context, containerID string,
}
}
// Join the namespace paths for the pod sandbox container.
podInfraState := s.runtime.ContainerStatus(sb.infraContainer)
podInfraState := s.runtime.ContainerStatus(sb.InfraContainer())
logrus.Debugf("pod container state %+v", podInfraState)
@ -475,7 +476,7 @@ func (s *Server) createSandboxContainer(ctx context.Context, containerID string,
return nil, err
}
netNsPath := sb.netNsPath()
netNsPath := sb.NetNsPath()
if netNsPath == "" {
// The sandbox does not have a permanent namespace,
// it's on the host one.
@ -497,15 +498,15 @@ func (s *Server) createSandboxContainer(ctx context.Context, containerID string,
}
// bind mount the pod shm
specgen.AddBindMount(sb.shmPath, "/dev/shm", []string{"rw"})
specgen.AddBindMount(sb.ShmPath(), "/dev/shm", []string{"rw"})
options := []string{"rw"}
if readOnlyRootfs {
options = []string{"ro"}
}
if sb.resolvPath != "" {
if sb.ResolvPath() != "" {
// bind mount the pod resolver file
specgen.AddBindMount(sb.resolvPath, "/etc/resolv.conf", options)
specgen.AddBindMount(sb.ResolvPath(), "/etc/resolv.conf", []string{"ro"})
}
// Bind mount /etc/hosts for host networking containers
@ -513,14 +514,14 @@ func (s *Server) createSandboxContainer(ctx context.Context, containerID string,
specgen.AddBindMount("/etc/hosts", "/etc/hosts", options)
}
if sb.hostname != "" {
specgen.SetHostname(sb.hostname)
if sb.Hostname() != "" {
specgen.SetHostname(sb.Hostname())
}
specgen.AddAnnotation(annotations.Name, containerName)
specgen.AddAnnotation(annotations.ContainerID, containerID)
specgen.AddAnnotation(annotations.SandboxID, sb.id)
specgen.AddAnnotation(annotations.SandboxName, sb.infraContainer.Name())
specgen.AddAnnotation(annotations.SandboxID, sb.ID())
specgen.AddAnnotation(annotations.SandboxName, sb.InfraContainer().Name())
specgen.AddAnnotation(annotations.ContainerType, annotations.ContainerTypeContainer)
specgen.AddAnnotation(annotations.LogPath, logPath)
specgen.AddAnnotation(annotations.TTY, fmt.Sprintf("%v", containerConfig.Tty))
@ -549,19 +550,19 @@ func (s *Server) createSandboxContainer(ctx context.Context, containerID string,
}
specgen.AddAnnotation(annotations.Annotations, string(kubeAnnotationsJSON))
if err = s.setupSeccomp(&specgen, containerName, sb.annotations); err != nil {
if err = s.setupSeccomp(&specgen, containerName, sb.Annotations()); err != nil {
return nil, err
}
metaname := metadata.Name
attempt := metadata.Attempt
containerInfo, err := s.storageRuntimeServer.CreateContainer(s.imageContext,
sb.name, sb.id,
sb.Name(), sb.ID(),
image, image,
containerName, containerID,
metaname,
attempt,
sb.mountLabel,
sb.MountLabel(),
nil)
if err != nil {
return nil, err
@ -662,7 +663,7 @@ func (s *Server) createSandboxContainer(ctx context.Context, containerID string,
return nil, err
}
container, err := oci.NewContainer(containerID, containerName, containerInfo.RunDir, logPath, sb.netNs(), labels, kubeAnnotations, imageSpec, metadata, sb.id, containerConfig.Tty, containerConfig.Stdin, containerConfig.StdinOnce, sb.privileged, sb.trusted, containerInfo.Dir, created, containerImageConfig.Config.StopSignal)
container, err := oci.NewContainer(containerID, containerName, containerInfo.RunDir, logPath, sb.NetNs(), labels, kubeAnnotations, imageSpec, metadata, sb.ID(), containerConfig.Tty, containerConfig.Stdin, containerConfig.StdinOnce, sb.Privileged(), sb.Trusted(), containerInfo.Dir, created, containerImageConfig.Config.StopSignal)
if err != nil {
return nil, err
}

2
server/container_list.go
View file

@ -57,7 +57,7 @@ func (s *Server) ListContainers(ctx context.Context, req *pb.ListContainersReque
if err != nil {
ctrList = []*oci.Container{}
} else {
ctrList = pod.containers.List()
ctrList = pod.Containers()
}
}
}

279
server/sandbox.go
View file

@ -1,279 +0,0 @@
package server
import (
"crypto/rand"
"errors"
"fmt"
"os"
"path/filepath"
"sync"
"github.com/Sirupsen/logrus"
"github.com/containernetworking/cni/pkg/ns"
"github.com/docker/docker/pkg/mount"
"github.com/docker/docker/pkg/symlink"
"github.com/kubernetes-incubator/cri-o/oci"
"golang.org/x/sys/unix"
"k8s.io/apimachinery/pkg/fields"
pb "k8s.io/kubernetes/pkg/kubelet/api/v1alpha1/runtime"
"k8s.io/kubernetes/pkg/kubelet/network/hostport"
)
type sandboxNetNs struct {
sync.Mutex
ns ns.NetNS
symlink *os.File
closed bool
restored bool
}
func (ns *sandboxNetNs) symlinkCreate(name string) error {
b := make([]byte, 4)
_, randErr := rand.Reader.Read(b)
if randErr != nil {
return randErr
}
nsName := fmt.Sprintf("%s-%x", name, b)
symlinkPath := filepath.Join(nsRunDir, nsName)
if err := os.Symlink(ns.ns.Path(), symlinkPath); err != nil {
return err
}
fd, err := os.Open(symlinkPath)
if err != nil {
if removeErr := os.RemoveAll(symlinkPath); removeErr != nil {
return removeErr
}
return err
}
ns.symlink = fd
return nil
}
func (ns *sandboxNetNs) symlinkRemove() error {
if err := ns.symlink.Close(); err != nil {
return err
}
return os.RemoveAll(ns.symlink.Name())
}
func isSymbolicLink(path string) (bool, error) {
fi, err := os.Lstat(path)
if err != nil {
return false, err
}
return fi.Mode()&os.ModeSymlink == os.ModeSymlink, nil
}
func netNsGet(nspath, name string) (*sandboxNetNs, error) {
if err := ns.IsNSorErr(nspath); err != nil {
return nil, errSandboxClosedNetNS
}
symlink, symlinkErr := isSymbolicLink(nspath)
if symlinkErr != nil {
return nil, symlinkErr
}
var resolvedNsPath string
if symlink {
path, err := os.Readlink(nspath)
if err != nil {
return nil, err
}
resolvedNsPath = path
} else {
resolvedNsPath = nspath
}
netNS, err := ns.GetNS(resolvedNsPath)
if err != nil {
return nil, err
}
netNs := &sandboxNetNs{ns: netNS, closed: false, restored: true}
if symlink {
fd, err := os.Open(nspath)
if err != nil {
return nil, err
}
netNs.symlink = fd
} else {
if err := netNs.symlinkCreate(name); err != nil {
return nil, err
}
}
return netNs, nil
}
func hostNetNsPath() (string, error) {
netNS, err := ns.GetCurrentNS()
if err != nil {
return "", err
}
defer netNS.Close()
return netNS.Path(), nil
}
type sandbox struct {
id string
namespace string
// OCI pod name (eg "<namespace>-<name>-<attempt>")
name string
// Kubernetes pod name (eg, "<name>")
kubeName string
logDir string
labels fields.Set
annotations map[string]string
infraContainer *oci.Container
containers oci.ContainerStorer
processLabel string
mountLabel string
netns *sandboxNetNs
metadata *pb.PodSandboxMetadata
shmPath string
cgroupParent string
privileged bool
trusted bool
resolvPath string
hostname string
portMappings []*hostport.PortMapping
}
const (
defaultShmSize = 64 * 1024 * 1024
nsRunDir = "/var/run/netns"
podInfraCommand = "/pause"
)
var (
errSandboxIDEmpty = errors.New("PodSandboxId should not be empty")
errSandboxClosedNetNS = errors.New("PodSandbox networking namespace is closed")
)
func (s *sandbox) addContainer(c *oci.Container) {
s.containers.Add(c.Name(), c)
}
func (s *sandbox) getContainer(name string) *oci.Container {
return s.containers.Get(name)
}
func (s *sandbox) removeContainer(c *oci.Container) {
s.containers.Delete(c.Name())
}
func (s *sandbox) netNs() ns.NetNS {
if s.netns == nil {
return nil
}
return s.netns.ns
}
func (s *sandbox) netNsPath() string {
if s.netns == nil {
return ""
}
return s.netns.symlink.Name()
}
func (s *sandbox) netNsCreate() error {
if s.netns != nil {
return fmt.Errorf("net NS already created")
}
netNS, err := ns.NewNS()
if err != nil {
return err
}
s.netns = &sandboxNetNs{
ns: netNS,
closed: false,
}
if err := s.netns.symlinkCreate(s.name); err != nil {
logrus.Warnf("Could not create nentns symlink %v", err)
if err1 := s.netns.ns.Close(); err1 != nil {
return err1
}
return err
}
return nil
}
func (s *sandbox) netNsRemove() error {
if s.netns == nil {
logrus.Warn("no networking namespace")
return nil
}
s.netns.Lock()
defer s.netns.Unlock()
if s.netns.closed {
// netNsRemove() can be called multiple
// times without returning an error.
return nil
}
if err := s.netns.symlinkRemove(); err != nil {
return err
}
if err := s.netns.ns.Close(); err != nil {
return err
}
if s.netns.restored {
// we got namespaces in the form of
// /var/run/netns/cni-0d08effa-06eb-a963-f51a-e2b0eceffc5d
// but /var/run on most system is symlinked to /run so we first resolve
// the symlink and then try and see if it's mounted
fp, err := symlink.FollowSymlinkInScope(s.netns.ns.Path(), "/")
if err != nil {
return err
}
if mounted, err := mount.Mounted(fp); err == nil && mounted {
if err := unix.Unmount(fp, unix.MNT_DETACH); err != nil {
return err
}
}
if err := os.RemoveAll(s.netns.ns.Path()); err != nil {
return err
}
}
s.netns.closed = true
return nil
}
func (s *Server) getPodSandboxFromRequest(podSandboxID string) (*sandbox, error) {
if podSandboxID == "" {
return nil, errSandboxIDEmpty
}
sb, err := s.state.LookupSandboxByID(podSandboxID)
if err != nil {
return nil, fmt.Errorf("could not retrieve pod sandbox with ID starting with %v: %v", podSandboxID, err)
}
return sb, nil
}

437
server/sandbox/sandbox.go Normal file
View file

@ -0,0 +1,437 @@
package sandbox
import (
"crypto/rand"
"errors"
"fmt"
"os"
"path/filepath"
"sync"
"github.com/Sirupsen/logrus"
"github.com/containernetworking/cni/pkg/ns"
"github.com/docker/docker/pkg/mount"
"github.com/docker/docker/pkg/symlink"
"github.com/kubernetes-incubator/cri-o/oci"
"golang.org/x/sys/unix"
"k8s.io/apimachinery/pkg/fields"
pb "k8s.io/kubernetes/pkg/kubelet/api/v1alpha1/runtime"
"k8s.io/kubernetes/pkg/kubelet/network/hostport"
)
type sandboxNetNs struct {
sync.Mutex
ns ns.NetNS
symlink *os.File
closed bool
restored bool
}
func (ns *sandboxNetNs) symlinkCreate(name string) error {
b := make([]byte, 4)
_, randErr := rand.Reader.Read(b)
if randErr != nil {
return randErr
}
nsName := fmt.Sprintf("%s-%x", name, b)
symlinkPath := filepath.Join(nsRunDir, nsName)
if err := os.Symlink(ns.ns.Path(), symlinkPath); err != nil {
return err
}
fd, err := os.Open(symlinkPath)
if err != nil {
if removeErr := os.RemoveAll(symlinkPath); removeErr != nil {
return removeErr
}
return err
}
ns.symlink = fd
return nil
}
func (ns *sandboxNetNs) symlinkRemove() error {
if err := ns.symlink.Close(); err != nil {
return err
}
return os.RemoveAll(ns.symlink.Name())
}
func isSymbolicLink(path string) (bool, error) {
fi, err := os.Lstat(path)
if err != nil {
return false, err
}
return fi.Mode()&os.ModeSymlink == os.ModeSymlink, nil
}
func netNsGet(nspath, name string) (*sandboxNetNs, error) {
if err := ns.IsNSorErr(nspath); err != nil {
return nil, ErrSandboxClosedNetNS
}
symlink, symlinkErr := isSymbolicLink(nspath)
if symlinkErr != nil {
return nil, symlinkErr
}
var resolvedNsPath string
if symlink {
path, err := os.Readlink(nspath)
if err != nil {
return nil, err
}
resolvedNsPath = path
} else {
resolvedNsPath = nspath
}
netNS, err := ns.GetNS(resolvedNsPath)
if err != nil {
return nil, err
}
netNs := &sandboxNetNs{ns: netNS, closed: false, restored: true}
if symlink {
fd, err := os.Open(nspath)
if err != nil {
return nil, err
}
netNs.symlink = fd
} else {
if err := netNs.symlinkCreate(name); err != nil {
return nil, err
}
}
return netNs, nil
}
// HostNetNsPath returns the path of the host's network namespace
func HostNetNsPath() (string, error) {
netNS, err := ns.GetCurrentNS()
if err != nil {
return "", err
}
defer netNS.Close()
return netNS.Path(), nil
}
// Sandbox represents a single pod sandbox
type Sandbox struct {
id string
namespace string
// OCI pod name (eg "<namespace>-<name>-<attempt>")
name string
// Kubernetes pod name (eg, "<name>")
kubeName string
logDir string
labels fields.Set
annotations map[string]string
infraContainer *oci.Container
containers oci.ContainerStorer
processLabel string
mountLabel string
netns *sandboxNetNs
metadata *pb.PodSandboxMetadata
shmPath string
cgroupParent string
privileged bool
trusted bool
resolvPath string
hostname string
portMappings []*hostport.PortMapping
}
const (
// DefaultShmSize is the default size of the SHM device for sandboxs
DefaultShmSize = 64 * 1024 * 1024
// PodInfraCommand is the default pause command for pods
PodInfraCommand = "/pause"
nsRunDir = "/var/run/netns"
)
var (
// ErrSandboxIDEmpty is the error returned when an operation passes "" instead of a sandbox ID
ErrSandboxIDEmpty = errors.New("PodSandboxId should not be empty")
// ErrSandboxClosedNetNS is the error returned when a network namespace is closed and cannot be joined
ErrSandboxClosedNetNS = errors.New("PodSandbox networking namespace is closed")
)
// New creates and populates a new sandbox
// New sandboxes have no containers, no infra container, and no network namespace associated with them.
// An infra container must be attached before the sandbox is added to the state
func New(id, namespace, name, kubeName, logDir string, labels, annotations map[string]string, processLabel, mountLabel string, metadata *pb.PodSandboxMetadata, shmPath, cgroupParent string, privileged bool, trusted bool, resolvPath, hostname string, portMappings []*hostport.PortMapping) (*Sandbox, error) {
sb := new(Sandbox)
sb.id = id
sb.namespace = namespace
sb.name = name
sb.kubeName = kubeName
sb.logDir = logDir
sb.labels = labels
sb.annotations = annotations
sb.containers = oci.NewMemoryStore()
sb.processLabel = processLabel
sb.mountLabel = mountLabel
sb.metadata = metadata
sb.shmPath = shmPath
sb.cgroupParent = cgroupParent
sb.privileged = privileged
sb.trusted = trusted
sb.resolvPath = resolvPath
sb.hostname = hostname
sb.portMappings = portMappings
return sb, nil
}
// ID returns the sandbox's ID
func (s *Sandbox) ID() string {
return s.id
}
// Namespace returns the sandbox's namespace
func (s *Sandbox) Namespace() string {
return s.namespace
}
// Name returns the sandbox's name
func (s *Sandbox) Name() string {
return s.name
}
// KubeName returns the name the sandbox was given by Kubernetes
// This is not a fully qualified, globally unique name and cannot be used to look up the sandbox
func (s *Sandbox) KubeName() string {
return s.kubeName
}
// LogDir returns the directory the sandbox logs to
func (s *Sandbox) LogDir() string {
return s.logDir
}
// Labels returns the sandbox's labels
func (s *Sandbox) Labels() map[string]string {
return s.labels
}
// Annotations returns the sandbox's annotations
func (s *Sandbox) Annotations() map[string]string {
return s.annotations
}
// InfraContainer returns the sandbox's infrastructure container
func (s *Sandbox) InfraContainer() *oci.Container {
return s.infraContainer
}
// Containers returns an array of all the containers in the sandbox
func (s *Sandbox) Containers() []*oci.Container {
return s.containers.List()
}
// ProcessLabel returns the SELinux process label of the sandbox
func (s *Sandbox) ProcessLabel() string {
return s.processLabel
}
// MountLabel returns the SELinux mount label of the sandbox
func (s *Sandbox) MountLabel() string {
return s.mountLabel
}
// Metadata returns Kubernetes metadata associated with the sandbox
func (s *Sandbox) Metadata() *pb.PodSandboxMetadata {
return s.metadata
}
// ShmPath returns the path to the sandbox's shared memory device
func (s *Sandbox) ShmPath() string {
return s.shmPath
}
// CgroupParent returns the sandbox's CGroup parent
func (s *Sandbox) CgroupParent() string {
return s.cgroupParent
}
// Privileged returns whether the sandbox can support privileged containers
func (s *Sandbox) Privileged() bool {
return s.privileged
}
// Trusted returns whether the sandbox is a trusted workload
func (s *Sandbox) Trusted() bool {
return s.trusted
}
// ResolvPath returns the path to the sandbox's DNS resolver configuration
func (s *Sandbox) ResolvPath() string {
return s.resolvPath
}
// Hostname returns the sandbox's hostname
func (s *Sandbox) Hostname() string {
return s.hostname
}
func (s *Sandbox) PortMappings() []*hostport.PortMapping {
return s.portMappings
}
// AddContainer adds a container to the sandbox
func (s *Sandbox) AddContainer(c *oci.Container) {
s.containers.Add(c.ID(), c)
}
// GetContainer retrieves the container with given ID from the sandbox
// Returns nil if no such container exists
func (s *Sandbox) GetContainer(id string) *oci.Container {
return s.containers.Get(id)
}
// RemoveContainer removes the container with given ID from the sandbox
// If no container with that ID exists in the sandbox, no action is taken
func (s *Sandbox) RemoveContainer(id string) {
s.containers.Delete(id)
}
// SetInfraContainer sets the infrastructure container of a sandbox
// Attempts to set the infrastructure container after one is already present will throw an error
func (s *Sandbox) SetInfraContainer(infraCtr *oci.Container) error {
if s.infraContainer != nil {
return fmt.Errorf("sandbox already has an infra container")
} else if infraCtr == nil {
return fmt.Errorf("must provide non-nil infra container")
}
s.infraContainer = infraCtr
return nil
}
// NetNs retrieves the network namespace of the sandbox
// If the sandbox uses the host namespace, nil is returned
func (s *Sandbox) NetNs() ns.NetNS {
if s.netns == nil {
return nil
}
return s.netns.ns
}
// NetNsPath returns the path to the network namespace
// If the sandbox uses the host namespace, "" is returned
func (s *Sandbox) NetNsPath() string {
if s.netns == nil {
return ""
}
return s.netns.symlink.Name()
}
// NetNsCreate creates a new network namespace for the sandbox
func (s *Sandbox) NetNsCreate() error {
if s.netns != nil {
return fmt.Errorf("net NS already created")
}
netNS, err := ns.NewNS()
if err != nil {
return err
}
s.netns = &sandboxNetNs{
ns: netNS,
closed: false,
}
if err := s.netns.symlinkCreate(s.name); err != nil {
logrus.Warnf("Could not create nentns symlink %v", err)
if err1 := s.netns.ns.Close(); err1 != nil {
return err1
}
return err
}
return nil
}
// NetNsJoin attempts to join the sandbox to an existing network namespace
// This will fail if the sandbox is already part of a network namespace
func (s *Sandbox) NetNsJoin(nspath, name string) error {
if s.netns != nil {
return fmt.Errorf("sandbox already has a network namespace, cannot join another")
}
netNS, err := netNsGet(nspath, name)
if err != nil {
return err
}
s.netns = netNS
return nil
}
// NetNsRemove removes the network namespace associated with the sandbox
func (s *Sandbox) NetNsRemove() error {
if s.netns == nil {
logrus.Warn("no networking namespace")
return nil
}
s.netns.Lock()
defer s.netns.Unlock()
if s.netns.closed {
// netNsRemove() can be called multiple
// times without returning an error.
return nil
}
if err := s.netns.symlinkRemove(); err != nil {
return err
}
if err := s.netns.ns.Close(); err != nil {
return err
}
if s.netns.restored {
// we got namespaces in the form of
// /var/run/netns/cni-0d08effa-06eb-a963-f51a-e2b0eceffc5d
// but /var/run on most system is symlinked to /run so we first resolve
// the symlink and then try and see if it's mounted
fp, err := symlink.FollowSymlinkInScope(s.netns.ns.Path(), "/")
if err != nil {
return err
}
if mounted, err := mount.Mounted(fp); err == nil && mounted {
if err := unix.Unmount(fp, unix.MNT_DETACH); err != nil {
return err
}
}
if err := os.RemoveAll(s.netns.ns.Path()); err != nil {
return err
}
}
s.netns.closed = true
return nil
}

21
server/sandbox_list.go
View file

@ -5,6 +5,7 @@ import (
"github.com/Sirupsen/logrus"
"github.com/kubernetes-incubator/cri-o/oci"
"github.com/kubernetes-incubator/cri-o/server/sandbox"
"golang.org/x/net/context"
"k8s.io/apimachinery/pkg/fields"
pb "k8s.io/kubernetes/pkg/kubelet/api/v1alpha1/runtime"
@ -32,16 +33,14 @@ func filterSandbox(p *pb.PodSandbox, filter *pb.PodSandboxFilter) bool {
func (s *Server) ListPodSandbox(ctx context.Context, req *pb.ListPodSandboxRequest) (*pb.ListPodSandboxResponse, error) {
logrus.Debugf("ListPodSandboxRequest %+v", req)
var pods []*pb.PodSandbox
var podList []*sandbox
var podList []*sandbox.Sandbox
sandboxes, err := s.state.GetAllSandboxes()
if err != nil {
return nil, fmt.Errorf("error retrieving sandboxes: %v", err)
}
for _, sb := range sandboxes {
podList = append(podList, sb)
}
podList = append(podList, sandboxes...)
filter := req.Filter
// Filter by pod id first.
@ -50,15 +49,15 @@ func (s *Server) ListPodSandbox(ctx context.Context, req *pb.ListPodSandboxReque
sb, err := s.state.LookupSandboxByID(filter.Id)
// TODO if we return something other than a No Such Sandbox should we throw an error instead?
if err != nil {
podList = []*sandbox{}
podList = []*sandbox.Sandbox{}
} else {
podList = []*sandbox{sb}
podList = []*sandbox.Sandbox{sb}
}
}
}
for _, sb := range podList {
podInfraContainer := sb.infraContainer
podInfraContainer := sb.InfraContainer()
if podInfraContainer == nil {
// this can't really happen, but if it does because of a bug
// it's better not to panic
@ -76,12 +75,12 @@ func (s *Server) ListPodSandbox(ctx context.Context, req *pb.ListPodSandboxReque
}
pod := &pb.PodSandbox{
Id: sb.id,
Id: sb.ID(),
CreatedAt: created,
State: rStatus,
Labels: sb.labels,
Annotations: sb.annotations,
Metadata: sb.metadata,
Labels: sb.Labels(),
Annotations: sb.Annotations(),
Metadata: sb.Metadata(),
}
// Filter by other criteria such as state and labels.

27
server/sandbox_remove.go
View file

@ -7,6 +7,7 @@ import (
"github.com/containers/storage"
"github.com/kubernetes-incubator/cri-o/oci"
pkgstorage "github.com/kubernetes-incubator/cri-o/pkg/storage"
"github.com/kubernetes-incubator/cri-o/server/sandbox"
"golang.org/x/net/context"
pb "k8s.io/kubernetes/pkg/kubelet/api/v1alpha1/runtime"
)
@ -17,7 +18,7 @@ func (s *Server) RemovePodSandbox(ctx context.Context, req *pb.RemovePodSandboxR
logrus.Debugf("RemovePodSandboxRequest %+v", req)
sb, err := s.getPodSandboxFromRequest(req.PodSandboxId)
if err != nil {
if err == errSandboxIDEmpty {
if err == sandbox.ErrSandboxIDEmpty {
return nil, err
}
@ -30,8 +31,8 @@ func (s *Server) RemovePodSandbox(ctx context.Context, req *pb.RemovePodSandboxR
return resp, nil
}
podInfraContainer := sb.infraContainer
containers := sb.containers.List()
podInfraContainer := sb.InfraContainer()
containers := sb.Containers()
containers = append(containers, podInfraContainer)
// Delete all the containers in the sandbox
@ -49,7 +50,7 @@ func (s *Server) RemovePodSandbox(ctx context.Context, req *pb.RemovePodSandboxR
}
if err := s.runtime.DeleteContainer(c); err != nil {
return nil, fmt.Errorf("failed to delete container %s in pod sandbox %s: %v", c.Name(), sb.id, err)
return nil, fmt.Errorf("failed to delete container %s in pod sandbox %s: %v", c.Name(), sb.ID(), err)
}
if c.ID() == podInfraContainer.ID() {
@ -58,27 +59,27 @@ func (s *Server) RemovePodSandbox(ctx context.Context, req *pb.RemovePodSandboxR
if err := s.storageRuntimeServer.StopContainer(c.ID()); err != nil && err != storage.ErrContainerUnknown {
// assume container already umounted
logrus.Warnf("failed to stop container %s in pod sandbox %s: %v", c.Name(), sb.id, err)
logrus.Warnf("failed to stop container %s in pod sandbox %s: %v", c.Name(), sb.ID(), err)
}
if err := s.storageRuntimeServer.DeleteContainer(c.ID()); err != nil && err != storage.ErrContainerUnknown {
return nil, fmt.Errorf("failed to delete container %s in pod sandbox %s: %v", c.Name(), sb.id, err)
return nil, fmt.Errorf("failed to delete container %s in pod sandbox %s: %v", c.Name(), sb.ID(), err)
}
if err := s.removeContainer(c); err != nil {
return nil, fmt.Errorf("failed to delete container %s in pod sandbox %s: %v", c.Name(), sb.id, err)
return nil, fmt.Errorf("failed to delete container %s in pod sandbox %s: %v", c.Name(), sb.ID(), err)
}
}
if err := s.removeSandbox(sb.id); err != nil {
return nil, fmt.Errorf("error removing sandbox %s: %v", sb.id, err)
if err := s.removeSandbox(sb.ID()); err != nil {
return nil, fmt.Errorf("error removing sandbox %s: %v", sb.ID(), err)
}
// Remove the files related to the sandbox
if err := s.storageRuntimeServer.StopContainer(sb.id); err != nil && err != storage.ErrContainerUnknown {
logrus.Warnf("failed to stop sandbox container in pod sandbox %s: %v", sb.id, err)
if err := s.storageRuntimeServer.StopContainer(sb.ID()); err != nil && err != storage.ErrContainerUnknown {
logrus.Warnf("failed to stop sandbox container in pod sandbox %s: %v", sb.ID(), err)
}
if err := s.storageRuntimeServer.RemovePodSandbox(sb.id); err != nil && err != pkgstorage.ErrInvalidSandboxID {
return nil, fmt.Errorf("failed to remove pod sandbox %s: %v", sb.id, err)
if err := s.storageRuntimeServer.RemovePodSandbox(sb.ID()); err != nil && err != pkgstorage.ErrInvalidSandboxID {
return nil, fmt.Errorf("failed to remove pod sandbox %s: %v", sb.ID(), err)
}
resp := &pb.RemovePodSandboxResponse{}

63
server/sandbox_run.go
View file

@ -16,6 +16,7 @@ import (
"github.com/containers/storage"
"github.com/kubernetes-incubator/cri-o/oci"
"github.com/kubernetes-incubator/cri-o/pkg/annotations"
"github.com/kubernetes-incubator/cri-o/server/sandbox"
"github.com/opencontainers/runc/libcontainer/cgroups/systemd"
"github.com/opencontainers/runtime-tools/generate"
"github.com/opencontainers/selinux/go-selinux/label"
@ -174,7 +175,7 @@ func (s *Server) RunPodSandbox(ctx context.Context, req *pb.RunPodSandboxRequest
if podContainer.Config != nil {
g.SetProcessArgs(podContainer.Config.Config.Cmd)
} else {
g.SetProcessArgs([]string{podInfraCommand})
g.SetProcessArgs([]string{sandbox.PodInfraCommand})
}
} else {
g.SetProcessArgs([]string{s.config.PauseCommand})
@ -296,6 +297,7 @@ func (s *Server) RunPodSandbox(ctx context.Context, req *pb.RunPodSandboxRequest
g.AddAnnotation(annotations.ResolvPath, resolvPath)
g.AddAnnotation(annotations.HostName, hostname)
g.AddAnnotation(annotations.KubeName, kubeName)
g.AddAnnotation(annotations.Namespace, namespace)
if podContainer.Config.Config.StopSignal != "" {
// this key is defined in image-spec conversion document at https://github.com/opencontainers/image-spec/pull/492/files#diff-8aafbe2c3690162540381b8cdb157112R57
g.AddAnnotation("org.opencontainers.image.stopSignal", podContainer.Config.Config.StopSignal)
@ -305,26 +307,11 @@ func (s *Server) RunPodSandbox(ctx context.Context, req *pb.RunPodSandboxRequest
g.AddAnnotation(annotations.Created, created.Format(time.RFC3339Nano))
portMappings := convertPortMappings(req.GetConfig().GetPortMappings())
sb := &sandbox{
id: id,
namespace: namespace,
name: name,
kubeName: kubeName,
logDir: logDir,
labels: labels,
annotations: kubeAnnotations,
containers: oci.NewMemoryStore(),
processLabel: processLabel,
mountLabel: mountLabel,
metadata: metadata,
shmPath: shmPath,
privileged: privileged,
trusted: trusted,
resolvPath: resolvPath,
hostname: hostname,
portMappings: portMappings,
mappingsJSON, err := json.Marshal(portMappings)
if err != nil {
return nil, err
}
g.AddAnnotation(annotations.PortMappings, string(mappingsJSON))
for k, v := range kubeAnnotations {
g.AddAnnotation(k, v)
@ -351,10 +338,9 @@ func (s *Server) RunPodSandbox(ctx context.Context, req *pb.RunPodSandboxRequest
return nil, err
}
g.SetLinuxCgroupsPath(cgPath + ":" + "crio" + ":" + id)
sb.cgroupParent = cgPath
cgroupParent = cgPath
} else {
g.SetLinuxCgroupsPath(cgroupParent + "/" + id)
sb.cgroupParent = cgroupParent
}
}
@ -362,6 +348,11 @@ func (s *Server) RunPodSandbox(ctx context.Context, req *pb.RunPodSandboxRequest
// so it doesn't get killed.
g.SetLinuxResourcesOOMScoreAdj(PodInfraOOMAdj)
sb, err := sandbox.New(id, namespace, name, kubeName, logDir, labels, kubeAnnotations, processLabel, mountLabel, metadata, shmPath, cgroupParent, privileged, trusted, resolvPath, hostname, portMappings)
if err != nil {
return nil, err
}
hostNetwork := req.GetConfig().GetLinux().GetSecurityContext().GetNamespaceOptions().HostNetwork
// set up namespaces
@ -371,13 +362,13 @@ func (s *Server) RunPodSandbox(ctx context.Context, req *pb.RunPodSandboxRequest
return nil, err
}
netNsPath, err = hostNetNsPath()
netNsPath, err = sandbox.HostNetNsPath()
if err != nil {
return nil, err
}
} else {
// Create the sandbox network namespace
if err = sb.netNsCreate(); err != nil {
if err = sb.NetNsCreate(); err != nil {
return nil, err
}
@ -386,18 +377,18 @@ func (s *Server) RunPodSandbox(ctx context.Context, req *pb.RunPodSandboxRequest
return
}
if netnsErr := sb.netNsRemove(); netnsErr != nil {
if netnsErr := sb.NetNsRemove(); netnsErr != nil {
logrus.Warnf("Failed to remove networking namespace: %v", netnsErr)
}
}()
// Pass the created namespace path to the runtime
err = g.AddOrReplaceLinuxNamespace("network", sb.netNsPath())
err = g.AddOrReplaceLinuxNamespace("network", sb.NetNsPath())
if err != nil {
return nil, err
}
netNsPath = sb.netNsPath()
netNsPath = sb.NetNsPath()
}
if req.GetConfig().GetLinux().GetSecurityContext().GetNamespaceOptions().HostPid {
@ -421,25 +412,25 @@ func (s *Server) RunPodSandbox(ctx context.Context, req *pb.RunPodSandboxRequest
saveOptions := generate.ExportOptions{}
mountPoint, err := s.storageRuntimeServer.StartContainer(id)
if err != nil {
return nil, fmt.Errorf("failed to mount container %s in pod sandbox %s(%s): %v", containerName, sb.name, id, err)
return nil, fmt.Errorf("failed to mount container %s in pod sandbox %s(%s): %v", containerName, name, id, err)
}
g.SetRootPath(mountPoint)
err = g.SaveToFile(filepath.Join(podContainer.Dir, "config.json"), saveOptions)
if err != nil {
return nil, fmt.Errorf("failed to save template configuration for pod sandbox %s(%s): %v", sb.name, id, err)
return nil, fmt.Errorf("failed to save template configuration for pod sandbox %s(%s): %v", name, id, err)
}
if err = g.SaveToFile(filepath.Join(podContainer.RunDir, "config.json"), saveOptions); err != nil {
return nil, fmt.Errorf("failed to write runtime configuration for pod sandbox %s(%s): %v", sb.name, id, err)
return nil, fmt.Errorf("failed to write runtime configuration for pod sandbox %s(%s): %v", name, id, err)
}
container, err := oci.NewContainer(id, containerName, podContainer.RunDir, logPath, sb.netNs(), labels, kubeAnnotations, nil, nil, id, false, false, false, sb.privileged, sb.trusted, podContainer.Dir, created, podContainer.Config.Config.StopSignal)
container, err := oci.NewContainer(id, containerName, podContainer.RunDir, logPath, sb.NetNs(), labels, kubeAnnotations, nil, nil, id, false, false, false, sb.Privileged(), sb.Trusted(), podContainer.Dir, created, podContainer.Config.Config.StopSignal)
if err != nil {
return nil, err
}
if err := sb.SetInfraContainer(container); err != nil {
return nil, err
}
sb.infraContainer = container
// Only register the sandbox after infra container has been added
if err = s.addSandbox(sb); err != nil {
return nil, err
}
@ -473,7 +464,7 @@ func (s *Server) RunPodSandbox(ctx context.Context, req *pb.RunPodSandboxRequest
}
}
if err = s.runContainer(container, sb.cgroupParent); err != nil {
if err = s.runContainer(container, sb.CgroupParent()); err != nil {
return nil, err
}
@ -541,7 +532,7 @@ func setupShm(podSandboxRunDir, mountLabel string) (shmPath string, err error) {
if err = os.Mkdir(shmPath, 0700); err != nil {
return "", err
}
shmOptions := "mode=1777,size=" + strconv.Itoa(defaultShmSize)
shmOptions := "mode=1777,size=" + strconv.Itoa(sandbox.DefaultShmSize)
if err = syscall.Mount("shm", shmPath, "tmpfs", uintptr(syscall.MS_NOEXEC|syscall.MS_NOSUID|syscall.MS_NODEV),
label.FormatMountLabel(shmOptions, mountLabel)); err != nil {
return "", fmt.Errorf("failed to mount shm tmpfs for pod: %v", err)

12
server/sandbox_status.go
View file

@ -15,7 +15,7 @@ func (s *Server) PodSandboxStatus(ctx context.Context, req *pb.PodSandboxStatusR
return nil, err
}
podInfraContainer := sb.infraContainer
podInfraContainer := sb.InfraContainer()
if err = s.runtime.UpdateStatus(podInfraContainer); err != nil {
return nil, err
}
@ -27,7 +27,7 @@ func (s *Server) PodSandboxStatus(ctx context.Context, req *pb.PodSandboxStatusR
if err != nil {
return nil, err
}
ip, err := s.netPlugin.GetContainerNetworkStatus(netNsPath, sb.namespace, sb.kubeName, sb.id)
ip, err := s.netPlugin.GetContainerNetworkStatus(netNsPath, sb.Namespace(), sb.KubeName(), sb.ID())
if err != nil {
// ignore the error on network status
ip = ""
@ -38,7 +38,7 @@ func (s *Server) PodSandboxStatus(ctx context.Context, req *pb.PodSandboxStatusR
rStatus = pb.PodSandboxState_SANDBOX_READY
}
sandboxID := sb.id
sandboxID := sb.ID()
resp := &pb.PodSandboxStatusResponse{
Status: &pb.PodSandboxStatus{
Id: sandboxID,
@ -50,9 +50,9 @@ func (s *Server) PodSandboxStatus(ctx context.Context, req *pb.PodSandboxStatusR
},
Network: &pb.PodSandboxNetworkStatus{Ip: ip},
State: rStatus,
Labels: sb.labels,
Annotations: sb.annotations,
Metadata: sb.metadata,
Labels: sb.Labels(),
Annotations: sb.Annotations(),
Metadata: sb.Metadata(),
},
}

41
server/sandbox_stop.go
View file

@ -9,6 +9,7 @@ import (
"github.com/docker/docker/pkg/mount"
"github.com/docker/docker/pkg/symlink"
"github.com/kubernetes-incubator/cri-o/oci"
"github.com/kubernetes-incubator/cri-o/server/sandbox"
"github.com/opencontainers/selinux/go-selinux/label"
"golang.org/x/net/context"
"golang.org/x/sys/unix"
@ -22,7 +23,7 @@ func (s *Server) StopPodSandbox(ctx context.Context, req *pb.StopPodSandboxReque
logrus.Debugf("StopPodSandboxRequest %+v", req)
sb, err := s.getPodSandboxFromRequest(req.PodSandboxId)
if err != nil {
if err == errSandboxIDEmpty {
if err == sandbox.ErrSandboxIDEmpty {
return nil, err
}
@ -35,36 +36,36 @@ func (s *Server) StopPodSandbox(ctx context.Context, req *pb.StopPodSandboxReque
return resp, nil
}
podInfraContainer := sb.infraContainer
podInfraContainer := sb.InfraContainer()
netnsPath, err := podInfraContainer.NetNsPath()
if err != nil {
return nil, err
}
if _, err := os.Stat(netnsPath); err == nil {
if err2 := s.hostportManager.Remove(sb.id, &hostport.PodPortMapping{
Name: sb.name,
PortMappings: sb.portMappings,
if err2 := s.hostportManager.Remove(sb.ID(), &hostport.PodPortMapping{
Name: sb.Name(),
PortMappings: sb.PortMappings(),
HostNetwork: false,
}); err2 != nil {
logrus.Warnf("failed to remove hostport for container %s in sandbox %s: %v",
podInfraContainer.Name(), sb.id, err2)
podInfraContainer.Name(), sb.ID(), err2)
}
if err2 := s.netPlugin.TearDownPod(netnsPath, sb.namespace, sb.kubeName, sb.id); err2 != nil {
if err2 := s.netPlugin.TearDownPod(netnsPath, sb.Namespace(), sb.KubeName(), sb.ID()); err2 != nil {
logrus.Warnf("failed to destroy network for container %s in sandbox %s: %v",
podInfraContainer.Name(), sb.id, err2)
podInfraContainer.Name(), sb.ID(), err2)
}
} else if !os.IsNotExist(err) { // it's ok for netnsPath to *not* exist
return nil, fmt.Errorf("failed to stat netns path for container %s in sandbox %s before tearing down the network: %v",
sb.name, sb.id, err)
podInfraContainer.Name(), sb.ID(), err)
}
// Close the sandbox networking namespace.
if err := sb.netNsRemove(); err != nil {
if err := sb.NetNsRemove(); err != nil {
return nil, err
}
containers := sb.containers.List()
containers := sb.Containers()
containers = append(containers, podInfraContainer)
for _, c := range containers {
@ -74,30 +75,30 @@ func (s *Server) StopPodSandbox(ctx context.Context, req *pb.StopPodSandboxReque
cStatus := s.runtime.ContainerStatus(c)
if cStatus.Status != oci.ContainerStateStopped {
if err := s.runtime.StopContainer(c, -1); err != nil {
return nil, fmt.Errorf("failed to stop container %s in pod sandbox %s: %v", c.Name(), sb.id, err)
return nil, fmt.Errorf("failed to stop container %s in pod sandbox %s: %v", c.Name(), sb.ID(), err)
}
if c.ID() == podInfraContainer.ID() {
continue
}
if err := s.storageRuntimeServer.StopContainer(c.ID()); err != nil && err != storage.ErrContainerUnknown {
// assume container already umounted
logrus.Warnf("failed to stop container %s in pod sandbox %s: %v", c.Name(), sb.id, err)
logrus.Warnf("failed to stop container %s in pod sandbox %s: %v", c.Name(), sb.ID(), err)
}
}
s.containerStateToDisk(c)
}
if err := label.ReleaseLabel(sb.processLabel); err != nil {
if err := label.ReleaseLabel(sb.ProcessLabel()); err != nil {
return nil, err
}
// unmount the shm for the pod
if sb.shmPath != "/dev/shm" {
if sb.ShmPath() != "/dev/shm" {
// we got namespaces in the form of
// /var/run/containers/storage/overlay-containers/CID/userdata/shm
// but /var/run on most system is symlinked to /run so we first resolve
// the symlink and then try and see if it's mounted
fp, err := symlink.FollowSymlinkInScope(sb.shmPath, "/")
fp, err := symlink.FollowSymlinkInScope(sb.ShmPath(), "/")
if err != nil {
return nil, err
}
@ -107,8 +108,8 @@ func (s *Server) StopPodSandbox(ctx context.Context, req *pb.StopPodSandboxReque
}
}
}
if err := s.storageRuntimeServer.StopContainer(sb.id); err != nil && err != storage.ErrContainerUnknown {
logrus.Warnf("failed to stop sandbox container in pod sandbox %s: %v", sb.id, err)
if err := s.storageRuntimeServer.StopContainer(sb.ID()); err != nil && err != storage.ErrContainerUnknown {
logrus.Warnf("failed to stop sandbox container in pod sandbox %s: %v", sb.ID(), err)
}
resp := &pb.StopPodSandboxResponse{}
@ -126,10 +127,10 @@ func (s *Server) StopAllPodSandboxes() {
}
for _, sb := range sandboxes {
pod := &pb.StopPodSandboxRequest{
PodSandboxId: sb.id,
PodSandboxId: sb.ID(),
}
if _, err := s.StopPodSandbox(nil, pod); err != nil {
logrus.Warnf("could not StopPodSandbox %s: %v", sb.id, err)
logrus.Warnf("could not StopPodSandbox %s: %v", sb.ID(), err)
}
}
}

79
server/server.go
View file

@ -19,7 +19,9 @@ import (
"github.com/kubernetes-incubator/cri-o/pkg/ocicni"
"github.com/kubernetes-incubator/cri-o/pkg/storage"
"github.com/kubernetes-incubator/cri-o/server/apparmor"
"github.com/kubernetes-incubator/cri-o/server/sandbox"
"github.com/kubernetes-incubator/cri-o/server/seccomp"
"github.com/kubernetes-incubator/cri-o/server/state"
rspec "github.com/opencontainers/runtime-spec/specs-go"
"github.com/opencontainers/selinux/go-selinux/label"
knet "k8s.io/apimachinery/pkg/util/net"
@ -56,7 +58,7 @@ type Server struct {
storageImageServer storage.ImageServer
storageRuntimeServer storage.RuntimeServer
updateLock sync.RWMutex
state StateStore
state state.Store
netPlugin ocicni.CNIPlugin
hostportManager hostport.HostPortManager
imageContext *types.SystemContext
@ -141,7 +143,7 @@ func (s *Server) loadContainer(id string) error {
return err
}
ctr, err := oci.NewContainer(id, name, containerPath, m.Annotations[annotations.LogPath], sb.netNs(), labels, kubeAnnotations, img, &metadata, sb.id, tty, stdin, stdinOnce, sb.privileged, sb.trusted, containerDir, created, m.Annotations["org.opencontainers.image.stopSignal"])
ctr, err := oci.NewContainer(id, name, containerPath, m.Annotations[annotations.LogPath], sb.NetNs(), labels, kubeAnnotations, img, &metadata, sb.ID(), tty, stdin, stdinOnce, sb.Privileged(), sb.Trusted(), containerDir, created, m.Annotations["org.opencontainers.image.stopSignal"])
if err != nil {
return err
}
@ -221,39 +223,30 @@ func (s *Server) loadSandbox(id string) error {
return err
}
portMappings := []*hostport.PortMapping{}
if err = json.Unmarshal([]byte(m.Annotations[annotations.PortMappings]), &portMappings); err != nil {
return err
}
privileged := isTrue(m.Annotations[annotations.PrivilegedRuntime])
trusted := isTrue(m.Annotations[annotations.TrustedSandbox])
sb := &sandbox{
id: id,
name: name,
kubeName: m.Annotations[annotations.KubeName],
logDir: filepath.Dir(m.Annotations[annotations.LogPath]),
labels: labels,
containers: oci.NewMemoryStore(),
processLabel: processLabel,
mountLabel: mountLabel,
annotations: kubeAnnotations,
metadata: &metadata,
shmPath: m.Annotations[annotations.ShmPath],
privileged: privileged,
trusted: trusted,
resolvPath: m.Annotations[annotations.ResolvPath],
sb, err := sandbox.New(id, m.Annotations[annotations.Namespace], name, m.Annotations[annotations.KubeName], filepath.Dir(m.Annotations[annotations.LogPath]), labels, kubeAnnotations, processLabel, mountLabel, &metadata, m.Annotations[annotations.ShmPath], m.Linux.CgroupsPath, privileged, trusted, m.Annotations[annotations.ResolvPath], m.Annotations[annotations.HostName], portMappings)
if err != nil {
return err
}
// We add a netNS only if we can load a permanent one.
// Otherwise, the sandbox will live in the host namespace.
netNsPath, err := configNetNsPath(m)
if err == nil {
netNS, nsErr := netNsGet(netNsPath, sb.name)
// If we can't load the networking namespace
// because it's closed, we just set the sb netns
// pointer to nil. Otherwise we return an error.
if nsErr != nil && nsErr != errSandboxClosedNetNS {
return nsErr
// because it's closed, just leave the sandbox's netns pointer as nil
if nsErr := sb.NetNsJoin(netNsPath, sb.Name()); err != nil {
if nsErr != sandbox.ErrSandboxClosedNetNS {
return nsErr
}
}
sb.netns = netNS
}
sandboxPath, err := s.store.ContainerRunDirectory(id)
@ -271,7 +264,7 @@ func (s *Server) loadSandbox(id string) error {
return err
}
scontainer, err := oci.NewContainer(m.Annotations[annotations.ContainerID], m.Annotations[annotations.ContainerName], sandboxPath, m.Annotations[annotations.LogPath], sb.netNs(), labels, kubeAnnotations, nil, nil, id, false, false, false, privileged, trusted, sandboxDir, created, m.Annotations["org.opencontainers.image.stopSignal"])
scontainer, err := oci.NewContainer(m.Annotations[annotations.ContainerID], m.Annotations[annotations.ContainerName], sandboxPath, m.Annotations[annotations.LogPath], sb.NetNs(), labels, kubeAnnotations, nil, nil, id, false, false, false, privileged, trusted, sandboxDir, created, m.Annotations["org.opencontainers.image.stopSignal"])
if err != nil {
return err
}
@ -281,7 +274,9 @@ func (s *Server) loadSandbox(id string) error {
if err = label.ReserveLabel(processLabel); err != nil {
return err
}
sb.infraContainer = scontainer
if err = sb.SetInfraContainer(scontainer); err != nil {
return err
}
return s.addSandbox(sb)
}
@ -397,9 +392,9 @@ func (s *Server) update() error {
return fmt.Errorf("error retrieving pods list: %v", err)
}
for _, pod := range pods {
if _, ok := oldPods[pod.id]; !ok {
if _, ok := oldPods[pod.ID()]; !ok {
// this pod's ID wasn't in the updated list -> removed
removedPods[pod.id] = pod.id
removedPods[pod.ID()] = pod.ID()
}
}
@ -410,11 +405,10 @@ func (s *Server) update() error {
logrus.Warnf("bad state when getting pod to remove %+v", removedPod)
continue
}
if err := s.removeSandbox(sb.id); err != nil {
return fmt.Errorf("error removing sandbox %s: %v", sb.id, err)
if err := s.removeSandbox(sb.ID()); err != nil {
return fmt.Errorf("error removing sandbox %s: %v", sb.ID(), err)
}
sb.infraContainer = nil
logrus.Debugf("forgetting removed pod %s", sb.id)
logrus.Debugf("forgetting removed pod %s", sb.ID())
}
for sandboxID := range newPods {
// load this pod
@ -505,7 +499,7 @@ func New(config *Config) (*Server, error) {
netPlugin: netPlugin,
hostportManager: hostportManager,
config: *config,
state: NewInMemoryState(),
state: state.NewInMemoryState(),
seccompEnabled: seccomp.IsEnabled(),
appArmorEnabled: apparmor.IsEnabled(),
appArmorProfile: config.ApparmorProfile,
@ -565,11 +559,11 @@ func New(config *Config) (*Server, error) {
return s, nil
}
func (s *Server) addSandbox(sb *sandbox) error {
func (s *Server) addSandbox(sb *sandbox.Sandbox) error {
return s.state.AddSandbox(sb)
}
func (s *Server) getSandbox(id string) (*sandbox, error) {
func (s *Server) getSandbox(id string) (*sandbox.Sandbox, error) {
return s.state.GetSandbox(id)
}
@ -601,7 +595,7 @@ func (s *Server) GetSandboxContainer(id string) (*oci.Container, error) {
return nil, err
}
return sb.infraContainer, nil
return sb.InfraContainer(), nil
}
// GetContainer returns a container by its ID
@ -612,3 +606,16 @@ func (s *Server) GetContainer(id string) (*oci.Container, error) {
func (s *Server) removeContainer(c *oci.Container) error {
return s.state.DeleteContainer(c.ID(), c.Sandbox())
}
func (s *Server) getPodSandboxFromRequest(podSandboxID string) (*sandbox.Sandbox, error) {
if podSandboxID == "" {
return nil, sandbox.ErrSandboxIDEmpty
}
sb, err := s.state.LookupSandboxByID(podSandboxID)
if err != nil {
return nil, fmt.Errorf("could not retrieve pod sandbox with ID starting with %v: %v", podSandboxID, err)
}
return sb, nil
}

62
server/in_memory_state.go → server/state/in_memory_state.go
View file

@ -1,4 +1,4 @@
package server
package state
import (
"fmt"
@ -7,6 +7,7 @@ import (
"github.com/docker/docker/pkg/registrar"
"github.com/docker/docker/pkg/truncindex"
"github.com/kubernetes-incubator/cri-o/oci"
"github.com/kubernetes-incubator/cri-o/server/sandbox"
)
// TODO: make operations atomic to greatest extent possible
@ -15,7 +16,7 @@ import (
// programs are expected to interact with the server
type InMemoryState struct {
lock sync.Mutex
sandboxes map[string]*sandbox
sandboxes map[string]*sandbox.Sandbox
containers oci.ContainerStorer
podNameIndex *registrar.Registrar
podIDIndex *truncindex.TruncIndex
@ -24,9 +25,9 @@ type InMemoryState struct {
}
// NewInMemoryState creates a new, empty server state
func NewInMemoryState() StateStore {
func NewInMemoryState() Store {
state := new(InMemoryState)
state.sandboxes = make(map[string]*sandbox)
state.sandboxes = make(map[string]*sandbox.Sandbox)
state.containers = oci.NewMemoryStore()
state.podNameIndex = registrar.NewRegistrar()
state.podIDIndex = truncindex.NewTruncIndex([]string{})
@ -37,39 +38,40 @@ func NewInMemoryState() StateStore {
}
// AddSandbox adds a sandbox and any containers in it to the state
func (s *InMemoryState) AddSandbox(sandbox *sandbox) error {
func (s *InMemoryState) AddSandbox(sandbox *sandbox.Sandbox) error {
s.lock.Lock()
defer s.lock.Unlock()
if _, exist := s.sandboxes[sandbox.id]; exist {
return fmt.Errorf("sandbox with ID %v already exists", sandbox.id)
if _, exist := s.sandboxes[sandbox.ID()]; exist {
return fmt.Errorf("sandbox with ID %v already exists", sandbox.ID())
}
// We shouldn't share ID with any containers, either
if ctrCheck := s.containers.Get(sandbox.id); ctrCheck != nil {
return fmt.Errorf("requested sandbox ID %v conflicts with existing container ID", sandbox.id)
// Our pod infra container will share our ID and we don't want it to conflict with anything
if ctrCheck := s.containers.Get(sandbox.ID()); ctrCheck != nil {
return fmt.Errorf("requested sandbox ID %v conflicts with existing container ID", sandbox.ID())
}
s.sandboxes[sandbox.id] = sandbox
if err := s.podNameIndex.Reserve(sandbox.name, sandbox.id); err != nil {
s.sandboxes[sandbox.ID()] = sandbox
if err := s.podNameIndex.Reserve(sandbox.Name(), sandbox.ID()); err != nil {
return fmt.Errorf("error registering sandbox name: %v", err)
}
if err := s.podIDIndex.Add(sandbox.id); err != nil {
if err := s.podIDIndex.Add(sandbox.ID()); err != nil {
return fmt.Errorf("error registering sandbox ID: %v", err)
}
// If there are containers in the sandbox add them to the mapping
containers := sandbox.containers.List()
containers := sandbox.Containers()
for _, ctr := range containers {
if err := s.addContainerMappings(ctr, true); err != nil {
return fmt.Errorf("error adding container %v mappings in sandbox %v", ctr.ID(), sandbox.id)
return fmt.Errorf("error adding container %v mappings in sandbox %v", ctr.ID(), sandbox.ID())
}
}
// Add the pod infrastructure container to mappings
// TODO: Right now, we don't add it to the all containers listing. We may want to change this.
if err := s.addContainerMappings(sandbox.infraContainer, false); err != nil {
return fmt.Errorf("error adding infrastructure container %v to mappings: %v", sandbox.infraContainer.ID(), err)
if err := s.addContainerMappings(sandbox.InfraContainer(), false); err != nil {
return fmt.Errorf("error adding infrastructure container %v to mappings: %v", sandbox.InfraContainer().ID(), err)
}
return nil
@ -94,9 +96,9 @@ func (s *InMemoryState) DeleteSandbox(id string) error {
return fmt.Errorf("no sandbox with ID %v exists, cannot delete", id)
}
name := s.sandboxes[id].name
containers := s.sandboxes[id].containers.List()
infraContainer := s.sandboxes[id].infraContainer
name := s.sandboxes[id].Name()
containers := s.sandboxes[id].Containers()
infraContainer := s.sandboxes[id].InfraContainer()
delete(s.sandboxes, id)
s.podNameIndex.Release(name)
@ -120,7 +122,7 @@ func (s *InMemoryState) DeleteSandbox(id string) error {
}
// GetSandbox returns a sandbox given its full ID
func (s *InMemoryState) GetSandbox(id string) (*sandbox, error) {
func (s *InMemoryState) GetSandbox(id string) (*sandbox.Sandbox, error) {
s.lock.Lock()
defer s.lock.Unlock()
@ -133,7 +135,7 @@ func (s *InMemoryState) GetSandbox(id string) (*sandbox, error) {
}
// LookupSandboxByName returns a sandbox given its full or partial name
func (s *InMemoryState) LookupSandboxByName(name string) (*sandbox, error) {
func (s *InMemoryState) LookupSandboxByName(name string) (*sandbox.Sandbox, error) {
s.lock.Lock()
defer s.lock.Unlock()
@ -153,7 +155,7 @@ func (s *InMemoryState) LookupSandboxByName(name string) (*sandbox, error) {
// LookupSandboxByID returns a sandbox given its full or partial ID
// An error will be returned if the partial ID given is not unique
func (s *InMemoryState) LookupSandboxByID(id string) (*sandbox, error) {
func (s *InMemoryState) LookupSandboxByID(id string) (*sandbox.Sandbox, error) {
s.lock.Lock()
defer s.lock.Unlock()
@ -172,11 +174,11 @@ func (s *InMemoryState) LookupSandboxByID(id string) (*sandbox, error) {
}
// GetAllSandboxes returns all sandboxes in the state
func (s *InMemoryState) GetAllSandboxes() ([]*sandbox, error) {
func (s *InMemoryState) GetAllSandboxes() ([]*sandbox.Sandbox, error) {
s.lock.Lock()
defer s.lock.Unlock()
sandboxes := make([]*sandbox, 0, len(s.sandboxes))
sandboxes := make([]*sandbox.Sandbox, 0, len(s.sandboxes))
for _, sb := range s.sandboxes {
sandboxes = append(sandboxes, sb)
}
@ -198,11 +200,11 @@ func (s *InMemoryState) AddContainer(c *oci.Container, sandboxID string) error {
return fmt.Errorf("sandbox with ID %v does not exist, cannot add container", sandboxID)
}
if ctr := sandbox.containers.Get(c.ID()); ctr != nil {
if ctr := sandbox.GetContainer(c.ID()); ctr != nil {
return fmt.Errorf("container with ID %v already exists in sandbox %v", c.ID(), sandboxID)
}
sandbox.containers.Add(c.ID(), c)
sandbox.AddContainer(c)
return s.addContainerMappings(c, true)
}
@ -242,7 +244,7 @@ func (s *InMemoryState) HasContainer(id, sandboxID string) bool {
return false
}
ctr := sandbox.containers.Get(id)
ctr := sandbox.GetContainer(id)
return ctr != nil
}
@ -257,12 +259,12 @@ func (s *InMemoryState) DeleteContainer(id, sandboxID string) error {
return fmt.Errorf("sandbox with ID %v does not exist", sandboxID)
}
ctr := sandbox.containers.Get(id)
ctr := sandbox.GetContainer(id)
if ctr == nil {
return fmt.Errorf("sandbox %v has no container with ID %v", sandboxID, id)
}
sandbox.containers.Delete(id)
sandbox.RemoveContainer(id)
return s.deleteContainerMappings(ctr, true)
}
@ -358,7 +360,7 @@ func (s *InMemoryState) getContainerFromSandbox(id, sandboxID string) (*oci.Cont
return nil, fmt.Errorf("sandbox with ID %v does not exist", sandboxID)
}
ctr := sandbox.containers.Get(id)
ctr := sandbox.GetContainer(id)
if ctr == nil {
return nil, fmt.Errorf("cannot find container %v in sandbox %v", id, sandboxID)
}

18
server/state_store.go → server/state/state_store.go
View file

@ -1,13 +1,13 @@
package server
package state
import (
"github.com/kubernetes-incubator/cri-o/oci"
"github.com/kubernetes-incubator/cri-o/server/sandbox"
)
// StateStore stores the state of the CRI-O server, including active pods and
// containers
type StateStore interface {
AddSandbox(s *sandbox) error
// Store stores the state of the CRI-O server, including active pods and containers
type Store interface {
AddSandbox(s *sandbox.Sandbox) error
HasSandbox(id string) bool
DeleteSandbox(id string) error
// These should modify the associated sandbox without prompting
@ -15,15 +15,15 @@ type StateStore interface {
HasContainer(id, sandboxID string) bool
DeleteContainer(id, sandboxID string) error
// These two require full, explicit ID
GetSandbox(id string) (*sandbox, error)
GetSandbox(id string) (*sandbox.Sandbox, error)
GetContainer(id, sandboxID string) (*oci.Container, error)
// Get ID of sandbox container belongs to
GetContainerSandbox(id string) (string, error)
// Following 4 should accept partial names as long as they are globally unique
LookupSandboxByName(name string) (*sandbox, error)
LookupSandboxByID(id string) (*sandbox, error)
LookupSandboxByName(name string) (*sandbox.Sandbox, error)
LookupSandboxByID(id string) (*sandbox.Sandbox, error)
LookupContainerByName(name string) (*oci.Container, error)
LookupContainerByID(id string) (*oci.Container, error)
GetAllSandboxes() ([]*sandbox, error)
GetAllSandboxes() ([]*sandbox.Sandbox, error)
GetAllContainers() ([]*oci.Container, error)
}