vbatts/cri-o
1
0
Fork 0
Code Issues Pull requests Releases Wiki Activity

Merge 1c3db75db5 into c9edee9af2

Browse source
This commit is contained in:
Matthew Heon 2017-07-11 23:21:09 +00:00 • committed by GitHub
commit 79b3caad81
22 changed files with 1316 additions and 730 deletions
Download patch file Download diff file Expand all files Collapse all files

6
pkg/annotations/annotations.go
View file

@ -25,6 +25,9 @@ const (
// KubeName is the kubernetes name annotation
KubeName = "io.kubernetes.cri-o.KubeName"
// Namespace is the kubernetes pod namespace annotation
Namespace = "io.kubernetes.cri-o.Namespace"
// Labels are the kubernetes labels annotation
Labels = "io.kubernetes.cri-o.Labels"
@ -63,6 +66,9 @@ const (
// StdinOnce is the stdin_once annotation
StdinOnce = "io.kubernetes.cri-o.StdinOnce"
// PortMappings is the annotation for sandbox port mappings
PortMappings = "io.kubernetes.cri-o.PortMappings"
)
// ContainerType values

8
server/container.go
View file

@ -11,14 +11,10 @@ func (s *Server) getContainerFromRequest(cid string) (*oci.Container, error) {
return nil, fmt.Errorf("container ID should not be empty")
}
containerID, err := s.ctrIDIndex.Get(cid)
c, err := s.state.LookupContainerByID(cid)
if err != nil {
return nil, fmt.Errorf("container with ID starting with %s not found: %v", cid, err)
return nil, fmt.Errorf("container with ID starting with %s could not be retrieved: %v", cid, err)
}
c := s.state.containers.Get(containerID)
if c == nil {
return nil, fmt.Errorf("specified container not found: %s", containerID)
}
return c, nil
}

7
server/container_attach.go
View file

@ -38,10 +38,9 @@ func (s *Server) Attach(ctx context.Context, req *pb.AttachRequest) (*pb.AttachR
// Attach endpoint for streaming.Runtime
func (ss streamService) Attach(containerID string, inputStream io.Reader, outputStream, errorStream io.WriteCloser, tty bool, resize <-chan term.Size) error {
c := ss.runtimeServer.GetContainer(containerID)
if c == nil {
return fmt.Errorf("could not find container %q", containerID)
c, err := ss.runtimeServer.GetContainer(containerID)
if err != nil {
return err
}
if err := ss.runtimeServer.runtime.UpdateStatus(c); err != nil {

77
server/container_create.go
View file

@ -17,6 +17,7 @@ import (
"github.com/kubernetes-incubator/cri-o/oci"
"github.com/kubernetes-incubator/cri-o/pkg/annotations"
"github.com/kubernetes-incubator/cri-o/server/apparmor"
"github.com/kubernetes-incubator/cri-o/server/sandbox"
"github.com/kubernetes-incubator/cri-o/server/seccomp"
"github.com/opencontainers/image-spec/specs-go/v1"
"github.com/opencontainers/runc/libcontainer/devices"
@ -34,7 +35,7 @@ const (
seccompLocalhostPrefix = "localhost/"
)
func addOCIBindMounts(sb *sandbox, containerConfig *pb.ContainerConfig, specgen *generate.Generator) error {
func addOCIBindMounts(sb *sandbox.Sandbox, containerConfig *pb.ContainerConfig, specgen *generate.Generator) error {
mounts := containerConfig.GetMounts()
for _, mount := range mounts {
dest := mount.ContainerPath
@ -60,7 +61,7 @@ func addOCIBindMounts(sb *sandbox, containerConfig *pb.ContainerConfig, specgen
if mount.SelinuxRelabel {
// Need a way in kubernetes to determine if the volume is shared or private
if err := label.Relabel(src, sb.mountLabel, true); err != nil && err != syscall.ENOTSUP {
if err := label.Relabel(src, sb.MountLabel(), true); err != nil && err != syscall.ENOTSUP {
return fmt.Errorf("relabel failed %s: %v", src, err)
}
}
@ -71,7 +72,7 @@ func addOCIBindMounts(sb *sandbox, containerConfig *pb.ContainerConfig, specgen
return nil
}
func addDevices(sb *sandbox, containerConfig *pb.ContainerConfig, specgen *generate.Generator) error {
func addDevices(sb *sandbox.Sandbox, containerConfig *pb.ContainerConfig, specgen *generate.Generator) error {
sp := specgen.Spec()
for _, device := range containerConfig.GetDevices() {
dev, err := devices.DeviceFromPath(device.HostPath, device.Permissions)
@ -236,14 +237,9 @@ func (s *Server) CreateContainer(ctx context.Context, req *pb.CreateContainerReq
return nil, fmt.Errorf("PodSandboxId should not be empty")
}
sandboxID, err := s.podIDIndex.Get(sbID)
sb, err := s.state.LookupSandboxByID(sbID)
if err != nil {
return nil, fmt.Errorf("PodSandbox with ID starting with %s not found: %v", sbID, err)
}
sb := s.getSandbox(sandboxID)
if sb == nil {
return nil, fmt.Errorf("specified sandbox not found: %s", sandboxID)
return nil, err
}
// The config of the container
@ -257,17 +253,11 @@ func (s *Server) CreateContainer(ctx context.Context, req *pb.CreateContainerReq
return nil, fmt.Errorf("CreateContainerRequest.ContainerConfig.Name is empty")
}
containerID, containerName, err := s.generateContainerIDandName(sb.metadata, containerConfig)
containerID, containerName, err := s.generateContainerIDandName(sb.Metadata(), containerConfig)
if err != nil {
return nil, err
}
defer func() {
if err != nil {
s.releaseContainerName(containerName)
}
}()
container, err := s.createSandboxContainer(ctx, containerID, containerName, sb, req.GetSandboxConfig(), containerConfig)
if err != nil {
return nil, err
@ -281,7 +271,7 @@ func (s *Server) CreateContainer(ctx context.Context, req *pb.CreateContainerReq
}
}()
if err = s.runtime.CreateContainer(container, sb.cgroupParent); err != nil {
if err = s.runtime.CreateContainer(container, sb.CgroupParent()); err != nil {
return nil, err
}
@ -289,10 +279,7 @@ func (s *Server) CreateContainer(ctx context.Context, req *pb.CreateContainerReq
return nil, err
}
s.addContainer(container)
if err = s.ctrIDIndex.Add(containerID); err != nil {
s.removeContainer(container)
if err := s.addContainer(container); err != nil {
return nil, err
}
@ -306,7 +293,7 @@ func (s *Server) CreateContainer(ctx context.Context, req *pb.CreateContainerReq
return resp, nil
}
func (s *Server) createSandboxContainer(ctx context.Context, containerID string, containerName string, sb *sandbox, SandboxConfig *pb.PodSandboxConfig, containerConfig *pb.ContainerConfig) (*oci.Container, error) {
func (s *Server) createSandboxContainer(ctx context.Context, containerID string, containerName string, sb *sandbox.Sandbox, SandboxConfig *pb.PodSandboxConfig, containerConfig *pb.ContainerConfig) (*oci.Container, error) {
if sb == nil {
return nil, errors.New("createSandboxContainer needs a sandbox")
}
@ -340,7 +327,7 @@ func (s *Server) createSandboxContainer(ctx context.Context, containerID string,
// set this container's apparmor profile if it is set by sandbox
if s.appArmorEnabled {
appArmorProfileName := s.getAppArmorProfileName(sb.annotations, metadata.Name)
appArmorProfileName := s.getAppArmorProfileName(sb.Annotations(), metadata.Name)
if appArmorProfileName != "" {
// reload default apparmor profile if it is unloaded.
if s.appArmorProfile == apparmor.DefaultApparmorProfile {
@ -367,12 +354,12 @@ func (s *Server) createSandboxContainer(ctx context.Context, containerID string,
logPath := containerConfig.LogPath
if logPath == "" {
// TODO: Should we use sandboxConfig.GetLogDirectory() here?
logPath = filepath.Join(sb.logDir, containerID+".log")
logPath = filepath.Join(sb.LogDir(), containerID+".log")
}
if !filepath.IsAbs(logPath) {
// XXX: It's not really clear what this should be versus the sbox logDirectory.
logrus.Warnf("requested logPath for ctr id %s is a relative path: %s", containerID, logPath)
logPath = filepath.Join(sb.logDir, logPath)
logPath = filepath.Join(sb.LogDir(), logPath)
}
// Handle https://issues.k8s.io/44043
@ -381,7 +368,7 @@ func (s *Server) createSandboxContainer(ctx context.Context, containerID string,
}
logrus.WithFields(logrus.Fields{
"sbox.logdir": sb.logDir,
"sbox.logdir": sb.LogDir(),
"ctr.logfile": containerConfig.LogPath,
"log_path": logPath,
}).Debugf("setting container's log_path")
@ -416,12 +403,12 @@ func (s *Server) createSandboxContainer(ctx context.Context, containerID string,
specgen.SetLinuxResourcesOOMScoreAdj(int(oomScoreAdj))
}
if sb.cgroupParent != "" {
if sb.CgroupParent() != "" {
if s.config.CgroupManager == "systemd" {
cgPath := sb.cgroupParent + ":" + "crio" + ":" + containerID
cgPath := sb.CgroupParent() + ":" + "crio" + ":" + containerID
specgen.SetLinuxCgroupsPath(cgPath)
} else {
specgen.SetLinuxCgroupsPath(sb.cgroupParent + "/" + containerID)
specgen.SetLinuxCgroupsPath(sb.CgroupParent() + "/" + containerID)
}
}
@ -452,8 +439,8 @@ func (s *Server) createSandboxContainer(ctx context.Context, containerID string,
}
}
specgen.SetProcessSelinuxLabel(sb.processLabel)
specgen.SetLinuxMountLabel(sb.mountLabel)
specgen.SetProcessSelinuxLabel(sb.ProcessLabel())
specgen.SetLinuxMountLabel(sb.MountLabel())
if containerConfig.GetLinux().GetSecurityContext() != nil &&
!containerConfig.GetLinux().GetSecurityContext().Privileged {
@ -481,7 +468,7 @@ func (s *Server) createSandboxContainer(ctx context.Context, containerID string,
}
}
// Join the namespace paths for the pod sandbox container.
podInfraState := s.runtime.ContainerStatus(sb.infraContainer)
podInfraState := s.runtime.ContainerStatus(sb.InfraContainer())
logrus.Debugf("pod container state %+v", podInfraState)
@ -490,7 +477,7 @@ func (s *Server) createSandboxContainer(ctx context.Context, containerID string,
return nil, err
}
netNsPath := sb.netNsPath()
netNsPath := sb.NetNsPath()
if netNsPath == "" {
// The sandbox does not have a permanent namespace,
// it's on the host one.
@ -512,15 +499,15 @@ func (s *Server) createSandboxContainer(ctx context.Context, containerID string,
}
// bind mount the pod shm
specgen.AddBindMount(sb.shmPath, "/dev/shm", []string{"rw"})
specgen.AddBindMount(sb.ShmPath(), "/dev/shm", []string{"rw"})
options := []string{"rw"}
if readOnlyRootfs {
options = []string{"ro"}
}
if sb.resolvPath != "" {
if sb.ResolvPath() != "" {
// bind mount the pod resolver file
specgen.AddBindMount(sb.resolvPath, "/etc/resolv.conf", options)
specgen.AddBindMount(sb.ResolvPath(), "/etc/resolv.conf", options)
}
// Bind mount /etc/hosts for host networking containers
@ -528,14 +515,14 @@ func (s *Server) createSandboxContainer(ctx context.Context, containerID string,
specgen.AddBindMount("/etc/hosts", "/etc/hosts", options)
}
if sb.hostname != "" {
specgen.SetHostname(sb.hostname)
if sb.Hostname() != "" {
specgen.SetHostname(sb.Hostname())
}
specgen.AddAnnotation(annotations.Name, containerName)
specgen.AddAnnotation(annotations.ContainerID, containerID)
specgen.AddAnnotation(annotations.SandboxID, sb.id)
specgen.AddAnnotation(annotations.SandboxName, sb.infraContainer.Name())
specgen.AddAnnotation(annotations.SandboxID, sb.ID())
specgen.AddAnnotation(annotations.SandboxName, sb.InfraContainer().Name())
specgen.AddAnnotation(annotations.ContainerType, annotations.ContainerTypeContainer)
specgen.AddAnnotation(annotations.LogPath, logPath)
specgen.AddAnnotation(annotations.TTY, fmt.Sprintf("%v", containerConfig.Tty))
@ -564,19 +551,19 @@ func (s *Server) createSandboxContainer(ctx context.Context, containerID string,
}
specgen.AddAnnotation(annotations.Annotations, string(kubeAnnotationsJSON))
if err = s.setupSeccomp(&specgen, containerName, sb.annotations); err != nil {
if err = s.setupSeccomp(&specgen, containerName, sb.Annotations()); err != nil {
return nil, err
}
metaname := metadata.Name
attempt := metadata.Attempt
containerInfo, err := s.storageRuntimeServer.CreateContainer(s.imageContext,
sb.name, sb.id,
sb.Name(), sb.ID(),
image, image,
containerName, containerID,
metaname,
attempt,
sb.mountLabel,
sb.MountLabel(),
nil)
if err != nil {
return nil, err
@ -684,7 +671,7 @@ func (s *Server) createSandboxContainer(ctx context.Context, containerID string,
return nil, err
}
container, err := oci.NewContainer(containerID, containerName, containerInfo.RunDir, logPath, sb.netNs(), labels, kubeAnnotations, imageSpec, metadata, sb.id, containerConfig.Tty, containerConfig.Stdin, containerConfig.StdinOnce, sb.privileged, sb.trusted, containerInfo.Dir, created, containerImageConfig.Config.StopSignal)
container, err := oci.NewContainer(containerID, containerName, containerInfo.RunDir, logPath, sb.NetNs(), labels, kubeAnnotations, imageSpec, metadata, sb.ID(), containerConfig.Tty, containerConfig.Stdin, containerConfig.StdinOnce, sb.Privileged(), sb.Trusted(), containerInfo.Dir, created, containerImageConfig.Config.StopSignal)
if err != nil {
return nil, err
}

10
server/container_exec.go
View file

@ -30,10 +30,14 @@ func (s *Server) Exec(ctx context.Context, req *pb.ExecRequest) (*pb.ExecRespons
// Exec endpoint for streaming.Runtime
func (ss streamService) Exec(containerID string, cmd []string, stdin io.Reader, stdout, stderr io.WriteCloser, tty bool, resize <-chan term.Size) error {
c := ss.runtimeServer.GetContainer(containerID)
sbID, err := ss.runtimeServer.state.GetContainerSandbox(containerID)
if err != nil {
return err
}
if c == nil {
return fmt.Errorf("could not find container %q", containerID)
c, err := ss.runtimeServer.state.GetContainer(containerID, sbID)
if err != nil {
return err
}
if err := ss.runtimeServer.runtime.UpdateStatus(c); err != nil {

25
server/container_list.go
View file

@ -3,6 +3,7 @@ package server
import (
"github.com/Sirupsen/logrus"
"github.com/kubernetes-incubator/cri-o/oci"
"github.com/kubernetes-incubator/cri-o/server/state"
"golang.org/x/net/context"
"k8s.io/apimachinery/pkg/fields"
pb "k8s.io/kubernetes/pkg/kubelet/api/v1alpha1/runtime"
@ -31,17 +32,19 @@ func (s *Server) ListContainers(ctx context.Context, req *pb.ListContainersReque
logrus.Debugf("ListContainersRequest %+v", req)
var ctrs []*pb.Container
filter := req.Filter
ctrList := s.state.containers.List()
ctrList, _ := s.state.GetAllContainers()
// Filter using container id and pod id first.
if filter != nil {
if filter.Id != "" {
id, err := s.ctrIDIndex.Get(filter.Id)
c, err := s.state.LookupContainerByID(filter.Id)
if err != nil {
return nil, err
}
c := s.state.containers.Get(id)
if c != nil {
if !state.IsCtrNotExist(err) {
return nil, err
}
ctrList = []*oci.Container{}
} else {
if filter.PodSandboxId != "" {
if c.Sandbox() == filter.PodSandboxId {
ctrList = []*oci.Container{c}
@ -55,11 +58,15 @@ func (s *Server) ListContainers(ctx context.Context, req *pb.ListContainersReque
}
} else {
if filter.PodSandboxId != "" {
pod := s.state.sandboxes[filter.PodSandboxId]
if pod == nil {
pod, err := s.state.GetSandbox(filter.PodSandboxId)
if err != nil {
if !state.IsSandboxNotExist(err) {
return nil, err
}
ctrList = []*oci.Container{}
} else {
ctrList = pod.containers.List()
ctrList = pod.Containers()
}
}
}

7
server/container_portforward.go
View file

@ -28,10 +28,9 @@ func (s *Server) PortForward(ctx context.Context, req *pb.PortForwardRequest) (*
}
func (ss streamService) PortForward(podSandboxID string, port int32, stream io.ReadWriteCloser) error {
c := ss.runtimeServer.GetSandboxContainer(podSandboxID)
if c == nil {
return fmt.Errorf("could not find container for sandbox %q", podSandboxID)
c, err := ss.runtimeServer.GetSandboxContainer(podSandboxID)
if err != nil {
return err
}
if err := ss.runtimeServer.runtime.UpdateStatus(c); err != nil {

10
server/container_remove.go
View file

@ -36,18 +36,14 @@ func (s *Server) RemoveContainer(ctx context.Context, req *pb.RemoveContainerReq
return nil, fmt.Errorf("failed to delete container %s: %v", c.ID(), err)
}
s.removeContainer(c)
if err := s.removeContainer(c); err != nil {
return nil, fmt.Errorf("failed to remove container %s: %v", c.ID(), err)
}
if err := s.storageRuntimeServer.DeleteContainer(c.ID()); err != nil {
return nil, fmt.Errorf("failed to delete storage for container %s: %v", c.ID(), err)
}
s.releaseContainerName(c.Name())
if err := s.ctrIDIndex.Delete(c.ID()); err != nil {
return nil, err
}
resp := &pb.RemoveContainerResponse{}
logrus.Debugf("RemoveContainerResponse: %+v", resp)
return resp, nil

27
server/naming.go
View file

@ -48,39 +48,30 @@ func makeContainerName(sandboxMetadata *pb.PodSandboxMetadata, containerConfig *
func (s *Server) generatePodIDandName(sandboxConfig *pb.PodSandboxConfig) (string, string, error) {
var (
err error
id = stringid.GenerateNonCryptoID()
err error
id = stringid.GenerateNonCryptoID()
name = makeSandboxName(sandboxConfig)
)
if sandboxConfig.Metadata.Namespace == "" {
return "", "", fmt.Errorf("cannot generate pod ID without namespace")
}
name, err := s.reservePodName(id, makeSandboxName(sandboxConfig))
if err != nil {
return "", "", err
}
return id, name, err
}
func (s *Server) generateContainerIDandNameForSandbox(sandboxConfig *pb.PodSandboxConfig) (string, string, error) {
var (
err error
id = stringid.GenerateNonCryptoID()
err error
id = stringid.GenerateNonCryptoID()
name = makeSandboxContainerName(sandboxConfig)
)
name, err := s.reserveContainerName(id, makeSandboxContainerName(sandboxConfig))
if err != nil {
return "", "", err
}
return id, name, err
}
func (s *Server) generateContainerIDandName(sandboxMetadata *pb.PodSandboxMetadata, containerConfig *pb.ContainerConfig) (string, string, error) {
var (
err error
id = stringid.GenerateNonCryptoID()
err error
id = stringid.GenerateNonCryptoID()
name = makeContainerName(sandboxMetadata, containerConfig)
)
name, err := s.reserveContainerName(id, makeContainerName(sandboxMetadata, containerConfig))
if err != nil {
return "", "", err
}
return id, name, err
}

283
server/sandbox.go
View file

@ -1,283 +0,0 @@
package server
import (
"crypto/rand"
"errors"
"fmt"
"os"
"path/filepath"
"sync"
"github.com/Sirupsen/logrus"
"github.com/containernetworking/cni/pkg/ns"
"github.com/docker/docker/pkg/mount"
"github.com/docker/docker/pkg/symlink"
"github.com/kubernetes-incubator/cri-o/oci"
"golang.org/x/sys/unix"
"k8s.io/apimachinery/pkg/fields"
pb "k8s.io/kubernetes/pkg/kubelet/api/v1alpha1/runtime"
"k8s.io/kubernetes/pkg/kubelet/network/hostport"
)
type sandboxNetNs struct {
sync.Mutex
ns ns.NetNS
symlink *os.File
closed bool
restored bool
}
func (ns *sandboxNetNs) symlinkCreate(name string) error {
b := make([]byte, 4)
_, randErr := rand.Reader.Read(b)
if randErr != nil {
return randErr
}
nsName := fmt.Sprintf("%s-%x", name, b)
symlinkPath := filepath.Join(nsRunDir, nsName)
if err := os.Symlink(ns.ns.Path(), symlinkPath); err != nil {
return err
}
fd, err := os.Open(symlinkPath)
if err != nil {
if removeErr := os.RemoveAll(symlinkPath); removeErr != nil {
return removeErr
}
return err
}
ns.symlink = fd
return nil
}
func (ns *sandboxNetNs) symlinkRemove() error {
if err := ns.symlink.Close(); err != nil {
return err
}
return os.RemoveAll(ns.symlink.Name())
}
func isSymbolicLink(path string) (bool, error) {
fi, err := os.Lstat(path)
if err != nil {
return false, err
}
return fi.Mode()&os.ModeSymlink == os.ModeSymlink, nil
}
func netNsGet(nspath, name string) (*sandboxNetNs, error) {
if err := ns.IsNSorErr(nspath); err != nil {
return nil, errSandboxClosedNetNS
}
symlink, symlinkErr := isSymbolicLink(nspath)
if symlinkErr != nil {
return nil, symlinkErr
}
var resolvedNsPath string
if symlink {
path, err := os.Readlink(nspath)
if err != nil {
return nil, err
}
resolvedNsPath = path
} else {
resolvedNsPath = nspath
}
netNS, err := ns.GetNS(resolvedNsPath)
if err != nil {
return nil, err
}
netNs := &sandboxNetNs{ns: netNS, closed: false, restored: true}
if symlink {
fd, err := os.Open(nspath)
if err != nil {
return nil, err
}
netNs.symlink = fd
} else {
if err := netNs.symlinkCreate(name); err != nil {
return nil, err
}
}
return netNs, nil
}
func hostNetNsPath() (string, error) {
netNS, err := ns.GetCurrentNS()
if err != nil {
return "", err
}
defer netNS.Close()
return netNS.Path(), nil
}
type sandbox struct {
id string
namespace string
// OCI pod name (eg "<namespace>-<name>-<attempt>")
name string
// Kubernetes pod name (eg, "<name>")
kubeName string
logDir string
labels fields.Set
annotations map[string]string
infraContainer *oci.Container
containers oci.ContainerStorer
processLabel string
mountLabel string
netns *sandboxNetNs
metadata *pb.PodSandboxMetadata
shmPath string
cgroupParent string
privileged bool
trusted bool
resolvPath string
hostname string
portMappings []*hostport.PortMapping
}
const (
defaultShmSize = 64 * 1024 * 1024
nsRunDir = "/var/run/netns"
podInfraCommand = "/pause"
)
var (
errSandboxIDEmpty = errors.New("PodSandboxId should not be empty")
errSandboxClosedNetNS = errors.New("PodSandbox networking namespace is closed")
)
func (s *sandbox) addContainer(c *oci.Container) {
s.containers.Add(c.Name(), c)
}
func (s *sandbox) getContainer(name string) *oci.Container {
return s.containers.Get(name)
}
func (s *sandbox) removeContainer(c *oci.Container) {
s.containers.Delete(c.Name())
}
func (s *sandbox) netNs() ns.NetNS {
if s.netns == nil {
return nil
}
return s.netns.ns
}
func (s *sandbox) netNsPath() string {
if s.netns == nil {
return ""
}
return s.netns.symlink.Name()
}
func (s *sandbox) netNsCreate() error {
if s.netns != nil {
return fmt.Errorf("net NS already created")
}
netNS, err := ns.NewNS()
if err != nil {
return err
}
s.netns = &sandboxNetNs{
ns: netNS,
closed: false,
}
if err := s.netns.symlinkCreate(s.name); err != nil {
logrus.Warnf("Could not create nentns symlink %v", err)
if err1 := s.netns.ns.Close(); err1 != nil {
return err1
}
return err
}
return nil
}
func (s *sandbox) netNsRemove() error {
if s.netns == nil {
logrus.Warn("no networking namespace")
return nil
}
s.netns.Lock()
defer s.netns.Unlock()
if s.netns.closed {
// netNsRemove() can be called multiple
// times without returning an error.
return nil
}
if err := s.netns.symlinkRemove(); err != nil {
return err
}
if err := s.netns.ns.Close(); err != nil {
return err
}
if s.netns.restored {
// we got namespaces in the form of
// /var/run/netns/cni-0d08effa-06eb-a963-f51a-e2b0eceffc5d
// but /var/run on most system is symlinked to /run so we first resolve
// the symlink and then try and see if it's mounted
fp, err := symlink.FollowSymlinkInScope(s.netns.ns.Path(), "/")
if err != nil {
return err
}
if mounted, err := mount.Mounted(fp); err == nil && mounted {
if err := unix.Unmount(fp, unix.MNT_DETACH); err != nil {
return err
}
}
if err := os.RemoveAll(s.netns.ns.Path()); err != nil {
return err
}
}
s.netns.closed = true
return nil
}
func (s *Server) getPodSandboxFromRequest(podSandboxID string) (*sandbox, error) {
if podSandboxID == "" {
return nil, errSandboxIDEmpty
}
sandboxID, err := s.podIDIndex.Get(podSandboxID)
if err != nil {
return nil, fmt.Errorf("PodSandbox with ID starting with %s not found: %v", podSandboxID, err)
}
sb := s.getSandbox(sandboxID)
if sb == nil {
return nil, fmt.Errorf("specified pod sandbox not found: %s", sandboxID)
}
return sb, nil
}

438
server/sandbox/sandbox.go Normal file
View file

@ -0,0 +1,438 @@
package sandbox
import (
"crypto/rand"
"errors"
"fmt"
"os"
"path/filepath"
"sync"
"github.com/Sirupsen/logrus"
"github.com/containernetworking/cni/pkg/ns"
"github.com/docker/docker/pkg/mount"
"github.com/docker/docker/pkg/symlink"
"github.com/kubernetes-incubator/cri-o/oci"
"golang.org/x/sys/unix"
"k8s.io/apimachinery/pkg/fields"
pb "k8s.io/kubernetes/pkg/kubelet/api/v1alpha1/runtime"
"k8s.io/kubernetes/pkg/kubelet/network/hostport"
)
type sandboxNetNs struct {
sync.Mutex
ns ns.NetNS
symlink *os.File
closed bool
restored bool
}
func (ns *sandboxNetNs) symlinkCreate(name string) error {
b := make([]byte, 4)
_, randErr := rand.Reader.Read(b)
if randErr != nil {
return randErr
}
nsName := fmt.Sprintf("%s-%x", name, b)
symlinkPath := filepath.Join(nsRunDir, nsName)
if err := os.Symlink(ns.ns.Path(), symlinkPath); err != nil {
return err
}
fd, err := os.Open(symlinkPath)
if err != nil {
if removeErr := os.RemoveAll(symlinkPath); removeErr != nil {
return removeErr
}
return err
}
ns.symlink = fd
return nil
}
func (ns *sandboxNetNs) symlinkRemove() error {
if err := ns.symlink.Close(); err != nil {
return err
}
return os.RemoveAll(ns.symlink.Name())
}
func isSymbolicLink(path string) (bool, error) {
fi, err := os.Lstat(path)
if err != nil {
return false, err
}
return fi.Mode()&os.ModeSymlink == os.ModeSymlink, nil
}
func netNsGet(nspath, name string) (*sandboxNetNs, error) {
if err := ns.IsNSorErr(nspath); err != nil {
return nil, ErrSandboxClosedNetNS
}
symlink, symlinkErr := isSymbolicLink(nspath)
if symlinkErr != nil {
return nil, symlinkErr
}
var resolvedNsPath string
if symlink {
path, err := os.Readlink(nspath)
if err != nil {
return nil, err
}
resolvedNsPath = path
} else {
resolvedNsPath = nspath
}
netNS, err := ns.GetNS(resolvedNsPath)
if err != nil {
return nil, err
}
netNs := &sandboxNetNs{ns: netNS, closed: false, restored: true}
if symlink {
fd, err := os.Open(nspath)
if err != nil {
return nil, err
}
netNs.symlink = fd
} else {
if err := netNs.symlinkCreate(name); err != nil {
return nil, err
}
}
return netNs, nil
}
// HostNetNsPath returns the path of the host's network namespace
func HostNetNsPath() (string, error) {
netNS, err := ns.GetCurrentNS()
if err != nil {
return "", err
}
defer netNS.Close()
return netNS.Path(), nil
}
// Sandbox represents a single pod sandbox
type Sandbox struct {
id string
namespace string
// OCI pod name (eg "<namespace>-<name>-<attempt>")
name string
// Kubernetes pod name (eg, "<name>")
kubeName string
logDir string
labels fields.Set
annotations map[string]string
infraContainer *oci.Container
containers oci.ContainerStorer
processLabel string
mountLabel string
netns *sandboxNetNs
metadata *pb.PodSandboxMetadata
shmPath string
cgroupParent string
privileged bool
trusted bool
resolvPath string
hostname string
portMappings []*hostport.PortMapping
}
const (
// DefaultShmSize is the default size of the SHM device for sandboxs
DefaultShmSize = 64 * 1024 * 1024
// PodInfraCommand is the default pause command for pods
PodInfraCommand = "/pause"
nsRunDir = "/var/run/netns"
)
var (
// ErrSandboxIDEmpty is the error returned when an operation passes "" instead of a sandbox ID
ErrSandboxIDEmpty = errors.New("PodSandboxId should not be empty")
// ErrSandboxClosedNetNS is the error returned when a network namespace is closed and cannot be joined
ErrSandboxClosedNetNS = errors.New("PodSandbox networking namespace is closed")
)
// New creates and populates a new sandbox
// New sandboxes have no containers, no infra container, and no network namespace associated with them.
// An infra container must be attached before the sandbox is added to the state
func New(id, namespace, name, kubeName, logDir string, labels, annotations map[string]string, processLabel, mountLabel string, metadata *pb.PodSandboxMetadata, shmPath, cgroupParent string, privileged bool, trusted bool, resolvPath, hostname string, portMappings []*hostport.PortMapping) (*Sandbox, error) {
sb := new(Sandbox)
sb.id = id
sb.namespace = namespace
sb.name = name
sb.kubeName = kubeName
sb.logDir = logDir
sb.labels = labels
sb.annotations = annotations
sb.containers = oci.NewMemoryStore()
sb.processLabel = processLabel
sb.mountLabel = mountLabel
sb.metadata = metadata
sb.shmPath = shmPath
sb.cgroupParent = cgroupParent
sb.privileged = privileged
sb.trusted = trusted
sb.resolvPath = resolvPath
sb.hostname = hostname
sb.portMappings = portMappings
return sb, nil
}
// ID returns the sandbox's ID
func (s *Sandbox) ID() string {
return s.id
}
// Namespace returns the sandbox's namespace
func (s *Sandbox) Namespace() string {
return s.namespace
}
// Name returns the sandbox's name
func (s *Sandbox) Name() string {
return s.name
}
// KubeName returns the name the sandbox was given by Kubernetes
// This is not a fully qualified, globally unique name and cannot be used to look up the sandbox
func (s *Sandbox) KubeName() string {
return s.kubeName
}
// LogDir returns the directory the sandbox logs to
func (s *Sandbox) LogDir() string {
return s.logDir
}
// Labels returns the sandbox's labels
func (s *Sandbox) Labels() map[string]string {
return s.labels
}
// Annotations returns the sandbox's annotations
func (s *Sandbox) Annotations() map[string]string {
return s.annotations
}
// InfraContainer returns the sandbox's infrastructure container
func (s *Sandbox) InfraContainer() *oci.Container {
return s.infraContainer
}
// Containers returns an array of all the containers in the sandbox
func (s *Sandbox) Containers() []*oci.Container {
return s.containers.List()
}
// ProcessLabel returns the SELinux process label of the sandbox
func (s *Sandbox) ProcessLabel() string {
return s.processLabel
}
// MountLabel returns the SELinux mount label of the sandbox
func (s *Sandbox) MountLabel() string {
return s.mountLabel
}
// Metadata returns Kubernetes metadata associated with the sandbox
func (s *Sandbox) Metadata() *pb.PodSandboxMetadata {
return s.metadata
}
// ShmPath returns the path to the sandbox's shared memory device
func (s *Sandbox) ShmPath() string {
return s.shmPath
}
// CgroupParent returns the sandbox's CGroup parent
func (s *Sandbox) CgroupParent() string {
return s.cgroupParent
}
// Privileged returns whether the sandbox can support privileged containers
func (s *Sandbox) Privileged() bool {
return s.privileged
}
// Trusted returns whether the sandbox is a trusted workload
func (s *Sandbox) Trusted() bool {
return s.trusted
}
// ResolvPath returns the path to the sandbox's DNS resolver configuration
func (s *Sandbox) ResolvPath() string {
return s.resolvPath
}
// Hostname returns the sandbox's hostname
func (s *Sandbox) Hostname() string {
return s.hostname
}
// PortMappings gets the sandbox's host port mappings
func (s *Sandbox) PortMappings() []*hostport.PortMapping {
return s.portMappings
}
// AddContainer adds a container to the sandbox
func (s *Sandbox) AddContainer(c *oci.Container) {
s.containers.Add(c.ID(), c)
}
// GetContainer retrieves the container with given ID from the sandbox
// Returns nil if no such container exists
func (s *Sandbox) GetContainer(id string) *oci.Container {
return s.containers.Get(id)
}
// RemoveContainer removes the container with given ID from the sandbox
// If no container with that ID exists in the sandbox, no action is taken
func (s *Sandbox) RemoveContainer(id string) {
s.containers.Delete(id)
}
// SetInfraContainer sets the infrastructure container of a sandbox
// Attempts to set the infrastructure container after one is already present will throw an error
func (s *Sandbox) SetInfraContainer(infraCtr *oci.Container) error {
if s.infraContainer != nil {
return fmt.Errorf("sandbox already has an infra container")
} else if infraCtr == nil {
return fmt.Errorf("must provide non-nil infra container")
}
s.infraContainer = infraCtr
return nil
}
// NetNs retrieves the network namespace of the sandbox
// If the sandbox uses the host namespace, nil is returned
func (s *Sandbox) NetNs() ns.NetNS {
if s.netns == nil {
return nil
}
return s.netns.ns
}
// NetNsPath returns the path to the network namespace
// If the sandbox uses the host namespace, "" is returned
func (s *Sandbox) NetNsPath() string {
if s.netns == nil {
return ""
}
return s.netns.symlink.Name()
}
// NetNsCreate creates a new network namespace for the sandbox
func (s *Sandbox) NetNsCreate() error {
if s.netns != nil {
return fmt.Errorf("net NS already created")
}
netNS, err := ns.NewNS()
if err != nil {
return err
}
s.netns = &sandboxNetNs{
ns: netNS,
closed: false,
}
if err := s.netns.symlinkCreate(s.name); err != nil {
logrus.Warnf("Could not create nentns symlink %v", err)
if err1 := s.netns.ns.Close(); err1 != nil {
return err1
}
return err
}
return nil
}
// NetNsJoin attempts to join the sandbox to an existing network namespace
// This will fail if the sandbox is already part of a network namespace
func (s *Sandbox) NetNsJoin(nspath, name string) error {
if s.netns != nil {
return fmt.Errorf("sandbox already has a network namespace, cannot join another")
}
netNS, err := netNsGet(nspath, name)
if err != nil {
return err
}
s.netns = netNS
return nil
}
// NetNsRemove removes the network namespace associated with the sandbox
func (s *Sandbox) NetNsRemove() error {
if s.netns == nil {
logrus.Warn("no networking namespace")
return nil
}
s.netns.Lock()
defer s.netns.Unlock()
if s.netns.closed {
// netNsRemove() can be called multiple
// times without returning an error.
return nil
}
if err := s.netns.symlinkRemove(); err != nil {
return err
}
if err := s.netns.ns.Close(); err != nil {
return err
}
if s.netns.restored {
// we got namespaces in the form of
// /var/run/netns/cni-0d08effa-06eb-a963-f51a-e2b0eceffc5d
// but /var/run on most system is symlinked to /run so we first resolve
// the symlink and then try and see if it's mounted
fp, err := symlink.FollowSymlinkInScope(s.netns.ns.Path(), "/")
if err != nil {
return err
}
if mounted, err := mount.Mounted(fp); err == nil && mounted {
if err := unix.Unmount(fp, unix.MNT_DETACH); err != nil {
return err
}
}
if err := os.RemoveAll(s.netns.ns.Path()); err != nil {
return err
}
}
s.netns.closed = true
return nil
}

38
server/sandbox_list.go
View file

@ -1,8 +1,12 @@
package server
import (
"fmt"
"github.com/Sirupsen/logrus"
"github.com/kubernetes-incubator/cri-o/oci"
"github.com/kubernetes-incubator/cri-o/server/sandbox"
"github.com/kubernetes-incubator/cri-o/server/state"
"golang.org/x/net/context"
"k8s.io/apimachinery/pkg/fields"
pb "k8s.io/kubernetes/pkg/kubelet/api/v1alpha1/runtime"
@ -30,30 +34,34 @@ func filterSandbox(p *pb.PodSandbox, filter *pb.PodSandboxFilter) bool {
func (s *Server) ListPodSandbox(ctx context.Context, req *pb.ListPodSandboxRequest) (*pb.ListPodSandboxResponse, error) {
logrus.Debugf("ListPodSandboxRequest %+v", req)
var pods []*pb.PodSandbox
var podList []*sandbox
for _, sb := range s.state.sandboxes {
podList = append(podList, sb)
var podList []*sandbox.Sandbox
sandboxes, err := s.state.GetAllSandboxes()
if err != nil {
return nil, fmt.Errorf("error retrieving sandboxes: %v", err)
}
podList = append(podList, sandboxes...)
filter := req.Filter
// Filter by pod id first.
if filter != nil {
if filter.Id != "" {
id, err := s.podIDIndex.Get(filter.Id)
sb, err := s.state.LookupSandboxByID(filter.Id)
if err != nil {
return nil, err
}
sb := s.getSandbox(id)
if sb == nil {
podList = []*sandbox{}
if !state.IsCtrNotExist(err) {
return nil, err
}
podList = []*sandbox.Sandbox{}
} else {
podList = []*sandbox{sb}
podList = []*sandbox.Sandbox{sb}
}
}
}
for _, sb := range podList {
podInfraContainer := sb.infraContainer
podInfraContainer := sb.InfraContainer()
if podInfraContainer == nil {
// this can't really happen, but if it does because of a bug
// it's better not to panic
@ -71,12 +79,12 @@ func (s *Server) ListPodSandbox(ctx context.Context, req *pb.ListPodSandboxReque
}
pod := &pb.PodSandbox{
Id: sb.id,
Id: sb.ID(),
CreatedAt: created,
State: rStatus,
Labels: sb.labels,
Annotations: sb.annotations,
Metadata: sb.metadata,
Labels: sb.Labels(),
Annotations: sb.Annotations(),
Metadata: sb.Metadata(),
}
// Filter by other criteria such as state and labels.

42
server/sandbox_remove.go
View file

@ -7,6 +7,7 @@ import (
"github.com/containers/storage"
"github.com/kubernetes-incubator/cri-o/oci"
pkgstorage "github.com/kubernetes-incubator/cri-o/pkg/storage"
"github.com/kubernetes-incubator/cri-o/server/sandbox"
"golang.org/x/net/context"
pb "k8s.io/kubernetes/pkg/kubelet/api/v1alpha1/runtime"
)
@ -17,7 +18,7 @@ func (s *Server) RemovePodSandbox(ctx context.Context, req *pb.RemovePodSandboxR
logrus.Debugf("RemovePodSandboxRequest %+v", req)
sb, err := s.getPodSandboxFromRequest(req.PodSandboxId)
if err != nil {
if err == errSandboxIDEmpty {
if err == sandbox.ErrSandboxIDEmpty {
return nil, err
}
@ -30,8 +31,8 @@ func (s *Server) RemovePodSandbox(ctx context.Context, req *pb.RemovePodSandboxR
return resp, nil
}
podInfraContainer := sb.infraContainer
containers := sb.containers.List()
podInfraContainer := sb.InfraContainer()
containers := sb.Containers()
containers = append(containers, podInfraContainer)
// Delete all the containers in the sandbox
@ -49,7 +50,7 @@ func (s *Server) RemovePodSandbox(ctx context.Context, req *pb.RemovePodSandboxR
}
if err := s.runtime.DeleteContainer(c); err != nil {
return nil, fmt.Errorf("failed to delete container %s in pod sandbox %s: %v", c.Name(), sb.id, err)
return nil, fmt.Errorf("failed to delete container %s in pod sandbox %s: %v", c.Name(), sb.ID(), err)
}
if c.ID() == podInfraContainer.ID() {
@ -58,38 +59,27 @@ func (s *Server) RemovePodSandbox(ctx context.Context, req *pb.RemovePodSandboxR
if err := s.storageRuntimeServer.StopContainer(c.ID()); err != nil && err != storage.ErrContainerUnknown {
// assume container already umounted
logrus.Warnf("failed to stop container %s in pod sandbox %s: %v", c.Name(), sb.id, err)
logrus.Warnf("failed to stop container %s in pod sandbox %s: %v", c.Name(), sb.ID(), err)
}
if err := s.storageRuntimeServer.DeleteContainer(c.ID()); err != nil && err != storage.ErrContainerUnknown {
return nil, fmt.Errorf("failed to delete container %s in pod sandbox %s: %v", c.Name(), sb.id, err)
return nil, fmt.Errorf("failed to delete container %s in pod sandbox %s: %v", c.Name(), sb.ID(), err)
}
s.releaseContainerName(c.Name())
s.removeContainer(c)
if err := s.ctrIDIndex.Delete(c.ID()); err != nil {
return nil, fmt.Errorf("failed to delete container %s in pod sandbox %s from index: %v", c.Name(), sb.id, err)
if err := s.removeContainer(c); err != nil {
return nil, err
}
}
s.removeContainer(podInfraContainer)
if err := s.removeSandbox(sb.ID()); err != nil {
return nil, fmt.Errorf("error removing sandbox %s: %v", sb.ID(), err)
}
// Remove the files related to the sandbox
if err := s.storageRuntimeServer.StopContainer(sb.id); err != nil && err != storage.ErrContainerUnknown {
logrus.Warnf("failed to stop sandbox container in pod sandbox %s: %v", sb.id, err)
if err := s.storageRuntimeServer.StopContainer(sb.ID()); err != nil && err != storage.ErrContainerUnknown {
logrus.Warnf("failed to stop sandbox container in pod sandbox %s: %v", sb.ID(), err)
}
if err := s.storageRuntimeServer.RemovePodSandbox(sb.id); err != nil && err != pkgstorage.ErrInvalidSandboxID {
return nil, fmt.Errorf("failed to remove pod sandbox %s: %v", sb.id, err)
}
s.releaseContainerName(podInfraContainer.Name())
if err := s.ctrIDIndex.Delete(podInfraContainer.ID()); err != nil {
return nil, fmt.Errorf("failed to delete infra container %s in pod sandbox %s from index: %v", podInfraContainer.ID(), sb.id, err)
}
s.releasePodName(sb.name)
s.removeSandbox(sb.id)
if err := s.podIDIndex.Delete(sb.id); err != nil {
return nil, fmt.Errorf("failed to delete pod sandbox %s from index: %v", sb.id, err)
if err := s.storageRuntimeServer.RemovePodSandbox(sb.ID()); err != nil && err != pkgstorage.ErrInvalidSandboxID {
return nil, fmt.Errorf("failed to remove pod sandbox %s: %v", sb.ID(), err)
}
resp := &pb.RemovePodSandboxResponse{}

105
server/sandbox_run.go
View file

@ -16,6 +16,7 @@ import (
"github.com/containers/storage"
"github.com/kubernetes-incubator/cri-o/oci"
"github.com/kubernetes-incubator/cri-o/pkg/annotations"
"github.com/kubernetes-incubator/cri-o/server/sandbox"
"github.com/opencontainers/runc/libcontainer/cgroups/systemd"
"github.com/opencontainers/runtime-tools/generate"
"github.com/opencontainers/selinux/go-selinux/label"
@ -135,23 +136,11 @@ func (s *Server) RunPodSandbox(ctx context.Context, req *pb.RunPodSandboxRequest
}
}
defer func() {
if err != nil {
s.releasePodName(name)
}
}()
_, containerName, err := s.generateContainerIDandNameForSandbox(req.GetConfig())
if err != nil {
return nil, err
}
defer func() {
if err != nil {
s.releaseContainerName(containerName)
}
}()
podContainer, err := s.storageRuntimeServer.CreatePodSandbox(s.imageContext,
name, id,
s.config.PauseImage, "",
@ -186,7 +175,7 @@ func (s *Server) RunPodSandbox(ctx context.Context, req *pb.RunPodSandboxRequest
if podContainer.Config != nil {
g.SetProcessArgs(podContainer.Config.Config.Cmd)
} else {
g.SetProcessArgs([]string{podInfraCommand})
g.SetProcessArgs([]string{sandbox.PodInfraCommand})
}
} else {
g.SetProcessArgs([]string{s.config.PauseCommand})
@ -283,18 +272,6 @@ func (s *Server) RunPodSandbox(ctx context.Context, req *pb.RunPodSandboxRequest
return nil, err
}
if err = s.ctrIDIndex.Add(id); err != nil {
return nil, err
}
defer func() {
if err != nil {
if err2 := s.ctrIDIndex.Delete(id); err2 != nil {
logrus.Warnf("couldn't delete ctr id %s from idIndex", id)
}
}
}()
// set log path inside log directory
logPath := filepath.Join(logDir, id+".log")
@ -320,6 +297,7 @@ func (s *Server) RunPodSandbox(ctx context.Context, req *pb.RunPodSandboxRequest
g.AddAnnotation(annotations.ResolvPath, resolvPath)
g.AddAnnotation(annotations.HostName, hostname)
g.AddAnnotation(annotations.KubeName, kubeName)
g.AddAnnotation(annotations.Namespace, namespace)
if podContainer.Config.Config.StopSignal != "" {
// this key is defined in image-spec conversion document at https://github.com/opencontainers/image-spec/pull/492/files#diff-8aafbe2c3690162540381b8cdb157112R57
g.AddAnnotation("org.opencontainers.image.stopSignal", podContainer.Config.Config.StopSignal)
@ -329,45 +307,11 @@ func (s *Server) RunPodSandbox(ctx context.Context, req *pb.RunPodSandboxRequest
g.AddAnnotation(annotations.Created, created.Format(time.RFC3339Nano))
portMappings := convertPortMappings(req.GetConfig().GetPortMappings())
sb := &sandbox{
id: id,
namespace: namespace,
name: name,
kubeName: kubeName,
logDir: logDir,
labels: labels,
annotations: kubeAnnotations,
containers: oci.NewMemoryStore(),
processLabel: processLabel,
mountLabel: mountLabel,
metadata: metadata,
shmPath: shmPath,
privileged: privileged,
trusted: trusted,
resolvPath: resolvPath,
hostname: hostname,
portMappings: portMappings,
}
s.addSandbox(sb)
defer func() {
if err != nil {
s.removeSandbox(id)
}
}()
if err = s.podIDIndex.Add(id); err != nil {
mappingsJSON, err := json.Marshal(portMappings)
if err != nil {
return nil, err
}
defer func() {
if err != nil {
if err := s.podIDIndex.Delete(id); err != nil {
logrus.Warnf("couldn't delete pod id %s from idIndex", id)
}
}
}()
g.AddAnnotation(annotations.PortMappings, string(mappingsJSON))
for k, v := range kubeAnnotations {
g.AddAnnotation(k, v)
@ -394,10 +338,9 @@ func (s *Server) RunPodSandbox(ctx context.Context, req *pb.RunPodSandboxRequest
return nil, err
}
g.SetLinuxCgroupsPath(cgPath + ":" + "crio" + ":" + id)
sb.cgroupParent = cgPath
cgroupParent = cgPath
} else {
g.SetLinuxCgroupsPath(cgroupParent + "/" + id)
sb.cgroupParent = cgroupParent
}
}
@ -405,6 +348,11 @@ func (s *Server) RunPodSandbox(ctx context.Context, req *pb.RunPodSandboxRequest
// so it doesn't get killed.
g.SetLinuxResourcesOOMScoreAdj(PodInfraOOMAdj)
sb, err := sandbox.New(id, namespace, name, kubeName, logDir, labels, kubeAnnotations, processLabel, mountLabel, metadata, shmPath, cgroupParent, privileged, trusted, resolvPath, hostname, portMappings)
if err != nil {
return nil, err
}
hostNetwork := req.GetConfig().GetLinux().GetSecurityContext().GetNamespaceOptions().HostNetwork
// set up namespaces
@ -414,13 +362,13 @@ func (s *Server) RunPodSandbox(ctx context.Context, req *pb.RunPodSandboxRequest
return nil, err
}
netNsPath, err = hostNetNsPath()
netNsPath, err = sandbox.HostNetNsPath()
if err != nil {
return nil, err
}
} else {
// Create the sandbox network namespace
if err = sb.netNsCreate(); err != nil {
if err = sb.NetNsCreate(); err != nil {
return nil, err
}
@ -429,18 +377,18 @@ func (s *Server) RunPodSandbox(ctx context.Context, req *pb.RunPodSandboxRequest
return
}
if netnsErr := sb.netNsRemove(); netnsErr != nil {
if netnsErr := sb.NetNsRemove(); netnsErr != nil {
logrus.Warnf("Failed to remove networking namespace: %v", netnsErr)
}
}()
// Pass the created namespace path to the runtime
err = g.AddOrReplaceLinuxNamespace("network", sb.netNsPath())
err = g.AddOrReplaceLinuxNamespace("network", sb.NetNsPath())
if err != nil {
return nil, err
}
netNsPath = sb.netNsPath()
netNsPath = sb.NetNsPath()
}
if req.GetConfig().GetLinux().GetSecurityContext().GetNamespaceOptions().HostPid {
@ -464,23 +412,28 @@ func (s *Server) RunPodSandbox(ctx context.Context, req *pb.RunPodSandboxRequest
saveOptions := generate.ExportOptions{}
mountPoint, err := s.storageRuntimeServer.StartContainer(id)
if err != nil {
return nil, fmt.Errorf("failed to mount container %s in pod sandbox %s(%s): %v", containerName, sb.name, id, err)
return nil, fmt.Errorf("failed to mount container %s in pod sandbox %s(%s): %v", containerName, name, id, err)
}
g.SetRootPath(mountPoint)
err = g.SaveToFile(filepath.Join(podContainer.Dir, "config.json"), saveOptions)
if err != nil {
return nil, fmt.Errorf("failed to save template configuration for pod sandbox %s(%s): %v", sb.name, id, err)
return nil, fmt.Errorf("failed to save template configuration for pod sandbox %s(%s): %v", name, id, err)
}
if err = g.SaveToFile(filepath.Join(podContainer.RunDir, "config.json"), saveOptions); err != nil {
return nil, fmt.Errorf("failed to write runtime configuration for pod sandbox %s(%s): %v", sb.name, id, err)
return nil, fmt.Errorf("failed to write runtime configuration for pod sandbox %s(%s): %v", name, id, err)
}
container, err := oci.NewContainer(id, containerName, podContainer.RunDir, logPath, sb.netNs(), labels, kubeAnnotations, nil, nil, id, false, false, false, sb.privileged, sb.trusted, podContainer.Dir, created, podContainer.Config.Config.StopSignal)
container, err := oci.NewContainer(id, containerName, podContainer.RunDir, logPath, sb.NetNs(), labels, kubeAnnotations, nil, nil, id, false, false, false, sb.Privileged(), sb.Trusted(), podContainer.Dir, created, podContainer.Config.Config.StopSignal)
if err != nil {
return nil, err
}
if err := sb.SetInfraContainer(container); err != nil {
return nil, err
}
sb.infraContainer = container
if err = s.addSandbox(sb); err != nil {
return nil, err
}
// setup the network
if !hostNetwork {
@ -511,7 +464,7 @@ func (s *Server) RunPodSandbox(ctx context.Context, req *pb.RunPodSandboxRequest
}
}
if err = s.runContainer(container, sb.cgroupParent); err != nil {
if err = s.runContainer(container, sb.CgroupParent()); err != nil {
return nil, err
}
@ -579,7 +532,7 @@ func setupShm(podSandboxRunDir, mountLabel string) (shmPath string, err error) {
if err = os.Mkdir(shmPath, 0700); err != nil {
return "", err
}
shmOptions := "mode=1777,size=" + strconv.Itoa(defaultShmSize)
shmOptions := "mode=1777,size=" + strconv.Itoa(sandbox.DefaultShmSize)
if err = syscall.Mount("shm", shmPath, "tmpfs", uintptr(syscall.MS_NOEXEC|syscall.MS_NOSUID|syscall.MS_NODEV),
label.FormatMountLabel(shmOptions, mountLabel)); err != nil {
return "", fmt.Errorf("failed to mount shm tmpfs for pod: %v", err)

12
server/sandbox_status.go
View file

@ -15,7 +15,7 @@ func (s *Server) PodSandboxStatus(ctx context.Context, req *pb.PodSandboxStatusR
return nil, err
}
podInfraContainer := sb.infraContainer
podInfraContainer := sb.InfraContainer()
if err = s.runtime.UpdateStatus(podInfraContainer); err != nil {
return nil, err
}
@ -27,7 +27,7 @@ func (s *Server) PodSandboxStatus(ctx context.Context, req *pb.PodSandboxStatusR
if err != nil {
return nil, err
}
ip, err := s.netPlugin.GetContainerNetworkStatus(netNsPath, sb.namespace, sb.kubeName, sb.id)
ip, err := s.netPlugin.GetContainerNetworkStatus(netNsPath, sb.Namespace(), sb.KubeName(), sb.ID())
if err != nil {
// ignore the error on network status
ip = ""
@ -38,7 +38,7 @@ func (s *Server) PodSandboxStatus(ctx context.Context, req *pb.PodSandboxStatusR
rStatus = pb.PodSandboxState_SANDBOX_READY
}
sandboxID := sb.id
sandboxID := sb.ID()
resp := &pb.PodSandboxStatusResponse{
Status: &pb.PodSandboxStatus{
Id: sandboxID,
@ -50,9 +50,9 @@ func (s *Server) PodSandboxStatus(ctx context.Context, req *pb.PodSandboxStatusR
},
Network: &pb.PodSandboxNetworkStatus{Ip: ip},
State: rStatus,
Labels: sb.labels,
Annotations: sb.annotations,
Metadata: sb.metadata,
Labels: sb.Labels(),
Annotations: sb.Annotations(),
Metadata: sb.Metadata(),
},
}

48
server/sandbox_stop.go
View file

@ -9,6 +9,7 @@ import (
"github.com/docker/docker/pkg/mount"
"github.com/docker/docker/pkg/symlink"
"github.com/kubernetes-incubator/cri-o/oci"
"github.com/kubernetes-incubator/cri-o/server/sandbox"
"github.com/opencontainers/selinux/go-selinux/label"
"golang.org/x/net/context"
"golang.org/x/sys/unix"
@ -22,7 +23,7 @@ func (s *Server) StopPodSandbox(ctx context.Context, req *pb.StopPodSandboxReque
logrus.Debugf("StopPodSandboxRequest %+v", req)
sb, err := s.getPodSandboxFromRequest(req.PodSandboxId)
if err != nil {
if err == errSandboxIDEmpty {
if err == sandbox.ErrSandboxIDEmpty {
return nil, err
}
@ -35,36 +36,36 @@ func (s *Server) StopPodSandbox(ctx context.Context, req *pb.StopPodSandboxReque
return resp, nil
}
podInfraContainer := sb.infraContainer
podInfraContainer := sb.InfraContainer()
netnsPath, err := podInfraContainer.NetNsPath()
if err != nil {
return nil, err
}
if _, err := os.Stat(netnsPath); err == nil {
if err2 := s.hostportManager.Remove(sb.id, &hostport.PodPortMapping{
Name: sb.name,
PortMappings: sb.portMappings,
if err2 := s.hostportManager.Remove(sb.ID(), &hostport.PodPortMapping{
Name: sb.Name(),
PortMappings: sb.PortMappings(),
HostNetwork: false,
}); err2 != nil {
logrus.Warnf("failed to remove hostport for container %s in sandbox %s: %v",
podInfraContainer.Name(), sb.id, err2)
podInfraContainer.Name(), sb.ID(), err2)
}
if err2 := s.netPlugin.TearDownPod(netnsPath, sb.namespace, sb.kubeName, sb.id); err2 != nil {
if err2 := s.netPlugin.TearDownPod(netnsPath, sb.Namespace(), sb.KubeName(), sb.ID()); err2 != nil {
logrus.Warnf("failed to destroy network for container %s in sandbox %s: %v",
podInfraContainer.Name(), sb.id, err2)
podInfraContainer.Name(), sb.ID(), err2)
}
} else if !os.IsNotExist(err) { // it's ok for netnsPath to *not* exist
return nil, fmt.Errorf("failed to stat netns path for container %s in sandbox %s before tearing down the network: %v",
sb.name, sb.id, err)
podInfraContainer.Name(), sb.ID(), err)
}
// Close the sandbox networking namespace.
if err := sb.netNsRemove(); err != nil {
if err := sb.NetNsRemove(); err != nil {
return nil, err
}
containers := sb.containers.List()
containers := sb.Containers()
containers = append(containers, podInfraContainer)
for _, c := range containers {
@ -74,30 +75,30 @@ func (s *Server) StopPodSandbox(ctx context.Context, req *pb.StopPodSandboxReque
cStatus := s.runtime.ContainerStatus(c)
if cStatus.Status != oci.ContainerStateStopped {
if err := s.runtime.StopContainer(c, -1); err != nil {
return nil, fmt.Errorf("failed to stop container %s in pod sandbox %s: %v", c.Name(), sb.id, err)
return nil, fmt.Errorf("failed to stop container %s in pod sandbox %s: %v", c.Name(), sb.ID(), err)
}
if c.ID() == podInfraContainer.ID() {
continue
}
if err := s.storageRuntimeServer.StopContainer(c.ID()); err != nil && err != storage.ErrContainerUnknown {
// assume container already umounted
logrus.Warnf("failed to stop container %s in pod sandbox %s: %v", c.Name(), sb.id, err)
logrus.Warnf("failed to stop container %s in pod sandbox %s: %v", c.Name(), sb.ID(), err)
}
}
s.containerStateToDisk(c)
}
if err := label.ReleaseLabel(sb.processLabel); err != nil {
if err := label.ReleaseLabel(sb.ProcessLabel()); err != nil {
return nil, err
}
// unmount the shm for the pod
if sb.shmPath != "/dev/shm" {
if sb.ShmPath() != "/dev/shm" {
// we got namespaces in the form of
// /var/run/containers/storage/overlay-containers/CID/userdata/shm
// but /var/run on most system is symlinked to /run so we first resolve
// the symlink and then try and see if it's mounted
fp, err := symlink.FollowSymlinkInScope(sb.shmPath, "/")
fp, err := symlink.FollowSymlinkInScope(sb.ShmPath(), "/")
if err != nil {
return nil, err
}
@ -107,8 +108,8 @@ func (s *Server) StopPodSandbox(ctx context.Context, req *pb.StopPodSandboxReque
}
}
}
if err := s.storageRuntimeServer.StopContainer(sb.id); err != nil && err != storage.ErrContainerUnknown {
logrus.Warnf("failed to stop sandbox container in pod sandbox %s: %v", sb.id, err)
if err := s.storageRuntimeServer.StopContainer(sb.ID()); err != nil && err != storage.ErrContainerUnknown {
logrus.Warnf("failed to stop sandbox container in pod sandbox %s: %v", sb.ID(), err)
}
resp := &pb.StopPodSandboxResponse{}
@ -119,12 +120,17 @@ func (s *Server) StopPodSandbox(ctx context.Context, req *pb.StopPodSandboxReque
// StopAllPodSandboxes removes all pod sandboxes
func (s *Server) StopAllPodSandboxes() {
logrus.Debugf("StopAllPodSandboxes")
for _, sb := range s.state.sandboxes {
sandboxes, err := s.state.GetAllSandboxes()
if err != nil {
logrus.Errorf("error retrieving sandboxes: %v", err)
return
}
for _, sb := range sandboxes {
pod := &pb.StopPodSandboxRequest{
PodSandboxId: sb.id,
PodSandboxId: sb.ID(),
}
if _, err := s.StopPodSandbox(nil, pod); err != nil {
logrus.Warnf("could not StopPodSandbox %s: %v", sb.id, err)
logrus.Warnf("could not StopPodSandbox %s: %v", sb.ID(), err)
}
}
}

303
server/server.go
View file

@ -14,14 +14,14 @@ import (
"github.com/containers/image/types"
sstorage "github.com/containers/storage"
"github.com/docker/docker/pkg/ioutils"
"github.com/docker/docker/pkg/registrar"
"github.com/docker/docker/pkg/truncindex"
"github.com/kubernetes-incubator/cri-o/oci"
"github.com/kubernetes-incubator/cri-o/pkg/annotations"
"github.com/kubernetes-incubator/cri-o/pkg/ocicni"
"github.com/kubernetes-incubator/cri-o/pkg/storage"
"github.com/kubernetes-incubator/cri-o/server/apparmor"
"github.com/kubernetes-incubator/cri-o/server/sandbox"
"github.com/kubernetes-incubator/cri-o/server/seccomp"
"github.com/kubernetes-incubator/cri-o/server/state"
rspec "github.com/opencontainers/runtime-spec/specs-go"
"github.com/opencontainers/selinux/go-selinux/label"
knet "k8s.io/apimachinery/pkg/util/net"
@ -57,15 +57,10 @@ type Server struct {
store sstorage.Store
storageImageServer storage.ImageServer
storageRuntimeServer storage.RuntimeServer
stateLock sync.Mutex
updateLock sync.RWMutex
state *serverState
state state.RuntimeStateStorer
netPlugin ocicni.CNIPlugin
hostportManager hostport.HostPortManager
podNameIndex *registrar.Registrar
podIDIndex *truncindex.TruncIndex
ctrNameIndex *registrar.Registrar
ctrIDIndex *truncindex.TruncIndex
imageContext *types.SystemContext
seccompEnabled bool
@ -106,23 +101,13 @@ func (s *Server) loadContainer(id string) error {
return err
}
name := m.Annotations[annotations.Name]
name, err = s.reserveContainerName(id, name)
if err != nil {
return err
}
defer func() {
if err != nil {
s.releaseContainerName(name)
}
}()
var metadata pb.ContainerMetadata
if err = json.Unmarshal([]byte(m.Annotations[annotations.Metadata]), &metadata); err != nil {
return err
}
sb := s.getSandbox(m.Annotations[annotations.SandboxID])
if sb == nil {
sb, err := s.getSandbox(m.Annotations[annotations.SandboxID])
if err != nil {
return fmt.Errorf("could not get sandbox with id %s, skipping", m.Annotations[annotations.SandboxID])
}
@ -158,15 +143,14 @@ func (s *Server) loadContainer(id string) error {
return err
}
ctr, err := oci.NewContainer(id, name, containerPath, m.Annotations[annotations.LogPath], sb.netNs(), labels, kubeAnnotations, img, &metadata, sb.id, tty, stdin, stdinOnce, sb.privileged, sb.trusted, containerDir, created, m.Annotations["org.opencontainers.image.stopSignal"])
ctr, err := oci.NewContainer(id, name, containerPath, m.Annotations[annotations.LogPath], sb.NetNs(), labels, kubeAnnotations, img, &metadata, sb.ID(), tty, stdin, stdinOnce, sb.Privileged(), sb.Trusted(), containerDir, created, m.Annotations["org.opencontainers.image.stopSignal"])
if err != nil {
return err
}
s.containerStateFromDisk(ctr)
s.addContainer(ctr)
return s.ctrIDIndex.Add(id)
return s.addContainer(ctr)
}
func (s *Server) containerStateFromDisk(c *oci.Container) error {
@ -224,15 +208,6 @@ func (s *Server) loadSandbox(id string) error {
return err
}
name := m.Annotations[annotations.Name]
name, err = s.reservePodName(id, name)
if err != nil {
return err
}
defer func() {
if err != nil {
s.releasePodName(name)
}
}()
var metadata pb.PodSandboxMetadata
if err = json.Unmarshal([]byte(m.Annotations[annotations.Metadata]), &metadata); err != nil {
return err
@ -248,49 +223,32 @@ func (s *Server) loadSandbox(id string) error {
return err
}
portMappings := []*hostport.PortMapping{}
if err = json.Unmarshal([]byte(m.Annotations[annotations.PortMappings]), &portMappings); err != nil {
return err
}
privileged := isTrue(m.Annotations[annotations.PrivilegedRuntime])
trusted := isTrue(m.Annotations[annotations.TrustedSandbox])
sb := &sandbox{
id: id,
name: name,
kubeName: m.Annotations[annotations.KubeName],
logDir: filepath.Dir(m.Annotations[annotations.LogPath]),
labels: labels,
containers: oci.NewMemoryStore(),
processLabel: processLabel,
mountLabel: mountLabel,
annotations: kubeAnnotations,
metadata: &metadata,
shmPath: m.Annotations[annotations.ShmPath],
privileged: privileged,
trusted: trusted,
resolvPath: m.Annotations[annotations.ResolvPath],
sb, err := sandbox.New(id, m.Annotations[annotations.Namespace], name, m.Annotations[annotations.KubeName], filepath.Dir(m.Annotations[annotations.LogPath]), labels, kubeAnnotations, processLabel, mountLabel, &metadata, m.Annotations[annotations.ShmPath], m.Linux.CgroupsPath, privileged, trusted, m.Annotations[annotations.ResolvPath], m.Annotations[annotations.HostName], portMappings)
if err != nil {
return err
}
// We add a netNS only if we can load a permanent one.
// Otherwise, the sandbox will live in the host namespace.
netNsPath, err := configNetNsPath(m)
if err == nil {
netNS, nsErr := netNsGet(netNsPath, sb.name)
// If we can't load the networking namespace
// because it's closed, we just set the sb netns
// pointer to nil. Otherwise we return an error.
if nsErr != nil && nsErr != errSandboxClosedNetNS {
return nsErr
// because it's closed, just leave the sandbox's netns pointer as nil
if nsErr := sb.NetNsJoin(netNsPath, sb.Name()); err != nil {
if nsErr != sandbox.ErrSandboxClosedNetNS {
return nsErr
}
}
sb.netns = netNS
}
s.addSandbox(sb)
defer func() {
if err != nil {
s.removeSandbox(sb.id)
}
}()
sandboxPath, err := s.store.ContainerRunDirectory(id)
if err != nil {
return err
@ -301,22 +259,12 @@ func (s *Server) loadSandbox(id string) error {
return err
}
cname, err := s.reserveContainerName(m.Annotations[annotations.ContainerID], m.Annotations[annotations.ContainerName])
if err != nil {
return err
}
defer func() {
if err != nil {
s.releaseContainerName(cname)
}
}()
created, err := time.Parse(time.RFC3339Nano, m.Annotations[annotations.Created])
if err != nil {
return err
}
scontainer, err := oci.NewContainer(m.Annotations[annotations.ContainerID], cname, sandboxPath, m.Annotations[annotations.LogPath], sb.netNs(), labels, kubeAnnotations, nil, nil, id, false, false, false, privileged, trusted, sandboxDir, created, m.Annotations["org.opencontainers.image.stopSignal"])
scontainer, err := oci.NewContainer(m.Annotations[annotations.ContainerID], m.Annotations[annotations.ContainerName], sandboxPath, m.Annotations[annotations.LogPath], sb.NetNs(), labels, kubeAnnotations, nil, nil, id, false, false, false, privileged, trusted, sandboxDir, created, m.Annotations["org.opencontainers.image.stopSignal"])
if err != nil {
return err
}
@ -326,14 +274,11 @@ func (s *Server) loadSandbox(id string) error {
if err = label.ReserveLabel(processLabel); err != nil {
return err
}
sb.infraContainer = scontainer
if err = s.ctrIDIndex.Add(scontainer.ID()); err != nil {
if err = sb.SetInfraContainer(scontainer); err != nil {
return err
}
if err = s.podIDIndex.Add(id); err != nil {
return err
}
return nil
return s.addSandbox(sb)
}
func (s *Server) restore() {
@ -399,7 +344,7 @@ func (s *Server) update() error {
oldPodContainers[container.ID] = container.ID
continue
}
if s.getContainer(container.ID) != nil {
if _, err := s.getContainer(container.ID); err == nil {
// FIXME: do we need to reload/update any info about the container?
oldPodContainers[container.ID] = container.ID
continue
@ -416,52 +361,54 @@ func (s *Server) update() error {
newPodContainers[container.ID] = &metadata
}
}
s.ctrIDIndex.Iterate(func(id string) {
if _, ok := oldPodContainers[id]; !ok {
// TODO this will not check pod infra containers - should we care about this?
stateContainers, err := s.state.GetAllContainers()
if err != nil {
return fmt.Errorf("error retrieving containers list: %v", err)
}
for _, ctr := range stateContainers {
if _, ok := oldPodContainers[ctr.ID()]; !ok {
// this container's ID wasn't in the updated list -> removed
removedPodContainers[id] = id
removedPodContainers[ctr.ID()] = ctr.ID()
}
})
}
for removedPodContainer := range removedPodContainers {
// forget this container
c := s.getContainer(removedPodContainer)
if c == nil {
c, err := s.getContainer(removedPodContainer)
if err != nil {
logrus.Warnf("bad state when getting container removed %+v", removedPodContainer)
continue
}
s.releaseContainerName(c.Name())
s.removeContainer(c)
if err = s.ctrIDIndex.Delete(c.ID()); err != nil {
return err
if err := s.removeContainer(c); err != nil {
return fmt.Errorf("error forgetting removed pod container %s: %v", c.ID(), err)
}
logrus.Debugf("forgetting removed pod container %s", c.ID())
}
s.podIDIndex.Iterate(func(id string) {
if _, ok := oldPods[id]; !ok {
pods, err := s.state.GetAllSandboxes()
if err != nil {
return fmt.Errorf("error retrieving pods list: %v", err)
}
for _, pod := range pods {
if _, ok := oldPods[pod.ID()]; !ok {
// this pod's ID wasn't in the updated list -> removed
removedPods[id] = id
removedPods[pod.ID()] = pod.ID()
}
})
}
for removedPod := range removedPods {
// forget this pod
sb := s.getSandbox(removedPod)
if sb == nil {
sb, err := s.getSandbox(removedPod)
if err != nil {
logrus.Warnf("bad state when getting pod to remove %+v", removedPod)
continue
}
podInfraContainer := sb.infraContainer
s.releaseContainerName(podInfraContainer.Name())
s.removeContainer(podInfraContainer)
if err = s.ctrIDIndex.Delete(podInfraContainer.ID()); err != nil {
return err
if err := s.removeSandbox(sb.ID()); err != nil {
return fmt.Errorf("error removing sandbox %s: %v", sb.ID(), err)
}
sb.infraContainer = nil
s.releasePodName(sb.name)
s.removeSandbox(sb.id)
if err = s.podIDIndex.Delete(sb.id); err != nil {
return err
}
logrus.Debugf("forgetting removed pod %s", sb.id)
logrus.Debugf("forgetting removed pod %s", sb.ID())
}
for sandboxID := range newPods {
// load this pod
@ -482,44 +429,6 @@ func (s *Server) update() error {
return nil
}
func (s *Server) reservePodName(id, name string) (string, error) {
if err := s.podNameIndex.Reserve(name, id); err != nil {
if err == registrar.ErrNameReserved {
id, err := s.podNameIndex.Get(name)
if err != nil {
logrus.Warnf("conflict, pod name %q already reserved", name)
return "", err
}
return "", fmt.Errorf("conflict, name %q already reserved for pod %q", name, id)
}
return "", fmt.Errorf("error reserving pod name %q", name)
}
return name, nil
}
func (s *Server) releasePodName(name string) {
s.podNameIndex.Release(name)
}
func (s *Server) reserveContainerName(id, name string) (string, error) {
if err := s.ctrNameIndex.Reserve(name, id); err != nil {
if err == registrar.ErrNameReserved {
id, err := s.ctrNameIndex.Get(name)
if err != nil {
logrus.Warnf("conflict, ctr name %q already reserved", name)
return "", err
}
return "", fmt.Errorf("conflict, name %q already reserved for ctr %q", name, id)
}
return "", fmt.Errorf("error reserving ctr name %s", name)
}
return name, nil
}
func (s *Server) releaseContainerName(name string) {
s.ctrNameIndex.Release(name)
}
// cleanupSandboxesOnShutdown Remove all running Sandboxes on system shutdown
func (s *Server) cleanupSandboxesOnShutdown() {
_, err := os.Stat(shutdownFile)
@ -575,8 +484,6 @@ func New(config *Config) (*Server, error) {
if err != nil {
return nil, err
}
sandboxes := make(map[string]*sandbox)
containers := oci.NewMemoryStore()
netPlugin, err := ocicni.InitCNI(config.NetworkDir, config.PluginDir)
if err != nil {
return nil, err
@ -592,13 +499,10 @@ func New(config *Config) (*Server, error) {
netPlugin: netPlugin,
hostportManager: hostportManager,
config: *config,
state: &serverState{
sandboxes: sandboxes,
containers: containers,
},
seccompEnabled: seccomp.IsEnabled(),
appArmorEnabled: apparmor.IsEnabled(),
appArmorProfile: config.ApparmorProfile,
state: state.NewInMemoryState(),
seccompEnabled: seccomp.IsEnabled(),
appArmorEnabled: apparmor.IsEnabled(),
appArmorProfile: config.ApparmorProfile,
}
if s.seccompEnabled {
seccompProfile, fileErr := ioutil.ReadFile(config.SeccompProfile)
@ -618,10 +522,6 @@ func New(config *Config) (*Server, error) {
}
}
s.podIDIndex = truncindex.NewTruncIndex([]string{})
s.podNameIndex = registrar.NewRegistrar()
s.ctrIDIndex = truncindex.NewTruncIndex([]string{})
s.ctrNameIndex = registrar.NewRegistrar()
s.imageContext = &types.SystemContext{
SignaturePolicyPath: config.ImageConfig.SignaturePolicyPath,
}
@ -656,73 +556,66 @@ func New(config *Config) (*Server, error) {
s.stream.streamServer.Start(true)
}()
logrus.Debugf("sandboxes: %v", s.state.sandboxes)
logrus.Debugf("containers: %v", s.state.containers)
return s, nil
}
type serverState struct {
sandboxes map[string]*sandbox
containers oci.ContainerStorer
func (s *Server) addSandbox(sb *sandbox.Sandbox) error {
return s.state.AddSandbox(sb)
}
func (s *Server) addSandbox(sb *sandbox) {
s.stateLock.Lock()
s.state.sandboxes[sb.id] = sb
s.stateLock.Unlock()
}
func (s *Server) getSandbox(id string) *sandbox {
s.stateLock.Lock()
sb := s.state.sandboxes[id]
s.stateLock.Unlock()
return sb
func (s *Server) getSandbox(id string) (*sandbox.Sandbox, error) {
return s.state.GetSandbox(id)
}
func (s *Server) hasSandbox(id string) bool {
s.stateLock.Lock()
_, ok := s.state.sandboxes[id]
s.stateLock.Unlock()
return ok
return s.state.HasSandbox(id)
}
func (s *Server) removeSandbox(id string) {
s.stateLock.Lock()
delete(s.state.sandboxes, id)
s.stateLock.Unlock()
func (s *Server) removeSandbox(id string) error {
return s.state.DeleteSandbox(id)
}
func (s *Server) addContainer(c *oci.Container) {
s.stateLock.Lock()
sandbox := s.state.sandboxes[c.Sandbox()]
// TODO(runcom): handle !ok above!!! otherwise it panics!
sandbox.addContainer(c)
s.state.containers.Add(c.ID(), c)
s.stateLock.Unlock()
func (s *Server) addContainer(c *oci.Container) error {
return s.state.AddContainer(c)
}
func (s *Server) getContainer(id string) *oci.Container {
s.stateLock.Lock()
c := s.state.containers.Get(id)
s.stateLock.Unlock()
return c
func (s *Server) getContainer(id string) (*oci.Container, error) {
sbID, err := s.state.GetContainerSandbox(id)
if err != nil {
return nil, err
}
return s.state.GetContainer(id, sbID)
}
// GetSandboxContainer returns the infra container for a given sandbox
func (s *Server) GetSandboxContainer(id string) *oci.Container {
sb := s.getSandbox(id)
return sb.infraContainer
func (s *Server) GetSandboxContainer(id string) (*oci.Container, error) {
sb, err := s.getSandbox(id)
if err != nil {
return nil, err
}
return sb.InfraContainer(), nil
}
// GetContainer returns a container by its ID
func (s *Server) GetContainer(id string) *oci.Container {
func (s *Server) GetContainer(id string) (*oci.Container, error) {
return s.getContainer(id)
}
func (s *Server) removeContainer(c *oci.Container) {
s.stateLock.Lock()
sandbox := s.state.sandboxes[c.Sandbox()]
sandbox.removeContainer(c)
s.state.containers.Delete(c.ID())
s.stateLock.Unlock()
func (s *Server) removeContainer(c *oci.Container) error {
return s.state.DeleteContainer(c.ID(), c.Sandbox())
}
func (s *Server) getPodSandboxFromRequest(podSandboxID string) (*sandbox.Sandbox, error) {
if podSandboxID == "" {
return nil, sandbox.ErrSandboxIDEmpty
}
sb, err := s.state.LookupSandboxByID(podSandboxID)
if err != nil {
return nil, err
}
return sb, nil
}

98
server/state/error.go Normal file
View file

@ -0,0 +1,98 @@
package state
import (
"fmt"
)
// NoSuchSandboxError is an error occurring when requested sandbox does not exist
type NoSuchSandboxError struct {
id string
name string
inner error
}
// Error produces an human-readable error message
func (e NoSuchSandboxError) Error() string {
if e.id != "" {
if e.inner == nil {
return fmt.Sprintf("no sandbox found with ID %s", e.id)
}
return fmt.Sprintf("no sandbox found with ID %s: %v", e.id, e.inner)
} else if e.name != "" {
if e.inner == nil {
return fmt.Sprintf("no sandbox found with name %s", e.name)
}
return fmt.Sprintf("no sandbox found with name %s: %v", e.name, e.inner)
} else if e.inner != nil {
return fmt.Sprintf("no such sandbox: %v", e.inner)
}
return "no such sandbox"
}
// NoSuchCtrError is an error occurring when requested container does not exist
type NoSuchCtrError struct {
id string
name string
sandbox string
inner error
}
// Error produces a human-readable error message
func (e NoSuchCtrError) Error() string {
if e.id != "" {
if e.sandbox != "" {
if e.inner == nil {
return fmt.Sprintf("no container found with ID %s in sandbox %s", e.id, e.sandbox)
}
return fmt.Sprintf("no container found with ID %s in sandbox %s: %v", e.id, e.sandbox, e.inner)
}
if e.inner == nil {
return fmt.Sprintf("no container found with ID %s", e.id)
}
return fmt.Sprintf("no container found with ID %s: %v", e.id, e.inner)
} else if e.name != "" {
if e.sandbox != "" {
if e.inner == nil {
return fmt.Sprintf("no container found with name %s in sandbox %s", e.name, e.sandbox)
}
return fmt.Sprintf("no container found with name %s in sandbox %s: %v", e.name, e.sandbox, e.inner)
}
if e.inner == nil {
return fmt.Sprintf("no container found with name %s", e.name)
}
return fmt.Sprintf("no container found with name %s: %v", e.name, e.inner)
} else if e.inner != nil {
return fmt.Sprintf("no such container: %v", e.inner)
}
return "no such container"
}
// Functions for verifying errors
// IsSandboxNotExist checks if an error indicated that given sandbox does not exist
func IsSandboxNotExist(err error) bool {
switch err.(type) {
case *NoSuchSandboxError:
return true
default:
return false
}
}
// IsCtrNotExist checks if an error indicates that given container does not exist
func IsCtrNotExist(err error) bool {
switch err.(type) {
case *NoSuchCtrError:
return true
default:
return false
}
}

469
server/state/in_memory_state.go Normal file
View file

@ -0,0 +1,469 @@
package state
import (
"fmt"
"sync"
"github.com/Sirupsen/logrus"
"github.com/docker/docker/pkg/registrar"
"github.com/docker/docker/pkg/truncindex"
"github.com/kubernetes-incubator/cri-o/oci"
"github.com/kubernetes-incubator/cri-o/server/sandbox"
)
// TODO: make operations atomic to greatest extent possible
// InMemoryState is an in-memory state store, suitable for use when no other
// programs are expected to interact with the server
type InMemoryState struct {
lock sync.Mutex
sandboxes map[string]*sandbox.Sandbox
containers map[string]*oci.Container
podNameIndex *registrar.Registrar
podIDIndex *truncindex.TruncIndex
ctrNameIndex *registrar.Registrar
ctrIDIndex *truncindex.TruncIndex
}
// NewInMemoryState creates a new, empty server state
func NewInMemoryState() RuntimeStateStorer {
state := new(InMemoryState)
state.sandboxes = make(map[string]*sandbox.Sandbox)
state.containers = make(map[string]*oci.Container)
state.podNameIndex = registrar.NewRegistrar()
state.podIDIndex = truncindex.NewTruncIndex([]string{})
state.ctrNameIndex = registrar.NewRegistrar()
state.ctrIDIndex = truncindex.NewTruncIndex([]string{})
return state
}
// AddSandbox adds a sandbox and any containers in it to the state
func (s *InMemoryState) AddSandbox(sandbox *sandbox.Sandbox) (err error) {
s.lock.Lock()
defer s.lock.Unlock()
if sandbox == nil {
return fmt.Errorf("nil passed as sandbox to AddSandbox")
}
// We shouldn't share ID with any containers
// Our pod infra container will share our ID and we don't want it to conflict with anything
// An equivalent check for sandbox IDs is done in addSandboxMappings()
if _, exist := s.containers[sandbox.ID()]; exist {
return fmt.Errorf("requested sandbox ID %v conflicts with existing container ID", sandbox.ID())
}
if err := s.addSandboxMappings(sandbox); err != nil {
return err
}
defer func() {
if err != nil {
if err2 := s.deleteSandboxMappings(sandbox); err2 != nil {
logrus.Errorf("Error removing mappings for incompletely-added sandbox %s: %v", sandbox.ID(), err2)
}
}
}()
// If there are containers in the sandbox add them to the mapping
for _, ctr := range sandbox.Containers() {
if err := s.addContainerMappings(ctr, true); err != nil {
return fmt.Errorf("error adding container %v mappings in sandbox %v", ctr.ID(), sandbox.ID())
}
defer func(c *oci.Container) {
if err != nil {
if err2 := s.deleteContainerMappings(c, true); err2 != nil {
logrus.Errorf("Error removing container %s mappings: %v", c.ID(), err2)
}
}
}(ctr)
}
// Add the pod infrastructure container to mappings
// TODO: Right now, we don't add it to the all containers listing. We may want to change this.
if err := s.addContainerMappings(sandbox.InfraContainer(), false); err != nil {
return fmt.Errorf("error adding infrastructure container %v to mappings: %v", sandbox.InfraContainer().ID(), err)
}
return nil
}
// Add sandbox name, ID to appropriate mappings
func (s *InMemoryState) addSandboxMappings(sb *sandbox.Sandbox) error {
if _, exist := s.sandboxes[sb.ID()]; exist {
return fmt.Errorf("sandbox with ID %s already exists in sandboxes map", sb.ID())
}
if err := s.podNameIndex.Reserve(sb.Name(), sb.ID()); err != nil {
return fmt.Errorf("error registering sandbox name: %v", err)
}
if err := s.podIDIndex.Add(sb.ID()); err != nil {
s.podNameIndex.Release(sb.Name())
return fmt.Errorf("error registering sandbox ID: %v", err)
}
s.sandboxes[sb.ID()] = sb
return nil
}
// HasSandbox determines if a given sandbox exists in the state
func (s *InMemoryState) HasSandbox(id string) bool {
s.lock.Lock()
defer s.lock.Unlock()
_, exist := s.sandboxes[id]
return exist
}
// DeleteSandbox removes a sandbox from the state
func (s *InMemoryState) DeleteSandbox(id string) (err error) {
s.lock.Lock()
defer s.lock.Unlock()
if _, exist := s.sandboxes[id]; !exist {
return &NoSuchSandboxError{id: id}
}
sb := s.sandboxes[id]
if err := s.deleteSandboxMappings(sb); err != nil {
return err
}
defer func() {
if err != nil {
if err2 := s.addSandboxMappings(sb); err2 != nil {
logrus.Errorf("Error re-adding sandbox mappings: %v", err2)
}
}
}()
// If there are containers left in the sandbox delete them from the mappings
for _, ctr := range sb.Containers() {
if err := s.deleteContainerMappings(ctr, true); err != nil {
return fmt.Errorf("error removing container %v mappings: %v", ctr.ID(), err)
}
defer func(c *oci.Container) {
if err != nil {
if err2 := s.addContainerMappings(c, true); err2 != nil {
logrus.Errorf("Error re-adding mappings for container %s: %v", c.ID(), err2)
}
}
}(ctr)
}
// Delete infra container from mappings
if err := s.deleteContainerMappings(sb.InfraContainer(), false); err != nil {
return fmt.Errorf("error removing infra container %v from mappings: %v", sb.InfraContainer().ID(), err)
}
return nil
}
// Remove sandbox name, ID to appropriate mappings
func (s *InMemoryState) deleteSandboxMappings(sb *sandbox.Sandbox) error {
if _, exist := s.sandboxes[sb.ID()]; !exist {
return fmt.Errorf("sandbox with ID %s does not exist in sandboxes map", sb.ID())
}
if err := s.podIDIndex.Delete(sb.ID()); err != nil {
return fmt.Errorf("error unregistering sandbox %s: %v", sb.ID(), err)
}
delete(s.sandboxes, sb.ID())
s.podNameIndex.Release(sb.Name())
return nil
}
// GetSandbox returns a sandbox given its full ID
func (s *InMemoryState) GetSandbox(id string) (*sandbox.Sandbox, error) {
s.lock.Lock()
defer s.lock.Unlock()
sandbox, ok := s.sandboxes[id]
if !ok {
return nil, &NoSuchSandboxError{id: id}
}
return sandbox, nil
}
// LookupSandboxByName returns a sandbox given its full or partial name
func (s *InMemoryState) LookupSandboxByName(name string) (*sandbox.Sandbox, error) {
s.lock.Lock()
defer s.lock.Unlock()
id, err := s.podNameIndex.Get(name)
if err != nil {
return nil, &NoSuchSandboxError{
name: name,
inner: err,
}
}
sandbox, ok := s.sandboxes[id]
if !ok {
// This should never happen - our internal state must be desynced
return nil, fmt.Errorf("cannot find sandbox %v in sandboxes map", id)
}
return sandbox, nil
}
// LookupSandboxByID returns a sandbox given its full or partial ID
// An error will be returned if the partial ID given is not unique
func (s *InMemoryState) LookupSandboxByID(id string) (*sandbox.Sandbox, error) {
s.lock.Lock()
defer s.lock.Unlock()
fullID, err := s.podIDIndex.Get(id)
if err != nil {
return nil, &NoSuchSandboxError{
id: id,
inner: err,
}
}
sandbox, ok := s.sandboxes[fullID]
if !ok {
// This should never happen, internal state must be desynced
return nil, fmt.Errorf("cannot find sandbox %v in sandboxes map", fullID)
}
return sandbox, nil
}
// GetAllSandboxes returns all sandboxes in the state
func (s *InMemoryState) GetAllSandboxes() ([]*sandbox.Sandbox, error) {
s.lock.Lock()
defer s.lock.Unlock()
sandboxes := make([]*sandbox.Sandbox, 0, len(s.sandboxes))
for _, sb := range s.sandboxes {
sandboxes = append(sandboxes, sb)
}
return sandboxes, nil
}
// AddContainer adds a single container to a given sandbox in the state
func (s *InMemoryState) AddContainer(c *oci.Container) error {
s.lock.Lock()
defer s.lock.Unlock()
if c == nil {
return fmt.Errorf("nil passed as container to AddContainer")
}
sandbox, ok := s.sandboxes[c.Sandbox()]
if !ok {
return &NoSuchSandboxError{id: c.Sandbox()}
}
if ctr := sandbox.GetContainer(c.ID()); ctr != nil {
return fmt.Errorf("container with ID %v already exists in sandbox %v", c.ID(), c.Sandbox())
}
if sandbox.InfraContainer().ID() == c.ID() {
return fmt.Errorf("container is infra container of sandbox %s, refusing to add to containers list", c.Sandbox())
}
if err := s.addContainerMappings(c, true); err != nil {
return err
}
sandbox.AddContainer(c)
return nil
}
// Add container ID, Name and Sandbox mappings
func (s *InMemoryState) addContainerMappings(c *oci.Container, addToContainers bool) error {
if addToContainers {
if _, exist := s.containers[c.ID()]; exist {
return fmt.Errorf("container with ID %v already exists in containers store", c.ID())
}
}
// TODO: if not a pod infra container, check if it conflicts with existing sandbox ID?
// Does this matter?
if err := s.ctrNameIndex.Reserve(c.Name(), c.ID()); err != nil {
return fmt.Errorf("error registering name for container %s: %v", c.ID(), err)
}
if err := s.ctrIDIndex.Add(c.ID()); err != nil {
s.ctrNameIndex.Release(c.ID())
return fmt.Errorf("error registering ID for container %s: %v", c.ID(), err)
}
if addToContainers {
s.containers[c.ID()] = c
}
return nil
}
// HasContainer checks if a container with the given ID exists in a given sandbox
func (s *InMemoryState) HasContainer(id, sandboxID string) bool {
s.lock.Lock()
defer s.lock.Unlock()
sandbox, ok := s.sandboxes[sandboxID]
if !ok {
return false
}
ctr := sandbox.GetContainer(id)
return ctr != nil
}
// DeleteContainer removes the container with given ID from the given sandbox
func (s *InMemoryState) DeleteContainer(id, sandboxID string) error {
s.lock.Lock()
defer s.lock.Unlock()
sandbox, ok := s.sandboxes[sandboxID]
if !ok {
return &NoSuchSandboxError{id: sandboxID}
}
ctr := sandbox.GetContainer(id)
if ctr == nil {
return &NoSuchCtrError{
id: id,
sandbox: id,
}
}
if err := s.deleteContainerMappings(ctr, true); err != nil {
return nil
}
sandbox.RemoveContainer(id)
return nil
}
// Deletes container from the ID and Name mappings and optionally from the global containers list
func (s *InMemoryState) deleteContainerMappings(ctr *oci.Container, deleteFromContainers bool) error {
if deleteFromContainers {
if _, exist := s.containers[ctr.ID()]; !exist {
return fmt.Errorf("container ID %v does not exist in containers store", ctr.ID())
}
}
if err := s.ctrIDIndex.Delete(ctr.ID()); err != nil {
return fmt.Errorf("error unregistering container ID for %s: %v", ctr.ID(), err)
}
s.ctrNameIndex.Release(ctr.Name())
if deleteFromContainers {
delete(s.containers, ctr.ID())
}
return nil
}
// GetContainer returns the container with given ID in the given sandbox
func (s *InMemoryState) GetContainer(id, sandboxID string) (*oci.Container, error) {
s.lock.Lock()
defer s.lock.Unlock()
return s.getContainerFromSandbox(id, sandboxID)
}
// GetContainerSandbox returns the ID of a container's sandbox from the full container ID
// May not find the ID of pod infrastructure containers
func (s *InMemoryState) GetContainerSandbox(id string) (string, error) {
s.lock.Lock()
defer s.lock.Unlock()
ctr, exist := s.containers[id]
if !exist {
return "", &NoSuchCtrError{id: id}
}
return ctr.Sandbox(), nil
}
// LookupContainerByName returns the full ID of a container given its full or partial name
func (s *InMemoryState) LookupContainerByName(name string) (*oci.Container, error) {
s.lock.Lock()
defer s.lock.Unlock()
fullID, err := s.ctrNameIndex.Get(name)
if err != nil {
return nil, &NoSuchCtrError{
name: name,
inner: err,
}
}
return s.getContainer(fullID)
}
// LookupContainerByID returns the full ID of a container given a full or partial ID
// If the given ID is not unique, an error is returned
func (s *InMemoryState) LookupContainerByID(id string) (*oci.Container, error) {
s.lock.Lock()
defer s.lock.Unlock()
fullID, err := s.ctrIDIndex.Get(id)
if err != nil {
return nil, &NoSuchCtrError{
id: id,
inner: err,
}
}
return s.getContainer(fullID)
}
// GetAllContainers returns all containers in the state, regardless of which sandbox they belong to
// Pod Infra containers are not included
func (s *InMemoryState) GetAllContainers() ([]*oci.Container, error) {
s.lock.Lock()
defer s.lock.Unlock()
containers := make([]*oci.Container, 0, len(s.containers))
for _, ctr := range s.containers {
containers = append(containers, ctr)
}
return containers, nil
}
// Returns a single container from any sandbox based on full ID
// TODO: is it worth making this public as an alternative to GetContainer
func (s *InMemoryState) getContainer(id string) (*oci.Container, error) {
ctr, exist := s.containers[id]
if !exist {
return nil, &NoSuchCtrError{id: id}
}
return s.getContainerFromSandbox(id, ctr.Sandbox())
}
// Returns a single container from a sandbox based on its full ID
// Internal implementation of GetContainer() but does not lock so it can be used in other functions
func (s *InMemoryState) getContainerFromSandbox(id, sandboxID string) (*oci.Container, error) {
sandbox, ok := s.sandboxes[sandboxID]
if !ok {
return nil, &NoSuchSandboxError{id: sandboxID}
}
ctr := sandbox.GetContainer(id)
if ctr == nil {
return nil, &NoSuchCtrError{
id: id,
sandbox: sandboxID,
}
}
return ctr, nil
}

29
server/state/state_store.go Normal file
View file

@ -0,0 +1,29 @@
package state
import (
"github.com/kubernetes-incubator/cri-o/oci"
"github.com/kubernetes-incubator/cri-o/server/sandbox"
)
// RuntimeStateStorer stores the state of the CRI-O server, including active pods and containers
type RuntimeStateStorer interface {
AddSandbox(s *sandbox.Sandbox) error
HasSandbox(id string) bool
DeleteSandbox(id string) error
// These should modify the associated sandbox without prompting
AddContainer(c *oci.Container) error
HasContainer(id, sandboxID string) bool
DeleteContainer(id, sandboxID string) error
// These two require full, explicit ID
GetSandbox(id string) (*sandbox.Sandbox, error)
GetContainer(id, sandboxID string) (*oci.Container, error)
// Get ID of sandbox container belongs to
GetContainerSandbox(id string) (string, error)
// Following 4 should accept partial names as long as they are globally unique
LookupSandboxByName(name string) (*sandbox.Sandbox, error)
LookupSandboxByID(id string) (*sandbox.Sandbox, error)
LookupContainerByName(name string) (*oci.Container, error)
LookupContainerByID(id string) (*oci.Container, error)
GetAllSandboxes() ([]*sandbox.Sandbox, error)
GetAllContainers() ([]*oci.Container, error)
}

2
test/ctr.bats
View file

@ -11,7 +11,7 @@ function teardown() {
run crioctl ctr status --id randomid
echo "$output"
[ "$status" -eq 1 ]
[[ "$output" =~ "container with ID starting with randomid not found" ]]
[[ "$output" =~ "container with ID starting with randomid could not be retrieved: no container found with ID randomid: ID does not exist" ]]
stop_crio
}

2
test/restore.bats
View file

@ -166,7 +166,7 @@ function teardown() {
run crioctl ctr stop --id "$ctr_id"
echo "$output"
[ "$status" -eq 1 ]
[[ "${output}" =~ "not found" ]]
[[ "${output}" =~ "no container found" ]]
cleanup_ctrs
cleanup_pods