Remove pre-existing state tracking in favor of new version

Signed-off-by: Matthew Heon <mheon@redhat.com>
This commit is contained in:
Matthew Heon 2017-04-06 13:26:48 -04:00
parent 1757f10c5b
commit 225f639a3f
12 changed files with 0 additions and 330 deletions

View file

@ -202,10 +202,6 @@ func ensureSaneLogPath(logPath string) error {
// CreateContainer creates a new container in specified PodSandbox // CreateContainer creates a new container in specified PodSandbox
func (s *Server) CreateContainer(ctx context.Context, req *pb.CreateContainerRequest) (res *pb.CreateContainerResponse, err error) { func (s *Server) CreateContainer(ctx context.Context, req *pb.CreateContainerRequest) (res *pb.CreateContainerResponse, err error) {
logrus.Debugf("CreateContainerRequest %+v", req) logrus.Debugf("CreateContainerRequest %+v", req)
s.Update()
s.updateLock.RLock()
defer s.updateLock.RUnlock()
sbID := req.PodSandboxId sbID := req.PodSandboxId
if sbID == "" { if sbID == "" {

View file

@ -29,7 +29,6 @@ func filterContainer(c *pb.Container, filter *pb.ContainerFilter) bool {
// ListContainers lists all containers by filters. // ListContainers lists all containers by filters.
func (s *Server) ListContainers(ctx context.Context, req *pb.ListContainersRequest) (*pb.ListContainersResponse, error) { func (s *Server) ListContainers(ctx context.Context, req *pb.ListContainersRequest) (*pb.ListContainersResponse, error) {
logrus.Debugf("ListContainersRequest %+v", req) logrus.Debugf("ListContainersRequest %+v", req)
s.Update()
var ctrs []*pb.Container var ctrs []*pb.Container
filter := req.Filter filter := req.Filter
ctrList, _ := s.state.GetAllContainers() ctrList, _ := s.state.GetAllContainers()

View file

@ -13,7 +13,6 @@ import (
// should be force removed. // should be force removed.
func (s *Server) RemoveContainer(ctx context.Context, req *pb.RemoveContainerRequest) (*pb.RemoveContainerResponse, error) { func (s *Server) RemoveContainer(ctx context.Context, req *pb.RemoveContainerRequest) (*pb.RemoveContainerResponse, error) {
logrus.Debugf("RemoveContainerRequest %+v", req) logrus.Debugf("RemoveContainerRequest %+v", req)
s.Update()
c, err := s.getContainerFromRequest(req.ContainerId) c, err := s.getContainerFromRequest(req.ContainerId)
if err != nil { if err != nil {
return nil, err return nil, err

View file

@ -11,7 +11,6 @@ import (
// StartContainer starts the container. // StartContainer starts the container.
func (s *Server) StartContainer(ctx context.Context, req *pb.StartContainerRequest) (*pb.StartContainerResponse, error) { func (s *Server) StartContainer(ctx context.Context, req *pb.StartContainerRequest) (*pb.StartContainerResponse, error) {
logrus.Debugf("StartContainerRequest %+v", req) logrus.Debugf("StartContainerRequest %+v", req)
s.Update()
c, err := s.getContainerFromRequest(req.ContainerId) c, err := s.getContainerFromRequest(req.ContainerId)
if err != nil { if err != nil {
return nil, err return nil, err

View file

@ -10,7 +10,6 @@ import (
// ContainerStatus returns status of the container. // ContainerStatus returns status of the container.
func (s *Server) ContainerStatus(ctx context.Context, req *pb.ContainerStatusRequest) (*pb.ContainerStatusResponse, error) { func (s *Server) ContainerStatus(ctx context.Context, req *pb.ContainerStatusRequest) (*pb.ContainerStatusResponse, error) {
logrus.Debugf("ContainerStatusRequest %+v", req) logrus.Debugf("ContainerStatusRequest %+v", req)
s.Update()
c, err := s.getContainerFromRequest(req.ContainerId) c, err := s.getContainerFromRequest(req.ContainerId)
if err != nil { if err != nil {
return nil, err return nil, err

View file

@ -12,7 +12,6 @@ import (
// StopContainer stops a running container with a grace period (i.e., timeout). // StopContainer stops a running container with a grace period (i.e., timeout).
func (s *Server) StopContainer(ctx context.Context, req *pb.StopContainerRequest) (*pb.StopContainerResponse, error) { func (s *Server) StopContainer(ctx context.Context, req *pb.StopContainerRequest) (*pb.StopContainerResponse, error) {
logrus.Debugf("StopContainerRequest %+v", req) logrus.Debugf("StopContainerRequest %+v", req)
s.Update()
c, err := s.getContainerFromRequest(req.ContainerId) c, err := s.getContainerFromRequest(req.ContainerId)
if err != nil { if err != nil {
return nil, err return nil, err

View file

@ -32,7 +32,6 @@ func filterSandbox(p *pb.PodSandbox, filter *pb.PodSandboxFilter) bool {
// ListPodSandbox returns a list of SandBoxes. // ListPodSandbox returns a list of SandBoxes.
func (s *Server) ListPodSandbox(ctx context.Context, req *pb.ListPodSandboxRequest) (*pb.ListPodSandboxResponse, error) { func (s *Server) ListPodSandbox(ctx context.Context, req *pb.ListPodSandboxRequest) (*pb.ListPodSandboxResponse, error) {
logrus.Debugf("ListPodSandboxRequest %+v", req) logrus.Debugf("ListPodSandboxRequest %+v", req)
s.Update()
var pods []*pb.PodSandbox var pods []*pb.PodSandbox
var podList []*sandbox.Sandbox var podList []*sandbox.Sandbox

View file

@ -16,7 +16,6 @@ import (
// sandbox, they should be force deleted. // sandbox, they should be force deleted.
func (s *Server) RemovePodSandbox(ctx context.Context, req *pb.RemovePodSandboxRequest) (*pb.RemovePodSandboxResponse, error) { func (s *Server) RemovePodSandbox(ctx context.Context, req *pb.RemovePodSandboxRequest) (*pb.RemovePodSandboxResponse, error) {
logrus.Debugf("RemovePodSandboxRequest %+v", req) logrus.Debugf("RemovePodSandboxRequest %+v", req)
s.Update()
sb, err := s.getPodSandboxFromRequest(req.PodSandboxId) sb, err := s.getPodSandboxFromRequest(req.PodSandboxId)
if err != nil { if err != nil {
if err == sandbox.ErrSandboxIDEmpty { if err == sandbox.ErrSandboxIDEmpty {

View file

@ -66,9 +66,6 @@ func (s *Server) runContainer(container *oci.Container, cgroupParent string) err
// RunPodSandbox creates and runs a pod-level sandbox. // RunPodSandbox creates and runs a pod-level sandbox.
func (s *Server) RunPodSandbox(ctx context.Context, req *pb.RunPodSandboxRequest) (resp *pb.RunPodSandboxResponse, err error) { func (s *Server) RunPodSandbox(ctx context.Context, req *pb.RunPodSandboxRequest) (resp *pb.RunPodSandboxResponse, err error) {
s.updateLock.RLock()
defer s.updateLock.RUnlock()
logrus.Debugf("RunPodSandboxRequest %+v", req) logrus.Debugf("RunPodSandboxRequest %+v", req)
var processLabel, mountLabel, netNsPath, resolvPath string var processLabel, mountLabel, netNsPath, resolvPath string
// process req.Name // process req.Name

View file

@ -10,7 +10,6 @@ import (
// PodSandboxStatus returns the Status of the PodSandbox. // PodSandboxStatus returns the Status of the PodSandbox.
func (s *Server) PodSandboxStatus(ctx context.Context, req *pb.PodSandboxStatusRequest) (*pb.PodSandboxStatusResponse, error) { func (s *Server) PodSandboxStatus(ctx context.Context, req *pb.PodSandboxStatusRequest) (*pb.PodSandboxStatusResponse, error) {
logrus.Debugf("PodSandboxStatusRequest %+v", req) logrus.Debugf("PodSandboxStatusRequest %+v", req)
s.Update()
sb, err := s.getPodSandboxFromRequest(req.PodSandboxId) sb, err := s.getPodSandboxFromRequest(req.PodSandboxId)
if err != nil { if err != nil {
return nil, err return nil, err

View file

@ -14,7 +14,6 @@ import (
// sandbox, they should be force terminated. // sandbox, they should be force terminated.
func (s *Server) StopPodSandbox(ctx context.Context, req *pb.StopPodSandboxRequest) (*pb.StopPodSandboxResponse, error) { func (s *Server) StopPodSandbox(ctx context.Context, req *pb.StopPodSandboxRequest) (*pb.StopPodSandboxResponse, error) {
logrus.Debugf("StopPodSandboxRequest %+v", req) logrus.Debugf("StopPodSandboxRequest %+v", req)
s.Update()
sb, err := s.getPodSandboxFromRequest(req.PodSandboxId) sb, err := s.getPodSandboxFromRequest(req.PodSandboxId)
if err != nil { if err != nil {
return nil, err return nil, err

View file

@ -4,11 +4,7 @@ import (
"encoding/json" "encoding/json"
"fmt" "fmt"
"io/ioutil" "io/ioutil"
"os"
"path/filepath"
"sync"
"github.com/Sirupsen/logrus"
"github.com/containers/image/types" "github.com/containers/image/types"
sstorage "github.com/containers/storage/storage" sstorage "github.com/containers/storage/storage"
"github.com/docker/docker/pkg/stringid" "github.com/docker/docker/pkg/stringid"
@ -19,9 +15,6 @@ import (
"github.com/kubernetes-incubator/cri-o/server/sandbox" "github.com/kubernetes-incubator/cri-o/server/sandbox"
"github.com/kubernetes-incubator/cri-o/server/seccomp" "github.com/kubernetes-incubator/cri-o/server/seccomp"
"github.com/kubernetes-incubator/cri-o/server/state" "github.com/kubernetes-incubator/cri-o/server/state"
rspec "github.com/opencontainers/runtime-spec/specs-go"
"github.com/opencontainers/selinux/go-selinux/label"
pb "k8s.io/kubernetes/pkg/kubelet/api/v1alpha1/runtime"
) )
const ( const (
@ -35,7 +28,6 @@ type Server struct {
store sstorage.Store store sstorage.Store
images storage.ImageServer images storage.ImageServer
storage storage.RuntimeServer storage storage.RuntimeServer
updateLock sync.RWMutex
state state.Store state state.Store
netPlugin ocicni.CNIPlugin netPlugin ocicni.CNIPlugin
imageContext *types.SystemContext imageContext *types.SystemContext
@ -47,310 +39,6 @@ type Server struct {
appArmorProfile string appArmorProfile string
} }
func (s *Server) loadContainer(id string) error {
config, err := s.store.GetFromContainerDirectory(id, "config.json")
if err != nil {
return err
}
var m rspec.Spec
if err = json.Unmarshal(config, &m); err != nil {
return err
}
labels := make(map[string]string)
if err = json.Unmarshal([]byte(m.Annotations["ocid/labels"]), &labels); err != nil {
return err
}
name := m.Annotations["ocid/name"]
var metadata pb.ContainerMetadata
if err = json.Unmarshal([]byte(m.Annotations["ocid/metadata"]), &metadata); err != nil {
return err
}
sb, err := s.getSandbox(m.Annotations["ocid/sandbox_id"])
if err != nil {
return fmt.Errorf("could not get sandbox with id %s, skipping", m.Annotations["ocid/sandbox_id"])
}
var tty bool
if v := m.Annotations["ocid/tty"]; v == "true" {
tty = true
}
containerPath, err := s.store.GetContainerRunDirectory(id)
if err != nil {
return err
}
var img *pb.ImageSpec
image, ok := m.Annotations["ocid/image"]
if ok {
img = &pb.ImageSpec{
Image: image,
}
}
annotations := make(map[string]string)
if err = json.Unmarshal([]byte(m.Annotations["ocid/annotations"]), &annotations); err != nil {
return err
}
ctr, err := oci.NewContainer(id, name, containerPath, m.Annotations["ocid/log_path"], sb.NetNs(), labels, annotations, img, &metadata, sb.ID(), tty, sb.Privileged())
if err != nil {
return err
}
if err = s.runtime.UpdateStatus(ctr); err != nil {
return fmt.Errorf("error updating status for container %s: %v", ctr.ID(), err)
}
if s.state.HasContainer(ctr.ID(), ctr.Sandbox()) {
logrus.Debugf("using on-disk version of container %s over version in state", ctr.ID())
if err := s.removeContainer(ctr); err != nil {
return fmt.Errorf("error updating container %s in state: %v", ctr.ID(), err)
}
}
return s.addContainer(ctr)
}
func configNetNsPath(spec rspec.Spec) (string, error) {
for _, ns := range spec.Linux.Namespaces {
if ns.Type != rspec.NetworkNamespace {
continue
}
if ns.Path == "" {
return "", fmt.Errorf("empty networking namespace")
}
return ns.Path, nil
}
return "", fmt.Errorf("missing networking namespace")
}
func (s *Server) loadSandbox(id string) error {
config, err := s.store.GetFromContainerDirectory(id, "config.json")
if err != nil {
return err
}
var m rspec.Spec
if err = json.Unmarshal(config, &m); err != nil {
return err
}
labels := make(map[string]string)
if err = json.Unmarshal([]byte(m.Annotations["ocid/labels"]), &labels); err != nil {
return err
}
name := m.Annotations["ocid/name"]
var metadata pb.PodSandboxMetadata
if err = json.Unmarshal([]byte(m.Annotations["ocid/metadata"]), &metadata); err != nil {
return err
}
processLabel, mountLabel, err := label.InitLabels(label.DupSecOpt(m.Process.SelinuxLabel))
if err != nil {
return err
}
annotations := make(map[string]string)
if err = json.Unmarshal([]byte(m.Annotations["ocid/annotations"]), &annotations); err != nil {
return err
}
privileged := m.Annotations["ocid/privileged_runtime"] == "true"
sb, err := sandbox.New(id, name, filepath.Dir(m.Annotations["ocid/log_path"]), labels, annotations, processLabel, mountLabel, &metadata, m.Annotations["ocid/shm_path"], *m.Linux.CgroupsPath, privileged, m.Annotations["ocid/resolv_path"], m.Annotations["ocid/hostname"])
if err != nil {
return err
}
// We add a netNS only if we can load a permanent one.
// Otherwise, the sandbox will live in the host namespace.
netNsPath, err := configNetNsPath(m)
if err == nil {
// If we can't load the networking namespace
// because it's closed, just leave the sandbox's netns pointer as nil
if nsErr := sb.NetNsJoin(netNsPath, sb.Name()); err != nil {
if nsErr != sandbox.ErrSandboxClosedNetNS {
return nsErr
}
}
}
sandboxPath, err := s.store.GetContainerRunDirectory(id)
if err != nil {
return err
}
scontainer, err := oci.NewContainer(m.Annotations["ocid/container_id"], m.Annotations["ocid/container_name"], sandboxPath, m.Annotations["ocid/log_path"], sb.NetNs(), labels, annotations, nil, nil, id, false, privileged)
if err != nil {
return err
}
if err = s.runtime.UpdateStatus(scontainer); err != nil {
return fmt.Errorf("error updating status for pod sandbox infra container %s: %v", scontainer.ID(), err)
}
if err = label.ReserveLabel(processLabel); err != nil {
return err
}
if err = sb.SetInfraContainer(scontainer); err != nil {
return err
}
if s.state.HasSandbox(sb.ID()) {
logrus.Debugf("using on-disk version of sandbox %s over version in state", sb.ID())
if err := s.removeSandbox(sb.ID()); err != nil {
return fmt.Errorf("error updating sandbox %s in state: %v", sb.ID(), err)
}
}
return s.addSandbox(sb)
}
func (s *Server) restore() {
containers, err := s.store.Containers()
if err != nil && !os.IsNotExist(err) {
logrus.Warnf("could not read containers and sandboxes: %v", err)
}
pods := map[string]*storage.RuntimeContainerMetadata{}
podContainers := map[string]*storage.RuntimeContainerMetadata{}
for _, container := range containers {
metadata, err2 := s.storage.GetContainerMetadata(container.ID)
if err2 != nil {
logrus.Warnf("error parsing metadata for %s: %v, ignoring", container.ID, err2)
continue
}
if metadata.Pod {
pods[container.ID] = &metadata
} else {
podContainers[container.ID] = &metadata
}
}
for containerID, metadata := range pods {
if err = s.loadSandbox(containerID); err != nil {
logrus.Warnf("could not restore sandbox %s container %s: %v", metadata.PodID, containerID, err)
}
}
for containerID := range podContainers {
if err := s.loadContainer(containerID); err != nil {
logrus.Warnf("could not restore container %s: %v", containerID, err)
}
}
}
// Update makes changes to the server's state (lists of pods and containers) to
// reflect the list of pods and containers that are stored on disk, possibly
// having been modified by other parties
func (s *Server) Update() {
logrus.Debugf("updating sandbox and container information")
if err := s.update(); err != nil {
logrus.Errorf("error updating sandbox and container information: %v", err)
}
}
func (s *Server) update() error {
s.updateLock.Lock()
defer s.updateLock.Unlock()
containers, err := s.store.Containers()
if err != nil && !os.IsNotExist(err) {
logrus.Warnf("could not read containers and sandboxes: %v", err)
return err
}
newPods := map[string]*storage.RuntimeContainerMetadata{}
oldPods := map[string]string{}
removedPods := map[string]string{}
newPodContainers := map[string]*storage.RuntimeContainerMetadata{}
oldPodContainers := map[string]string{}
removedPodContainers := map[string]string{}
for _, container := range containers {
if s.hasSandbox(container.ID) {
// FIXME: do we need to reload/update any info about the sandbox?
oldPods[container.ID] = container.ID
oldPodContainers[container.ID] = container.ID
continue
}
if _, err := s.getContainer(container.ID); err == nil {
// FIXME: do we need to reload/update any info about the container?
oldPodContainers[container.ID] = container.ID
continue
}
// not previously known, so figure out what it is
metadata, err2 := s.storage.GetContainerMetadata(container.ID)
if err2 != nil {
logrus.Errorf("error parsing metadata for %s: %v, ignoring", container.ID, err2)
continue
}
if metadata.Pod {
newPods[container.ID] = &metadata
} else {
newPodContainers[container.ID] = &metadata
}
}
// TODO this will not check pod infra containers - should we care about this?
stateContainers, err := s.state.GetAllContainers()
if err != nil {
return fmt.Errorf("error retrieving containers list: %v", err)
}
for _, ctr := range stateContainers {
if _, ok := oldPodContainers[ctr.ID()]; !ok {
// this container's ID wasn't in the updated list -> removed
removedPodContainers[ctr.ID()] = ctr.ID()
}
}
for removedPodContainer := range removedPodContainers {
// forget this container
c, err := s.getContainer(removedPodContainer)
if err != nil {
logrus.Warnf("bad state when getting container removed %+v", removedPodContainer)
continue
}
if err := s.removeContainer(c); err != nil {
return fmt.Errorf("error forgetting removed pod container %s: %v", c.ID(), err)
}
logrus.Debugf("forgetting removed pod container %s", c.ID())
}
pods, err := s.state.GetAllSandboxes()
if err != nil {
return fmt.Errorf("error retrieving pods list: %v", err)
}
for _, pod := range pods {
if _, ok := oldPods[pod.ID()]; !ok {
// this pod's ID wasn't in the updated list -> removed
removedPods[pod.ID()] = pod.ID()
}
}
for removedPod := range removedPods {
// forget this pod
sb, err := s.getSandbox(removedPod)
if err != nil {
logrus.Warnf("bad state when getting pod to remove %+v", removedPod)
continue
}
if err := s.removeSandbox(sb.ID()); err != nil {
return fmt.Errorf("error removing sandbox %s: %v", sb.ID(), err)
}
logrus.Debugf("forgetting removed pod %s", sb.ID())
}
for sandboxID := range newPods {
// load this pod
if err = s.loadSandbox(sandboxID); err != nil {
logrus.Warnf("could not load new pod sandbox %s: %v, ignoring", sandboxID, err)
} else {
logrus.Debugf("loaded new pod sandbox %s", sandboxID, err)
}
}
for containerID := range newPodContainers {
// load this container
if err = s.loadContainer(containerID); err != nil {
logrus.Warnf("could not load new sandbox container %s: %v, ignoring", containerID, err)
} else {
logrus.Debugf("loaded new pod container %s", containerID, err)
}
}
return nil
}
// Shutdown attempts to shut down the server's storage cleanly // Shutdown attempts to shut down the server's storage cleanly
func (s *Server) Shutdown() error { func (s *Server) Shutdown() error {
_, err := s.store.Shutdown(false) _, err := s.store.Shutdown(false)
@ -426,8 +114,6 @@ func New(config *Config) (*Server, error) {
SignaturePolicyPath: config.ImageConfig.SignaturePolicyPath, SignaturePolicyPath: config.ImageConfig.SignaturePolicyPath,
} }
s.restore()
return s, nil return s, nil
} }