From 80a789bce37d232feaf5dfc1c07a4a3bf33cb724 Mon Sep 17 00:00:00 2001 From: Antonio Murdaca Date: Thu, 11 May 2017 10:43:50 +0200 Subject: [PATCH 01/17] server: store sandbox creation time Signed-off-by: Antonio Murdaca --- server/sandbox.go | 2 ++ server/sandbox_run.go | 5 +++++ 2 files changed, 7 insertions(+) diff --git a/server/sandbox.go b/server/sandbox.go index 0f57f557..ffcef13f 100644 --- a/server/sandbox.go +++ b/server/sandbox.go @@ -7,6 +7,7 @@ import ( "os" "path/filepath" "sync" + "time" "github.com/Sirupsen/logrus" "github.com/containernetworking/cni/pkg/ns" @@ -145,6 +146,7 @@ type sandbox struct { privileged bool resolvPath string hostname string + created time.Time } const ( diff --git a/server/sandbox_run.go b/server/sandbox_run.go index 9fa6bc7e..fb0abe42 100644 --- a/server/sandbox_run.go +++ b/server/sandbox_run.go @@ -8,6 +8,7 @@ import ( "strconv" "strings" "syscall" + "time" "github.com/Sirupsen/logrus" "github.com/containers/storage" @@ -269,6 +270,9 @@ func (s *Server) RunPodSandbox(ctx context.Context, req *pb.RunPodSandboxRequest g.AddAnnotation("crio/hostname", hostname) g.AddAnnotation("crio/kube_name", kubeName) + created := time.Now() + g.AddAnnotation("ocid/created", created.Format(time.RFC3339Nano)) + sb := &sandbox{ id: id, namespace: namespace, @@ -285,6 +289,7 @@ func (s *Server) RunPodSandbox(ctx context.Context, req *pb.RunPodSandboxRequest privileged: privileged, resolvPath: resolvPath, hostname: hostname, + created: created, } defer func() { From 59e3f98b6dbfcdb8ae27950304d25ee7d9ce48a2 Mon Sep 17 00:00:00 2001 From: Antonio Murdaca Date: Thu, 11 May 2017 11:00:55 +0200 Subject: [PATCH 02/17] server: return sandbox's own created time Signed-off-by: Antonio Murdaca --- server/sandbox_status.go | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/server/sandbox_status.go b/server/sandbox_status.go index 15d35260..6be2ebb0 100644 --- a/server/sandbox_status.go +++ b/server/sandbox_status.go @@ -21,7 +21,6 @@ func (s *Server) PodSandboxStatus(ctx context.Context, req *pb.PodSandboxStatusR } cState := s.runtime.ContainerStatus(podInfraContainer) - created := cState.Created.UnixNano() netNsPath, err := podInfraContainer.NetNsPath() if err != nil { @@ -42,7 +41,7 @@ func (s *Server) PodSandboxStatus(ctx context.Context, req *pb.PodSandboxStatusR resp := &pb.PodSandboxStatusResponse{ Status: &pb.PodSandboxStatus{ Id: sandboxID, - CreatedAt: created, + CreatedAt: sb.created.UnixNano(), Linux: &pb.LinuxPodSandboxStatus{ Namespaces: &pb.Namespace{ Network: netNsPath, From 3bd4811b3b430b0e56f34c2426a052291b520668 Mon Sep 17 00:00:00 2001 From: Antonio Murdaca Date: Thu, 11 May 2017 11:04:43 +0200 Subject: [PATCH 03/17] server: restore sandbox created time from disk Signed-off-by: Antonio Murdaca --- server/sandbox_run.go | 2 +- server/server.go | 7 +++++++ 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/server/sandbox_run.go b/server/sandbox_run.go index fb0abe42..6359be17 100644 --- a/server/sandbox_run.go +++ b/server/sandbox_run.go @@ -271,7 +271,7 @@ func (s *Server) RunPodSandbox(ctx context.Context, req *pb.RunPodSandboxRequest g.AddAnnotation("crio/kube_name", kubeName) created := time.Now() - g.AddAnnotation("ocid/created", created.Format(time.RFC3339Nano)) + g.AddAnnotation("crio/created", created.Format(time.RFC3339Nano)) sb := &sandbox{ id: id, diff --git a/server/server.go b/server/server.go index 16643996..ce9564c1 100644 --- a/server/server.go +++ b/server/server.go @@ -7,6 +7,7 @@ import ( "os" "path/filepath" "sync" + "time" "github.com/Sirupsen/logrus" "github.com/containers/image/types" @@ -200,6 +201,11 @@ func (s *Server) loadSandbox(id string) error { privileged := m.Annotations["crio/privileged_runtime"] == "true" + created, err := time.Parse(time.RFC3339Nano, m.Annotations["crio/created"]) + if err != nil { + return err + } + sb := &sandbox{ id: id, name: name, @@ -214,6 +220,7 @@ func (s *Server) loadSandbox(id string) error { shmPath: m.Annotations["crio/shm_path"], privileged: privileged, resolvPath: m.Annotations["crio/resolv_path"], + created: created, } // We add a netNS only if we can load a permanent one. From 1f4a4742cb19c2398857a35d8d3a1130725dc993 Mon Sep 17 00:00:00 2001 From: Antonio Murdaca Date: Thu, 11 May 2017 11:12:37 +0200 Subject: [PATCH 04/17] oci: add container directory to Container struct Signed-off-by: Antonio Murdaca --- oci/container.go | 8 ++++++-- server/container_create.go | 2 +- server/sandbox_run.go | 2 +- server/server.go | 14 ++++++++++++-- 4 files changed, 20 insertions(+), 6 deletions(-) diff --git a/oci/container.go b/oci/container.go index 9ac624e4..ea8856c0 100644 --- a/oci/container.go +++ b/oci/container.go @@ -15,7 +15,6 @@ import ( type Container struct { id string name string - bundlePath string logPath string labels fields.Set annotations fields.Set @@ -27,6 +26,10 @@ type Container struct { state *ContainerState metadata *pb.ContainerMetadata opLock sync.Mutex + // this is the /var/run/storage/... directory, erased on reboot + bundlePath string + // this is the /var/lib/storage/... directory + dir string } // ContainerState represents the status of a container. @@ -39,7 +42,7 @@ type ContainerState struct { } // NewContainer creates a container object. -func NewContainer(id string, name string, bundlePath string, logPath string, netns ns.NetNS, labels map[string]string, annotations map[string]string, image *pb.ImageSpec, metadata *pb.ContainerMetadata, sandbox string, terminal bool, privileged bool) (*Container, error) { +func NewContainer(id string, name string, bundlePath string, logPath string, netns ns.NetNS, labels map[string]string, annotations map[string]string, image *pb.ImageSpec, metadata *pb.ContainerMetadata, sandbox string, terminal bool, privileged bool, dir string) (*Container, error) { c := &Container{ id: id, name: name, @@ -53,6 +56,7 @@ func NewContainer(id string, name string, bundlePath string, logPath string, net metadata: metadata, annotations: annotations, image: image, + dir: dir, } return c, nil } diff --git a/server/container_create.go b/server/container_create.go index d9f8dbf8..28eb3f75 100644 --- a/server/container_create.go +++ b/server/container_create.go @@ -648,7 +648,7 @@ func (s *Server) createSandboxContainer(ctx context.Context, containerID string, return nil, err } - container, err := oci.NewContainer(containerID, containerName, containerInfo.RunDir, logPath, sb.netNs(), labels, annotations, imageSpec, metadata, sb.id, containerConfig.Tty, sb.privileged) + container, err := oci.NewContainer(containerID, containerName, containerInfo.RunDir, logPath, sb.netNs(), labels, annotations, imageSpec, metadata, sb.id, containerConfig.Tty, sb.privileged, containerInfo.Dir) if err != nil { return nil, err } diff --git a/server/sandbox_run.go b/server/sandbox_run.go index 6359be17..77204a52 100644 --- a/server/sandbox_run.go +++ b/server/sandbox_run.go @@ -408,7 +408,7 @@ func (s *Server) RunPodSandbox(ctx context.Context, req *pb.RunPodSandboxRequest return nil, fmt.Errorf("failed to write runtime configuration for pod sandbox %s(%s): %v", sb.name, id, err) } - container, err := oci.NewContainer(id, containerName, podContainer.RunDir, logPath, sb.netNs(), labels, annotations, nil, nil, id, false, sb.privileged) + container, err := oci.NewContainer(id, containerName, podContainer.RunDir, logPath, sb.netNs(), labels, annotations, nil, nil, id, false, sb.privileged, podContainer.Dir) if err != nil { return nil, err } diff --git a/server/server.go b/server/server.go index ce9564c1..647f6892 100644 --- a/server/server.go +++ b/server/server.go @@ -121,6 +121,11 @@ func (s *Server) loadContainer(id string) error { return err } + containerDir, err := s.store.GetContainerDirectory(id) + if err != nil { + return err + } + var img *pb.ImageSpec image, ok := m.Annotations["crio/image"] if ok { @@ -134,7 +139,7 @@ func (s *Server) loadContainer(id string) error { return err } - ctr, err := oci.NewContainer(id, name, containerPath, m.Annotations["crio/log_path"], sb.netNs(), labels, annotations, img, &metadata, sb.id, tty, sb.privileged) + ctr, err := oci.NewContainer(id, name, containerPath, m.Annotations["crio/log_path"], sb.netNs(), labels, annotations, img, &metadata, sb.id, tty, sb.privileged, containerDir) if err != nil { return err } @@ -251,6 +256,11 @@ func (s *Server) loadSandbox(id string) error { return err } + sandboxDir, err := s.store.GetContainerDirectory(id) + if err != nil { + return err + } + cname, err := s.reserveContainerName(m.Annotations["crio/container_id"], m.Annotations["crio/container_name"]) if err != nil { return err @@ -261,7 +271,7 @@ func (s *Server) loadSandbox(id string) error { } }() - scontainer, err := oci.NewContainer(m.Annotations["crio/container_id"], cname, sandboxPath, m.Annotations["crio/log_path"], sb.netNs(), labels, annotations, nil, nil, id, false, privileged) + scontainer, err := oci.NewContainer(m.Annotations["crio/container_id"], cname, sandboxPath, m.Annotations["crio/log_path"], sb.netNs(), labels, annotations, nil, nil, id, false, privileged, sandboxDir) if err != nil { return err } From 790c6d891a4a9c05074b49a18f8fa39394c235f9 Mon Sep 17 00:00:00 2001 From: Antonio Murdaca Date: Thu, 11 May 2017 11:22:47 +0200 Subject: [PATCH 05/17] server: store creation in containers Signed-off-by: Antonio Murdaca --- oci/container.go | 10 +++++++++- server/container_create.go | 6 +++++- server/sandbox.go | 2 -- server/sandbox_run.go | 3 +-- server/sandbox_status.go | 2 +- server/server.go | 20 ++++++++++++-------- 6 files changed, 28 insertions(+), 15 deletions(-) diff --git a/oci/container.go b/oci/container.go index ea8856c0..cd020fcb 100644 --- a/oci/container.go +++ b/oci/container.go @@ -42,7 +42,9 @@ type ContainerState struct { } // NewContainer creates a container object. -func NewContainer(id string, name string, bundlePath string, logPath string, netns ns.NetNS, labels map[string]string, annotations map[string]string, image *pb.ImageSpec, metadata *pb.ContainerMetadata, sandbox string, terminal bool, privileged bool, dir string) (*Container, error) { +func NewContainer(id string, name string, bundlePath string, logPath string, netns ns.NetNS, labels map[string]string, annotations map[string]string, image *pb.ImageSpec, metadata *pb.ContainerMetadata, sandbox string, terminal bool, privileged bool, dir string, created time.Time) (*Container, error) { + state := &ContainerState{} + state.Created = created c := &Container{ id: id, name: name, @@ -57,10 +59,16 @@ func NewContainer(id string, name string, bundlePath string, logPath string, net annotations: annotations, image: image, dir: dir, + state: state, } return c, nil } +// CreatedAt returns the container creation time +func (c *Container) CreatedAt() time.Time { + return c.state.Created +} + // Name returns the name of the container. func (c *Container) Name() string { return c.name diff --git a/server/container_create.go b/server/container_create.go index 28eb3f75..f4037a51 100644 --- a/server/container_create.go +++ b/server/container_create.go @@ -10,6 +10,7 @@ import ( "strconv" "strings" "syscall" + "time" "github.com/Sirupsen/logrus" "github.com/docker/docker/pkg/stringid" @@ -540,6 +541,9 @@ func (s *Server) createSandboxContainer(ctx context.Context, containerID string, specgen.AddAnnotation("crio/tty", fmt.Sprintf("%v", containerConfig.Tty)) specgen.AddAnnotation("crio/image", image) + created := time.Now() + specgen.AddAnnotation("crio/created", created.Format(time.RFC3339Nano)) + metadataJSON, err := json.Marshal(metadata) if err != nil { return nil, err @@ -648,7 +652,7 @@ func (s *Server) createSandboxContainer(ctx context.Context, containerID string, return nil, err } - container, err := oci.NewContainer(containerID, containerName, containerInfo.RunDir, logPath, sb.netNs(), labels, annotations, imageSpec, metadata, sb.id, containerConfig.Tty, sb.privileged, containerInfo.Dir) + container, err := oci.NewContainer(containerID, containerName, containerInfo.RunDir, logPath, sb.netNs(), labels, annotations, imageSpec, metadata, sb.id, containerConfig.Tty, sb.privileged, containerInfo.Dir, created) if err != nil { return nil, err } diff --git a/server/sandbox.go b/server/sandbox.go index ffcef13f..0f57f557 100644 --- a/server/sandbox.go +++ b/server/sandbox.go @@ -7,7 +7,6 @@ import ( "os" "path/filepath" "sync" - "time" "github.com/Sirupsen/logrus" "github.com/containernetworking/cni/pkg/ns" @@ -146,7 +145,6 @@ type sandbox struct { privileged bool resolvPath string hostname string - created time.Time } const ( diff --git a/server/sandbox_run.go b/server/sandbox_run.go index 77204a52..bb7e41ec 100644 --- a/server/sandbox_run.go +++ b/server/sandbox_run.go @@ -289,7 +289,6 @@ func (s *Server) RunPodSandbox(ctx context.Context, req *pb.RunPodSandboxRequest privileged: privileged, resolvPath: resolvPath, hostname: hostname, - created: created, } defer func() { @@ -408,7 +407,7 @@ func (s *Server) RunPodSandbox(ctx context.Context, req *pb.RunPodSandboxRequest return nil, fmt.Errorf("failed to write runtime configuration for pod sandbox %s(%s): %v", sb.name, id, err) } - container, err := oci.NewContainer(id, containerName, podContainer.RunDir, logPath, sb.netNs(), labels, annotations, nil, nil, id, false, sb.privileged, podContainer.Dir) + container, err := oci.NewContainer(id, containerName, podContainer.RunDir, logPath, sb.netNs(), labels, annotations, nil, nil, id, false, sb.privileged, podContainer.Dir, created) if err != nil { return nil, err } diff --git a/server/sandbox_status.go b/server/sandbox_status.go index 6be2ebb0..9a8c864f 100644 --- a/server/sandbox_status.go +++ b/server/sandbox_status.go @@ -41,7 +41,7 @@ func (s *Server) PodSandboxStatus(ctx context.Context, req *pb.PodSandboxStatusR resp := &pb.PodSandboxStatusResponse{ Status: &pb.PodSandboxStatus{ Id: sandboxID, - CreatedAt: sb.created.UnixNano(), + CreatedAt: podInfraContainer.CreatedAt().UnixNano(), Linux: &pb.LinuxPodSandboxStatus{ Namespaces: &pb.Namespace{ Network: netNsPath, diff --git a/server/server.go b/server/server.go index 647f6892..e2f9e4df 100644 --- a/server/server.go +++ b/server/server.go @@ -139,7 +139,12 @@ func (s *Server) loadContainer(id string) error { return err } - ctr, err := oci.NewContainer(id, name, containerPath, m.Annotations["crio/log_path"], sb.netNs(), labels, annotations, img, &metadata, sb.id, tty, sb.privileged, containerDir) + created, err := time.Parse(time.RFC3339Nano, m.Annotations["crio/created"]) + if err != nil { + return err + } + + ctr, err := oci.NewContainer(id, name, containerPath, m.Annotations["crio/log_path"], sb.netNs(), labels, annotations, img, &metadata, sb.id, tty, sb.privileged, containerDir, created) if err != nil { return err } @@ -206,11 +211,6 @@ func (s *Server) loadSandbox(id string) error { privileged := m.Annotations["crio/privileged_runtime"] == "true" - created, err := time.Parse(time.RFC3339Nano, m.Annotations["crio/created"]) - if err != nil { - return err - } - sb := &sandbox{ id: id, name: name, @@ -225,7 +225,6 @@ func (s *Server) loadSandbox(id string) error { shmPath: m.Annotations["crio/shm_path"], privileged: privileged, resolvPath: m.Annotations["crio/resolv_path"], - created: created, } // We add a netNS only if we can load a permanent one. @@ -271,7 +270,12 @@ func (s *Server) loadSandbox(id string) error { } }() - scontainer, err := oci.NewContainer(m.Annotations["crio/container_id"], cname, sandboxPath, m.Annotations["crio/log_path"], sb.netNs(), labels, annotations, nil, nil, id, false, privileged, sandboxDir) + created, err := time.Parse(time.RFC3339Nano, m.Annotations["crio/created"]) + if err != nil { + return err + } + + scontainer, err := oci.NewContainer(m.Annotations["crio/container_id"], cname, sandboxPath, m.Annotations["crio/log_path"], sb.netNs(), labels, annotations, nil, nil, id, false, privileged, sandboxDir, created) if err != nil { return err } From fbc5e49a60c49b28e9cde012eab5c8549d76b0c5 Mon Sep 17 00:00:00 2001 From: Antonio Murdaca Date: Thu, 11 May 2017 11:35:31 +0200 Subject: [PATCH 06/17] oci: save container's finished time Signed-off-by: Antonio Murdaca --- oci/oci.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/oci/oci.go b/oci/oci.go index 913f1211..2e4d1644 100644 --- a/oci/oci.go +++ b/oci/oci.go @@ -501,6 +501,8 @@ func (r *Runtime) StopContainer(c *Container, timeout int64) error { } } + c.state.Finished = time.Now() + return nil } From 2ddc062bbe3d6fcdba390efcbcb2117982649d9f Mon Sep 17 00:00:00 2001 From: Antonio Murdaca Date: Thu, 11 May 2017 11:45:31 +0200 Subject: [PATCH 07/17] oci: ignore non existing containers on delete Signed-off-by: Antonio Murdaca --- Dockerfile | 2 +- oci/oci.go | 3 ++- utils/utils.go | 2 +- 3 files changed, 4 insertions(+), 3 deletions(-) diff --git a/Dockerfile b/Dockerfile index f3495a59..ff14ef7b 100644 --- a/Dockerfile +++ b/Dockerfile @@ -44,7 +44,7 @@ RUN mkdir -p /usr/src/criu \ && rm -rf /usr/src/criu # Install runc -ENV RUNC_COMMIT v1.0.0-rc3 +ENV RUNC_COMMIT 639454475cb9c8b861cc599f8bcd5c8c790ae402 RUN set -x \ && export GOPATH="$(mktemp -d)" \ && git clone https://github.com/opencontainers/runc.git "$GOPATH/src/github.com/opencontainers/runc" \ diff --git a/oci/oci.go b/oci/oci.go index 2e4d1644..b04c60bd 100644 --- a/oci/oci.go +++ b/oci/oci.go @@ -510,7 +510,8 @@ func (r *Runtime) StopContainer(c *Container, timeout int64) error { func (r *Runtime) DeleteContainer(c *Container) error { c.opLock.Lock() defer c.opLock.Unlock() - return utils.ExecCmdWithStdStreams(os.Stdin, os.Stdout, os.Stderr, r.Path(c), "delete", c.name) + _, err := utils.ExecCmd(r.Path(c), "delete", "--force", c.name) + return err } // UpdateStatus refreshes the status of the container. diff --git a/utils/utils.go b/utils/utils.go index 340e1ba9..1635e4b4 100644 --- a/utils/utils.go +++ b/utils/utils.go @@ -23,7 +23,7 @@ func ExecCmd(name string, args ...string) (string, error) { err := cmd.Run() if err != nil { - return "", fmt.Errorf("`%v %v` failed: %v (%v)", name, strings.Join(args, " "), stderr.String(), err) + return "", fmt.Errorf("`%v %v` failed: %v %v (%v)", name, strings.Join(args, " "), stderr.String(), stdout.String(), err) } return stdout.String(), nil From da0b8a6157bce748192cc5b511fa03ef392813a4 Mon Sep 17 00:00:00 2001 From: Antonio Murdaca Date: Thu, 11 May 2017 12:03:59 +0200 Subject: [PATCH 08/17] server: store containers state on disk Signed-off-by: Antonio Murdaca --- oci/container.go | 6 ++++++ server/container_create.go | 2 ++ server/container_start.go | 2 ++ server/container_stop.go | 2 ++ server/sandbox_run.go | 2 ++ server/sandbox_stop.go | 1 + server/server.go | 15 +++++++++++++++ 7 files changed, 30 insertions(+) diff --git a/oci/container.go b/oci/container.go index cd020fcb..7a5ba81e 100644 --- a/oci/container.go +++ b/oci/container.go @@ -2,6 +2,7 @@ package oci import ( "fmt" + "path/filepath" "sync" "time" @@ -64,6 +65,11 @@ func NewContainer(id string, name string, bundlePath string, logPath string, net return c, nil } +// StatePath returns the containers state.json path +func (c *Container) StatePath() string { + return filepath.Join(c.dir, "state.json") +} + // CreatedAt returns the container creation time func (c *Container) CreatedAt() time.Time { return c.state.Created diff --git a/server/container_create.go b/server/container_create.go index f4037a51..c15985e9 100644 --- a/server/container_create.go +++ b/server/container_create.go @@ -310,6 +310,8 @@ func (s *Server) CreateContainer(ctx context.Context, req *pb.CreateContainerReq return nil, err } + s.containerStateToDisk(container) + resp := &pb.CreateContainerResponse{ ContainerId: containerID, } diff --git a/server/container_start.go b/server/container_start.go index 128cc5fc..a426def9 100644 --- a/server/container_start.go +++ b/server/container_start.go @@ -20,6 +20,8 @@ func (s *Server) StartContainer(ctx context.Context, req *pb.StartContainerReque return nil, fmt.Errorf("failed to start container %s: %v", c.ID(), err) } + s.containerStateToDisk(c) + resp := &pb.StartContainerResponse{} logrus.Debugf("StartContainerResponse %+v", resp) return resp, nil diff --git a/server/container_stop.go b/server/container_stop.go index 6503b3fa..5ea53bf7 100644 --- a/server/container_stop.go +++ b/server/container_stop.go @@ -27,6 +27,8 @@ func (s *Server) StopContainer(ctx context.Context, req *pb.StopContainerRequest } } + s.containerStateToDisk(c) + resp := &pb.StopContainerResponse{} logrus.Debugf("StopContainerResponse: %+v", resp) return resp, nil diff --git a/server/sandbox_run.go b/server/sandbox_run.go index bb7e41ec..71662c9e 100644 --- a/server/sandbox_run.go +++ b/server/sandbox_run.go @@ -425,6 +425,8 @@ func (s *Server) RunPodSandbox(ctx context.Context, req *pb.RunPodSandboxRequest return nil, err } + s.containerStateToDisk(container) + resp = &pb.RunPodSandboxResponse{PodSandboxId: id} logrus.Debugf("RunPodSandboxResponse: %+v", resp) return resp, nil diff --git a/server/sandbox_stop.go b/server/sandbox_stop.go index 76a0cadf..b932afb2 100644 --- a/server/sandbox_stop.go +++ b/server/sandbox_stop.go @@ -52,6 +52,7 @@ func (s *Server) StopPodSandbox(ctx context.Context, req *pb.StopPodSandboxReque return nil, fmt.Errorf("failed to stop container %s in pod sandbox %s: %v", c.Name(), sb.id, err) } } + s.containerStateToDisk(c) } resp := &pb.StopPodSandboxResponse{} diff --git a/server/server.go b/server/server.go index e2f9e4df..2b0db7af 100644 --- a/server/server.go +++ b/server/server.go @@ -19,6 +19,7 @@ import ( "github.com/kubernetes-incubator/cri-o/pkg/storage" "github.com/kubernetes-incubator/cri-o/server/apparmor" "github.com/kubernetes-incubator/cri-o/server/seccomp" + "github.com/moby/moby/pkg/ioutils" rspec "github.com/opencontainers/runtime-spec/specs-go" "github.com/opencontainers/selinux/go-selinux/label" pb "k8s.io/kubernetes/pkg/kubelet/api/v1alpha1/runtime" @@ -155,6 +156,20 @@ func (s *Server) loadContainer(id string) error { return s.ctrIDIndex.Add(id) } +func (s *Server) containerStateToDisk(c *oci.Container) error { + // ignore errors, this is a best effort to have up-to-date info about + // a given container before its state gets stored + s.runtime.UpdateStatus(c) + + jsonSource, err := ioutils.NewAtomicFileWriter(c.StatePath(), 0644) + if err != nil { + return err + } + defer jsonSource.Close() + enc := json.NewEncoder(jsonSource) + return enc.Encode(s.runtime.ContainerStatus(c)) +} + func configNetNsPath(spec rspec.Spec) (string, error) { for _, ns := range spec.Linux.Namespaces { if ns.Type != rspec.NetworkNamespace { From a41ca975c1eb1a9ab526d635421416fb208cfae2 Mon Sep 17 00:00:00 2001 From: Antonio Murdaca Date: Thu, 11 May 2017 12:10:18 +0200 Subject: [PATCH 09/17] server: restore containers state from disk on startup Signed-off-by: Antonio Murdaca --- oci/container.go | 14 ++++++++++++++ server/server.go | 29 ++++++++++++++++++++--------- 2 files changed, 34 insertions(+), 9 deletions(-) diff --git a/oci/container.go b/oci/container.go index 7a5ba81e..4717b9ea 100644 --- a/oci/container.go +++ b/oci/container.go @@ -1,7 +1,9 @@ package oci import ( + "encoding/json" "fmt" + "os" "path/filepath" "sync" "time" @@ -65,6 +67,18 @@ func NewContainer(id string, name string, bundlePath string, logPath string, net return c, nil } +// FromDisk restores container's state from disk +func (c *Container) FromDisk() error { + jsonSource, err := os.Open(c.StatePath()) + if err != nil { + return err + } + defer jsonSource.Close() + + dec := json.NewDecoder(jsonSource) + return dec.Decode(c.state) +} + // StatePath returns the containers state.json path func (c *Container) StatePath() string { return filepath.Join(c.dir, "state.json") diff --git a/server/server.go b/server/server.go index 2b0db7af..533149ae 100644 --- a/server/server.go +++ b/server/server.go @@ -12,6 +12,7 @@ import ( "github.com/Sirupsen/logrus" "github.com/containers/image/types" sstorage "github.com/containers/storage" + "github.com/docker/docker/pkg/ioutils" "github.com/docker/docker/pkg/registrar" "github.com/docker/docker/pkg/truncindex" "github.com/kubernetes-incubator/cri-o/oci" @@ -19,7 +20,6 @@ import ( "github.com/kubernetes-incubator/cri-o/pkg/storage" "github.com/kubernetes-incubator/cri-o/server/apparmor" "github.com/kubernetes-incubator/cri-o/server/seccomp" - "github.com/moby/moby/pkg/ioutils" rspec "github.com/opencontainers/runtime-spec/specs-go" "github.com/opencontainers/selinux/go-selinux/label" pb "k8s.io/kubernetes/pkg/kubelet/api/v1alpha1/runtime" @@ -122,7 +122,7 @@ func (s *Server) loadContainer(id string) error { return err } - containerDir, err := s.store.GetContainerDirectory(id) + containerDir, err := s.store.ContainerDirectory(id) if err != nil { return err } @@ -149,13 +149,24 @@ func (s *Server) loadContainer(id string) error { if err != nil { return err } - if err = s.runtime.UpdateStatus(ctr); err != nil { - return fmt.Errorf("error updating status for container %s: %v", ctr.ID(), err) - } + + s.containerStateFromDisk(ctr) + s.addContainer(ctr) return s.ctrIDIndex.Add(id) } +func (s *Server) containerStateFromDisk(c *oci.Container) error { + if err := c.FromDisk(); err != nil { + return err + } + // ignore errors, this is a best effort to have up-to-date info about + // a given container before its state gets stored + s.runtime.UpdateStatus(c) + + return nil +} + func (s *Server) containerStateToDisk(c *oci.Container) error { // ignore errors, this is a best effort to have up-to-date info about // a given container before its state gets stored @@ -270,7 +281,7 @@ func (s *Server) loadSandbox(id string) error { return err } - sandboxDir, err := s.store.GetContainerDirectory(id) + sandboxDir, err := s.store.ContainerDirectory(id) if err != nil { return err } @@ -294,9 +305,9 @@ func (s *Server) loadSandbox(id string) error { if err != nil { return err } - if err = s.runtime.UpdateStatus(scontainer); err != nil { - return fmt.Errorf("error updating status for pod sandbox infra container %s: %v", scontainer.ID(), err) - } + + s.containerStateFromDisk(scontainer) + if err = label.ReserveLabel(processLabel); err != nil { return err } From 358dac96d45ac3d7404259c313105e88b367c932 Mon Sep 17 00:00:00 2001 From: Antonio Murdaca Date: Thu, 11 May 2017 12:41:08 +0200 Subject: [PATCH 10/17] server: ignore runc not exist errors Signed-off-by: Antonio Murdaca --- oci/oci.go | 3 +++ server/container_status.go | 1 + server/sandbox_list.go | 1 + server/sandbox_status.go | 1 + 4 files changed, 6 insertions(+) diff --git a/oci/oci.go b/oci/oci.go index b04c60bd..bd3cac3b 100644 --- a/oci/oci.go +++ b/oci/oci.go @@ -520,6 +520,9 @@ func (r *Runtime) UpdateStatus(c *Container) error { defer c.opLock.Unlock() out, err := exec.Command(r.Path(c), "state", c.name).CombinedOutput() if err != nil { + if err := unix.Kill(c.state.Pid, 0); err == syscall.ESRCH { + return nil + } return fmt.Errorf("error getting container state for %s: %s: %q", c.name, err, out) } if err := json.NewDecoder(bytes.NewBuffer(out)).Decode(&c.state); err != nil { diff --git a/server/container_status.go b/server/container_status.go index d946e9ae..febe7060 100644 --- a/server/container_status.go +++ b/server/container_status.go @@ -21,6 +21,7 @@ func (s *Server) ContainerStatus(ctx context.Context, req *pb.ContainerStatusReq if err = s.runtime.UpdateStatus(c); err != nil { return nil, err } + s.containerStateToDisk(c) containerID := c.ID() image := c.Image() diff --git a/server/sandbox_list.go b/server/sandbox_list.go index 9e4b3562..155c3d59 100644 --- a/server/sandbox_list.go +++ b/server/sandbox_list.go @@ -62,6 +62,7 @@ func (s *Server) ListPodSandbox(ctx context.Context, req *pb.ListPodSandboxReque if err := s.runtime.UpdateStatus(podInfraContainer); err != nil { return nil, err } + cState := s.runtime.ContainerStatus(podInfraContainer) created := cState.Created.UnixNano() rStatus := pb.PodSandboxState_SANDBOX_NOTREADY diff --git a/server/sandbox_status.go b/server/sandbox_status.go index 9a8c864f..d8e2453f 100644 --- a/server/sandbox_status.go +++ b/server/sandbox_status.go @@ -19,6 +19,7 @@ func (s *Server) PodSandboxStatus(ctx context.Context, req *pb.PodSandboxStatusR if err = s.runtime.UpdateStatus(podInfraContainer); err != nil { return nil, err } + s.containerStateToDisk(podInfraContainer) cState := s.runtime.ContainerStatus(podInfraContainer) From 6622feb4801b4bfbe786186d0f33ab713e653afe Mon Sep 17 00:00:00 2001 From: Antonio Murdaca Date: Thu, 11 May 2017 14:14:15 +0200 Subject: [PATCH 11/17] server: still update status on container not found in runc Signed-off-by: Antonio Murdaca --- oci/oci.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/oci/oci.go b/oci/oci.go index bd3cac3b..85841f1d 100644 --- a/oci/oci.go +++ b/oci/oci.go @@ -521,6 +521,9 @@ func (r *Runtime) UpdateStatus(c *Container) error { out, err := exec.Command(r.Path(c), "state", c.name).CombinedOutput() if err != nil { if err := unix.Kill(c.state.Pid, 0); err == syscall.ESRCH { + c.state.Status = ContainerStateStopped + c.state.Finished = time.Now() + c.state.ExitCode = 255 return nil } return fmt.Errorf("error getting container state for %s: %s: %q", c.name, err, out) From 4a8debe6c5525cce4d5b28cb5273b8b5012943f6 Mon Sep 17 00:00:00 2001 From: Antonio Murdaca Date: Mon, 15 May 2017 12:12:06 +0200 Subject: [PATCH 12/17] oci: do not serialize empty fields on disk Signed-off-by: Antonio Murdaca --- oci/container.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/oci/container.go b/oci/container.go index 4717b9ea..9ffa13ca 100644 --- a/oci/container.go +++ b/oci/container.go @@ -39,9 +39,9 @@ type Container struct { type ContainerState struct { specs.State Created time.Time `json:"created"` - Started time.Time `json:"started"` - Finished time.Time `json:"finished"` - ExitCode int32 `json:"exitCode"` + Started time.Time `json:"started,omitempty"` + Finished time.Time `json:"finished,omitempty"` + ExitCode int32 `json:"exitCode,omitempty"` } // NewContainer creates a container object. From f00edc2b2dcc584d86b607922cf06e3ed351b1d8 Mon Sep 17 00:00:00 2001 From: Antonio Murdaca Date: Mon, 15 May 2017 12:12:29 +0200 Subject: [PATCH 13/17] server: do not remove but stop sandboxes on shutdown Signed-off-by: Antonio Murdaca --- server/sandbox_remove.go | 13 ++++++------- server/server.go | 2 +- 2 files changed, 7 insertions(+), 8 deletions(-) diff --git a/server/sandbox_remove.go b/server/sandbox_remove.go index e2502cba..855df465 100644 --- a/server/sandbox_remove.go +++ b/server/sandbox_remove.go @@ -107,16 +107,15 @@ func (s *Server) RemovePodSandbox(ctx context.Context, req *pb.RemovePodSandboxR return resp, nil } -// RemoveAllPodSandboxes removes all pod sandboxes -func (s *Server) RemoveAllPodSandboxes() { - logrus.Debugf("RemoveAllPodSandboxes") - s.Update() +// StopAllPodSandboxes removes all pod sandboxes +func (s *Server) StopAllPodSandboxes() { + logrus.Debugf("StopAllPodSandboxes") for _, sb := range s.state.sandboxes { - pod := &pb.RemovePodSandboxRequest{ + pod := &pb.StopPodSandboxRequest{ PodSandboxId: sb.id, } - if _, err := s.RemovePodSandbox(nil, pod); err != nil { - logrus.Warnf("could not RemovePodSandbox %s: %v", sb.id, err) + if _, err := s.StopPodSandbox(nil, pod); err != nil { + logrus.Warnf("could not StopPodSandbox %s: %v", sb.id, err) } } } diff --git a/server/server.go b/server/server.go index 533149ae..1b0570c8 100644 --- a/server/server.go +++ b/server/server.go @@ -510,7 +510,7 @@ func (s *Server) cleanupSandboxesOnShutdown() { _, err := os.Stat(shutdownFile) if err == nil || !os.IsNotExist(err) { logrus.Debugf("shutting down all sandboxes, on shutdown") - s.RemoveAllPodSandboxes() + s.StopAllPodSandboxes() err = os.Remove(shutdownFile) if err != nil { logrus.Warnf("Failed to remove %q", shutdownFile) From 3b2e5aa5c22d61ae13704598b9a064fe952258a7 Mon Sep 17 00:00:00 2001 From: Antonio Murdaca Date: Mon, 15 May 2017 15:50:41 +0200 Subject: [PATCH 14/17] add a note on server shutdown... Signed-off-by: Antonio Murdaca --- server/server.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/server/server.go b/server/server.go index 1b0570c8..fb00dc54 100644 --- a/server/server.go +++ b/server/server.go @@ -521,6 +521,10 @@ func (s *Server) cleanupSandboxesOnShutdown() { // Shutdown attempts to shut down the server's storage cleanly func (s *Server) Shutdown() error { + // why do this on clean shutdown! we want containers left running when ocid + // is down for whatever reason no?! + // notice this won't trigger just on system halt but also on normal + // ocid.service restart!!! s.cleanupSandboxesOnShutdown() _, err := s.store.Shutdown(false) return err From de96e54230b6921f04142abca40605911f1ca0ee Mon Sep 17 00:00:00 2001 From: Antonio Murdaca Date: Mon, 15 May 2017 16:30:50 +0200 Subject: [PATCH 15/17] server: move StopAllSandboxes in sandbox_stop.go Signed-off-by: Antonio Murdaca --- server/sandbox_remove.go | 13 ------------- server/sandbox_stop.go | 13 +++++++++++++ 2 files changed, 13 insertions(+), 13 deletions(-) diff --git a/server/sandbox_remove.go b/server/sandbox_remove.go index 855df465..b4a5b66e 100644 --- a/server/sandbox_remove.go +++ b/server/sandbox_remove.go @@ -106,16 +106,3 @@ func (s *Server) RemovePodSandbox(ctx context.Context, req *pb.RemovePodSandboxR logrus.Debugf("RemovePodSandboxResponse %+v", resp) return resp, nil } - -// StopAllPodSandboxes removes all pod sandboxes -func (s *Server) StopAllPodSandboxes() { - logrus.Debugf("StopAllPodSandboxes") - for _, sb := range s.state.sandboxes { - pod := &pb.StopPodSandboxRequest{ - PodSandboxId: sb.id, - } - if _, err := s.StopPodSandbox(nil, pod); err != nil { - logrus.Warnf("could not StopPodSandbox %s: %v", sb.id, err) - } - } -} diff --git a/server/sandbox_stop.go b/server/sandbox_stop.go index b932afb2..d18cf51f 100644 --- a/server/sandbox_stop.go +++ b/server/sandbox_stop.go @@ -59,3 +59,16 @@ func (s *Server) StopPodSandbox(ctx context.Context, req *pb.StopPodSandboxReque logrus.Debugf("StopPodSandboxResponse: %+v", resp) return resp, nil } + +// StopAllPodSandboxes removes all pod sandboxes +func (s *Server) StopAllPodSandboxes() { + logrus.Debugf("StopAllPodSandboxes") + for _, sb := range s.state.sandboxes { + pod := &pb.StopPodSandboxRequest{ + PodSandboxId: sb.id, + } + if _, err := s.StopPodSandbox(nil, pod); err != nil { + logrus.Warnf("could not StopPodSandbox %s: %v", sb.id, err) + } + } +} From 22b922727b7437b2cc0f8f4e3c5e4ec86a1b10bc Mon Sep 17 00:00:00 2001 From: Antonio Murdaca Date: Tue, 16 May 2017 12:50:21 +0200 Subject: [PATCH 16/17] test: add bad state restore test Signed-off-by: Antonio Murdaca --- test/restore.bats | 56 +++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 56 insertions(+) diff --git a/test/restore.bats b/test/restore.bats index c50344a1..1351cab5 100644 --- a/test/restore.bats +++ b/test/restore.bats @@ -77,3 +77,59 @@ function teardown() { cleanup_pods stop_crio } + +@test "crio restore with bad state" { + start_crio + run crioctl pod run --config "$TESTDATA"/sandbox_config.json + echo "$output" + [ "$status" -eq 0 ] + pod_id="$output" + + run crioctl pod status --id "$pod_id" + echo "$output" + [ "$status" -eq 0 ] + [[ "${output}" =~ "SANDBOX_READY" ]] + + run crioctl ctr create --config "$TESTDATA"/container_config.json --pod "$pod_id" + echo "$output" + [ "$status" -eq 0 ] + ctr_id="$output" + + run crioctl ctr status --id "$ctr_id" + echo "$output" + [ "$status" -eq 0 ] + [[ "${output}" =~ "CONTAINER_CREATED" ]] + + stop_crio + + # simulate reboot with runc state going away + for i in $("$RUNTIME" list -q | xargs); do "$RUNTIME" delete -f $i; done + + start_crio + run crioctl pod list + echo "$output" + [ "$status" -eq 0 ] + [[ "${output}" != "" ]] + [[ "${output}" =~ "${pod_id}" ]] + + run crioctl pod status --id "$pod_id" + echo "$output" + [ "$status" -eq 0 ] + [[ "${output}" =~ "SANDBOX_NOTREADY" ]] + + run crioctl ctr list + echo "$output" + [ "$status" -eq 0 ] + [[ "${output}" != "" ]] + [[ "${output}" =~ "${ctr_id}" ]] + + run crioctl ctr status --id "$ctr_id" + echo "$output" + [ "$status" -eq 0 ] + [[ "${output}" =~ "CONTAINER_EXITED" ]] + [[ "${output}" =~ "Exit Code: 255" ]] + + cleanup_ctrs + cleanup_pods + stop_crio +} From 9b74e546ca463f19c5ac91090c85a75cd90eae8a Mon Sep 17 00:00:00 2001 From: Antonio Murdaca Date: Tue, 16 May 2017 12:50:41 +0200 Subject: [PATCH 17/17] test: fix restore test Signed-off-by: Antonio Murdaca --- test/restore.bats | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/restore.bats b/test/restore.bats index 1351cab5..78e826e2 100644 --- a/test/restore.bats +++ b/test/restore.bats @@ -61,7 +61,7 @@ function teardown() { echo "$output" [ "$status" -eq 0 ] [[ "${output}" != "" ]] - [[ "${output}" =~ "${pod_id}" ]] + [[ "${output}" =~ "${ctr_id}" ]] run crioctl ctr list --id "$ctr_id" echo "$output"