Need to cleanup all pods on service poweroff

When powering off the system, we want the ocid service, to shutdown
all containers running on the system so they can cleanup properly
This patch will cleanup all pods on poweroff.

The ocid-shutdown.service drops a file /var/run/ocid.shutdown when the system
is shutting down. The ocid-shutdown.service should only be executed at system
shutdown.

On bootup sequence should be
start ocid.service
start ocid-shutdown.service (This is a NO-OP)

On system shutdown
stop ocid-shutdown.service (Creates /var/run/ocid.shutdown)
stop ocid.service (Notices /var/run/ocid.service and stops all pods before exiting.)

Signed-off-by: Daniel J Walsh <dwalsh@redhat.com>

Makefile

@@ -134,6 +134,7 @@ install.completions:
 
 install.systemd:
 	install -D -m 644 contrib/systemd/ocid.service $(PREFIX)/lib/systemd/system/ocid.service
+	install -D -m 644 contrib/systemd/ocid-shutdown.service $(PREFIX)/lib/systemd/system/ocid-shutdown.service
 
 uninstall:
 	rm -f $(BINDIR)/ocid

contrib/systemd/ocid-shutdown.service

@@ -0,0 +1,14 @@
+[Unit]
+Description=Shutdown OCID containers before shutting down the system
+Wants=ocid.service
+After=ocid.service
+Documentation=man:ocid(8)
+
+[Service]
+Type=oneshot
+ExecStart=/usr/bin/true
+ExecStop=mkdir -p /var/lib/ocid; touch /var/lib/ocid/ocid.shutdown
+RemainAfterExit=yes
+
+[Install]
+WantedBy=multi-user.target

server/sandbox_remove.go

@@ -40,7 +40,8 @@ func (s *Server) RemovePodSandbox(ctx context.Context, req *pb.RemovePodSandboxR
 		cState := s.runtime.ContainerStatus(c)
 		if cState.Status == oci.ContainerStateCreated || cState.Status == oci.ContainerStateRunning {
 			if err := s.runtime.StopContainer(c); err != nil {
-				return nil, fmt.Errorf("failed to stop container %s: %v", c.Name(), err)
+				// Assume container is already stopped
+				logrus.Warnf("failed to stop container %s: %v", c.Name(), err)
 			}
 		}
 
@@ -107,3 +108,17 @@ func (s *Server) RemovePodSandbox(ctx context.Context, req *pb.RemovePodSandboxR
 	logrus.Debugf("RemovePodSandboxResponse %+v", resp)
 	return resp, nil
 }
+
+// RemoveAllPodSandboxes removes all pod sandboxes
+func (s *Server) RemoveAllPodSandboxes() {
+	logrus.Debugf("RemoveAllPodSandboxes")
+	s.Update()
+	for _, sb := range s.state.sandboxes {
+		pod := &pb.RemovePodSandboxRequest{
+			PodSandboxId: sb.id,
+		}
+		if _, err := s.RemovePodSandbox(nil, pod); err != nil {
+			logrus.Warnf("could not RemovePodSandbox %s: %v", sb.id, err)
+		}
+	}
+}

server/server.go

@@ -25,6 +25,7 @@ import (
 
 const (
 	runtimeAPIVersion = "v1alpha1"
+	shutdownFile      = "/var/lib/ocid/ocid.shutdown"
 )
 
 // Server implements the RuntimeService and ImageService
@@ -431,8 +432,23 @@ func (s *Server) releaseContainerName(name string) {
 	s.ctrNameIndex.Release(name)
 }
 
+// cleanupSandboxesOnShutdown Remove all running Sandboxes on system shutdown
+func (s *Server) cleanupSandboxesOnShutdown() {
+	_, err := os.Stat(shutdownFile)
+	if err == nil || !os.IsNotExist(err) {
+		logrus.Debugf("shutting down all sandboxes, on shutdown")
+		s.RemoveAllPodSandboxes()
+		err = os.Remove(shutdownFile)
+		if err != nil {
+			logrus.Warnf("Failed to remove %q", shutdownFile)
+		}
+
+	}
+}
+
 // Shutdown attempts to shut down the server's storage cleanly
 func (s *Server) Shutdown() error {
+	s.cleanupSandboxesOnShutdown()
 	_, err := s.store.Shutdown(false)
 	return err
 }
@@ -511,6 +527,7 @@ func New(config *Config) (*Server, error) {
 	}
 
 	s.restore()
+	s.cleanupSandboxesOnShutdown()
 
 	logrus.Debugf("sandboxes: %v", s.state.sandboxes)
 	logrus.Debugf("containers: %v", s.state.containers)