Merge pull request #409 from sameo/topic/fat-lock

Serialize Update and Sandbox/Container creation operations

server/container_create.go

@@ -230,6 +230,10 @@ func ensureSaneLogPath(logPath string) error {
 func (s *Server) CreateContainer(ctx context.Context, req *pb.CreateContainerRequest) (res *pb.CreateContainerResponse, err error) {
 	logrus.Debugf("CreateContainerRequest %+v", req)
 	s.Update()
+
+	s.updateLock.RLock()
+	defer s.updateLock.RUnlock()
+
 	sbID := req.PodSandboxId
 	if sbID == "" {
 		return nil, fmt.Errorf("PodSandboxId should not be empty")

server/sandbox_run.go

@@ -65,6 +65,9 @@ func (s *Server) runContainer(container *oci.Container, cgroupParent string) err
 
 // RunPodSandbox creates and runs a pod-level sandbox.
 func (s *Server) RunPodSandbox(ctx context.Context, req *pb.RunPodSandboxRequest) (resp *pb.RunPodSandboxResponse, err error) {
+	s.updateLock.RLock()
+	defer s.updateLock.RUnlock()
+
 	logrus.Debugf("RunPodSandboxRequest %+v", req)
 	var processLabel, mountLabel, netNsPath, resolvPath string
 	// process req.Name
@@ -91,18 +94,6 @@ func (s *Server) RunPodSandbox(ctx context.Context, req *pb.RunPodSandboxRequest
 		}
 	}()
 
-	if err = s.podIDIndex.Add(id); err != nil {
-		return nil, err
-	}
-
-	defer func() {
-		if err != nil {
-			if err2 := s.podIDIndex.Delete(id); err2 != nil {
-				logrus.Warnf("couldn't delete pod id %s from idIndex", id)
-			}
-		}
-	}()
-
 	podContainer, err := s.storage.CreatePodSandbox(s.imageContext,
 		name, id,
 		s.config.PauseImage, "",
@@ -292,6 +283,17 @@ func (s *Server) RunPodSandbox(ctx context.Context, req *pb.RunPodSandboxRequest
 	}
 
 	s.addSandbox(sb)
+	if err = s.podIDIndex.Add(id); err != nil {
+		return nil, err
+	}
+
+	defer func() {
+		if err != nil {
+			if err2 := s.podIDIndex.Delete(id); err2 != nil {
+				logrus.Warnf("couldn't delete pod id %s from idIndex", id)
+			}
+		}
+	}()
 
 	for k, v := range annotations {
 		g.AddAnnotation(k, v)

server/server.go

@@ -35,6 +35,7 @@ type Server struct {
 	images       storage.ImageServer
 	storage      storage.RuntimeServer
 	stateLock    sync.Mutex
+	updateLock   sync.RWMutex
 	state        *serverState
 	netPlugin    ocicni.CNIPlugin
 	podNameIndex *registrar.Registrar
@@ -288,6 +289,9 @@ func (s *Server) Update() {
 }
 
 func (s *Server) update() error {
+	s.updateLock.Lock()
+	defer s.updateLock.Unlock()
+
 	containers, err := s.store.Containers()
 	if err != nil && !os.IsNotExist(err) {
 		logrus.Warnf("could not read containers and sandboxes: %v", err)