server: Serialize container/pod creation with updates

Interleaving asynchronous updates with pod or container creations can
lead to unrecoverable races and corruptions of the pod or container hash
tables. This is fixed by serializing update against pod or container
creation operations, while pod and container creation operations can
run in parallel.

Signed-off-by: Samuel Ortiz <sameo@linux.intel.com>

server/container_create.go

@@ -207,6 +207,10 @@ func setupContainerUser(specgen *generate.Generator, rootfs string, sc *pb.Linux
 func (s *Server) CreateContainer(ctx context.Context, req *pb.CreateContainerRequest) (res *pb.CreateContainerResponse, err error) {
 	logrus.Debugf("CreateContainerRequest %+v", req)
 	s.Update()
+
+	s.updateLock.RLock()
+	defer s.updateLock.RUnlock()
+
 	sbID := req.PodSandboxId
 	if sbID == "" {
 		return nil, fmt.Errorf("PodSandboxId should not be empty")

server/sandbox_run.go

@@ -65,6 +65,9 @@ func (s *Server) runContainer(container *oci.Container, cgroupParent string) err
 
 // RunPodSandbox creates and runs a pod-level sandbox.
 func (s *Server) RunPodSandbox(ctx context.Context, req *pb.RunPodSandboxRequest) (resp *pb.RunPodSandboxResponse, err error) {
+	s.updateLock.RLock()
+	defer s.updateLock.RUnlock()
+
 	logrus.Debugf("RunPodSandboxRequest %+v", req)
 	var processLabel, mountLabel, netNsPath, resolvPath string
 	// process req.Name

server/server.go

@@ -34,6 +34,7 @@ type Server struct {
 	images       storage.ImageServer
 	storage      storage.RuntimeServer
 	stateLock    sync.Mutex
+	updateLock   sync.RWMutex
 	state        *serverState
 	netPlugin    ocicni.CNIPlugin
 	podNameIndex *registrar.Registrar
@@ -287,6 +288,9 @@ func (s *Server) Update() {
 }
 
 func (s *Server) update() error {
+	s.updateLock.Lock()
+	defer s.updateLock.Unlock()
+
 	containers, err := s.store.Containers()
 	if err != nil && !os.IsNotExist(err) {
 		logrus.Warnf("could not read containers and sandboxes: %v", err)