Fix container removal state tracking

Signed-off-by: Mrunal Patel <mrunalp@gmail.com>

server/runtime.go

@@ -97,7 +97,7 @@ func (s *Server) CreatePodSandbox(ctx context.Context, req *pb.CreatePodSandboxR
 		name:       name,
 		logDir:     logDir,
 		labels:     labels,
-		containers: []*oci.Container{},
+		containers: make(map[string]*oci.Container),
 	})
 
 	annotations := req.GetConfig().GetAnnotations()
@@ -516,6 +516,8 @@ func (s *Server) RemoveContainer(ctx context.Context, req *pb.RemoveContainerReq
 		return nil, fmt.Errorf("failed to remove container %s directory: %v", *containerName, err)
 	}
 
+	s.removeContainer(c)
+
 	return &pb.RemoveContainerResponse{}, nil
 }