diff --git a/oci/oci.go b/oci/oci.go index b72b4357..27400843 100644 --- a/oci/oci.go +++ b/oci/oci.go @@ -147,7 +147,7 @@ func (r *Runtime) UpdateStatus(c *Container) error { defer c.stateLock.Unlock() out, err := exec.Command(r.path, "state", c.name).Output() if err != nil { - return fmt.Errorf("error getting container state for %s: %s", c.name, err) + return fmt.Errorf("error getting container state for %s: %s: %v", c.name, err, out) } stateReader := bytes.NewReader(out) if err := json.NewDecoder(stateReader).Decode(&c.state); err != nil { diff --git a/server/container.go b/server/container.go index 91842ae8..9fae2a5b 100644 --- a/server/container.go +++ b/server/container.go @@ -1,6 +1,7 @@ package server import ( + "errors" "fmt" "os" "path/filepath" @@ -17,10 +18,10 @@ import ( func (s *Server) CreateContainer(ctx context.Context, req *pb.CreateContainerRequest) (*pb.CreateContainerResponse, error) { // The id of the PodSandbox podSandboxID := req.GetPodSandboxId() - if !s.hasSandbox(podSandboxID) { + sb := s.getSandbox(podSandboxID) + if sb == nil { return nil, fmt.Errorf("the pod sandbox (%s) does not exist", podSandboxID) } - // The config of the container containerConfig := req.GetConfig() if containerConfig == nil { @@ -43,7 +44,7 @@ func (s *Server) CreateContainer(ctx context.Context, req *pb.CreateContainerReq return nil, err } - container, err := s.createSandboxContainer(name, podSandboxID, req.GetSandboxConfig(), containerDir, containerConfig) + container, err := s.createSandboxContainer(name, sb, req.GetSandboxConfig(), containerDir, containerConfig) if err != nil { return nil, err } @@ -63,7 +64,10 @@ func (s *Server) CreateContainer(ctx context.Context, req *pb.CreateContainerReq }, nil } -func (s *Server) createSandboxContainer(name, podSandboxID string, SandboxConfig *pb.PodSandboxConfig, containerDir string, containerConfig *pb.ContainerConfig) (*oci.Container, error) { +func (s *Server) createSandboxContainer(name string, sb *sandbox, SandboxConfig *pb.PodSandboxConfig, containerDir string, containerConfig *pb.ContainerConfig) (*oci.Container, error) { + if sb == nil { + return nil, errors.New("createSandboxContainer needs a sandbox") + } // creates a spec Generator with the default spec. specgen := generate.New() @@ -129,8 +133,6 @@ func (s *Server) createSandboxContainer(name, podSandboxID string, SandboxConfig } } - specgen.AddAnnotation("pod_sandbox_id", podSandboxID) - if containerConfig.GetPrivileged() { specgen.SetupPrivileged(true) } @@ -235,9 +237,8 @@ func (s *Server) createSandboxContainer(name, podSandboxID string, SandboxConfig } } } - // Join the namespace paths for the pod sandbox container. - podContainerName := podSandboxID + "-infra" + podContainerName := sb.name + "-infra" podInfraContainer := s.state.containers.Get(podContainerName) podInfraState := s.runtime.ContainerStatus(podInfraContainer) @@ -274,7 +275,7 @@ func (s *Server) createSandboxContainer(name, podSandboxID string, SandboxConfig return nil, err } - container, err := oci.NewContainer(name, containerDir, logPath, labels, podSandboxID, containerConfig.GetTty()) + container, err := oci.NewContainer(name, containerDir, logPath, labels, sb.id, containerConfig.GetTty()) if err != nil { return nil, err } diff --git a/server/sandbox.go b/server/sandbox.go index 50d7c090..f1920fb6 100644 --- a/server/sandbox.go +++ b/server/sandbox.go @@ -3,11 +3,11 @@ package server import ( "encoding/json" "fmt" - "io/ioutil" "os" "path/filepath" "github.com/Sirupsen/logrus" + "github.com/docker/docker/pkg/stringid" "github.com/kubernetes-incubator/ocid/oci" "github.com/kubernetes-incubator/ocid/utils" pb "github.com/kubernetes/kubernetes/pkg/kubelet/api/v1alpha1/runtime" @@ -16,18 +16,13 @@ import ( ) type sandbox struct { + id string name string logDir string labels map[string]string containers oci.Store } -type metadata struct { - LogDir string `json:"log_dir"` - ContainerName string `json:"container_name"` - Labels map[string]string `json:"labels"` -} - const ( podInfraRootfs = "/var/lib/ocid/graph/vfs/pause" ) @@ -44,6 +39,17 @@ func (s *sandbox) removeContainer(c *oci.Container) { s.containers.Delete(c.Name()) } +func (s *Server) generatePodIDandName(name string) (string, string, error) { + var ( + err error + id = stringid.GenerateNonCryptoID() + ) + if name, err = s.reservePodName(id, name); err != nil { + return "", "", err + } + return id, name, err +} + // CreatePodSandbox creates a pod-level sandbox. // The definition of PodSandbox is at https://github.com/kubernetes/kubernetes/pull/25899 func (s *Server) CreatePodSandbox(ctx context.Context, req *pb.CreatePodSandboxRequest) (*pb.CreatePodSandboxResponse, error) { @@ -53,16 +59,20 @@ func (s *Server) CreatePodSandbox(ctx context.Context, req *pb.CreatePodSandboxR return nil, fmt.Errorf("PodSandboxConfig.Name should not be empty") } - podSandboxDir := filepath.Join(s.sandboxDir, name) - if _, err := os.Stat(podSandboxDir); err == nil { + var err error + id, name, err := s.generatePodIDandName(name) + if err != nil { + return nil, err + } + podSandboxDir := filepath.Join(s.sandboxDir, id) + if _, err = os.Stat(podSandboxDir); err == nil { return nil, fmt.Errorf("pod sandbox (%s) already exists", podSandboxDir) } - if err := os.MkdirAll(podSandboxDir, 0755); err != nil { + if err = os.MkdirAll(podSandboxDir, 0755); err != nil { return nil, err } - var err error defer func() { if err != nil { if err2 := os.RemoveAll(podSandboxDir); err2 != nil { @@ -88,7 +98,7 @@ func (s *Server) CreatePodSandbox(ctx context.Context, req *pb.CreatePodSandboxR // process req.LogDirectory logDir := req.GetConfig().GetLogDirectory() if logDir == "" { - logDir = fmt.Sprintf("/var/log/ocid/pods/%s", name) + logDir = fmt.Sprintf("/var/log/ocid/pods/%s", id) } dnsServers := req.GetConfig().GetDnsOptions().GetServers() @@ -107,7 +117,17 @@ func (s *Server) CreatePodSandbox(ctx context.Context, req *pb.CreatePodSandboxR g.AddBindMount(resolvPath, "/etc/resolv.conf", "ro") labels := req.GetConfig().GetLabels() + labelsJSON, err := json.Marshal(labels) + if err != nil { + return nil, err + } + g.AddAnnotation("ocid/labels", string(labelsJSON)) + g.AddAnnotation("ocid/log_path", logDir) + g.AddAnnotation("ocid/name", name) + containerName := name + "-infra" + g.AddAnnotation("ocid/container_name", containerName) s.addSandbox(&sandbox{ + id: id, name: name, logDir: logDir, labels: labels, @@ -162,8 +182,7 @@ func (s *Server) CreatePodSandbox(ctx context.Context, req *pb.CreatePodSandboxR } } - containerName := name + "-infra" - container, err := oci.NewContainer(containerName, podSandboxDir, podSandboxDir, labels, name, false) + container, err := oci.NewContainer(containerName, podSandboxDir, podSandboxDir, labels, id, false) if err != nil { return nil, err } @@ -182,8 +201,8 @@ func (s *Server) CreatePodSandbox(ctx context.Context, req *pb.CreatePodSandboxR if err != nil { return nil, err } - if err = s.netPlugin.SetUpPod(netnsPath, podNamespace, name, containerName); err != nil { - return nil, fmt.Errorf("failed to create network for container %s in sandbox %s: %v", containerName, name, err) + if err = s.netPlugin.SetUpPod(netnsPath, podNamespace, id, containerName); err != nil { + return nil, fmt.Errorf("failed to create network for container %s in sandbox %s: %v", containerName, id, err) } if err = s.runtime.StartContainer(container); err != nil { @@ -192,24 +211,7 @@ func (s *Server) CreatePodSandbox(ctx context.Context, req *pb.CreatePodSandboxR s.addContainer(container) - meta := &metadata{ - LogDir: logDir, - ContainerName: containerName, - Labels: labels, - } - - b, err := json.Marshal(meta) - if err != nil { - return nil, err - } - - // TODO: eventually we would track all containers in this pod so on server start - // we can repopulate the structs in memory properly... - // e.g. each container can write itself in podSandboxDir - err = ioutil.WriteFile(filepath.Join(podSandboxDir, "metadata.json"), b, 0644) - if err != nil { - return nil, err - } + s.podIDIndex.Add(id) if err = s.runtime.UpdateStatus(container); err != nil { return nil, err diff --git a/server/server.go b/server/server.go index c2d8c2d8..988a0a4e 100644 --- a/server/server.go +++ b/server/server.go @@ -9,8 +9,11 @@ import ( "sync" "github.com/Sirupsen/logrus" + "github.com/docker/docker/pkg/registrar" + "github.com/docker/docker/pkg/truncindex" "github.com/kubernetes-incubator/ocid/oci" "github.com/kubernetes-incubator/ocid/utils" + rspec "github.com/opencontainers/runtime-spec/specs-go" "github.com/rajatchopra/ocicni" ) @@ -21,30 +24,38 @@ const ( // Server implements the RuntimeService and ImageService type Server struct { - runtime *oci.Runtime - sandboxDir string - stateLock sync.Mutex - state *serverState - netPlugin ocicni.CNIPlugin + runtime *oci.Runtime + sandboxDir string + stateLock sync.Mutex + state *serverState + netPlugin ocicni.CNIPlugin + podNameIndex *registrar.Registrar + podIDIndex *truncindex.TruncIndex } func (s *Server) loadSandbox(id string) error { - metaJSON, err := ioutil.ReadFile(filepath.Join(s.sandboxDir, id, "metadata.json")) + config, err := ioutil.ReadFile(filepath.Join(s.sandboxDir, id, "config.json")) if err != nil { return err } - var m metadata - if err = json.Unmarshal(metaJSON, &m); err != nil { + var m rspec.Spec + if err = json.Unmarshal(config, &m); err != nil { return err } + labels := make(map[string]string) + if err = json.Unmarshal([]byte(m.Annotations["ocid/labels"]), &labels); err != nil { + return err + } + name := m.Annotations["ocid/name"] s.addSandbox(&sandbox{ - name: id, - logDir: m.LogDir, - labels: m.Labels, + id: id, + name: name, + logDir: m.Annotations["ocid/log_path"], + labels: labels, containers: oci.NewMemoryStore(), }) sandboxPath := filepath.Join(s.sandboxDir, id) - scontainer, err := oci.NewContainer(m.ContainerName, sandboxPath, sandboxPath, m.Labels, id, false) + scontainer, err := oci.NewContainer(m.Annotations["ocid/container_name"], sandboxPath, sandboxPath, labels, id, false) if err != nil { return err } @@ -52,6 +63,8 @@ func (s *Server) loadSandbox(id string) error { if err = s.runtime.UpdateStatus(scontainer); err != nil { logrus.Warnf("error updating status for container %s: %v", scontainer, err) } + s.podIDIndex.Add(id) + s.reservePodName(id, name) return nil } @@ -68,6 +81,21 @@ func (s *Server) restore() error { return nil } +func (s *Server) reservePodName(id, name string) (string, error) { + if err := s.podNameIndex.Reserve(name, id); err != nil { + if err == registrar.ErrNameReserved { + id, err := s.podNameIndex.Get(name) + if err != nil { + logrus.Warnf("name %s already reserved for %s", name, id) + return "", err + } + return "", fmt.Errorf("conflict, name %s already reserver", name) + } + return "", fmt.Errorf("error reserving name %s", name) + } + return name, nil +} + // New creates a new Server with options provided func New(runtimePath, sandboxDir, containerDir string) (*Server, error) { // TODO: This will go away later when we have wrapper process or systemd acting as @@ -105,6 +133,8 @@ func New(runtimePath, sandboxDir, containerDir string) (*Server, error) { containers: containers, }, } + s.podIDIndex = truncindex.NewTruncIndex([]string{}) + s.podNameIndex = registrar.NewRegistrar() if err := s.restore(); err != nil { logrus.Warnf("couldn't restore: %v", err) } @@ -120,20 +150,20 @@ type serverState struct { func (s *Server) addSandbox(sb *sandbox) { s.stateLock.Lock() - s.state.sandboxes[sb.name] = sb + s.state.sandboxes[sb.id] = sb s.stateLock.Unlock() } -func (s *Server) getSandbox(name string) *sandbox { +func (s *Server) getSandbox(id string) *sandbox { s.stateLock.Lock() - sb := s.state.sandboxes[name] + sb := s.state.sandboxes[id] s.stateLock.Unlock() return sb } -func (s *Server) hasSandbox(name string) bool { +func (s *Server) hasSandbox(id string) bool { s.stateLock.Lock() - _, ok := s.state.sandboxes[name] + _, ok := s.state.sandboxes[id] s.stateLock.Unlock() return ok } @@ -141,6 +171,7 @@ func (s *Server) hasSandbox(name string) bool { func (s *Server) addContainer(c *oci.Container) { s.stateLock.Lock() sandbox := s.state.sandboxes[c.Sandbox()] + // TODO(runcom): handle !ok above!!! otherwise it panics! sandbox.addContainer(c) s.state.containers.Add(c.Name(), c) s.stateLock.Unlock()