Load runtimes dynamically via go1.8 plugins

Signed-off-by: Michael Crosby <crosbymichael@gmail.com>

Add registration for more subsystems via plugins

Signed-off-by: Michael Crosby <crosbymichael@gmail.com>

Move content service to separate package

Signed-off-by: Michael Crosby <crosbymichael@gmail.com>

services/content/service.go

@@ -0,0 +1,249 @@
+package content
+
+import (
+	"errors"
+	"io"
+
+	"github.com/docker/containerd"
+	api "github.com/docker/containerd/api/services/content"
+	"github.com/docker/containerd/content"
+	"golang.org/x/net/context"
+	"google.golang.org/grpc"
+	"google.golang.org/grpc/codes"
+)
+
+type Service struct {
+	store *content.Store
+}
+
+var _ api.ContentServer = &Service{}
+
+func init() {
+	containerd.Register("content-grpc", &containerd.Registration{
+		Type: containerd.GRPCPlugin,
+		Init: NewService,
+	})
+}
+
+func NewService(ic *containerd.InitContext) (interface{}, error) {
+	return &Service{
+		store: ic.Store,
+	}, nil
+}
+
+func (s *Service) Register(server *grpc.Server) error {
+	api.RegisterContentServer(server, s)
+	return nil
+}
+
+func (s *Service) Info(ctx context.Context, req *api.InfoRequest) (*api.InfoResponse, error) {
+	if err := req.Digest.Validate(); err != nil {
+		return nil, grpc.Errorf(codes.InvalidArgument, "%q failed validation", req.Digest)
+	}
+
+	bi, err := s.store.Info(req.Digest)
+	if err != nil {
+		return nil, maybeNotFoundGRPC(err, req.Digest.String())
+	}
+
+	return &api.InfoResponse{
+		Digest:      req.Digest,
+		Size_:       bi.Size,
+		CommittedAt: bi.CommittedAt,
+	}, nil
+}
+
+func (s *Service) Read(req *api.ReadRequest, session api.Content_ReadServer) error {
+	if err := req.Digest.Validate(); err != nil {
+		return grpc.Errorf(codes.InvalidArgument, "%v: %v", req.Digest, err)
+	}
+
+	oi, err := s.store.Info(req.Digest)
+	if err != nil {
+		return maybeNotFoundGRPC(err, req.Digest.String())
+	}
+
+	rc, err := s.store.Reader(session.Context(), req.Digest)
+	if err != nil {
+		return maybeNotFoundGRPC(err, req.Digest.String())
+	}
+	defer rc.Close() // TODO(stevvooe): Cache these file descriptors for performance.
+
+	ra, ok := rc.(io.ReaderAt)
+	if !ok {
+		// TODO(stevvooe): Need to set this up to get correct behavior across
+		// board. May change interface to store to just return ReaderAtCloser.
+		// Possibly, we could just return io.ReaderAt and handle file
+		// descriptors internally.
+		return errors.New("content service only supports content stores that return ReaderAt")
+	}
+
+	var (
+		offset = req.Offset
+		size   = req.Size_
+
+		// TODO(stevvooe): Using the global buffer pool. At 32KB, it is probably
+		// little inefficient for work over a fast network. We can tune this later.
+		p = content.BufPool.Get().([]byte)
+	)
+	defer content.BufPool.Put(p)
+
+	if offset < 0 {
+		offset = 0
+	}
+
+	if size <= 0 {
+		size = oi.Size - offset
+	}
+
+	if offset+size > oi.Size {
+		return grpc.Errorf(codes.OutOfRange, "read past object length %v bytes", oi.Size)
+	}
+
+	if _, err := io.CopyBuffer(
+		&readResponseWriter{session: session},
+		io.NewSectionReader(ra, offset, size), p); err != nil {
+		return err
+	}
+
+	return nil
+}
+
+type readResponseWriter struct {
+	offset  int64
+	session api.Content_ReadServer
+}
+
+func (rw *readResponseWriter) Write(p []byte) (n int, err error) {
+	if err := rw.session.Send(&api.ReadResponse{
+		Offset: rw.offset,
+		Data:   p,
+	}); err != nil {
+		return 0, err
+	}
+
+	rw.offset += int64(len(p))
+	return len(p), nil
+}
+
+func (s *Service) Write(session api.Content_WriteServer) (err error) {
+	var (
+		ref string
+		msg api.WriteResponse
+		req *api.WriteRequest
+	)
+
+	defer func(msg *api.WriteResponse) {
+		// pump through the last message if no error was encountered
+		if err != nil {
+			return
+		}
+
+		err = session.Send(msg)
+	}(&msg)
+
+	// handle the very first request!
+	req, err = session.Recv()
+	if err != nil {
+		return err
+	}
+
+	ref = req.Ref
+	if ref == "" {
+		return grpc.Errorf(codes.InvalidArgument, "first message must have a reference")
+	}
+
+	// this action locks the writer for the session.
+	wr, err := s.store.Writer(session.Context(), ref)
+	if err != nil {
+		return err
+	}
+	defer wr.Close()
+
+	for {
+		// TODO(stevvooe): We need to study this behavior in containerd a
+		// little better to decide where to put this. We may be able to make
+		// this determination elsewhere and avoid even creating the writer.
+		//
+		// Ideally, we just use the expected digest on commit to abandon the
+		// cost of the move when they collide.
+		if req.ExpectedDigest != "" {
+			if _, err := s.store.Info(req.ExpectedDigest); err != nil {
+				if !content.IsNotFound(err) {
+					return err
+				}
+
+				return grpc.Errorf(codes.AlreadyExists, "blob with expected digest %v exists", req.ExpectedDigest)
+			}
+		}
+
+		msg.Action = req.Action
+		ws, err := wr.Status()
+		if err != nil {
+			return err
+		}
+
+		msg.Offset = ws.Offset
+		msg.StartedAt = ws.StartedAt
+		msg.UpdatedAt = ws.UpdatedAt
+
+		switch req.Action {
+		case api.WriteActionStat:
+			msg.Digest = wr.Digest()
+		case api.WriteActionWrite, api.WriteActionCommit:
+			if req.Offset > 0 {
+				// validate the offset if provided
+				if req.Offset != ws.Offset {
+					return grpc.Errorf(codes.OutOfRange, "write @%v must occur at current offset %v", req.Offset, ws.Offset)
+				}
+			}
+
+			// issue the write if we actually have data.
+			if len(req.Data) > 0 {
+				// While this looks like we could use io.WriterAt here, because we
+				// maintain the offset as append only, we just issue the write.
+				n, err := wr.Write(req.Data)
+				if err != nil {
+					return err
+				}
+
+				if n != len(req.Data) {
+					// TODO(stevvooe): Perhaps, we can recover this by including it
+					// in the offset on the write return.
+					return grpc.Errorf(codes.DataLoss, "wrote %v of %v bytes", n, len(req.Data))
+				}
+
+				msg.Offset += int64(n)
+			}
+
+			if req.Action == api.WriteActionCommit {
+				return wr.Commit(req.ExpectedSize, req.ExpectedDigest)
+			}
+		case api.WriteActionAbort:
+			return s.store.Abort(ref)
+		}
+
+		if err := session.Send(&msg); err != nil {
+			return err
+		}
+
+		req, err = session.Recv()
+		if err != nil {
+			return err
+		}
+	}
+
+	return nil
+}
+
+func (s *Service) Status(*api.StatusRequest, api.Content_StatusServer) error {
+	return grpc.Errorf(codes.Unimplemented, "not implemented")
+}
+
+func maybeNotFoundGRPC(err error, id string) error {
+	if content.IsNotFound(err) {
+		return grpc.Errorf(codes.NotFound, "%v: not found", id)
+	}
+
+	return err
+}

services/execution/collector.go

@@ -0,0 +1,89 @@
+package execution
+
+import (
+	"sync"
+
+	"github.com/docker/containerd"
+
+	"golang.org/x/net/context"
+)
+
+func newCollector(ctx context.Context, runtimes map[string]containerd.Runtime) (*collector, error) {
+	c := &collector{
+		context:      ctx,
+		ch:           make(chan *containerd.Event, 2048),
+		eventClients: make(map[*eventClient]struct{}),
+	}
+	for _, r := range runtimes {
+		if err := c.collect(r); err != nil {
+			return nil, err
+		}
+	}
+	// run the publisher
+	go c.publisher()
+	// run a goroutine that waits for the context to be done
+	// and closes the channel after all writes have finished
+	go c.waitDone()
+	return c, nil
+}
+
+type eventClient struct {
+	eCh chan error
+	w   *grpcEventWriter
+}
+
+type collector struct {
+	mu sync.Mutex
+	wg sync.WaitGroup
+
+	context      context.Context
+	ch           chan *containerd.Event
+	eventClients map[*eventClient]struct{}
+}
+
+// collect collects events from the provided runtime
+func (c *collector) collect(r containerd.Runtime) error {
+	c.wg.Add(1)
+	go func() {
+		defer c.wg.Done()
+		for e := range r.Events(c.context) {
+			c.ch <- e
+		}
+	}()
+	return nil
+}
+
+func (c *collector) forward(w *grpcEventWriter) error {
+	client := &eventClient{
+		w:   w,
+		eCh: make(chan error, 1),
+	}
+	c.mu.Lock()
+	c.eventClients[client] = struct{}{}
+	c.mu.Unlock()
+	err := <-client.eCh
+	c.mu.Lock()
+	delete(c.eventClients, client)
+	c.mu.Unlock()
+	return err
+}
+
+func (c *collector) publisher() {
+	for e := range c.ch {
+		c.mu.Lock()
+		for client := range c.eventClients {
+			if err := client.w.Write(e); err != nil {
+				client.eCh <- err
+			}
+		}
+		c.mu.Unlock()
+	}
+}
+
+// waitDone waits for the context to finish, waits for all the goroutines to finish
+// collecting grpc events from the shim, and closes the output channel
+func (c *collector) waitDone() {
+	<-c.context.Done()
+	c.wg.Wait()
+	close(c.ch)
+}

services/execution/service.go

@@ -1,11 +1,14 @@
 package execution
 
 import (
+	"sync"
+
 	"github.com/docker/containerd"
 	api "github.com/docker/containerd/api/services/execution"
 	"github.com/docker/containerd/api/types/container"
 	google_protobuf "github.com/golang/protobuf/ptypes/empty"
 	"golang.org/x/net/context"
+	"google.golang.org/grpc"
 )
 
 var (
@@ -13,15 +16,46 @@ var (
 	empty = &google_protobuf.Empty{}
 )
 
-// New creates a new GRPC service for the ContainerService
-func New(s *containerd.Supervisor) *Service {
-	return &Service{
-		s: s,
+func init() {
+	containerd.Register("runtime-grpc", &containerd.Registration{
+		Type: containerd.GRPCPlugin,
+		Init: New,
+	})
+}
+
+func New(ic *containerd.InitContext) (interface{}, error) {
+	c, err := newCollector(ic.Context, ic.Runtimes)
+	if err != nil {
+		return nil, err
 	}
+	return &Service{
+		runtimes:   ic.Runtimes,
+		containers: make(map[string]containerd.Container),
+		collector:  c,
+	}, nil
 }
 
 type Service struct {
-	s *containerd.Supervisor
+	mu sync.Mutex
+
+	runtimes   map[string]containerd.Runtime
+	containers map[string]containerd.Container
+	collector  *collector
+}
+
+func (s *Service) Register(server *grpc.Server) error {
+	api.RegisterContainerServiceServer(server, s)
+	// load all containers
+	for _, r := range s.runtimes {
+		containers, err := r.Containers()
+		if err != nil {
+			return err
+		}
+		for _, c := range containers {
+			s.containers[c.Info().ID] = c
+		}
+	}
+	return nil
 }
 
 func (s *Service) Create(ctx context.Context, r *api.CreateRequest) (*api.CreateResponse, error) {
@@ -41,13 +75,28 @@ func (s *Service) Create(ctx context.Context, r *api.CreateRequest) (*api.Create
 			Options: m.Options,
 		})
 	}
-	c, err := s.s.Create(ctx, r.ID, r.Runtime, opts)
+	runtime, err := s.getRuntime(r.Runtime)
 	if err != nil {
 		return nil, err
 	}
+	s.mu.Lock()
+	if _, ok := s.containers[r.ID]; ok {
+		s.mu.Unlock()
+		return nil, containerd.ErrContainerExists
+	}
+	c, err := runtime.Create(ctx, r.ID, opts)
+	if err != nil {
+		s.mu.Unlock()
+		return nil, err
+	}
+	s.containers[r.ID] = c
+	s.mu.Unlock()
 	state, err := c.State(ctx)
 	if err != nil {
-		s.s.Delete(ctx, r.ID)
+		s.mu.Lock()
+		delete(s.containers, r.ID)
+		runtime.Delete(ctx, c)
+		s.mu.Unlock()
 		return nil, err
 	}
 	return &api.CreateResponse{
@@ -57,7 +106,7 @@ func (s *Service) Create(ctx context.Context, r *api.CreateRequest) (*api.Create
 }
 
 func (s *Service) Start(ctx context.Context, r *api.StartRequest) (*google_protobuf.Empty, error) {
-	c, err := s.s.Get(r.ID)
+	c, err := s.getContainer(r.ID)
 	if err != nil {
 		return nil, err
 	}
@@ -68,7 +117,15 @@ func (s *Service) Start(ctx context.Context, r *api.StartRequest) (*google_proto
 }
 
 func (s *Service) Delete(ctx context.Context, r *api.DeleteRequest) (*google_protobuf.Empty, error) {
-	if err := s.s.Delete(ctx, r.ID); err != nil {
+	c, err := s.getContainer(r.ID)
+	if err != nil {
+		return nil, err
+	}
+	runtime, err := s.getRuntime(c.Info().Runtime)
+	if err != nil {
+		return nil, err
+	}
+	if err := runtime.Delete(ctx, c); err != nil {
 		return nil, err
 	}
 	return empty, nil
@@ -76,7 +133,7 @@ func (s *Service) Delete(ctx context.Context, r *api.DeleteRequest) (*google_pro
 
 func (s *Service) List(ctx context.Context, r *api.ListRequest) (*api.ListResponse, error) {
 	resp := &api.ListResponse{}
-	for _, c := range s.s.Containers() {
+	for _, c := range s.containers {
 		state, err := c.State(ctx)
 		if err != nil {
 			return nil, err
@@ -105,7 +162,25 @@ func (s *Service) Events(r *api.EventsRequest, server api.ContainerService_Event
 	w := &grpcEventWriter{
 		server: server,
 	}
-	return s.s.ForwardEvents(w)
+	return s.collector.forward(w)
+}
+
+func (s *Service) getContainer(id string) (containerd.Container, error) {
+	s.mu.Lock()
+	c, ok := s.containers[id]
+	s.mu.Unlock()
+	if !ok {
+		return nil, containerd.ErrContainerNotExist
+	}
+	return c, nil
+}
+
+func (s *Service) getRuntime(name string) (containerd.Runtime, error) {
+	runtime, ok := s.runtimes[name]
+	if !ok {
+		return nil, containerd.ErrUnknownRuntime
+	}
+	return runtime, nil
 }
 
 type grpcEventWriter struct {