From 1ea809dc2aa6d37f040b5b1c350c8aaeecc9c8da Mon Sep 17 00:00:00 2001 From: Stephen J Day Date: Mon, 3 Apr 2017 18:33:24 -0700 Subject: [PATCH] images, services/images: implement image service Server and Client images of the image store are now provided. We have created an image metadata interface and converted the bolt functions to implement that interface over an transaction. A remote client implementation is provided that implements the same interface. Signed-off-by: Stephen J Day --- images/image.go | 16 +++---- images/storage.go | 77 ++++++++++++++++++++++----------- services/images/client.go | 60 ++++++++++++++++++++++++++ services/images/helpers.go | 87 ++++++++++++++++++++++++++++++++++++++ services/images/service.go | 85 +++++++++++++++++++++++++++++++------ 5 files changed, 280 insertions(+), 45 deletions(-) create mode 100644 services/images/client.go create mode 100644 services/images/helpers.go diff --git a/images/image.go b/images/image.go index 3ffc056..832b441 100644 --- a/images/image.go +++ b/images/image.go @@ -14,8 +14,8 @@ import ( // Image provides the model for how containerd views container images. type Image struct { - Name string - Descriptor ocispec.Descriptor + Name string + Target ocispec.Descriptor } // TODO(stevvooe): Many of these functions make strong platform assumptions, @@ -29,9 +29,9 @@ type Image struct { func (image *Image) Config(ctx context.Context, provider content.Provider) (ocispec.Descriptor, error) { var configDesc ocispec.Descriptor return configDesc, Walk(ctx, HandlerFunc(func(ctx context.Context, desc ocispec.Descriptor) ([]ocispec.Descriptor, error) { - switch image.Descriptor.MediaType { + switch image.Target.MediaType { case MediaTypeDockerSchema2Manifest, ocispec.MediaTypeImageManifest: - rc, err := provider.Reader(ctx, image.Descriptor.Digest) + rc, err := provider.Reader(ctx, image.Target.Digest) if err != nil { return nil, err } @@ -54,7 +54,7 @@ func (image *Image) Config(ctx context.Context, provider content.Provider) (ocis return nil, errors.New("could not resolve config") } - }), image.Descriptor) + }), image.Target) } // RootFS returns the unpacked diffids that make up and images rootfs. @@ -91,10 +91,10 @@ func (image *Image) RootFS(ctx context.Context, provider content.Provider) ([]di func (image *Image) Size(ctx context.Context, provider content.Provider) (int64, error) { var size int64 return size, Walk(ctx, HandlerFunc(func(ctx context.Context, desc ocispec.Descriptor) ([]ocispec.Descriptor, error) { - switch image.Descriptor.MediaType { + switch image.Target.MediaType { case MediaTypeDockerSchema2Manifest, ocispec.MediaTypeImageManifest: size += desc.Size - rc, err := provider.Reader(ctx, image.Descriptor.Digest) + rc, err := provider.Reader(ctx, image.Target.Digest) if err != nil { return nil, err } @@ -121,5 +121,5 @@ func (image *Image) Size(ctx context.Context, provider content.Provider) (int64, return nil, errors.New("unsupported type") } - }), image.Descriptor) + }), image.Target) } diff --git a/images/storage.go b/images/storage.go index 9cb124b..61bb1a6 100644 --- a/images/storage.go +++ b/images/storage.go @@ -1,6 +1,7 @@ package images import ( + "context" "encoding/binary" "fmt" @@ -8,12 +9,30 @@ import ( "github.com/containerd/containerd/log" digest "github.com/opencontainers/go-digest" ocispec "github.com/opencontainers/image-spec/specs-go/v1" + "github.com/pkg/errors" ) var ( - errImageUnknown = fmt.Errorf("image: unknown") + ErrExists = errors.New("images: exists") + ErrNotFound = errors.New("images: not found") ) +type Store interface { + Put(ctx context.Context, name string, desc ocispec.Descriptor) error + Get(ctx context.Context, name string) (Image, error) + List(ctx context.Context) ([]Image, error) + Delete(ctx context.Context, name string) error +} + +// IsNotFound returns true if the error is due to a missing image. +func IsNotFound(err error) bool { + return errors.Cause(err) == ErrNotFound +} + +func IsExists(err error) bool { + return errors.Cause(err) == ErrExists +} + var ( bucketKeyStorageVersion = []byte("v1") bucketKeyImages = []byte("images") @@ -26,6 +45,8 @@ var ( // "metadata" store. For now, it is bound tightly to the local machine and bolt // but we can take this and use it to define a service interface. +// InitDB will initialize the database for use. The database must be opened for +// write and the caller must not be holding an open transaction. func InitDB(db *bolt.DB) error { log.L.Debug("init db") return db.Update(func(tx *bolt.Tx) error { @@ -34,8 +55,28 @@ func InitDB(db *bolt.DB) error { }) } -func Register(tx *bolt.Tx, name string, desc ocispec.Descriptor) error { - return withImagesBucket(tx, func(bkt *bolt.Bucket) error { +func NewImageStore(tx *bolt.Tx) Store { + return &storage{tx: tx} +} + +type storage struct { + tx *bolt.Tx +} + +func (s *storage) Get(ctx context.Context, name string) (Image, error) { + var image Image + if err := withImageBucket(s.tx, name, func(bkt *bolt.Bucket) error { + image.Name = name + return readImage(&image, bkt) + }); err != nil { + return Image{}, err + } + + return image, nil +} + +func (s *storage) Put(ctx context.Context, name string, desc ocispec.Descriptor) error { + return withImagesBucket(s.tx, func(bkt *bolt.Bucket) error { ibkt, err := bkt.CreateBucketIfNotExists([]byte(name)) if err != nil { return err @@ -65,22 +106,10 @@ func Register(tx *bolt.Tx, name string, desc ocispec.Descriptor) error { }) } -func Get(tx *bolt.Tx, name string) (Image, error) { - var image Image - if err := withImageBucket(tx, name, func(bkt *bolt.Bucket) error { - image.Name = name - return readImage(&image, bkt) - }); err != nil { - return Image{}, err - } - - return image, nil -} - -func List(tx *bolt.Tx) ([]Image, error) { +func (s *storage) List(ctx context.Context) ([]Image, error) { var images []Image - if err := withImagesBucket(tx, func(bkt *bolt.Bucket) error { + if err := withImagesBucket(s.tx, func(bkt *bolt.Bucket) error { return bkt.ForEach(func(k, v []byte) error { var ( image = Image{ @@ -103,8 +132,8 @@ func List(tx *bolt.Tx) ([]Image, error) { return images, nil } -func Delete(tx *bolt.Tx, name string) error { - return withImagesBucket(tx, func(bkt *bolt.Bucket) error { +func (s *storage) Delete(ctx context.Context, name string) error { + return withImagesBucket(s.tx, func(bkt *bolt.Bucket) error { return bkt.DeleteBucket([]byte(name)) }) } @@ -119,11 +148,11 @@ func readImage(image *Image, bkt *bolt.Bucket) error { // keys, rather than full arrays. switch string(k) { case string(bucketKeyDigest): - image.Descriptor.Digest = digest.Digest(v) + image.Target.Digest = digest.Digest(v) case string(bucketKeyMediaType): - image.Descriptor.MediaType = string(v) + image.Target.MediaType = string(v) case string(bucketKeySize): - image.Descriptor.Size, _ = binary.Varint(v) + image.Target.Size, _ = binary.Varint(v) } return nil @@ -149,7 +178,7 @@ func createBucketIfNotExists(tx *bolt.Tx, keys ...[]byte) (*bolt.Bucket, error) func withImagesBucket(tx *bolt.Tx, fn func(bkt *bolt.Bucket) error) error { bkt := getImagesBucket(tx) if bkt == nil { - return errImageUnknown + return ErrNotFound } return fn(bkt) @@ -158,7 +187,7 @@ func withImagesBucket(tx *bolt.Tx, fn func(bkt *bolt.Bucket) error) error { func withImageBucket(tx *bolt.Tx, name string, fn func(bkt *bolt.Bucket) error) error { bkt := getImageBucket(tx, name) if bkt == nil { - return errImageUnknown + return ErrNotFound } return fn(bkt) diff --git a/services/images/client.go b/services/images/client.go new file mode 100644 index 0000000..3e47f03 --- /dev/null +++ b/services/images/client.go @@ -0,0 +1,60 @@ +package images + +import ( + "context" + + imagesapi "github.com/containerd/containerd/api/services/images" + "github.com/containerd/containerd/images" + ocispec "github.com/opencontainers/image-spec/specs-go/v1" +) + +type remoteStore struct { + client imagesapi.ImagesClient +} + +func NewStoreFromClient(client imagesapi.ImagesClient) images.Store { + return &remoteStore{ + client: client, + } +} + +func (s *remoteStore) Put(ctx context.Context, name string, desc ocispec.Descriptor) error { + // TODO(stevvooe): Consider that the remote may want to augment and return + // a modified image. + _, err := s.client.Put(ctx, &imagesapi.PutRequest{ + Image: imagesapi.Image{ + Name: name, + Target: descToProto(&desc), + }, + }) + + return rewriteGRPCError(err) +} + +func (s *remoteStore) Get(ctx context.Context, name string) (images.Image, error) { + resp, err := s.client.Get(ctx, &imagesapi.GetRequest{ + Name: name, + }) + if err != nil { + return images.Image{}, rewriteGRPCError(err) + } + + return imageFromProto(resp.Image), nil +} + +func (s *remoteStore) List(ctx context.Context) ([]images.Image, error) { + resp, err := s.client.List(ctx, &imagesapi.ListRequest{}) + if err != nil { + return nil, rewriteGRPCError(err) + } + + return imagesFromProto(resp.Images), nil +} + +func (s *remoteStore) Delete(ctx context.Context, name string) error { + _, err := s.client.Delete(ctx, &imagesapi.DeleteRequest{ + Name: name, + }) + + return rewriteGRPCError(err) +} diff --git a/services/images/helpers.go b/services/images/helpers.go new file mode 100644 index 0000000..539a918 --- /dev/null +++ b/services/images/helpers.go @@ -0,0 +1,87 @@ +package images + +import ( + imagesapi "github.com/containerd/containerd/api/services/images" + "github.com/containerd/containerd/api/types/descriptor" + "github.com/containerd/containerd/images" + ocispec "github.com/opencontainers/image-spec/specs-go/v1" + "github.com/pkg/errors" + "google.golang.org/grpc" + "google.golang.org/grpc/codes" +) + +func imagesToProto(images []images.Image) []imagesapi.Image { + var imagespb []imagesapi.Image + + for _, image := range images { + imagespb = append(imagespb, imageToProto(&image)) + } + + return imagespb +} + +func imagesFromProto(imagespb []imagesapi.Image) []images.Image { + var images []images.Image + + for _, image := range imagespb { + images = append(images, imageFromProto(&image)) + } + + return images +} + +func imageToProto(image *images.Image) imagesapi.Image { + return imagesapi.Image{ + Name: image.Name, + Target: descToProto(&image.Target), + } +} + +func imageFromProto(imagepb *imagesapi.Image) images.Image { + return images.Image{ + Name: imagepb.Name, + Target: descFromProto(&imagepb.Target), + } +} + +func descFromProto(desc *descriptor.Descriptor) ocispec.Descriptor { + return ocispec.Descriptor{ + MediaType: desc.MediaType, + Size: desc.Size_, + Digest: desc.Digest, + } +} + +func descToProto(desc *ocispec.Descriptor) descriptor.Descriptor { + return descriptor.Descriptor{ + MediaType: desc.MediaType, + Size_: desc.Size, + Digest: desc.Digest, + } +} + +func rewriteGRPCError(err error) error { + if err == nil { + return err + } + + switch grpc.Code(errors.Cause(err)) { + case codes.AlreadyExists: + return images.ErrExists + case codes.NotFound: + return images.ErrNotFound + } + + return err +} + +func mapGRPCError(err error, id string) error { + switch { + case images.IsNotFound(err): + return grpc.Errorf(codes.NotFound, "image %v not found", id) + case images.IsExists(err): + return grpc.Errorf(codes.AlreadyExists, "image %v already exists", id) + } + + return err +} diff --git a/services/images/service.go b/services/images/service.go index 1a07bbc..8f2f765 100644 --- a/services/images/service.go +++ b/services/images/service.go @@ -1,32 +1,91 @@ package images import ( - imagesapi "github.com/docker/containerd/api/services/images" - "github.com/docker/containerd/images" + "github.com/boltdb/bolt" + imagesapi "github.com/containerd/containerd/api/services/images" + "github.com/containerd/containerd/images" + "github.com/containerd/containerd/plugin" "github.com/golang/protobuf/ptypes/empty" "golang.org/x/net/context" + "google.golang.org/grpc" ) +func init() { + plugin.Register("images-grpc", &plugin.Registration{ + Type: plugin.GRPCPlugin, + Init: func(ic *plugin.InitContext) (interface{}, error) { + return NewService(ic.Meta), nil + }, + }) +} + type Service struct { - store images.Store + db *bolt.DB } -func NewService(store images.Store) imagesapi.ImagesServer { - return &Service{store: store} +func NewService(db *bolt.DB) imagesapi.ImagesServer { + return &Service{db: db} } -func (s *Service) Get(context.Context, *imagesapi.GetRequest) (*imagesapi.GetResponse, error) { - panic("not implemented") +func (s *Service) Register(server *grpc.Server) error { + imagesapi.RegisterImagesServer(server, s) + return nil } -func (s *Service) Register(context.Context, *imagesapi.RegisterRequest) (*imagesapi.RegisterRequest, error) { - panic("not implemented") +func (s *Service) Get(ctx context.Context, req *imagesapi.GetRequest) (*imagesapi.GetResponse, error) { + var resp imagesapi.GetResponse + + return &resp, s.withStoreTx(ctx, req.Name, false, func(ctx context.Context, store images.Store) error { + image, err := store.Get(ctx, req.Name) + if err != nil { + return mapGRPCError(err, req.Name) + } + imagepb := imageToProto(&image) + resp.Image = &imagepb + return nil + }) } -func (s *Service) List(context.Context, *imagesapi.ListRequest) (*imagesapi.ListResponse, error) { - panic("not implemented") +func (s *Service) Put(ctx context.Context, req *imagesapi.PutRequest) (*empty.Empty, error) { + return &empty.Empty{}, s.withStoreTx(ctx, req.Image.Name, true, func(ctx context.Context, store images.Store) error { + return mapGRPCError(store.Put(ctx, req.Image.Name, descFromProto(&req.Image.Target)), req.Image.Name) + }) } -func (s *Service) Delete(context.Context, *imagesapi.DeleteRequest) (*empty.Empty, error) { - panic("not implemented") +func (s *Service) List(ctx context.Context, _ *imagesapi.ListRequest) (*imagesapi.ListResponse, error) { + var resp imagesapi.ListResponse + + return &resp, s.withStoreTx(ctx, "", false, func(ctx context.Context, store images.Store) error { + images, err := store.List(ctx) + if err != nil { + return mapGRPCError(err, "") + } + + resp.Images = imagesToProto(images) + return nil + }) +} + +func (s *Service) Delete(ctx context.Context, req *imagesapi.DeleteRequest) (*empty.Empty, error) { + return &empty.Empty{}, s.withStoreTx(ctx, req.Name, true, func(ctx context.Context, store images.Store) error { + return mapGRPCError(store.Delete(ctx, req.Name), req.Name) + }) +} + +func (s *Service) withStoreTx(ctx context.Context, id string, writable bool, fn func(ctx context.Context, store images.Store) error) error { + tx, err := s.db.Begin(writable) + if err != nil { + return mapGRPCError(err, id) + } + defer tx.Rollback() + + if err := fn(ctx, images.NewImageStore(tx)); err != nil { + return err + } + + if writable { + return tx.Commit() + } + + return nil }