diff --git a/snapshot/storage/boltdb/bolt.go b/snapshot/storage/boltdb/bolt.go new file mode 100644 index 0000000..061aa0d --- /dev/null +++ b/snapshot/storage/boltdb/bolt.go @@ -0,0 +1,395 @@ +package boltdb + +import ( + "context" + "encoding/binary" + "fmt" + + "github.com/boltdb/bolt" + "github.com/docker/containerd/snapshot" + "github.com/docker/containerd/snapshot/storage" + "github.com/gogo/protobuf/proto" + "github.com/pkg/errors" +) + +var ( + bucketKeyStorageVersion = []byte("v1") + bucketKeySnapshot = []byte("snapshots") + bucketKeyParents = []byte("parents") +) + +type boltFileTransactor struct { + db *bolt.DB + tx *bolt.Tx +} + +type boltMetastore struct { + dbfile string +} + +// NewMetaStore returns a snapshot MetaStore for storage of metadata related to +// a snapshot driver backed by a bolt file database. This implementation is +// strongly consistent and does all metadata changes in a transaction to prevent +// against process crashes causing inconsistent metadata state. +func NewMetaStore(ctx context.Context, dbfile string) (storage.MetaStore, error) { + return &boltMetastore{ + dbfile: dbfile, + }, nil +} + +func (ms *boltFileTransactor) Rollback() error { + defer ms.db.Close() + return ms.tx.Rollback() +} + +func (ms *boltFileTransactor) Commit() error { + defer ms.db.Close() + return ms.tx.Commit() +} + +type transactionKey struct{} + +func (ms *boltMetastore) TransactionContext(ctx context.Context, writable bool) (context.Context, storage.Transactor, error) { + db, err := bolt.Open(ms.dbfile, 0600, nil) + if err != nil { + return ctx, nil, errors.Wrap(err, "failed to open database file") + } + + tx, err := db.Begin(writable) + if err != nil { + return ctx, nil, errors.Wrap(err, "failed to start transaction") + } + + t := &boltFileTransactor{ + db: db, + tx: tx, + } + + ctx = context.WithValue(ctx, transactionKey{}, t) + + return ctx, t, nil +} + +func (ms *boltMetastore) withBucket(ctx context.Context, fn func(ctx context.Context, tx *bolt.Bucket) error) error { + t, ok := ctx.Value(transactionKey{}).(*boltFileTransactor) + if !ok { + return errors.Errorf("no transaction in context") + } + bkt := t.tx.Bucket(bucketKeyStorageVersion).Bucket(bucketKeySnapshot) + return fn(ctx, bkt) +} + +func (ms *boltMetastore) createBucketIfNotExists(ctx context.Context, fn func(ctx context.Context, tx *bolt.Bucket) error) error { + t, ok := ctx.Value(transactionKey{}).(*boltFileTransactor) + if !ok { + return errors.Errorf("no transaction in context") + } + + bkt, err := t.tx.CreateBucketIfNotExists(bucketKeyStorageVersion) + if err != nil { + return errors.Wrap(err, "failed to create version bucket") + } + bkt, err = bkt.CreateBucketIfNotExists(bucketKeySnapshot) + if err != nil { + return errors.Wrap(err, "failed to create snapshots bucket") + } + return fn(ctx, bkt) +} + +func fromProtoActive(active bool) snapshot.Kind { + if active { + return snapshot.KindActive + } + return snapshot.KindCommitted +} + +func parentKey(parent, child uint64) []byte { + b := make([]byte, binary.Size([]uint64{parent, child})) + i := binary.PutUvarint(b, parent) + j := binary.PutUvarint(b[i:], child) + return b[0 : i+j] +} + +func getParent(b []byte) uint64 { + parent, _ := binary.Uvarint(b) + return parent +} + +func (ms *boltMetastore) Stat(ctx context.Context, key string) (snapshot.Info, error) { + var ss Snapshot + err := ms.withBucket(ctx, func(ctx context.Context, bkt *bolt.Bucket) error { + return getSnapshot(bkt, key, &ss) + }) + if err != nil { + return snapshot.Info{}, err + } + + return snapshot.Info{ + Name: key, + Parent: ss.Parent, + Kind: fromProtoActive(ss.Active), + Readonly: ss.Readonly, + }, nil +} + +func (ms *boltMetastore) Walk(ctx context.Context, fn func(context.Context, snapshot.Info) error) error { + return ms.withBucket(ctx, func(ctx context.Context, bkt *bolt.Bucket) error { + return bkt.ForEach(func(k, v []byte) error { + // skip nested buckets + if v == nil { + return nil + } + var ss Snapshot + if err := proto.Unmarshal(v, &ss); err != nil { + return errors.Wrap(err, "failed to unmarshal snapshot") + } + + info := snapshot.Info{ + Name: string(k), + Parent: ss.Parent, + Kind: fromProtoActive(ss.Active), + Readonly: ss.Readonly, + } + return fn(ctx, info) + }) + }) +} + +func (ms *boltMetastore) CreateActive(ctx context.Context, key, parent string, readonly bool) (a storage.Active, err error) { + err = ms.createBucketIfNotExists(ctx, func(ctx context.Context, bkt *bolt.Bucket) error { + var ( + parentS *Snapshot + ) + if parent != "" { + parentS = new(Snapshot) + if err := getSnapshot(bkt, parent, parentS); err != nil { + return errors.Wrap(err, "failed to get parent snapshot") + } + + if parentS.Active { + return errors.Errorf("cannot create active from active") + } + } + b := bkt.Get([]byte(key)) + if len(b) != 0 { + return errors.Errorf("key already exists") + } + + id, err := bkt.NextSequence() + if err != nil { + return errors.Wrap(err, "unable to get identifier") + } + + ss := Snapshot{ + ID: id, + Parent: parent, + Active: true, + Readonly: readonly, + } + if err := putSnapshot(bkt, key, &ss); err != nil { + return err + } + + if parentS != nil { + pbkt, err := bkt.CreateBucketIfNotExists(bucketKeyParents) + if err != nil { + return errors.Wrap(err, "failed to create parent bucket") + } + if err := pbkt.Put(parentKey(parentS.ID, ss.ID), nil); err != nil { + return errors.Wrap(err, "failed to write parent link") + } + + a.ParentIDs, err = ms.parents(bkt, parentS) + if err != nil { + return errors.Wrap(err, "failed to get parent chain") + } + } + + a.ID = fmt.Sprintf("%d", id) + a.Readonly = readonly + + return nil + }) + if err != nil { + return storage.Active{}, err + } + + return +} + +func (ms *boltMetastore) GetActive(ctx context.Context, key string) (a storage.Active, err error) { + err = ms.withBucket(ctx, func(ctx context.Context, bkt *bolt.Bucket) error { + b := bkt.Get([]byte(key)) + if len(b) == 0 { + return errors.Errorf("active not found") + } + + var ss Snapshot + if err := proto.Unmarshal(b, &ss); err != nil { + return errors.Wrap(err, "failed to unmarshal snapshot") + } + if !ss.Active { + return errors.Errorf("active not found") + } + + a.ID = fmt.Sprintf("%d", ss.ID) + a.Readonly = ss.Readonly + + if ss.Parent != "" { + var parent Snapshot + if err := getSnapshot(bkt, ss.Parent, &parent); err != nil { + return errors.Wrap(err, "failed to get parent snapshot") + } + + a.ParentIDs, err = ms.parents(bkt, &parent) + if err != nil { + return errors.Wrap(err, "failed to get parent chain") + } + } + return nil + }) + if err != nil { + return storage.Active{}, err + } + + return +} + +func (ms *boltMetastore) parents(bkt *bolt.Bucket, parent *Snapshot) (parents []string, err error) { + for { + parents = append(parents, fmt.Sprintf("%d", parent.ID)) + + if parent.Parent == "" { + return + } + + var ps Snapshot + if err := getSnapshot(bkt, parent.Parent, &ps); err != nil { + return nil, errors.Wrap(err, "failed to get parent snapshot") + } + parent = &ps + } + + return +} + +func (ms *boltMetastore) Remove(ctx context.Context, key string) (id string, err error) { + err = ms.withBucket(ctx, func(ctx context.Context, bkt *bolt.Bucket) error { + var ss Snapshot + b := bkt.Get([]byte(key)) + if len(b) == 0 { + return errors.Errorf("key does not exist") + } + + if err := proto.Unmarshal(b, &ss); err != nil { + return errors.Wrap(err, "failed to unmarshal snapshot") + } + + pbkt := bkt.Bucket(bucketKeyParents) + if pbkt != nil { + k, _ := pbkt.Cursor().Seek(parentKey(ss.ID, 0)) + if getParent(k) == ss.ID { + return errors.Errorf("cannot remove snapshot with child") + } + + if ss.Parent != "" { + var ps Snapshot + if err := getSnapshot(bkt, ss.Parent, &ps); err != nil { + return errors.Wrap(err, "failed to get parent snapshot") + } + + if err := pbkt.Delete(parentKey(ps.ID, ss.ID)); err != nil { + return errors.Wrap(err, "failed to delte parent link") + } + } + } + + if err := bkt.Delete([]byte(key)); err != nil { + return errors.Wrap(err, "failed to delete snapshot") + } + + id = fmt.Sprintf("%d", ss.ID) + + return nil + }) + if err != nil { + return "", err + } + + return +} + +func (ms *boltMetastore) Commit(ctx context.Context, key, name string) (id string, err error) { + err = ms.withBucket(ctx, func(ctx context.Context, bkt *bolt.Bucket) error { + b := bkt.Get([]byte(name)) + if len(b) != 0 { + return errors.Errorf("key already exists") + } + + var ss Snapshot + if err := getSnapshot(bkt, key, &ss); err != nil { + return errors.Wrap(err, "failed to get active snapshot") + } + if !ss.Active { + return errors.Errorf("key is not active snapshot") + } + if ss.Readonly { + return errors.Errorf("active snapshot is readonly") + } + + ss.Active = false + ss.Readonly = true + + if err := putSnapshot(bkt, name, &ss); err != nil { + return err + } + if err := bkt.Delete([]byte(key)); err != nil { + return errors.Wrap(err, "failed to delete active") + } + + id = fmt.Sprintf("%d", ss.ID) + + return nil + }) + if err != nil { + return "", err + } + + return +} + +func getSnapshot(bkt *bolt.Bucket, key string, ss *Snapshot) error { + b := bkt.Get([]byte(key)) + if len(b) == 0 { + return errors.Errorf("snapshot not found") + } + if err := proto.Unmarshal(b, ss); err != nil { + return errors.Wrap(err, "failed to unmarshal snapshot") + } + return nil +} + +func putSnapshot(bkt *bolt.Bucket, key string, ss *Snapshot) error { + b, err := proto.Marshal(ss) + if err != nil { + return errors.Wrap(err, "failed to marshal snapshot") + } + + if err := bkt.Put([]byte(key), b); err != nil { + return errors.Wrap(err, "failed to save snapshot") + } + return nil +} + +func getBucket(tx *bolt.Tx, keys ...[]byte) *bolt.Bucket { + bkt := tx.Bucket(keys[0]) + + for _, key := range keys[1:] { + if bkt == nil { + break + } + bkt = bkt.Bucket(key) + } + + return bkt +} diff --git a/snapshot/storage/boltdb/bolt_test.go b/snapshot/storage/boltdb/bolt_test.go new file mode 100644 index 0000000..5e4d379 --- /dev/null +++ b/snapshot/storage/boltdb/bolt_test.go @@ -0,0 +1,16 @@ +package boltdb + +import ( + "context" + "path/filepath" + "testing" + + "github.com/docker/containerd/snapshot/storage" + "github.com/docker/containerd/snapshot/storage/testsuite" +) + +func BenchmarkSuite(b *testing.B) { + testsuite.Benchmarks(b, "BoltDB", func(ctx context.Context, root string) (storage.MetaStore, error) { + return NewMetaStore(ctx, filepath.Join(root, "metadata.db")) + }) +} diff --git a/snapshot/storage/boltdb/record.pb.go b/snapshot/storage/boltdb/record.pb.go new file mode 100644 index 0000000..65fa524 --- /dev/null +++ b/snapshot/storage/boltdb/record.pb.go @@ -0,0 +1,444 @@ +// Code generated by protoc-gen-gogo. +// source: github.com/docker/containerd/snapshot/storage/boltdb/record.proto +// DO NOT EDIT! + +/* + Package boltdb is a generated protocol buffer package. + + It is generated from these files: + github.com/docker/containerd/snapshot/storage/boltdb/record.proto + + It has these top-level messages: + Snapshot +*/ +package boltdb + +import proto "github.com/gogo/protobuf/proto" +import fmt "fmt" +import math "math" +import _ "github.com/gogo/protobuf/gogoproto" + +import strings "strings" +import reflect "reflect" + +import io "io" + +// Reference imports to suppress errors if they are not otherwise used. +var _ = proto.Marshal +var _ = fmt.Errorf +var _ = math.Inf + +// This is a compile-time assertion to ensure that this generated file +// is compatible with the proto package it is being compiled against. +// A compilation error at this line likely means your copy of the +// proto package needs to be updated. +const _ = proto.GoGoProtoPackageIsVersion2 // please upgrade the proto package + +type Snapshot struct { + ID uint64 `protobuf:"varint,1,opt,name=id,proto3" json:"id,omitempty"` + Parent string `protobuf:"bytes,2,opt,name=parent,proto3" json:"parent,omitempty"` + Active bool `protobuf:"varint,4,opt,name=active,proto3" json:"active,omitempty"` + Readonly bool `protobuf:"varint,5,opt,name=readonly,proto3" json:"readonly,omitempty"` +} + +func (m *Snapshot) Reset() { *m = Snapshot{} } +func (*Snapshot) ProtoMessage() {} +func (*Snapshot) Descriptor() ([]byte, []int) { return fileDescriptorRecord, []int{0} } + +func init() { + proto.RegisterType((*Snapshot)(nil), "containerd.v1.Snapshot") +} +func (m *Snapshot) Marshal() (dAtA []byte, err error) { + size := m.Size() + dAtA = make([]byte, size) + n, err := m.MarshalTo(dAtA) + if err != nil { + return nil, err + } + return dAtA[:n], nil +} + +func (m *Snapshot) MarshalTo(dAtA []byte) (int, error) { + var i int + _ = i + var l int + _ = l + if m.ID != 0 { + dAtA[i] = 0x8 + i++ + i = encodeVarintRecord(dAtA, i, uint64(m.ID)) + } + if len(m.Parent) > 0 { + dAtA[i] = 0x12 + i++ + i = encodeVarintRecord(dAtA, i, uint64(len(m.Parent))) + i += copy(dAtA[i:], m.Parent) + } + if m.Active { + dAtA[i] = 0x20 + i++ + if m.Active { + dAtA[i] = 1 + } else { + dAtA[i] = 0 + } + i++ + } + if m.Readonly { + dAtA[i] = 0x28 + i++ + if m.Readonly { + dAtA[i] = 1 + } else { + dAtA[i] = 0 + } + i++ + } + return i, nil +} + +func encodeFixed64Record(dAtA []byte, offset int, v uint64) int { + dAtA[offset] = uint8(v) + dAtA[offset+1] = uint8(v >> 8) + dAtA[offset+2] = uint8(v >> 16) + dAtA[offset+3] = uint8(v >> 24) + dAtA[offset+4] = uint8(v >> 32) + dAtA[offset+5] = uint8(v >> 40) + dAtA[offset+6] = uint8(v >> 48) + dAtA[offset+7] = uint8(v >> 56) + return offset + 8 +} +func encodeFixed32Record(dAtA []byte, offset int, v uint32) int { + dAtA[offset] = uint8(v) + dAtA[offset+1] = uint8(v >> 8) + dAtA[offset+2] = uint8(v >> 16) + dAtA[offset+3] = uint8(v >> 24) + return offset + 4 +} +func encodeVarintRecord(dAtA []byte, offset int, v uint64) int { + for v >= 1<<7 { + dAtA[offset] = uint8(v&0x7f | 0x80) + v >>= 7 + offset++ + } + dAtA[offset] = uint8(v) + return offset + 1 +} +func (m *Snapshot) Size() (n int) { + var l int + _ = l + if m.ID != 0 { + n += 1 + sovRecord(uint64(m.ID)) + } + l = len(m.Parent) + if l > 0 { + n += 1 + l + sovRecord(uint64(l)) + } + if m.Active { + n += 2 + } + if m.Readonly { + n += 2 + } + return n +} + +func sovRecord(x uint64) (n int) { + for { + n++ + x >>= 7 + if x == 0 { + break + } + } + return n +} +func sozRecord(x uint64) (n int) { + return sovRecord(uint64((x << 1) ^ uint64((int64(x) >> 63)))) +} +func (this *Snapshot) String() string { + if this == nil { + return "nil" + } + s := strings.Join([]string{`&Snapshot{`, + `ID:` + fmt.Sprintf("%v", this.ID) + `,`, + `Parent:` + fmt.Sprintf("%v", this.Parent) + `,`, + `Active:` + fmt.Sprintf("%v", this.Active) + `,`, + `Readonly:` + fmt.Sprintf("%v", this.Readonly) + `,`, + `}`, + }, "") + return s +} +func valueToStringRecord(v interface{}) string { + rv := reflect.ValueOf(v) + if rv.IsNil() { + return "nil" + } + pv := reflect.Indirect(rv).Interface() + return fmt.Sprintf("*%v", pv) +} +func (m *Snapshot) Unmarshal(dAtA []byte) error { + l := len(dAtA) + iNdEx := 0 + for iNdEx < l { + preIndex := iNdEx + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowRecord + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + fieldNum := int32(wire >> 3) + wireType := int(wire & 0x7) + if wireType == 4 { + return fmt.Errorf("proto: Snapshot: wiretype end group for non-group") + } + if fieldNum <= 0 { + return fmt.Errorf("proto: Snapshot: illegal tag %d (wire type %d)", fieldNum, wire) + } + switch fieldNum { + case 1: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field ID", wireType) + } + m.ID = 0 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowRecord + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + m.ID |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + case 2: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Parent", wireType) + } + var stringLen uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowRecord + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLen |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + intStringLen := int(stringLen) + if intStringLen < 0 { + return ErrInvalidLengthRecord + } + postIndex := iNdEx + intStringLen + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.Parent = string(dAtA[iNdEx:postIndex]) + iNdEx = postIndex + case 4: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field Active", wireType) + } + var v int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowRecord + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + v |= (int(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + m.Active = bool(v != 0) + case 5: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field Readonly", wireType) + } + var v int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowRecord + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + v |= (int(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + m.Readonly = bool(v != 0) + default: + iNdEx = preIndex + skippy, err := skipRecord(dAtA[iNdEx:]) + if err != nil { + return err + } + if skippy < 0 { + return ErrInvalidLengthRecord + } + if (iNdEx + skippy) > l { + return io.ErrUnexpectedEOF + } + iNdEx += skippy + } + } + + if iNdEx > l { + return io.ErrUnexpectedEOF + } + return nil +} +func skipRecord(dAtA []byte) (n int, err error) { + l := len(dAtA) + iNdEx := 0 + for iNdEx < l { + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return 0, ErrIntOverflowRecord + } + if iNdEx >= l { + return 0, io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + wireType := int(wire & 0x7) + switch wireType { + case 0: + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return 0, ErrIntOverflowRecord + } + if iNdEx >= l { + return 0, io.ErrUnexpectedEOF + } + iNdEx++ + if dAtA[iNdEx-1] < 0x80 { + break + } + } + return iNdEx, nil + case 1: + iNdEx += 8 + return iNdEx, nil + case 2: + var length int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return 0, ErrIntOverflowRecord + } + if iNdEx >= l { + return 0, io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + length |= (int(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + iNdEx += length + if length < 0 { + return 0, ErrInvalidLengthRecord + } + return iNdEx, nil + case 3: + for { + var innerWire uint64 + var start int = iNdEx + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return 0, ErrIntOverflowRecord + } + if iNdEx >= l { + return 0, io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + innerWire |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + innerWireType := int(innerWire & 0x7) + if innerWireType == 4 { + break + } + next, err := skipRecord(dAtA[start:]) + if err != nil { + return 0, err + } + iNdEx = start + next + } + return iNdEx, nil + case 4: + return iNdEx, nil + case 5: + iNdEx += 4 + return iNdEx, nil + default: + return 0, fmt.Errorf("proto: illegal wireType %d", wireType) + } + } + panic("unreachable") +} + +var ( + ErrInvalidLengthRecord = fmt.Errorf("proto: negative length found during unmarshaling") + ErrIntOverflowRecord = fmt.Errorf("proto: integer overflow") +) + +func init() { + proto.RegisterFile("github.com/docker/containerd/snapshot/storage/boltdb/record.proto", fileDescriptorRecord) +} + +var fileDescriptorRecord = []byte{ + // 225 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x44, 0x8e, 0xb1, 0x4e, 0xc3, 0x30, + 0x10, 0x86, 0x7b, 0x51, 0x89, 0x82, 0x25, 0x96, 0x08, 0x55, 0x56, 0x07, 0x13, 0x31, 0x65, 0x8a, + 0x85, 0x78, 0x02, 0x2a, 0x16, 0xd6, 0xf0, 0x04, 0x8e, 0x6d, 0xa5, 0x16, 0xc5, 0x17, 0x5d, 0x8e, + 0x4a, 0x6c, 0x3c, 0x5e, 0x47, 0x46, 0x26, 0x44, 0xfd, 0x24, 0xa8, 0x21, 0xa2, 0xdb, 0x7d, 0xf7, + 0xdd, 0xfd, 0xfa, 0xc5, 0x43, 0x1f, 0x78, 0xfb, 0xd6, 0x35, 0x16, 0x5f, 0xb5, 0x43, 0xfb, 0xe2, + 0x49, 0x5b, 0x8c, 0x6c, 0x42, 0xf4, 0xe4, 0xf4, 0x18, 0xcd, 0x30, 0x6e, 0x91, 0xf5, 0xc8, 0x48, + 0xa6, 0xf7, 0xba, 0xc3, 0x1d, 0xbb, 0x4e, 0x93, 0xb7, 0x48, 0xae, 0x19, 0x08, 0x19, 0xcb, 0xab, + 0xf3, 0x43, 0xb3, 0xbf, 0x5b, 0x5f, 0xf7, 0xd8, 0xe3, 0x64, 0xf4, 0x69, 0xfa, 0x3b, 0xba, 0x8d, + 0xa2, 0x78, 0x9e, 0xc3, 0xca, 0x95, 0xc8, 0x82, 0x93, 0x50, 0x41, 0xbd, 0xdc, 0xe4, 0xe9, 0xfb, + 0x26, 0x7b, 0x7a, 0x6c, 0xb3, 0xe0, 0xca, 0x95, 0xc8, 0x07, 0x43, 0x3e, 0xb2, 0xcc, 0x2a, 0xa8, + 0x2f, 0xdb, 0x99, 0x4e, 0x7b, 0x63, 0x39, 0xec, 0xbd, 0x5c, 0x56, 0x50, 0x17, 0xed, 0x4c, 0xe5, + 0x5a, 0x14, 0xe4, 0x8d, 0xc3, 0xb8, 0x7b, 0x97, 0x17, 0x93, 0xf9, 0xe7, 0x8d, 0x3c, 0x1c, 0xd5, + 0xe2, 0xeb, 0xa8, 0x16, 0x1f, 0x49, 0xc1, 0x21, 0x29, 0xf8, 0x4c, 0x0a, 0x7e, 0x92, 0x82, 0x2e, + 0x9f, 0x0a, 0xdd, 0xff, 0x06, 0x00, 0x00, 0xff, 0xff, 0xd3, 0xbe, 0x14, 0x8b, 0xfa, 0x00, 0x00, + 0x00, +} diff --git a/snapshot/storage/boltdb/record.proto b/snapshot/storage/boltdb/record.proto new file mode 100644 index 0000000..b7ecb64 --- /dev/null +++ b/snapshot/storage/boltdb/record.proto @@ -0,0 +1,12 @@ +syntax = "proto3"; + +package containerd.v1; + +import "gogoproto/gogo.proto"; + +message Snapshot { + uint64 id = 1 [(gogoproto.customname) = "ID"]; + string parent = 2; + bool active = 4; + bool readonly = 5; +} diff --git a/snapshot/storage/testsuite/bench.go b/snapshot/storage/testsuite/bench.go new file mode 100644 index 0000000..608bef5 --- /dev/null +++ b/snapshot/storage/testsuite/bench.go @@ -0,0 +1,215 @@ +package testsuite + +import ( + "context" + "fmt" + "io/ioutil" + "os" + "testing" + + "github.com/docker/containerd/snapshot/storage" +) + +func Benchmarks(b *testing.B, name string, metaFn func(context.Context, string) (storage.MetaStore, error)) { + b.Run("StatActive", makeBench(b, name, metaFn, statActiveBenchmark)) + b.Run("StatCommitted", makeBench(b, name, metaFn, statCommittedBenchmark)) + b.Run("CreateActive", makeBench(b, name, metaFn, createActiveBenchmark)) + b.Run("Remove", makeBench(b, name, metaFn, removeBenchmark)) + b.Run("Commit", makeBench(b, name, metaFn, commitBenchmark)) + b.Run("GetActive", makeBench(b, name, metaFn, getActiveBenchmark)) + b.Run("WriteTransaction", openCloseWritable(b, name, metaFn)) + b.Run("ReadTransaction", openCloseReadonly(b, name, metaFn)) +} + +// makeBench creates a benchmark with a writable transaction +func makeBench(b *testing.B, name string, metaFn func(context.Context, string) (storage.MetaStore, error), fn func(context.Context, *testing.B, storage.MetaStore)) func(b *testing.B) { + return func(b *testing.B) { + ctx := context.Background() + tmpDir, err := ioutil.TempDir("", "metastore-bench-"+name+"-") + if err != nil { + b.Fatal(err) + } + defer os.RemoveAll(tmpDir) + + ms, err := metaFn(ctx, tmpDir) + if err != nil { + b.Fatal(err) + } + + ctx, t, err := ms.TransactionContext(ctx, true) + if err != nil { + b.Fatal(err) + } + defer t.Commit() + + b.ResetTimer() + fn(ctx, b, ms) + } +} + +func openCloseWritable(b *testing.B, name string, metaFn func(context.Context, string) (storage.MetaStore, error)) func(b *testing.B) { + return func(b *testing.B) { + ctx := context.Background() + tmpDir, err := ioutil.TempDir("", "metastore-bench-"+name+"-") + if err != nil { + b.Fatal(err) + } + defer os.RemoveAll(tmpDir) + + ms, err := metaFn(ctx, tmpDir) + if err != nil { + b.Fatal(err) + } + + b.ResetTimer() + + for i := 0; i < b.N; i++ { + _, t, err := ms.TransactionContext(ctx, true) + if err != nil { + b.Fatal(err) + } + if err := t.Commit(); err != nil { + b.Fatal(err) + } + } + } +} + +func openCloseReadonly(b *testing.B, name string, metaFn func(context.Context, string) (storage.MetaStore, error)) func(b *testing.B) { + return func(b *testing.B) { + ctx := context.Background() + tmpDir, err := ioutil.TempDir("", "metastore-bench-"+name+"-") + if err != nil { + b.Fatal(err) + } + defer os.RemoveAll(tmpDir) + + ms, err := metaFn(ctx, tmpDir) + if err != nil { + b.Fatal(err) + } + + b.ResetTimer() + + for i := 0; i < b.N; i++ { + _, t, err := ms.TransactionContext(ctx, false) + if err != nil { + b.Fatal(err) + } + if err := t.Rollback(); err != nil { + b.Fatal(err) + } + } + } +} + +func createActiveFromBase(ctx context.Context, ms storage.MetaStore, active, base string) error { + if _, err := ms.CreateActive(ctx, "bottom", "", false); err != nil { + return err + } + if _, err := ms.Commit(ctx, "bottom", base); err != nil { + return err + } + + _, err := ms.CreateActive(ctx, active, base, false) + return err +} + +func statActiveBenchmark(ctx context.Context, b *testing.B, ms storage.MetaStore) { + if err := createActiveFromBase(ctx, ms, "active", "base"); err != nil { + b.Fatal(err) + } + + b.ResetTimer() + for i := 0; i < b.N; i++ { + _, err := ms.Stat(ctx, "active") + if err != nil { + b.Fatal(err) + } + } +} + +func statCommittedBenchmark(ctx context.Context, b *testing.B, ms storage.MetaStore) { + if err := createActiveFromBase(ctx, ms, "active", "base"); err != nil { + b.Fatal(err) + } + if _, err := ms.Commit(ctx, "active", "committed"); err != nil { + b.Fatal(err) + } + + b.ResetTimer() + for i := 0; i < b.N; i++ { + _, err := ms.Stat(ctx, "committed") + if err != nil { + b.Fatal(err) + } + } +} + +func createActiveBenchmark(ctx context.Context, b *testing.B, ms storage.MetaStore) { + for i := 0; i < b.N; i++ { + if _, err := ms.CreateActive(ctx, "active", "", false); err != nil { + b.Fatal(err) + } + b.StopTimer() + if _, err := ms.Remove(ctx, "active"); err != nil { + b.Fatal(err) + } + b.StartTimer() + } +} + +func removeBenchmark(ctx context.Context, b *testing.B, ms storage.MetaStore) { + for i := 0; i < b.N; i++ { + b.StopTimer() + if _, err := ms.CreateActive(ctx, "active", "", false); err != nil { + b.Fatal(err) + } + b.StartTimer() + if _, err := ms.Remove(ctx, "active"); err != nil { + b.Fatal(err) + } + } +} + +func commitBenchmark(ctx context.Context, b *testing.B, ms storage.MetaStore) { + b.StopTimer() + for i := 0; i < b.N; i++ { + if _, err := ms.CreateActive(ctx, "active", "", false); err != nil { + b.Fatal(err) + } + b.StartTimer() + if _, err := ms.Commit(ctx, "active", "committed"); err != nil { + b.Fatal(err) + } + b.StopTimer() + if _, err := ms.Remove(ctx, "committed"); err != nil { + b.Fatal(err) + } + } +} + +func getActiveBenchmark(ctx context.Context, b *testing.B, ms storage.MetaStore) { + var base string + for i := 1; i <= 10; i++ { + if _, err := ms.CreateActive(ctx, "tmp", base, false); err != nil { + b.Fatalf("create active failed: %+v", err) + } + base = fmt.Sprintf("base-%d", i) + if _, err := ms.Commit(ctx, "tmp", base); err != nil { + b.Fatalf("commit failed: %+v", err) + } + + } + + if _, err := ms.CreateActive(ctx, "active", base, false); err != nil { + b.Fatalf("create active failed: %+v", err) + } + b.ResetTimer() + + for i := 0; i < b.N; i++ { + if _, err := ms.GetActive(ctx, "active"); err != nil { + b.Fatal(err) + } + } +}