vbatts
/
containerd
1
0
Fork 0
Code Pull Requests Releases Activity

Merge pull request #663 from dmcgowan/snapshot-storage 

Refactor snapshot storage package
This commit is contained in:
Michael Crosby 2017-04-04 16:46:09 -07:00 committed by GitHub
parent e2b042e7c1 1c2f5fe203
commit 2048f891d2
12 changed files with 422 additions and 432 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

54
snapshot/btrfs/btrfs.go
View File

@ -12,7 +12,6 @@ import (
"github.com/containerd/containerd/plugin" "github.com/containerd/containerd/plugin"
"github.com/containerd/containerd/snapshot" "github.com/containerd/containerd/snapshot"
"github.com/containerd/containerd/snapshot/storage" "github.com/containerd/containerd/snapshot/storage"
"github.com/containerd/containerd/snapshot/storage/boltdb"
"github.com/pkg/errors" "github.com/pkg/errors"
"github.com/stevvooe/go-btrfs" "github.com/stevvooe/go-btrfs"
) )
@ -32,24 +31,21 @@ func init() {
// TODO: check device for root // TODO: check device for root
return nil, errors.Errorf("btrfs requires \"device\" configuration") return nil, errors.Errorf("btrfs requires \"device\" configuration")
} }
return NewSnapshotter(conf.Device, root)
ms, err := boltdb.NewMetaStore(ic.Context, filepath.Join(root, "metadata.db"))
if err != nil {
return nil, err
}
return NewSnapshotter(conf.Device, root, ms)
}, },
}) })
} }
type Snapshotter struct { type snapshotter struct {
device string // maybe we can resolve it with path? device string // maybe we can resolve it with path?
root string // root provides paths for internal storage. root string // root provides paths for internal storage.
ms storage.MetaStore ms *storage.MetaStore
} }
func NewSnapshotter(device, root string, ms storage.MetaStore) (snapshot.Snapshotter, error) { // NewSnapshotter returns a Snapshotter using btrfs. Uses the provided
// device and root directory for snapshots and stores the metadata in
// a file in the provided root.
func NewSnapshotter(device, root string) (snapshot.Snapshotter, error) {
var ( var (
active = filepath.Join(root, "active") active = filepath.Join(root, "active")
snapshots = filepath.Join(root, "snapshots") snapshots = filepath.Join(root, "snapshots")
@ -63,8 +59,12 @@ func NewSnapshotter(device, root string, ms storage.MetaStore) (snapshot.Snapsho
return nil, err return nil, err
} }
} }
ms, err := storage.NewMetaStore(filepath.Join(root, "metadata.db"))
if err != nil {
return nil, err
}
return &Snapshotter{ return &snapshotter{
device: device, device: device,
root: root, root: root,
ms: ms, ms: ms,
@ -76,34 +76,34 @@ func NewSnapshotter(device, root string, ms storage.MetaStore) (snapshot.Snapsho
// //
// Should be used for parent resolution, existence checks and to discern // Should be used for parent resolution, existence checks and to discern
// the kind of snapshot. // the kind of snapshot.
func (b *Snapshotter) Stat(ctx context.Context, key string) (snapshot.Info, error) { func (b *snapshotter) Stat(ctx context.Context, key string) (snapshot.Info, error) {
ctx, t, err := b.ms.TransactionContext(ctx, false) ctx, t, err := b.ms.TransactionContext(ctx, false)
if err != nil { if err != nil {
return snapshot.Info{}, err return snapshot.Info{}, err
} }
defer t.Rollback() defer t.Rollback()
return b.ms.Stat(ctx, key) return storage.GetInfo(ctx, key)
} }
// Walk the committed snapshots. // Walk the committed snapshots.
func (b *Snapshotter) Walk(ctx context.Context, fn func(context.Context, snapshot.Info) error) error { func (b *snapshotter) Walk(ctx context.Context, fn func(context.Context, snapshot.Info) error) error {
ctx, t, err := b.ms.TransactionContext(ctx, false) ctx, t, err := b.ms.TransactionContext(ctx, false)
if err != nil { if err != nil {
return err return err
} }
defer t.Rollback() defer t.Rollback()
return b.ms.Walk(ctx, fn) return storage.WalkInfo(ctx, fn)
} }
func (b *Snapshotter) Prepare(ctx context.Context, key, parent string) ([]containerd.Mount, error) { func (b *snapshotter) Prepare(ctx context.Context, key, parent string) ([]containerd.Mount, error) {
return b.makeActive(ctx, key, parent, false) return b.makeActive(ctx, key, parent, false)
} }
func (b *Snapshotter) View(ctx context.Context, key, parent string) ([]containerd.Mount, error) { func (b *snapshotter) View(ctx context.Context, key, parent string) ([]containerd.Mount, error) {
return b.makeActive(ctx, key, parent, true) return b.makeActive(ctx, key, parent, true)
} }
func (b *Snapshotter) makeActive(ctx context.Context, key, parent string, readonly bool) ([]containerd.Mount, error) { func (b *snapshotter) makeActive(ctx context.Context, key, parent string, readonly bool) ([]containerd.Mount, error) {
ctx, t, err := b.ms.TransactionContext(ctx, true) ctx, t, err := b.ms.TransactionContext(ctx, true)
if err != nil { if err != nil {
return nil, err return nil, err
@ -116,7 +116,7 @@ func (b *Snapshotter) makeActive(ctx context.Context, key, parent string, readon
} }
}() }()
a, err := b.ms.CreateActive(ctx, key, parent, readonly) a, err := storage.CreateActive(ctx, key, parent, readonly)
if err != nil { if err != nil {
return nil, err return nil, err
} }
@ -148,7 +148,7 @@ func (b *Snapshotter) makeActive(ctx context.Context, key, parent string, readon
return b.mounts(target) return b.mounts(target)
} }
func (b *Snapshotter) mounts(dir string) ([]containerd.Mount, error) { func (b *snapshotter) mounts(dir string) ([]containerd.Mount, error) {
var options []string var options []string
// get the subvolume id back out for the mount // get the subvolume id back out for the mount
@ -174,7 +174,7 @@ func (b *Snapshotter) mounts(dir string) ([]containerd.Mount, error) {
}, nil }, nil
} }
func (b *Snapshotter) Commit(ctx context.Context, name, key string) (err error) { func (b *snapshotter) Commit(ctx context.Context, name, key string) (err error) {
ctx, t, err := b.ms.TransactionContext(ctx, true) ctx, t, err := b.ms.TransactionContext(ctx, true)
if err != nil { if err != nil {
return err return err
@ -187,7 +187,7 @@ func (b *Snapshotter) Commit(ctx context.Context, name, key string) (err error)
} }
}() }()
id, err := b.ms.Commit(ctx, key, name) id, err := storage.CommitActive(ctx, key, name)
if err != nil { if err != nil {
return errors.Wrap(err, "failed to commit") return errors.Wrap(err, "failed to commit")
} }
@ -220,12 +220,12 @@ func (b *Snapshotter) Commit(ctx context.Context, name, key string) (err error)
// called on an read-write or readonly transaction. // called on an read-write or readonly transaction.
// //
// This can be used to recover mounts after calling View or Prepare. // This can be used to recover mounts after calling View or Prepare.
func (b *Snapshotter) Mounts(ctx context.Context, key string) ([]containerd.Mount, error) { func (b *snapshotter) Mounts(ctx context.Context, key string) ([]containerd.Mount, error) {
ctx, t, err := b.ms.TransactionContext(ctx, false) ctx, t, err := b.ms.TransactionContext(ctx, false)
if err != nil { if err != nil {
return nil, err return nil, err
} }
a, err := b.ms.GetActive(ctx, key) a, err := storage.GetActive(ctx, key)
t.Rollback() t.Rollback()
if err != nil { if err != nil {
return nil, errors.Wrap(err, "failed to get active snapshot") return nil, errors.Wrap(err, "failed to get active snapshot")
@ -236,7 +236,7 @@ func (b *Snapshotter) Mounts(ctx context.Context, key string) ([]containerd.Moun
// Remove abandons the transaction identified by key. All resources // Remove abandons the transaction identified by key. All resources
// associated with the key will be removed. // associated with the key will be removed.
func (b *Snapshotter) Remove(ctx context.Context, key string) (err error) { func (b *snapshotter) Remove(ctx context.Context, key string) (err error) {
var ( var (
source, removed string source, removed string
readonly bool readonly bool
@ -260,7 +260,7 @@ func (b *Snapshotter) Remove(ctx context.Context, key string) (err error) {
} }
}() }()
id, k, err := b.ms.Remove(ctx, key) id, k, err := storage.Remove(ctx, key)
if err != nil { if err != nil {
return errors.Wrap(err, "failed to remove snapshot") return errors.Wrap(err, "failed to remove snapshot")
} }

7
snapshot/btrfs/btrfs_test.go
View File

@ -11,7 +11,6 @@ import (
"github.com/containerd/containerd" "github.com/containerd/containerd"
"github.com/containerd/containerd/snapshot" "github.com/containerd/containerd/snapshot"
"github.com/containerd/containerd/snapshot/storage/boltdb"
"github.com/containerd/containerd/snapshot/testsuite" "github.com/containerd/containerd/snapshot/testsuite"
"github.com/containerd/containerd/testutil" "github.com/containerd/containerd/testutil"
) )
@ -23,11 +22,7 @@ const (
func boltSnapshotter(t *testing.T) func(context.Context, string) (snapshot.Snapshotter, func(), error) { func boltSnapshotter(t *testing.T) func(context.Context, string) (snapshot.Snapshotter, func(), error) {
return func(ctx context.Context, root string) (snapshot.Snapshotter, func(), error) { return func(ctx context.Context, root string) (snapshot.Snapshotter, func(), error) {
device := setupBtrfsLoopbackDevice(t, root) device := setupBtrfsLoopbackDevice(t, root)
store, err := boltdb.NewMetaStore(ctx, filepath.Join(root, "metadata.db")) snapshotter, err := NewSnapshotter(device.deviceName, root)
if err != nil {
return nil, nil, err
}
snapshotter, err := NewSnapshotter(device.deviceName, root, store)
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
} }

64
snapshot/overlay/overlay.go
View File

@ -13,7 +13,6 @@ import (
"github.com/containerd/containerd/plugin" "github.com/containerd/containerd/plugin"
"github.com/containerd/containerd/snapshot" "github.com/containerd/containerd/snapshot"
"github.com/containerd/containerd/snapshot/storage" "github.com/containerd/containerd/snapshot/storage"
"github.com/containerd/containerd/snapshot/storage/boltdb"
"github.com/pkg/errors" "github.com/pkg/errors"
) )
@ -21,19 +20,14 @@ func init() {
plugin.Register("snapshot-overlay", &plugin.Registration{ plugin.Register("snapshot-overlay", &plugin.Registration{
Type: plugin.SnapshotPlugin, Type: plugin.SnapshotPlugin,
Init: func(ic *plugin.InitContext) (interface{}, error) { Init: func(ic *plugin.InitContext) (interface{}, error) {
root := filepath.Join(ic.Root, "snapshot", "overlay") return NewSnapshotter(filepath.Join(ic.Root, "snapshot", "overlay"))
ms, err := boltdb.NewMetaStore(ic.Context, filepath.Join(root, "metadata.db"))
if err != nil {
return nil, err
}
return NewSnapshotter(root, ms)
}, },
}) })
} }
type Snapshotter struct { type snapshotter struct {
root string root string
ms storage.MetaStore ms *storage.MetaStore
} }
type activeSnapshot struct { type activeSnapshot struct {
@ -43,15 +37,23 @@ type activeSnapshot struct {
readonly bool readonly bool
} }
func NewSnapshotter(root string, ms storage.MetaStore) (snapshot.Snapshotter, error) { // NewSnapshotter returns a Snapshotter which uses overlayfs. The overlayfs
// diffs are stored under the provided root. A metadata file is stored under
// the root.
func NewSnapshotter(root string) (snapshot.Snapshotter, error) {
if err := os.MkdirAll(root, 0700); err != nil { if err := os.MkdirAll(root, 0700); err != nil {
return nil, err return nil, err
} }
if err := os.MkdirAll(filepath.Join(root, "snapshots"), 0700); err != nil { ms, err := storage.NewMetaStore(filepath.Join(root, "metadata.db"))
if err != nil {
return nil, err return nil, err
} }
return &Snapshotter{ if err := os.Mkdir(filepath.Join(root, "snapshots"), 0700); err != nil {
return nil, err
}
return &snapshotter{
root: root, root: root,
ms: ms, ms: ms,
}, nil }, nil
@ -62,20 +64,20 @@ func NewSnapshotter(root string, ms storage.MetaStore) (snapshot.Snapshotter, er
// //
// Should be used for parent resolution, existence checks and to discern // Should be used for parent resolution, existence checks and to discern
// the kind of snapshot. // the kind of snapshot.
func (o *Snapshotter) Stat(ctx context.Context, key string) (snapshot.Info, error) { func (o *snapshotter) Stat(ctx context.Context, key string) (snapshot.Info, error) {
ctx, t, err := o.ms.TransactionContext(ctx, false) ctx, t, err := o.ms.TransactionContext(ctx, false)
if err != nil { if err != nil {
return snapshot.Info{}, err return snapshot.Info{}, err
} }
defer t.Rollback() defer t.Rollback()
return o.ms.Stat(ctx, key) return storage.GetInfo(ctx, key)
} }
func (o *Snapshotter) Prepare(ctx context.Context, key, parent string) ([]containerd.Mount, error) { func (o *snapshotter) Prepare(ctx context.Context, key, parent string) ([]containerd.Mount, error) {
return o.createActive(ctx, key, parent, false) return o.createActive(ctx, key, parent, false)
} }
func (o *Snapshotter) View(ctx context.Context, key, parent string) ([]containerd.Mount, error) { func (o *snapshotter) View(ctx context.Context, key, parent string) ([]containerd.Mount, error) {
return o.createActive(ctx, key, parent, true) return o.createActive(ctx, key, parent, true)
} }
@ -83,12 +85,12 @@ func (o *Snapshotter) View(ctx context.Context, key, parent string) ([]container
// called on an read-write or readonly transaction. // called on an read-write or readonly transaction.
// //
// This can be used to recover mounts after calling View or Prepare. // This can be used to recover mounts after calling View or Prepare.
func (o *Snapshotter) Mounts(ctx context.Context, key string) ([]containerd.Mount, error) { func (o *snapshotter) Mounts(ctx context.Context, key string) ([]containerd.Mount, error) {
ctx, t, err := o.ms.TransactionContext(ctx, false) ctx, t, err := o.ms.TransactionContext(ctx, false)
if err != nil { if err != nil {
return nil, err return nil, err
} }
active, err := o.ms.GetActive(ctx, key) active, err := storage.GetActive(ctx, key)
t.Rollback() t.Rollback()
if err != nil { if err != nil {
return nil, errors.Wrap(err, "failed to get active mount") return nil, errors.Wrap(err, "failed to get active mount")
@ -96,12 +98,12 @@ func (o *Snapshotter) Mounts(ctx context.Context, key string) ([]containerd.Moun
return o.mounts(active), nil return o.mounts(active), nil
} }
func (o *Snapshotter) Commit(ctx context.Context, name, key string) error { func (o *snapshotter) Commit(ctx context.Context, name, key string) error {
ctx, t, err := o.ms.TransactionContext(ctx, true) ctx, t, err := o.ms.TransactionContext(ctx, true)
if err != nil { if err != nil {
return err return err
} }
if _, err := o.ms.Commit(ctx, key, name); err != nil { if _, err := storage.CommitActive(ctx, key, name); err != nil {
if rerr := t.Rollback(); rerr != nil { if rerr := t.Rollback(); rerr != nil {
log.G(ctx).WithError(rerr).Warn("Failure rolling back transaction") log.G(ctx).WithError(rerr).Warn("Failure rolling back transaction")
} }
@ -112,7 +114,7 @@ func (o *Snapshotter) Commit(ctx context.Context, name, key string) error {
// Remove abandons the transaction identified by key. All resources // Remove abandons the transaction identified by key. All resources
// associated with the key will be removed. // associated with the key will be removed.
func (o *Snapshotter) Remove(ctx context.Context, key string) (err error) { func (o *snapshotter) Remove(ctx context.Context, key string) (err error) {
ctx, t, err := o.ms.TransactionContext(ctx, true) ctx, t, err := o.ms.TransactionContext(ctx, true)
if err != nil { if err != nil {
return err return err
@ -125,7 +127,7 @@ func (o *Snapshotter) Remove(ctx context.Context, key string) (err error) {
} }
}() }()
id, _, err := o.ms.Remove(ctx, key) id, _, err := storage.Remove(ctx, key)
if err != nil { if err != nil {
return errors.Wrap(err, "failed to remove") return errors.Wrap(err, "failed to remove")
} }
@ -154,16 +156,16 @@ func (o *Snapshotter) Remove(ctx context.Context, key string) (err error) {
} }
// Walk the committed snapshots. // Walk the committed snapshots.
func (o *Snapshotter) Walk(ctx context.Context, fn func(context.Context, snapshot.Info) error) error { func (o *snapshotter) Walk(ctx context.Context, fn func(context.Context, snapshot.Info) error) error {
ctx, t, err := o.ms.TransactionContext(ctx, false) ctx, t, err := o.ms.TransactionContext(ctx, false)
if err != nil { if err != nil {
return err return err
} }
defer t.Rollback() defer t.Rollback()
return o.ms.Walk(ctx, fn) return storage.WalkInfo(ctx, fn)
} }
func (o *Snapshotter) createActive(ctx context.Context, key, parent string, readonly bool) ([]containerd.Mount, error) { func (o *snapshotter) createActive(ctx context.Context, key, parent string, readonly bool) ([]containerd.Mount, error) {
var ( var (
path string path string
snapshotDir = filepath.Join(o.root, "snapshots") snapshotDir = filepath.Join(o.root, "snapshots")
@ -202,7 +204,7 @@ func (o *Snapshotter) createActive(ctx context.Context, key, parent string, read
return nil, err return nil, err
} }
active, err := o.ms.CreateActive(ctx, key, parent, readonly) active, err := storage.CreateActive(ctx, key, parent, readonly)
if err != nil { if err != nil {
if rerr := t.Rollback(); rerr != nil { if rerr := t.Rollback(); rerr != nil {
log.G(ctx).WithError(rerr).Warn("Failure rolling back transaction") log.G(ctx).WithError(rerr).Warn("Failure rolling back transaction")
@ -211,7 +213,7 @@ func (o *Snapshotter) createActive(ctx context.Context, key, parent string, read
} }
path = filepath.Join(snapshotDir, active.ID) path = filepath.Join(snapshotDir, active.ID)
if err := os.Rename(td, path); err != nil { if err = os.Rename(td, path); err != nil {
if rerr := t.Rollback(); rerr != nil { if rerr := t.Rollback(); rerr != nil {
log.G(ctx).WithError(rerr).Warn("Failure rolling back transaction") log.G(ctx).WithError(rerr).Warn("Failure rolling back transaction")
} }
@ -219,14 +221,14 @@ func (o *Snapshotter) createActive(ctx context.Context, key, parent string, read
} }
td = "" td = ""
if err := t.Commit(); err != nil { if err = t.Commit(); err != nil {
return nil, errors.Wrap(err, "commit failed") return nil, errors.Wrap(err, "commit failed")
} }
return o.mounts(active), nil return o.mounts(active), nil
} }
func (o *Snapshotter) mounts(active storage.Active) []containerd.Mount { func (o *snapshotter) mounts(active storage.Active) []containerd.Mount {
if len(active.ParentIDs) == 0 { if len(active.ParentIDs) == 0 {
// if we only have one layer/no parents then just return a bind mount as overlay // if we only have one layer/no parents then just return a bind mount as overlay
// will not work // will not work
@ -282,10 +284,10 @@ func (o *Snapshotter) mounts(active storage.Active) []containerd.Mount {
} }
func (o *Snapshotter) upperPath(id string) string { func (o *snapshotter) upperPath(id string) string {
return filepath.Join(o.root, "snapshots", id, "fs") return filepath.Join(o.root, "snapshots", id, "fs")
} }
func (o *Snapshotter) workPath(id string) string { func (o *snapshotter) workPath(id string) string {
return filepath.Join(o.root, "snapshots", id, "work") return filepath.Join(o.root, "snapshots", id, "work")
} }

30
snapshot/overlay/overlay_test.go
View File

@ -11,17 +11,13 @@ import (
"github.com/containerd/containerd" "github.com/containerd/containerd"
"github.com/containerd/containerd/snapshot" "github.com/containerd/containerd/snapshot"
"github.com/containerd/containerd/snapshot/storage/boltdb" "github.com/containerd/containerd/snapshot/storage"
"github.com/containerd/containerd/snapshot/testsuite" "github.com/containerd/containerd/snapshot/testsuite"
"github.com/containerd/containerd/testutil" "github.com/containerd/containerd/testutil"
) )
func boltSnapshotter(ctx context.Context, root string) (snapshot.Snapshotter, func(), error) { func newSnapshotter(ctx context.Context, root string) (snapshot.Snapshotter, func(), error) {
store, err := boltdb.NewMetaStore(ctx, filepath.Join(root, "metadata.db")) snapshotter, err := NewSnapshotter(root)
if err != nil {
return nil, nil, err
}
snapshotter, err := NewSnapshotter(root, store)
if err != nil { if err != nil {
return nil, nil, err return nil, nil, err
} }
@ -31,7 +27,7 @@ func boltSnapshotter(ctx context.Context, root string) (snapshot.Snapshotter, fu
func TestOverlay(t *testing.T) { func TestOverlay(t *testing.T) {
testutil.RequiresRoot(t) testutil.RequiresRoot(t)
testsuite.SnapshotterSuite(t, "Overlay", boltSnapshotter) testsuite.SnapshotterSuite(t, "Overlay", newSnapshotter)
} }
func TestOverlayMounts(t *testing.T) { func TestOverlayMounts(t *testing.T) {
@ -41,7 +37,7 @@ func TestOverlayMounts(t *testing.T) {
t.Fatal(err) t.Fatal(err)
} }
defer os.RemoveAll(root) defer os.RemoveAll(root)
o, _, err := boltSnapshotter(ctx, root) o, _, err := newSnapshotter(ctx, root)
if err != nil { if err != nil {
t.Error(err) t.Error(err)
return return
@ -77,7 +73,7 @@ func TestOverlayCommit(t *testing.T) {
t.Fatal(err) t.Fatal(err)
} }
defer os.RemoveAll(root) defer os.RemoveAll(root)
o, _, err := boltSnapshotter(ctx, root) o, _, err := newSnapshotter(ctx, root)
if err != nil { if err != nil {
t.Error(err) t.Error(err)
return return
@ -106,7 +102,7 @@ func TestOverlayOverlayMount(t *testing.T) {
t.Fatal(err) t.Fatal(err)
} }
defer os.RemoveAll(root) defer os.RemoveAll(root)
o, _, err := boltSnapshotter(ctx, root) o, _, err := newSnapshotter(ctx, root)
if err != nil { if err != nil {
t.Error(err) t.Error(err)
return return
@ -153,14 +149,14 @@ func TestOverlayOverlayMount(t *testing.T) {
} }
func getBasePath(ctx context.Context, sn snapshot.Snapshotter, root, key string) string { func getBasePath(ctx context.Context, sn snapshot.Snapshotter, root, key string) string {
o := sn.(*Snapshotter) o := sn.(*snapshotter)
ctx, t, err := o.ms.TransactionContext(ctx, false) ctx, t, err := o.ms.TransactionContext(ctx, false)
if err != nil { if err != nil {
panic(err) panic(err)
} }
defer t.Rollback() defer t.Rollback()
active, err := o.ms.GetActive(ctx, key) active, err := storage.GetActive(ctx, key)
if err != nil { if err != nil {
panic(err) panic(err)
} }
@ -169,13 +165,13 @@ func getBasePath(ctx context.Context, sn snapshot.Snapshotter, root, key string)
} }
func getParents(ctx context.Context, sn snapshot.Snapshotter, root, key string) []string { func getParents(ctx context.Context, sn snapshot.Snapshotter, root, key string) []string {
o := sn.(*Snapshotter) o := sn.(*snapshotter)
ctx, t, err := o.ms.TransactionContext(ctx, false) ctx, t, err := o.ms.TransactionContext(ctx, false)
if err != nil { if err != nil {
panic(err) panic(err)
} }
defer t.Rollback() defer t.Rollback()
active, err := o.ms.GetActive(ctx, key) active, err := storage.GetActive(ctx, key)
if err != nil { if err != nil {
panic(err) panic(err)
} }
@ -194,7 +190,7 @@ func TestOverlayOverlayRead(t *testing.T) {
t.Fatal(err) t.Fatal(err)
} }
defer os.RemoveAll(root) defer os.RemoveAll(root)
o, _, err := boltSnapshotter(ctx, root) o, _, err := newSnapshotter(ctx, root)
if err != nil { if err != nil {
t.Error(err) t.Error(err)
return return
@ -246,7 +242,7 @@ func TestOverlayView(t *testing.T) {
t.Fatal(err) t.Fatal(err)
} }
defer os.RemoveAll(root) defer os.RemoveAll(root)
o, _, err := boltSnapshotter(ctx, root) o, _, err := newSnapshotter(ctx, root)
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
} }

260
snapshot/storage/boltdb/bolt.go → snapshot/storage/bolt.go
View File

@ -1,4 +1,4 @@
package boltdb package storage
import ( import (
"context" "context"
@ -7,7 +7,7 @@ import (
"github.com/boltdb/bolt" "github.com/boltdb/bolt"
"github.com/containerd/containerd/snapshot" "github.com/containerd/containerd/snapshot"
"github.com/containerd/containerd/snapshot/storage" db "github.com/containerd/containerd/snapshot/storage/proto"
"github.com/gogo/protobuf/proto" "github.com/gogo/protobuf/proto"
"github.com/pkg/errors" "github.com/pkg/errors"
) )
@ -16,6 +16,10 @@ var (
bucketKeyStorageVersion = []byte("v1") bucketKeyStorageVersion = []byte("v1")
bucketKeySnapshot = []byte("snapshots") bucketKeySnapshot = []byte("snapshots")
bucketKeyParents = []byte("parents") bucketKeyParents = []byte("parents")
// ErrNoTransaction is returned when an operation is attempted with
// a context which is not inside of a transaction.
ErrNoTransaction = errors.New("no transaction in context")
) )
type boltFileTransactor struct { type boltFileTransactor struct {
@ -23,91 +27,14 @@ type boltFileTransactor struct {
tx *bolt.Tx tx *bolt.Tx
} }
type boltMetastore struct { func (bft *boltFileTransactor) Rollback() error {
dbfile string defer bft.db.Close()
return bft.tx.Rollback()
} }
// NewMetaStore returns a snapshot MetaStore for storage of metadata related to func (bft *boltFileTransactor) Commit() error {
// a snapshot driver backed by a bolt file database. This implementation is defer bft.db.Close()
// strongly consistent and does all metadata changes in a transaction to prevent return bft.tx.Commit()
// against process crashes causing inconsistent metadata state.
func NewMetaStore(ctx context.Context, dbfile string) (storage.MetaStore, error) {
return &boltMetastore{
dbfile: dbfile,
}, nil
}
func (ms *boltFileTransactor) Rollback() error {
defer ms.db.Close()
return ms.tx.Rollback()
}
func (ms *boltFileTransactor) Commit() error {
defer ms.db.Close()
return ms.tx.Commit()
}
type transactionKey struct{}
func (ms *boltMetastore) TransactionContext(ctx context.Context, writable bool) (context.Context, storage.Transactor, error) {
db, err := bolt.Open(ms.dbfile, 0600, nil)
if err != nil {
return ctx, nil, errors.Wrap(err, "failed to open database file")
}
tx, err := db.Begin(writable)
if err != nil {
return ctx, nil, errors.Wrap(err, "failed to start transaction")
}
t := &boltFileTransactor{
db: db,
tx: tx,
}
ctx = context.WithValue(ctx, transactionKey{}, t)
return ctx, t, nil
}
func (ms *boltMetastore) withBucket(ctx context.Context, fn func(context.Context, *bolt.Bucket, *bolt.Bucket) error) error {
t, ok := ctx.Value(transactionKey{}).(*boltFileTransactor)
if !ok {
return errors.Errorf("no transaction in context")
}
bkt := t.tx.Bucket(bucketKeyStorageVersion)
if bkt == nil {
return errors.Wrap(snapshot.ErrSnapshotNotExist, "bucket does not exist")
}
return fn(ctx, bkt.Bucket(bucketKeySnapshot), bkt.Bucket(bucketKeyParents))
}
func (ms *boltMetastore) createBucketIfNotExists(ctx context.Context, fn func(context.Context, *bolt.Bucket, *bolt.Bucket) error) error {
t, ok := ctx.Value(transactionKey{}).(*boltFileTransactor)
if !ok {
return errors.Errorf("no transaction in context")
}
bkt, err := t.tx.CreateBucketIfNotExists(bucketKeyStorageVersion)
if err != nil {
return errors.Wrap(err, "failed to create version bucket")
}
sbkt, err := bkt.CreateBucketIfNotExists(bucketKeySnapshot)
if err != nil {
return errors.Wrap(err, "failed to create snapshots bucket")
}
pbkt, err := bkt.CreateBucketIfNotExists(bucketKeyParents)
if err != nil {
return errors.Wrap(err, "failed to create snapshots bucket")
}
return fn(ctx, sbkt, pbkt)
}
func fromProtoKind(k Kind) snapshot.Kind {
if k == KindActive {
return snapshot.KindActive
}
return snapshot.KindCommitted
} }
// parentKey returns a composite key of the parent and child identifiers. The // parentKey returns a composite key of the parent and child identifiers. The
@ -134,9 +61,11 @@ func getParentPrefix(b []byte) uint64 {
return parent return parent
} }
func (ms *boltMetastore) Stat(ctx context.Context, key string) (snapshot.Info, error) { // GetInfo returns the snapshot Info directly from the metadata. Requires a
var ss Snapshot // context with a storage transaction.
err := ms.withBucket(ctx, func(ctx context.Context, bkt, pbkt *bolt.Bucket) error { func GetInfo(ctx context.Context, key string) (snapshot.Info, error) {
var ss db.Snapshot
err := withBucket(ctx, func(ctx context.Context, bkt, pbkt *bolt.Bucket) error {
return getSnapshot(bkt, key, &ss) return getSnapshot(bkt, key, &ss)
}) })
if err != nil { if err != nil {
@ -151,14 +80,17 @@ func (ms *boltMetastore) Stat(ctx context.Context, key string) (snapshot.Info, e
}, nil }, nil
} }
func (ms *boltMetastore) Walk(ctx context.Context, fn func(context.Context, snapshot.Info) error) error { // WalkInfo iterates through all metadata Info for the stored snapshots and
return ms.withBucket(ctx, func(ctx context.Context, bkt, pbkt *bolt.Bucket) error { // calls the provided function for each. Requires a context with a storage
// transaction.
func WalkInfo(ctx context.Context, fn func(context.Context, snapshot.Info) error) error {
return withBucket(ctx, func(ctx context.Context, bkt, pbkt *bolt.Bucket) error {
return bkt.ForEach(func(k, v []byte) error { return bkt.ForEach(func(k, v []byte) error {
// skip nested buckets // skip nested buckets
if v == nil { if v == nil {
return nil return nil
} }
var ss Snapshot var ss db.Snapshot
if err := proto.Unmarshal(v, &ss); err != nil { if err := proto.Unmarshal(v, &ss); err != nil {
return errors.Wrap(err, "failed to unmarshal snapshot") return errors.Wrap(err, "failed to unmarshal snapshot")
} }
@ -174,18 +106,23 @@ func (ms *boltMetastore) Walk(ctx context.Context, fn func(context.Context, snap
}) })
} }
func (ms *boltMetastore) CreateActive(ctx context.Context, key, parent string, readonly bool) (a storage.Active, err error) { // CreateActive creates a new active snapshot transaction referenced by
err = ms.createBucketIfNotExists(ctx, func(ctx context.Context, bkt, pbkt *bolt.Bucket) error { // the provided key. The new active snapshot will have the provided
// parent. If the readonly option is given, the active snapshot will be
// marked as readonly and can only be removed, and not committed. The
// provided context must contain a writable transaction.
func CreateActive(ctx context.Context, key, parent string, readonly bool) (a Active, err error) {
err = createBucketIfNotExists(ctx, func(ctx context.Context, bkt, pbkt *bolt.Bucket) error {
var ( var (
parentS *Snapshot parentS *db.Snapshot
) )
if parent != "" { if parent != "" {
parentS = new(Snapshot) parentS = new(db.Snapshot)
if err := getSnapshot(bkt, parent, parentS); err != nil { if err := getSnapshot(bkt, parent, parentS); err != nil {
return errors.Wrap(err, "failed to get parent snapshot") return errors.Wrap(err, "failed to get parent snapshot")
} }
if parentS.Kind != KindCommitted { if parentS.Kind != db.KindCommitted {
return errors.Wrap(snapshot.ErrSnapshotNotCommitted, "parent is not committed snapshot") return errors.Wrap(snapshot.ErrSnapshotNotCommitted, "parent is not committed snapshot")
} }
} }
@ -199,10 +136,10 @@ func (ms *boltMetastore) CreateActive(ctx context.Context, key, parent string, r
return errors.Wrap(err, "unable to get identifier") return errors.Wrap(err, "unable to get identifier")
} }
ss := Snapshot{ ss := db.Snapshot{
ID: id, ID: id,
Parent: parent, Parent: parent,
Kind: KindActive, Kind: db.KindActive,
Readonly: readonly, Readonly: readonly,
} }
if err := putSnapshot(bkt, key, &ss); err != nil { if err := putSnapshot(bkt, key, &ss); err != nil {
@ -216,7 +153,7 @@ func (ms *boltMetastore) CreateActive(ctx context.Context, key, parent string, r
return errors.Wrap(err, "failed to write parent link") return errors.Wrap(err, "failed to write parent link")
} }
a.ParentIDs, err = ms.parents(bkt, parentS) a.ParentIDs, err = parents(bkt, parentS)
if err != nil { if err != nil {
return errors.Wrap(err, "failed to get parent chain") return errors.Wrap(err, "failed to get parent chain")
} }
@ -228,24 +165,26 @@ func (ms *boltMetastore) CreateActive(ctx context.Context, key, parent string, r
return nil return nil
}) })
if err != nil { if err != nil {
return storage.Active{}, err return Active{}, err
} }
return return
} }
func (ms *boltMetastore) GetActive(ctx context.Context, key string) (a storage.Active, err error) { // GetActive returns the metadata for the active snapshot transaction referenced
err = ms.withBucket(ctx, func(ctx context.Context, bkt, pbkt *bolt.Bucket) error { // by the given key. Requires a context with a storage transaction.
func GetActive(ctx context.Context, key string) (a Active, err error) {
err = withBucket(ctx, func(ctx context.Context, bkt, pbkt *bolt.Bucket) error {
b := bkt.Get([]byte(key)) b := bkt.Get([]byte(key))
if len(b) == 0 { if len(b) == 0 {
return snapshot.ErrSnapshotNotExist return snapshot.ErrSnapshotNotExist
} }
var ss Snapshot var ss db.Snapshot
if err := proto.Unmarshal(b, &ss); err != nil { if err := proto.Unmarshal(b, &ss); err != nil {
return errors.Wrap(err, "failed to unmarshal snapshot") return errors.Wrap(err, "failed to unmarshal snapshot")
} }
if ss.Kind != KindActive { if ss.Kind != db.KindActive {
return snapshot.ErrSnapshotNotActive return snapshot.ErrSnapshotNotActive
} }
@ -253,12 +192,12 @@ func (ms *boltMetastore) GetActive(ctx context.Context, key string) (a storage.A
a.Readonly = ss.Readonly a.Readonly = ss.Readonly
if ss.Parent != "" { if ss.Parent != "" {
var parent Snapshot var parent db.Snapshot
if err := getSnapshot(bkt, ss.Parent, &parent); err != nil { if err := getSnapshot(bkt, ss.Parent, &parent); err != nil {
return errors.Wrap(err, "failed to get parent snapshot") return errors.Wrap(err, "failed to get parent snapshot")
} }
a.ParentIDs, err = ms.parents(bkt, &parent) a.ParentIDs, err = parents(bkt, &parent)
if err != nil { if err != nil {
return errors.Wrap(err, "failed to get parent chain") return errors.Wrap(err, "failed to get parent chain")
} }
@ -266,33 +205,18 @@ func (ms *boltMetastore) GetActive(ctx context.Context, key string) (a storage.A
return nil return nil
}) })
if err != nil { if err != nil {
return storage.Active{}, err return Active{}, err
} }
return return
} }
func (ms *boltMetastore) parents(bkt *bolt.Bucket, parent *Snapshot) (parents []string, err error) { // Remove removes a snapshot from the metastore. The string identifier for the
for { // snapshot is returned as well as the kind. The provided context must contain a
parents = append(parents, fmt.Sprintf("%d", parent.ID)) // writable transaction.
func Remove(ctx context.Context, key string) (id string, k snapshot.Kind, err error) {
if parent.Parent == "" { err = withBucket(ctx, func(ctx context.Context, bkt, pbkt *bolt.Bucket) error {
return var ss db.Snapshot
}
var ps Snapshot
if err := getSnapshot(bkt, parent.Parent, &ps); err != nil {
return nil, errors.Wrap(err, "failed to get parent snapshot")
}
parent = &ps
}
return
}
func (ms *boltMetastore) Remove(ctx context.Context, key string) (id string, k snapshot.Kind, err error) {
err = ms.withBucket(ctx, func(ctx context.Context, bkt, pbkt *bolt.Bucket) error {
var ss Snapshot
b := bkt.Get([]byte(key)) b := bkt.Get([]byte(key))
if len(b) == 0 { if len(b) == 0 {
return snapshot.ErrSnapshotNotExist return snapshot.ErrSnapshotNotExist
@ -309,7 +233,7 @@ func (ms *boltMetastore) Remove(ctx context.Context, key string) (id string, k s
} }
if ss.Parent != "" { if ss.Parent != "" {
var ps Snapshot var ps db.Snapshot
if err := getSnapshot(bkt, ss.Parent, &ps); err != nil { if err := getSnapshot(bkt, ss.Parent, &ps); err != nil {
return errors.Wrap(err, "failed to get parent snapshot") return errors.Wrap(err, "failed to get parent snapshot")
} }
@ -333,25 +257,31 @@ func (ms *boltMetastore) Remove(ctx context.Context, key string) (id string, k s
return return
} }
func (ms *boltMetastore) Commit(ctx context.Context, key, name string) (id string, err error) { // CommitActive renames the active snapshot transaction referenced by `key`
err = ms.withBucket(ctx, func(ctx context.Context, bkt, pbkt *bolt.Bucket) error { // as a committed snapshot referenced by `Name`. The resulting snapshot will be
// committed and readonly. The `key` reference will no longer be available for
// lookup or removal. The returned string identifier for the committed snapshot
// is the same identifier of the original active snapshot. The provided context
// must contain a writable transaction.
func CommitActive(ctx context.Context, key, name string) (id string, err error) {
err = withBucket(ctx, func(ctx context.Context, bkt, pbkt *bolt.Bucket) error {
b := bkt.Get([]byte(name)) b := bkt.Get([]byte(name))
if len(b) != 0 { if len(b) != 0 {
return errors.Wrap(snapshot.ErrSnapshotExist, "committed name already exists") return errors.Wrap(snapshot.ErrSnapshotExist, "committed name already exists")
} }
var ss Snapshot var ss db.Snapshot
if err := getSnapshot(bkt, key, &ss); err != nil { if err := getSnapshot(bkt, key, &ss); err != nil {
return errors.Wrap(err, "failed to get active snapshot") return errors.Wrap(err, "failed to get active snapshot")
} }
if ss.Kind != KindActive { if ss.Kind != db.KindActive {
return snapshot.ErrSnapshotNotActive return snapshot.ErrSnapshotNotActive
} }
if ss.Readonly { if ss.Readonly {
return errors.Errorf("active snapshot is readonly") return errors.Errorf("active snapshot is readonly")
} }
ss.Kind = KindCommitted ss.Kind = db.KindCommitted
ss.Readonly = true ss.Readonly = true
if err := putSnapshot(bkt, name, &ss); err != nil { if err := putSnapshot(bkt, name, &ss); err != nil {
@ -372,7 +302,65 @@ func (ms *boltMetastore) Commit(ctx context.Context, key, name string) (id strin
return return
} }
func getSnapshot(bkt *bolt.Bucket, key string, ss *Snapshot) error { func withBucket(ctx context.Context, fn func(context.Context, *bolt.Bucket, *bolt.Bucket) error) error {
t, ok := ctx.Value(transactionKey{}).(*boltFileTransactor)
if !ok {
return ErrNoTransaction
}
bkt := t.tx.Bucket(bucketKeyStorageVersion)
if bkt == nil {
return errors.Wrap(snapshot.ErrSnapshotNotExist, "bucket does not exist")
}
return fn(ctx, bkt.Bucket(bucketKeySnapshot), bkt.Bucket(bucketKeyParents))
}
func createBucketIfNotExists(ctx context.Context, fn func(context.Context, *bolt.Bucket, *bolt.Bucket) error) error {
t, ok := ctx.Value(transactionKey{}).(*boltFileTransactor)
if !ok {
return ErrNoTransaction
}
bkt, err := t.tx.CreateBucketIfNotExists(bucketKeyStorageVersion)
if err != nil {
return errors.Wrap(err, "failed to create version bucket")
}
sbkt, err := bkt.CreateBucketIfNotExists(bucketKeySnapshot)
if err != nil {
return errors.Wrap(err, "failed to create snapshots bucket")
}
pbkt, err := bkt.CreateBucketIfNotExists(bucketKeyParents)
if err != nil {
return errors.Wrap(err, "failed to create snapshots bucket")
}
return fn(ctx, sbkt, pbkt)
}
func fromProtoKind(k db.Kind) snapshot.Kind {
if k == db.KindActive {
return snapshot.KindActive
}
return snapshot.KindCommitted
}
func parents(bkt *bolt.Bucket, parent *db.Snapshot) (parents []string, err error) {
for {
parents = append(parents, fmt.Sprintf("%d", parent.ID))
if parent.Parent == "" {
return
}
var ps db.Snapshot
if err := getSnapshot(bkt, parent.Parent, &ps); err != nil {
return nil, errors.Wrap(err, "failed to get parent snapshot")
}
parent = &ps
}
return
}
func getSnapshot(bkt *bolt.Bucket, key string, ss *db.Snapshot) error {
b := bkt.Get([]byte(key)) b := bkt.Get([]byte(key))
if len(b) == 0 { if len(b) == 0 {
return snapshot.ErrSnapshotNotExist return snapshot.ErrSnapshotNotExist
@ -383,7 +371,7 @@ func getSnapshot(bkt *bolt.Bucket, key string, ss *Snapshot) error {
return nil return nil
} }
func putSnapshot(bkt *bolt.Bucket, key string, ss *Snapshot) error { func putSnapshot(bkt *bolt.Bucket, key string, ss *db.Snapshot) error {
b, err := proto.Marshal(ss) b, err := proto.Marshal(ss)
if err != nil { if err != nil {
return errors.Wrap(err, "failed to marshal snapshot") return errors.Wrap(err, "failed to marshal snapshot")

21
snapshot/storage/bolt_test.go Normal file
View File

@ -0,0 +1,21 @@
package storage
import (
"path/filepath"
"testing"
// Does not require root but flag must be defined for snapshot tests
_ "github.com/containerd/containerd/testutil"
)
func TestMetastore(t *testing.T) {
MetaStoreSuite(t, "Metastore", func(root string) (*MetaStore, error) {
return NewMetaStore(filepath.Join(root, "metadata.db"))
})
}
func BenchmarkSuite(b *testing.B) {
Benchmarks(b, "BoltDBBench", func(root string) (*MetaStore, error) {
return NewMetaStore(filepath.Join(root, "metadata.db"))
})
}

25
snapshot/storage/boltdb/bolt_test.go
View File

@ -1,25 +0,0 @@
package boltdb
import (
"context"
"path/filepath"
"testing"
"github.com/containerd/containerd/snapshot/storage"
"github.com/containerd/containerd/snapshot/storage/testsuite"
// Does not require root but flag must be defined for snapshot tests
_ "github.com/containerd/containerd/testutil"
)
func TestBoltDB(t *testing.T) {
testsuite.MetaStoreSuite(t, "BoltDB", func(ctx context.Context, root string) (storage.MetaStore, error) {
return NewMetaStore(ctx, filepath.Join(root, "metadata.db"))
})
}
func BenchmarkSuite(b *testing.B) {
testsuite.Benchmarks(b, "BoltDBBench", func(ctx context.Context, root string) (storage.MetaStore, error) {
return NewMetaStore(ctx, filepath.Join(root, "metadata.db"))
})
}

103
snapshot/storage/metastore.go
View File

@ -1,59 +1,28 @@
// Package storage provides a metadata storage implementation for snapshot
// drivers. Drive implementations are responsible for starting and managing
// transactions using the defined context creator. This storage package uses
// BoltDB for storing metadata. Access to the raw boltdb transaction is not
// provided, but the stored object is provided by the proto subpackage.
package storage package storage
import ( import (
"context" "context"
"github.com/containerd/containerd/snapshot" "github.com/boltdb/bolt"
"github.com/pkg/errors"
) )
// MetaStore is used to store metadata related to a snapshot driver. The
// MetaStore is intended to store metadata related to name, state and
// parentage. Using the MetaStore is not required to implement a snapshot
// driver but can be used to handle the persistence and transactional
// complexities of a driver implementation.
type MetaStore interface {
// TransactionContext creates a new transaction context.
TransactionContext(ctx context.Context, writable bool) (context.Context, Transactor, error)
// Stat returns the snapshot stat Info directly from
// the metadata.
Stat(ctx context.Context, key string) (snapshot.Info, error)
// Walk iterates through all metadata for the stored
// snapshots and calls the provided function for each.
Walk(ctx context.Context, fn func(context.Context, snapshot.Info) error) error
// CreateActive creates a new active snapshot transaction referenced by
// the provided key. The new active snapshot will have the provided
// parent. If the readonly option is given, the active snapshot will be
// marked as readonly and can only be removed, and not committed. The
// provided context must contain a writable transaction.
CreateActive(ctx context.Context, key, parent string, readonly bool) (Active, error)
// GetActive returns the metadata for the active snapshot transaction
// referenced by the given key.
GetActive(ctx context.Context, key string) (Active, error)
// Remove removes a snapshot from the metastore. The provided context
// must contain a writable transaction. The string identifier for the
// snapshot is returned as well as the kind.
Remove(ctx context.Context, key string) (string, snapshot.Kind, error)
// Commit renames the active snapshot transaction referenced by `key`
// as a committed snapshot referenced by `Name`. The resulting snapshot
// will be committed and readonly. The `key` reference will no longer
// be available for lookup or removal. The returned string identifier
// for the committed snapshot is the same identifier of the original
// active snapshot.
Commit(ctx context.Context, key, name string) (string, error)
}
// Transactor is used to finalize an active transaction. // Transactor is used to finalize an active transaction.
type Transactor interface { type Transactor interface {
// Commit commits any changes made during the transaction. // Commit commits any changes made during the transaction. On error a
// caller is expected to clean up any resources which would have relied
// on data mutated as part of this transaction. Only writable
// transactions can commit, non-writable must call Rollback.
Commit() error Commit() error
// Rollback rolls back any changes made during the transaction. // Rollback rolls back any changes made during the transaction. This
// must be called on all non-writable transactions and aborted writable
// transaction.
Rollback() error Rollback() error
} }
@ -67,3 +36,47 @@ type Active struct {
ParentIDs []string ParentIDs []string
Readonly bool Readonly bool
} }
// MetaStore is used to store metadata related to a snapshot driver. The
// MetaStore is intended to store metadata related to name, state and
// parentage. Using the MetaStore is not required to implement a snapshot
// driver but can be used to handle the persistence and transactional
// complexities of a driver implementation.
type MetaStore struct {
dbfile string
}
// NewMetaStore returns a snapshot MetaStore for storage of metadata related to
// a snapshot driver backed by a bolt file database. This implementation is
// strongly consistent and does all metadata changes in a transaction to prevent
// against process crashes causing inconsistent metadata state.
func NewMetaStore(dbfile string) (*MetaStore, error) {
return &MetaStore{
dbfile: dbfile,
}, nil
}
type transactionKey struct{}
// TransactionContext creates a new transaction context. The writable value
// should be set to true for transactions which are expected to mutate data.
func (ms *MetaStore) TransactionContext(ctx context.Context, writable bool) (context.Context, Transactor, error) {
db, err := bolt.Open(ms.dbfile, 0600, nil)
if err != nil {
return ctx, nil, errors.Wrap(err, "failed to open database file")
}
tx, err := db.Begin(writable)
if err != nil {
return ctx, nil, errors.Wrap(err, "failed to start transaction")
}
t := &boltFileTransactor{
db: db,
tx: tx,
}
ctx = context.WithValue(ctx, transactionKey{}, t)
return ctx, t, nil
}

66
snapshot/storage/testsuite/bench.go → snapshot/storage/metastore_bench_test.go
View File

@ -1,4 +1,4 @@
package testsuite package storage
import ( import (
"context" "context"
@ -6,13 +6,11 @@ import (
"io/ioutil" "io/ioutil"
"os" "os"
"testing" "testing"
"github.com/containerd/containerd/snapshot/storage"
) )
// Benchmarks returns a benchmark suite using the provided metadata store // Benchmarks returns a benchmark suite using the provided metadata store
// creation method // creation method
func Benchmarks(b *testing.B, name string, metaFn func(context.Context, string) (storage.MetaStore, error)) { func Benchmarks(b *testing.B, name string, metaFn metaFactory) {
b.Run("StatActive", makeBench(b, name, metaFn, statActiveBenchmark)) b.Run("StatActive", makeBench(b, name, metaFn, statActiveBenchmark))
b.Run("StatCommitted", makeBench(b, name, metaFn, statCommittedBenchmark)) b.Run("StatCommitted", makeBench(b, name, metaFn, statCommittedBenchmark))
b.Run("CreateActive", makeBench(b, name, metaFn, createActiveBenchmark)) b.Run("CreateActive", makeBench(b, name, metaFn, createActiveBenchmark))
@ -24,7 +22,7 @@ func Benchmarks(b *testing.B, name string, metaFn func(context.Context, string)
} }
// makeBench creates a benchmark with a writable transaction // makeBench creates a benchmark with a writable transaction
func makeBench(b *testing.B, name string, metaFn func(context.Context, string) (storage.MetaStore, error), fn func(context.Context, *testing.B, storage.MetaStore)) func(b *testing.B) { func makeBench(b *testing.B, name string, metaFn metaFactory, fn func(context.Context, *testing.B, *MetaStore)) func(b *testing.B) {
return func(b *testing.B) { return func(b *testing.B) {
ctx := context.Background() ctx := context.Background()
tmpDir, err := ioutil.TempDir("", "metastore-bench-"+name+"-") tmpDir, err := ioutil.TempDir("", "metastore-bench-"+name+"-")
@ -33,7 +31,7 @@ func makeBench(b *testing.B, name string, metaFn func(context.Context, string) (
} }
defer os.RemoveAll(tmpDir) defer os.RemoveAll(tmpDir)
ms, err := metaFn(ctx, tmpDir) ms, err := metaFn(tmpDir)
if err != nil { if err != nil {
b.Fatal(err) b.Fatal(err)
} }
@ -49,7 +47,7 @@ func makeBench(b *testing.B, name string, metaFn func(context.Context, string) (
} }
} }
func openCloseWritable(b *testing.B, name string, metaFn func(context.Context, string) (storage.MetaStore, error)) func(b *testing.B) { func openCloseWritable(b *testing.B, name string, metaFn metaFactory) func(b *testing.B) {
return func(b *testing.B) { return func(b *testing.B) {
ctx := context.Background() ctx := context.Background()
tmpDir, err := ioutil.TempDir("", "metastore-bench-"+name+"-") tmpDir, err := ioutil.TempDir("", "metastore-bench-"+name+"-")
@ -58,7 +56,7 @@ func openCloseWritable(b *testing.B, name string, metaFn func(context.Context, s
} }
defer os.RemoveAll(tmpDir) defer os.RemoveAll(tmpDir)
ms, err := metaFn(ctx, tmpDir) ms, err := metaFn(tmpDir)
if err != nil { if err != nil {
b.Fatal(err) b.Fatal(err)
} }
@ -77,7 +75,7 @@ func openCloseWritable(b *testing.B, name string, metaFn func(context.Context, s
} }
} }
func openCloseReadonly(b *testing.B, name string, metaFn func(context.Context, string) (storage.MetaStore, error)) func(b *testing.B) { func openCloseReadonly(b *testing.B, name string, metaFn metaFactory) func(b *testing.B) {
return func(b *testing.B) { return func(b *testing.B) {
ctx := context.Background() ctx := context.Background()
tmpDir, err := ioutil.TempDir("", "metastore-bench-"+name+"-") tmpDir, err := ioutil.TempDir("", "metastore-bench-"+name+"-")
@ -86,7 +84,7 @@ func openCloseReadonly(b *testing.B, name string, metaFn func(context.Context, s
} }
defer os.RemoveAll(tmpDir) defer os.RemoveAll(tmpDir)
ms, err := metaFn(ctx, tmpDir) ms, err := metaFn(tmpDir)
if err != nil { if err != nil {
b.Fatal(err) b.Fatal(err)
} }
@ -105,112 +103,112 @@ func openCloseReadonly(b *testing.B, name string, metaFn func(context.Context, s
} }
} }
func createActiveFromBase(ctx context.Context, ms storage.MetaStore, active, base string) error { func createActiveFromBase(ctx context.Context, ms *MetaStore, active, base string) error {
if _, err := ms.CreateActive(ctx, "bottom", "", false); err != nil { if _, err := CreateActive(ctx, "bottom", "", false); err != nil {
return err return err
} }
if _, err := ms.Commit(ctx, "bottom", base); err != nil { if _, err := CommitActive(ctx, "bottom", base); err != nil {
return err return err
} }
_, err := ms.CreateActive(ctx, active, base, false) _, err := CreateActive(ctx, active, base, false)
return err return err
} }
func statActiveBenchmark(ctx context.Context, b *testing.B, ms storage.MetaStore) { func statActiveBenchmark(ctx context.Context, b *testing.B, ms *MetaStore) {
if err := createActiveFromBase(ctx, ms, "active", "base"); err != nil { if err := createActiveFromBase(ctx, ms, "active", "base"); err != nil {
b.Fatal(err) b.Fatal(err)
} }
b.ResetTimer() b.ResetTimer()
for i := 0; i < b.N; i++ { for i := 0; i < b.N; i++ {
_, err := ms.Stat(ctx, "active") _, err := GetInfo(ctx, "active")
if err != nil { if err != nil {
b.Fatal(err) b.Fatal(err)
} }
} }
} }
func statCommittedBenchmark(ctx context.Context, b *testing.B, ms storage.MetaStore) { func statCommittedBenchmark(ctx context.Context, b *testing.B, ms *MetaStore) {
if err := createActiveFromBase(ctx, ms, "active", "base"); err != nil { if err := createActiveFromBase(ctx, ms, "active", "base"); err != nil {
b.Fatal(err) b.Fatal(err)
} }
if _, err := ms.Commit(ctx, "active", "committed"); err != nil { if _, err := CommitActive(ctx, "active", "committed"); err != nil {
b.Fatal(err) b.Fatal(err)
} }
b.ResetTimer() b.ResetTimer()
for i := 0; i < b.N; i++ { for i := 0; i < b.N; i++ {
_, err := ms.Stat(ctx, "committed") _, err := GetInfo(ctx, "committed")
if err != nil { if err != nil {
b.Fatal(err) b.Fatal(err)
} }
} }
} }
func createActiveBenchmark(ctx context.Context, b *testing.B, ms storage.MetaStore) { func createActiveBenchmark(ctx context.Context, b *testing.B, ms *MetaStore) {
for i := 0; i < b.N; i++ { for i := 0; i < b.N; i++ {
if _, err := ms.CreateActive(ctx, "active", "", false); err != nil { if _, err := CreateActive(ctx, "active", "", false); err != nil {
b.Fatal(err) b.Fatal(err)
} }
b.StopTimer() b.StopTimer()
if _, _, err := ms.Remove(ctx, "active"); err != nil { if _, _, err := Remove(ctx, "active"); err != nil {
b.Fatal(err) b.Fatal(err)
} }
b.StartTimer() b.StartTimer()
} }
} }
func removeBenchmark(ctx context.Context, b *testing.B, ms storage.MetaStore) { func removeBenchmark(ctx context.Context, b *testing.B, ms *MetaStore) {
for i := 0; i < b.N; i++ { for i := 0; i < b.N; i++ {
b.StopTimer() b.StopTimer()
if _, err := ms.CreateActive(ctx, "active", "", false); err != nil { if _, err := CreateActive(ctx, "active", "", false); err != nil {
b.Fatal(err) b.Fatal(err)
} }
b.StartTimer() b.StartTimer()
if _, _, err := ms.Remove(ctx, "active"); err != nil { if _, _, err := Remove(ctx, "active"); err != nil {
b.Fatal(err) b.Fatal(err)
} }
} }
} }
func commitBenchmark(ctx context.Context, b *testing.B, ms storage.MetaStore) { func commitBenchmark(ctx context.Context, b *testing.B, ms *MetaStore) {
b.StopTimer() b.StopTimer()
for i := 0; i < b.N; i++ { for i := 0; i < b.N; i++ {
if _, err := ms.CreateActive(ctx, "active", "", false); err != nil { if _, err := CreateActive(ctx, "active", "", false); err != nil {
b.Fatal(err) b.Fatal(err)
} }
b.StartTimer() b.StartTimer()
if _, err := ms.Commit(ctx, "active", "committed"); err != nil { if _, err := CommitActive(ctx, "active", "committed"); err != nil {
b.Fatal(err) b.Fatal(err)
} }
b.StopTimer() b.StopTimer()
if _, _, err := ms.Remove(ctx, "committed"); err != nil { if _, _, err := Remove(ctx, "committed"); err != nil {
b.Fatal(err) b.Fatal(err)
} }
} }
} }
func getActiveBenchmark(ctx context.Context, b *testing.B, ms storage.MetaStore) { func getActiveBenchmark(ctx context.Context, b *testing.B, ms *MetaStore) {
var base string var base string
for i := 1; i <= 10; i++ { for i := 1; i <= 10; i++ {
if _, err := ms.CreateActive(ctx, "tmp", base, false); err != nil { if _, err := CreateActive(ctx, "tmp", base, false); err != nil {
b.Fatalf("create active failed: %+v", err) b.Fatalf("create active failed: %+v", err)
} }
base = fmt.Sprintf("base-%d", i) base = fmt.Sprintf("base-%d", i)
if _, err := ms.Commit(ctx, "tmp", base); err != nil { if _, err := CommitActive(ctx, "tmp", base); err != nil {
b.Fatalf("commit failed: %+v", err) b.Fatalf("commit failed: %+v", err)
} }
} }
if _, err := ms.CreateActive(ctx, "active", base, false); err != nil { if _, err := CreateActive(ctx, "active", base, false); err != nil {
b.Fatalf("create active failed: %+v", err) b.Fatalf("create active failed: %+v", err)
} }
b.ResetTimer() b.ResetTimer()
for i := 0; i < b.N; i++ { for i := 0; i < b.N; i++ {
if _, err := ms.GetActive(ctx, "active"); err != nil { if _, err := GetActive(ctx, "active"); err != nil {
b.Fatal(err) b.Fatal(err)
} }
} }

161
snapshot/storage/testsuite/testsuite.go → snapshot/storage/metastore_test.go
View File

@ -1,4 +1,4 @@
package testsuite package storage
import ( import (
"context" "context"
@ -7,22 +7,21 @@ import (
"testing" "testing"
"github.com/containerd/containerd/snapshot" "github.com/containerd/containerd/snapshot"
"github.com/containerd/containerd/snapshot/storage"
"github.com/pkg/errors" "github.com/pkg/errors"
"github.com/stretchr/testify/assert" "github.com/stretchr/testify/assert"
) )
type testFunc func(context.Context, *testing.T, storage.MetaStore) type testFunc func(context.Context, *testing.T, *MetaStore)
type metaFactory func(context.Context, string) (storage.MetaStore, error) type metaFactory func(string) (*MetaStore, error)
type populateFunc func(context.Context, storage.MetaStore) error type populateFunc func(context.Context, *MetaStore) error
// MetaStoreSuite runs a test suite on the metastore given a factory function. // MetaStoreSuite runs a test suite on the metastore given a factory function.
func MetaStoreSuite(t *testing.T, name string, meta func(ctx context.Context, root string) (storage.MetaStore, error)) { func MetaStoreSuite(t *testing.T, name string, meta func(root string) (*MetaStore, error)) {
t.Run("Stat", makeTest(t, name, meta, inReadTransaction(testStat, basePopulate))) t.Run("GetInfo", makeTest(t, name, meta, inReadTransaction(testGetInfo, basePopulate)))
t.Run("StatNotExist", makeTest(t, name, meta, inReadTransaction(testStatNotExist, basePopulate))) t.Run("GetInfoNotExist", makeTest(t, name, meta, inReadTransaction(testGetInfoNotExist, basePopulate)))
t.Run("StatEmptyDB", makeTest(t, name, meta, inReadTransaction(testStatNotExist, nil))) t.Run("GetInfoEmptyDB", makeTest(t, name, meta, inReadTransaction(testGetInfoNotExist, nil)))
t.Run("Walk", makeTest(t, name, meta, inReadTransaction(testWalk, basePopulate))) t.Run("Walk", makeTest(t, name, meta, inReadTransaction(testWalk, basePopulate)))
t.Run("GetActive", makeTest(t, name, meta, testGetActive)) t.Run("GetActive", makeTest(t, name, meta, testGetActive))
t.Run("GetActiveNotExist", makeTest(t, name, meta, inReadTransaction(testGetActiveNotExist, basePopulate))) t.Run("GetActiveNotExist", makeTest(t, name, meta, inReadTransaction(testGetActiveNotExist, basePopulate)))
@ -52,7 +51,7 @@ func makeTest(t *testing.T, name string, metaFn metaFactory, fn testFunc) func(t
} }
defer os.RemoveAll(tmpDir) defer os.RemoveAll(tmpDir)
ms, err := metaFn(ctx, tmpDir) ms, err := metaFn(tmpDir)
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
} }
@ -62,7 +61,7 @@ func makeTest(t *testing.T, name string, metaFn metaFactory, fn testFunc) func(t
} }
func inReadTransaction(fn testFunc, pf populateFunc) testFunc { func inReadTransaction(fn testFunc, pf populateFunc) testFunc {
return func(ctx context.Context, t *testing.T, ms storage.MetaStore) { return func(ctx context.Context, t *testing.T, ms *MetaStore) {
if pf != nil { if pf != nil {
ctx, tx, err := ms.TransactionContext(ctx, true) ctx, tx, err := ms.TransactionContext(ctx, true)
if err != nil { if err != nil {
@ -97,7 +96,7 @@ func inReadTransaction(fn testFunc, pf populateFunc) testFunc {
} }
func inWriteTransaction(fn testFunc) testFunc { func inWriteTransaction(fn testFunc) testFunc {
return func(ctx context.Context, t *testing.T, ms storage.MetaStore) { return func(ctx context.Context, t *testing.T, ms *MetaStore) {
ctx, tx, err := ms.TransactionContext(ctx, true) ctx, tx, err := ms.TransactionContext(ctx, true)
if err != nil { if err != nil {
t.Fatal("Failed to start transaction: %+v", err) t.Fatal("Failed to start transaction: %+v", err)
@ -125,32 +124,32 @@ func inWriteTransaction(fn testFunc) testFunc {
// - "active-3": active with parent "committed-2" // - "active-3": active with parent "committed-2"
// - "active-4": readonly active without parent" // - "active-4": readonly active without parent"
// - "active-5": readonly active with parent "committed-2" // - "active-5": readonly active with parent "committed-2"
func basePopulate(ctx context.Context, ms storage.MetaStore) error { func basePopulate(ctx context.Context, ms *MetaStore) error {
if _, err := ms.CreateActive(ctx, "committed-tmp-1", "", false); err != nil { if _, err := CreateActive(ctx, "committed-tmp-1", "", false); err != nil {
return errors.Wrap(err, "failed to create active") return errors.Wrap(err, "failed to create active")
} }
if _, err := ms.Commit(ctx, "committed-tmp-1", "committed-1"); err != nil { if _, err := CommitActive(ctx, "committed-tmp-1", "committed-1"); err != nil {
return errors.Wrap(err, "failed to create active") return errors.Wrap(err, "failed to create active")
} }
if _, err := ms.CreateActive(ctx, "committed-tmp-2", "committed-1", false); err != nil { if _, err := CreateActive(ctx, "committed-tmp-2", "committed-1", false); err != nil {
return errors.Wrap(err, "failed to create active") return errors.Wrap(err, "failed to create active")
} }
if _, err := ms.Commit(ctx, "committed-tmp-2", "committed-2"); err != nil { if _, err := CommitActive(ctx, "committed-tmp-2", "committed-2"); err != nil {
return errors.Wrap(err, "failed to create active") return errors.Wrap(err, "failed to create active")
} }
if _, err := ms.CreateActive(ctx, "active-1", "", false); err != nil { if _, err := CreateActive(ctx, "active-1", "", false); err != nil {
return errors.Wrap(err, "failed to create active") return errors.Wrap(err, "failed to create active")
} }
if _, err := ms.CreateActive(ctx, "active-2", "committed-1", false); err != nil { if _, err := CreateActive(ctx, "active-2", "committed-1", false); err != nil {
return errors.Wrap(err, "failed to create active") return errors.Wrap(err, "failed to create active")
} }
if _, err := ms.CreateActive(ctx, "active-3", "committed-2", false); err != nil { if _, err := CreateActive(ctx, "active-3", "committed-2", false); err != nil {
return errors.Wrap(err, "failed to create active") return errors.Wrap(err, "failed to create active")
} }
if _, err := ms.CreateActive(ctx, "active-4", "", true); err != nil { if _, err := CreateActive(ctx, "active-4", "", true); err != nil {
return errors.Wrap(err, "failed to create active") return errors.Wrap(err, "failed to create active")
} }
if _, err := ms.CreateActive(ctx, "active-5", "committed-2", true); err != nil { if _, err := CreateActive(ctx, "active-5", "committed-2", true); err != nil {
return errors.Wrap(err, "failed to create active") return errors.Wrap(err, "failed to create active")
} }
return nil return nil
@ -237,24 +236,24 @@ func assertExist(t *testing.T, err error) {
} }
} }
func testStat(ctx context.Context, t *testing.T, ms storage.MetaStore) { func testGetInfo(ctx context.Context, t *testing.T, ms *MetaStore) {
for key, expected := range baseInfo { for key, expected := range baseInfo {
info, err := ms.Stat(ctx, key) info, err := GetInfo(ctx, key)
if err != nil { if err != nil {
t.Fatalf("Stat on %v failed: %+v", key, err) t.Fatalf("GetInfo on %v failed: %+v", key, err)
} }
assert.Equal(t, expected, info) assert.Equal(t, expected, info)
} }
} }
func testStatNotExist(ctx context.Context, t *testing.T, ms storage.MetaStore) { func testGetInfoNotExist(ctx context.Context, t *testing.T, ms *MetaStore) {
_, err := ms.Stat(ctx, "active-not-exist") _, err := GetInfo(ctx, "active-not-exist")
assertNotExist(t, err) assertNotExist(t, err)
} }
func testWalk(ctx context.Context, t *testing.T, ms storage.MetaStore) { func testWalk(ctx context.Context, t *testing.T, ms *MetaStore) {
found := map[string]snapshot.Info{} found := map[string]snapshot.Info{}
err := ms.Walk(ctx, func(ctx context.Context, info snapshot.Info) error { err := WalkInfo(ctx, func(ctx context.Context, info snapshot.Info) error {
if _, ok := found[info.Name]; ok { if _, ok := found[info.Name]; ok {
return errors.Errorf("entry already encountered") return errors.Errorf("entry already encountered")
} }
@ -267,13 +266,13 @@ func testWalk(ctx context.Context, t *testing.T, ms storage.MetaStore) {
assert.Equal(t, baseInfo, found) assert.Equal(t, baseInfo, found)
} }
func testGetActive(ctx context.Context, t *testing.T, ms storage.MetaStore) { func testGetActive(ctx context.Context, t *testing.T, ms *MetaStore) {
activeMap := map[string]storage.Active{} activeMap := map[string]Active{}
populate := func(ctx context.Context, ms storage.MetaStore) error { populate := func(ctx context.Context, ms *MetaStore) error {
if _, err := ms.CreateActive(ctx, "committed-tmp-1", "", false); err != nil { if _, err := CreateActive(ctx, "committed-tmp-1", "", false); err != nil {
return errors.Wrap(err, "failed to create active") return errors.Wrap(err, "failed to create active")
} }
if _, err := ms.Commit(ctx, "committed-tmp-1", "committed-1"); err != nil { if _, err := CommitActive(ctx, "committed-tmp-1", "committed-1"); err != nil {
return errors.Wrap(err, "failed to create active") return errors.Wrap(err, "failed to create active")
} }
@ -299,7 +298,7 @@ func testGetActive(ctx context.Context, t *testing.T, ms storage.MetaStore) {
Readonly: true, Readonly: true,
}, },
} { } {
active, err := ms.CreateActive(ctx, opts.Name, opts.Parent, opts.Readonly) active, err := CreateActive(ctx, opts.Name, opts.Parent, opts.Readonly)
if err != nil { if err != nil {
return errors.Wrap(err, "failed to create active") return errors.Wrap(err, "failed to create active")
} }
@ -308,9 +307,9 @@ func testGetActive(ctx context.Context, t *testing.T, ms storage.MetaStore) {
return nil return nil
} }
test := func(ctx context.Context, t *testing.T, ms storage.MetaStore) { test := func(ctx context.Context, t *testing.T, ms *MetaStore) {
for key, expected := range activeMap { for key, expected := range activeMap {
active, err := ms.GetActive(ctx, key) active, err := GetActive(ctx, key)
if err != nil { if err != nil {
t.Fatal("Failed to get active: %+v", err) t.Fatal("Failed to get active: %+v", err)
} }
@ -321,18 +320,18 @@ func testGetActive(ctx context.Context, t *testing.T, ms storage.MetaStore) {
inReadTransaction(test, populate)(ctx, t, ms) inReadTransaction(test, populate)(ctx, t, ms)
} }
func testGetActiveCommitted(ctx context.Context, t *testing.T, ms storage.MetaStore) { func testGetActiveCommitted(ctx context.Context, t *testing.T, ms *MetaStore) {
_, err := ms.GetActive(ctx, "committed-1") _, err := GetActive(ctx, "committed-1")
assertNotActive(t, err) assertNotActive(t, err)
} }
func testGetActiveNotExist(ctx context.Context, t *testing.T, ms storage.MetaStore) { func testGetActiveNotExist(ctx context.Context, t *testing.T, ms *MetaStore) {
_, err := ms.GetActive(ctx, "active-not-exist") _, err := GetActive(ctx, "active-not-exist")
assertNotExist(t, err) assertNotExist(t, err)
} }
func testCreateActive(ctx context.Context, t *testing.T, ms storage.MetaStore) { func testCreateActive(ctx context.Context, t *testing.T, ms *MetaStore) {
a1, err := ms.CreateActive(ctx, "active-1", "", false) a1, err := CreateActive(ctx, "active-1", "", false)
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
} }
@ -340,7 +339,7 @@ func testCreateActive(ctx context.Context, t *testing.T, ms storage.MetaStore) {
t.Fatal("Expected writable active") t.Fatal("Expected writable active")
} }
a2, err := ms.CreateActive(ctx, "active-2", "", true) a2, err := CreateActive(ctx, "active-2", "", true)
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
} }
@ -351,7 +350,7 @@ func testCreateActive(ctx context.Context, t *testing.T, ms storage.MetaStore) {
t.Fatal("Expected readonly active") t.Fatal("Expected readonly active")
} }
commitID, err := ms.Commit(ctx, "active-1", "committed-1") commitID, err := CommitActive(ctx, "active-1", "committed-1")
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
} }
@ -359,7 +358,7 @@ func testCreateActive(ctx context.Context, t *testing.T, ms storage.MetaStore) {
t.Fatal("Snapshot identifier must not change on commit") t.Fatal("Snapshot identifier must not change on commit")
} }
a3, err := ms.CreateActive(ctx, "active-3", "committed-1", false) a3, err := CreateActive(ctx, "active-3", "committed-1", false)
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
} }
@ -376,7 +375,7 @@ func testCreateActive(ctx context.Context, t *testing.T, ms storage.MetaStore) {
t.Fatal("Expected writable active") t.Fatal("Expected writable active")
} }
a4, err := ms.CreateActive(ctx, "active-4", "committed-1", true) a4, err := CreateActive(ctx, "active-4", "committed-1", true)
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
} }
@ -394,31 +393,31 @@ func testCreateActive(ctx context.Context, t *testing.T, ms storage.MetaStore) {
} }
} }
func testCreateActiveExist(ctx context.Context, t *testing.T, ms storage.MetaStore) { func testCreateActiveExist(ctx context.Context, t *testing.T, ms *MetaStore) {
if err := basePopulate(ctx, ms); err != nil { if err := basePopulate(ctx, ms); err != nil {
t.Fatalf("Populate failed: %+v", err) t.Fatalf("Populate failed: %+v", err)
} }
_, err := ms.CreateActive(ctx, "active-1", "", false) _, err := CreateActive(ctx, "active-1", "", false)
assertExist(t, err) assertExist(t, err)
_, err = ms.CreateActive(ctx, "committed-1", "", false) _, err = CreateActive(ctx, "committed-1", "", false)
assertExist(t, err) assertExist(t, err)
} }
func testCreateActiveNotExist(ctx context.Context, t *testing.T, ms storage.MetaStore) { func testCreateActiveNotExist(ctx context.Context, t *testing.T, ms *MetaStore) {
_, err := ms.CreateActive(ctx, "active-1", "does-not-exist", false) _, err := CreateActive(ctx, "active-1", "does-not-exist", false)
assertNotExist(t, err) assertNotExist(t, err)
} }
func testCreateActiveFromActive(ctx context.Context, t *testing.T, ms storage.MetaStore) { func testCreateActiveFromActive(ctx context.Context, t *testing.T, ms *MetaStore) {
if err := basePopulate(ctx, ms); err != nil { if err := basePopulate(ctx, ms); err != nil {
t.Fatalf("Populate failed: %+v", err) t.Fatalf("Populate failed: %+v", err)
} }
_, err := ms.CreateActive(ctx, "active-new", "active-1", false) _, err := CreateActive(ctx, "active-new", "active-1", false)
assertNotCommitted(t, err) assertNotCommitted(t, err)
} }
func testCommit(ctx context.Context, t *testing.T, ms storage.MetaStore) { func testCommit(ctx context.Context, t *testing.T, ms *MetaStore) {
a1, err := ms.CreateActive(ctx, "active-1", "", false) a1, err := CreateActive(ctx, "active-1", "", false)
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
} }
@ -426,7 +425,7 @@ func testCommit(ctx context.Context, t *testing.T, ms storage.MetaStore) {
t.Fatal("Expected writable active") t.Fatal("Expected writable active")
} }
commitID, err := ms.Commit(ctx, "active-1", "committed-1") commitID, err := CommitActive(ctx, "active-1", "committed-1")
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
} }
@ -434,50 +433,50 @@ func testCommit(ctx context.Context, t *testing.T, ms storage.MetaStore) {
t.Fatal("Snapshot identifier must not change on commit") t.Fatal("Snapshot identifier must not change on commit")
} }
_, err = ms.GetActive(ctx, "active-1") _, err = GetActive(ctx, "active-1")
assertNotExist(t, err) assertNotExist(t, err)
_, err = ms.GetActive(ctx, "committed-1") _, err = GetActive(ctx, "committed-1")
assertNotActive(t, err) assertNotActive(t, err)
} }
func testCommitNotExist(ctx context.Context, t *testing.T, ms storage.MetaStore) { func testCommitNotExist(ctx context.Context, t *testing.T, ms *MetaStore) {
_, err := ms.Commit(ctx, "active-not-exist", "committed-1") _, err := CommitActive(ctx, "active-not-exist", "committed-1")
assertNotExist(t, err) assertNotExist(t, err)
} }
func testCommitExist(ctx context.Context, t *testing.T, ms storage.MetaStore) { func testCommitExist(ctx context.Context, t *testing.T, ms *MetaStore) {
if err := basePopulate(ctx, ms); err != nil { if err := basePopulate(ctx, ms); err != nil {
t.Fatalf("Populate failed: %+v", err) t.Fatalf("Populate failed: %+v", err)
} }
_, err := ms.Commit(ctx, "active-1", "committed-1") _, err := CommitActive(ctx, "active-1", "committed-1")
assertExist(t, err) assertExist(t, err)
} }
func testCommitCommitted(ctx context.Context, t *testing.T, ms storage.MetaStore) { func testCommitCommitted(ctx context.Context, t *testing.T, ms *MetaStore) {
if err := basePopulate(ctx, ms); err != nil { if err := basePopulate(ctx, ms); err != nil {
t.Fatalf("Populate failed: %+v", err) t.Fatalf("Populate failed: %+v", err)
} }
_, err := ms.Commit(ctx, "committed-1", "committed-3") _, err := CommitActive(ctx, "committed-1", "committed-3")
assertNotActive(t, err) assertNotActive(t, err)
} }
func testCommitReadonly(ctx context.Context, t *testing.T, ms storage.MetaStore) { func testCommitReadonly(ctx context.Context, t *testing.T, ms *MetaStore) {
if err := basePopulate(ctx, ms); err != nil { if err := basePopulate(ctx, ms); err != nil {
t.Fatalf("Populate failed: %+v", err) t.Fatalf("Populate failed: %+v", err)
} }
_, err := ms.Commit(ctx, "active-5", "committed-3") _, err := CommitActive(ctx, "active-5", "committed-3")
if err == nil { if err == nil {
t.Fatal("Expected error committing readonly active") t.Fatal("Expected error committing readonly active")
} }
} }
func testRemove(ctx context.Context, t *testing.T, ms storage.MetaStore) { func testRemove(ctx context.Context, t *testing.T, ms *MetaStore) {
a1, err := ms.CreateActive(ctx, "active-1", "", false) a1, err := CreateActive(ctx, "active-1", "", false)
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
} }
commitID, err := ms.Commit(ctx, "active-1", "committed-1") commitID, err := CommitActive(ctx, "active-1", "committed-1")
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
} }
@ -485,20 +484,20 @@ func testRemove(ctx context.Context, t *testing.T, ms storage.MetaStore) {
t.Fatal("Snapshot identifier must not change on commit") t.Fatal("Snapshot identifier must not change on commit")
} }
a2, err := ms.CreateActive(ctx, "active-2", "committed-1", true) a2, err := CreateActive(ctx, "active-2", "committed-1", true)
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
} }
a3, err := ms.CreateActive(ctx, "active-3", "committed-1", true) a3, err := CreateActive(ctx, "active-3", "committed-1", true)
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
} }
_, _, err = ms.Remove(ctx, "active-1") _, _, err = Remove(ctx, "active-1")
assertNotExist(t, err) assertNotExist(t, err)
r3, k3, err := ms.Remove(ctx, "active-3") r3, k3, err := Remove(ctx, "active-3")
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
} }
@ -509,7 +508,7 @@ func testRemove(ctx context.Context, t *testing.T, ms storage.MetaStore) {
t.Fatal("Expected active kind, got %v", k3) t.Fatal("Expected active kind, got %v", k3)
} }
r2, k2, err := ms.Remove(ctx, "active-2") r2, k2, err := Remove(ctx, "active-2")
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
} }
@ -520,7 +519,7 @@ func testRemove(ctx context.Context, t *testing.T, ms storage.MetaStore) {
t.Fatal("Expected active kind, got %v", k2) t.Fatal("Expected active kind, got %v", k2)
} }
r1, k1, err := ms.Remove(ctx, "committed-1") r1, k1, err := Remove(ctx, "committed-1")
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
} }
@ -532,21 +531,21 @@ func testRemove(ctx context.Context, t *testing.T, ms storage.MetaStore) {
} }
} }
func testRemoveWithChildren(ctx context.Context, t *testing.T, ms storage.MetaStore) { func testRemoveWithChildren(ctx context.Context, t *testing.T, ms *MetaStore) {
if err := basePopulate(ctx, ms); err != nil { if err := basePopulate(ctx, ms); err != nil {
t.Fatalf("Populate failed: %+v", err) t.Fatalf("Populate failed: %+v", err)
} }
_, _, err := ms.Remove(ctx, "committed-1") _, _, err := Remove(ctx, "committed-1")
if err == nil { if err == nil {
t.Fatalf("Expected removal of snapshot with children to error") t.Fatalf("Expected removal of snapshot with children to error")
} }
_, _, err = ms.Remove(ctx, "committed-1") _, _, err = Remove(ctx, "committed-1")
if err == nil { if err == nil {
t.Fatalf("Expected removal of snapshot with children to error") t.Fatalf("Expected removal of snapshot with children to error")
} }
} }
func testRemoveNotExist(ctx context.Context, t *testing.T, ms storage.MetaStore) { func testRemoveNotExist(ctx context.Context, t *testing.T, ms *MetaStore) {
_, _, err := ms.Remove(ctx, "does-not-exist") _, _, err := Remove(ctx, "does-not-exist")
assertNotExist(t, err) assertNotExist(t, err)
} }

61
snapshot/storage/boltdb/record.pb.go → snapshot/storage/proto/record.pb.go
View File

@ -1,19 +1,19 @@
// Code generated by protoc-gen-gogo. // Code generated by protoc-gen-gogo.
// source: github.com/containerd/containerd/snapshot/storage/boltdb/record.proto // source: github.com/containerd/containerd/snapshot/storage/proto/record.proto
// DO NOT EDIT! // DO NOT EDIT!
/* /*
Package boltdb is a generated protocol buffer package. Package proto is a generated protocol buffer package.
It is generated from these files: It is generated from these files:
github.com/containerd/containerd/snapshot/storage/boltdb/record.proto github.com/containerd/containerd/snapshot/storage/proto/record.proto
It has these top-level messages: It has these top-level messages:
Snapshot Snapshot
*/ */
package boltdb package proto
import proto "github.com/gogo/protobuf/proto" import proto1 "github.com/gogo/protobuf/proto"
import fmt "fmt" import fmt "fmt"
import math "math" import math "math"
import _ "github.com/gogo/protobuf/gogoproto" import _ "github.com/gogo/protobuf/gogoproto"
@ -24,7 +24,7 @@ import reflect "reflect"
import io "io" import io "io"
// Reference imports to suppress errors if they are not otherwise used. // Reference imports to suppress errors if they are not otherwise used.
var _ = proto.Marshal var _ = proto1.Marshal
var _ = fmt.Errorf var _ = fmt.Errorf
var _ = math.Inf var _ = math.Inf
@ -32,7 +32,7 @@ var _ = math.Inf
// is compatible with the proto package it is being compiled against. // is compatible with the proto package it is being compiled against.
// A compilation error at this line likely means your copy of the // A compilation error at this line likely means your copy of the
// proto package needs to be updated. // proto package needs to be updated.
const _ = proto.GoGoProtoPackageIsVersion2 // please upgrade the proto package const _ = proto1.GoGoProtoPackageIsVersion2 // please upgrade the proto package
// Kind defines the kind of snapshot. // Kind defines the kind of snapshot.
type Kind int32 type Kind int32
@ -54,10 +54,12 @@ var Kind_value = map[string]int32{
} }
func (x Kind) String() string { func (x Kind) String() string {
return proto.EnumName(Kind_name, int32(x)) return proto1.EnumName(Kind_name, int32(x))
} }
func (Kind) EnumDescriptor() ([]byte, []int) { return fileDescriptorRecord, []int{0} } func (Kind) EnumDescriptor() ([]byte, []int) { return fileDescriptorRecord, []int{0} }
// Snapshot defines the storage type for a snapshot in the
// metadata store.
type Snapshot struct { type Snapshot struct {
ID uint64 `protobuf:"varint,1,opt,name=id,proto3" json:"id,omitempty"` ID uint64 `protobuf:"varint,1,opt,name=id,proto3" json:"id,omitempty"`
Parent string `protobuf:"bytes,2,opt,name=parent,proto3" json:"parent,omitempty"` Parent string `protobuf:"bytes,2,opt,name=parent,proto3" json:"parent,omitempty"`
@ -70,8 +72,8 @@ func (*Snapshot) ProtoMessage() {}
func (*Snapshot) Descriptor() ([]byte, []int) { return fileDescriptorRecord, []int{0} } func (*Snapshot) Descriptor() ([]byte, []int) { return fileDescriptorRecord, []int{0} }
func init() { func init() {
proto.RegisterType((*Snapshot)(nil), "containerd.v1.Snapshot") proto1.RegisterType((*Snapshot)(nil), "containerd.v1.Snapshot")
proto.RegisterEnum("containerd.v1.Kind", Kind_name, Kind_value) proto1.RegisterEnum("containerd.v1.Kind", Kind_name, Kind_value)
} }
func (m *Snapshot) Marshal() (dAtA []byte, err error) { func (m *Snapshot) Marshal() (dAtA []byte, err error) {
size := m.Size() size := m.Size()
@ -440,29 +442,28 @@ var (
) )
func init() { func init() {
proto.RegisterFile("github.com/containerd/containerd/snapshot/storage/boltdb/record.proto", fileDescriptorRecord) proto1.RegisterFile("github.com/containerd/containerd/snapshot/storage/proto/record.proto", fileDescriptorRecord)
} }
var fileDescriptorRecord = []byte{ var fileDescriptorRecord = []byte{
// 309 bytes of a gzipped FileDescriptorProto // 304 bytes of a gzipped FileDescriptorProto
0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xe2, 0x72, 0x4d, 0xcf, 0x2c, 0xc9, 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xe2, 0x72, 0x49, 0xcf, 0x2c, 0xc9,
0x28, 0x4d, 0xd2, 0x4b, 0xce, 0xcf, 0xd5, 0x4f, 0xce, 0xcf, 0x2b, 0x49, 0xcc, 0xcc, 0x4b, 0x2d, 0x28, 0x4d, 0xd2, 0x4b, 0xce, 0xcf, 0xd5, 0x4f, 0xce, 0xcf, 0x2b, 0x49, 0xcc, 0xcc, 0x4b, 0x2d,
0x4a, 0x41, 0x66, 0x16, 0xe7, 0x25, 0x16, 0x14, 0x67, 0xe4, 0x97, 0xe8, 0x17, 0x97, 0xe4, 0x17, 0x4a, 0x41, 0x66, 0x16, 0xe7, 0x25, 0x16, 0x14, 0x67, 0xe4, 0x97, 0xe8, 0x17, 0x97, 0xe4, 0x17,
0x25, 0xa6, 0xa7, 0xea, 0x27, 0xe5, 0xe7, 0x94, 0xa4, 0x24, 0xe9, 0x17, 0xa5, 0x26, 0xe7, 0x17, 0x25, 0xa6, 0xa7, 0xea, 0x17, 0x14, 0xe5, 0x97, 0xe4, 0xeb, 0x17, 0xa5, 0x26, 0xe7, 0x17, 0xa5,
0xa5, 0xe8, 0x15, 0x14, 0xe5, 0x97, 0xe4, 0x0b, 0xf1, 0x22, 0x34, 0xe8, 0x95, 0x19, 0x4a, 0x89, 0xe8, 0x81, 0x39, 0x42, 0xbc, 0x08, 0xf5, 0x7a, 0x65, 0x86, 0x52, 0x22, 0xe9, 0xf9, 0xe9, 0xf9,
0xa4, 0xe7, 0xa7, 0xe7, 0x83, 0x65, 0xf4, 0x41, 0x2c, 0x88, 0x22, 0xa5, 0x7a, 0x2e, 0x8e, 0x60, 0x10, 0x65, 0x20, 0x16, 0x44, 0x91, 0x52, 0x3d, 0x17, 0x47, 0x30, 0xd4, 0x2c, 0x21, 0x31, 0x2e,
0xa8, 0x61, 0x42, 0x62, 0x5c, 0x4c, 0x99, 0x29, 0x12, 0x8c, 0x0a, 0x8c, 0x1a, 0x2c, 0x4e, 0x6c, 0xa6, 0xcc, 0x14, 0x09, 0x46, 0x05, 0x46, 0x0d, 0x16, 0x27, 0xb6, 0x47, 0xf7, 0xe4, 0x99, 0x3c,
0x8f, 0xee, 0xc9, 0x33, 0x79, 0xba, 0x04, 0x31, 0x65, 0xa6, 0x08, 0x89, 0x71, 0xb1, 0x15, 0x24, 0x5d, 0x82, 0x98, 0x32, 0x53, 0x84, 0xc4, 0xb8, 0xd8, 0x0a, 0x12, 0x8b, 0x52, 0xf3, 0x4a, 0x24,
0x16, 0xa5, 0xe6, 0x95, 0x48, 0x30, 0x29, 0x30, 0x6a, 0x70, 0x06, 0x41, 0x79, 0x42, 0xea, 0x5c, 0x98, 0x14, 0x18, 0x35, 0x38, 0x83, 0xa0, 0x3c, 0x21, 0x75, 0x2e, 0x96, 0xec, 0xcc, 0xbc, 0x14,
0x2c, 0xd9, 0x99, 0x79, 0x29, 0x12, 0x2c, 0x0a, 0x8c, 0x1a, 0x7c, 0x46, 0xc2, 0x7a, 0x28, 0xf6, 0x09, 0x16, 0x05, 0x46, 0x0d, 0x3e, 0x23, 0x61, 0x3d, 0x14, 0xfb, 0xf4, 0xbc, 0x33, 0xf3, 0x52,
0xe9, 0x79, 0x67, 0xe6, 0xa5, 0x04, 0x81, 0x15, 0x08, 0x49, 0x71, 0x71, 0x14, 0xa5, 0x26, 0xa6, 0x82, 0xc0, 0x0a, 0x84, 0xa4, 0xb8, 0x38, 0x8a, 0x52, 0x13, 0x53, 0xf2, 0xf3, 0x72, 0x2a, 0x25,
0xe4, 0xe7, 0xe5, 0x54, 0x4a, 0xb0, 0x2a, 0x30, 0x6a, 0x70, 0x04, 0xc1, 0xf9, 0x5a, 0x41, 0x5c, 0x58, 0x15, 0x18, 0x35, 0x38, 0x82, 0xe0, 0x7c, 0xad, 0x20, 0x2e, 0x16, 0x6f, 0x88, 0x1a, 0x36,
0x2c, 0xde, 0x10, 0x35, 0x6c, 0x8e, 0xce, 0x21, 0x9e, 0x61, 0xae, 0x02, 0x0c, 0x52, 0x7c, 0x5d, 0x47, 0xe7, 0x10, 0xcf, 0x30, 0x57, 0x01, 0x06, 0x29, 0xbe, 0xae, 0xb9, 0x0a, 0x5c, 0x20, 0x51,
0x73, 0x15, 0xb8, 0x40, 0xa2, 0x8e, 0xc9, 0x25, 0x99, 0x65, 0xa9, 0x42, 0x0a, 0x5c, 0x9c, 0xce, 0xc7, 0xe4, 0x92, 0xcc, 0xb2, 0x54, 0x21, 0x05, 0x2e, 0x4e, 0x67, 0x7f, 0x5f, 0x5f, 0xcf, 0x90,
0xfe, 0xbe, 0xbe, 0x9e, 0x21, 0x21, 0xae, 0x2e, 0x02, 0x8c, 0x52, 0x82, 0x5d, 0x73, 0x15, 0x78, 0x10, 0x57, 0x17, 0x01, 0x46, 0x29, 0xc1, 0xae, 0xb9, 0x0a, 0xbc, 0x20, 0x69, 0xe7, 0xfc, 0xdc,
0x41, 0xd2, 0xce, 0xf9, 0xb9, 0xb9, 0x99, 0x25, 0x25, 0xa9, 0x29, 0x52, 0x3c, 0x1d, 0x8b, 0xe5, 0xdc, 0xcc, 0x92, 0x92, 0xd4, 0x14, 0x29, 0x9e, 0x8e, 0xc5, 0x72, 0x0c, 0xbb, 0x96, 0xc8, 0x81,
0x18, 0x76, 0x2d, 0x91, 0x03, 0x9b, 0xe5, 0x24, 0x71, 0xe2, 0xa1, 0x1c, 0xc3, 0x8d, 0x87, 0x72, 0xcd, 0x72, 0x92, 0x38, 0xf1, 0x50, 0x8e, 0xe1, 0xc6, 0x43, 0x39, 0x86, 0x86, 0x47, 0x72, 0x8c,
0x0c, 0x0d, 0x8f, 0xe4, 0x18, 0x4f, 0x3c, 0x92, 0x63, 0xbc, 0xf0, 0x48, 0x8e, 0xf1, 0xc1, 0x23, 0x27, 0x1e, 0xc9, 0x31, 0x5e, 0x78, 0x24, 0xc7, 0xf8, 0xe0, 0x91, 0x1c, 0x63, 0x12, 0x1b, 0xd8,
0x39, 0xc6, 0x24, 0x36, 0xb0, 0xaf, 0x8d, 0x01, 0x01, 0x00, 0x00, 0xff, 0xff, 0x73, 0x89, 0x2b, 0xd7, 0xc6, 0x80, 0x00, 0x00, 0x00, 0xff, 0xff, 0xbb, 0x89, 0x2c, 0x37, 0x62, 0x01, 0x00, 0x00,
0x02, 0x63, 0x01, 0x00, 0x00,
} }

2
snapshot/storage/boltdb/record.proto → snapshot/storage/proto/record.proto
View File

@ -16,6 +16,8 @@ enum Kind {
COMMITTED = 1 [(gogoproto.enumvalue_customname) = "KindCommitted"]; COMMITTED = 1 [(gogoproto.enumvalue_customname) = "KindCommitted"];
} }
// Snapshot defines the storage type for a snapshot in the
// metadata store.
message Snapshot { message Snapshot {
uint64 id = 1 [(gogoproto.customname) = "ID"]; uint64 id = 1 [(gogoproto.customname) = "ID"];
string parent = 2; string parent = 2;