From ca56448263fd5efb72a85ac9ccdb2734cc7e1606 Mon Sep 17 00:00:00 2001 From: Derek McGowan Date: Fri, 17 Feb 2017 16:34:21 -0800 Subject: [PATCH 1/4] snapshot/overlay: implement view Allow creating actives without an upper directory for capturing changes. Actives without the upper directory will always be mounted read only. Read only actives must have a parent. Signed-off-by: Derek McGowan (github: dmcgowan) --- snapshot/overlay/overlay.go | 49 +++++++++++++++++++++++--------- snapshot/overlay/overlay_test.go | 47 ++++++++++++++++++++++++++++++ 2 files changed, 83 insertions(+), 13 deletions(-) diff --git a/snapshot/overlay/overlay.go b/snapshot/overlay/overlay.go index 2aba97c..69f1c88 100644 --- a/snapshot/overlay/overlay.go +++ b/snapshot/overlay/overlay.go @@ -102,7 +102,7 @@ func (o *Snapshotter) stat(path string) (snapshot.Info, error) { } func (o *Snapshotter) Prepare(ctx context.Context, key, parent string) ([]containerd.Mount, error) { - active, err := o.newActiveDir(key) + active, err := o.newActiveDir(key, false) if err != nil { return nil, err } @@ -115,7 +115,16 @@ func (o *Snapshotter) Prepare(ctx context.Context, key, parent string) ([]contai } func (o *Snapshotter) View(ctx context.Context, key, parent string) ([]containerd.Mount, error) { - panic("not implemented") + active, err := o.newActiveDir(key, true) + if err != nil { + return nil, err + } + if parent != "" { + if err := active.setParent(parent); err != nil { + return nil, err + } + } + return active.mounts(o.links) } // Mounts returns the mounts for the transaction identified by key. Can be @@ -175,7 +184,7 @@ func (o *Snapshotter) Walk(ctx context.Context, fn func(context.Context, snapsho }) } -func (o *Snapshotter) newActiveDir(key string) (*activeDir, error) { +func (o *Snapshotter) newActiveDir(key string, readonly bool) (*activeDir, error) { var ( path = filepath.Join(o.root, "active", hash(key)) name = filepath.Join(path, "name") @@ -186,11 +195,18 @@ func (o *Snapshotter) newActiveDir(key string) (*activeDir, error) { committedDir: filepath.Join(o.root, "committed"), indexlink: indexlink, } - for _, p := range []string{ - "work", - "fs", - } { - if err := os.MkdirAll(filepath.Join(path, p), 0700); err != nil { + if !readonly { + for _, p := range []string{ + "work", + "fs", + } { + if err := os.MkdirAll(filepath.Join(path, p), 0700); err != nil { + a.delete() + return nil, err + } + } + } else { + if err := os.MkdirAll(path, 0700); err != nil { a.delete() return nil, err } @@ -286,7 +302,7 @@ func (a *activeDir) mounts(c *cache) ([]containerd.Mount, error) { } if len(parents) == 0 { // if we only have one layer/no parents then just return a bind mount as overlay - // will not work + // will not work, readonly always has parent return []containerd.Mount{ { Source: filepath.Join(a.path, "fs"), @@ -298,11 +314,18 @@ func (a *activeDir) mounts(c *cache) ([]containerd.Mount, error) { }, }, nil } - options := []string{ - fmt.Sprintf("workdir=%s", filepath.Join(a.path, "work")), - fmt.Sprintf("upperdir=%s", filepath.Join(a.path, "fs")), - fmt.Sprintf("lowerdir=%s", strings.Join(parents, ":")), + var options []string + + if _, err := os.Stat(filepath.Join(a.path, "fs")); err == nil { + options = append(options, + fmt.Sprintf("workdir=%s", filepath.Join(a.path, "work")), + fmt.Sprintf("upperdir=%s", filepath.Join(a.path, "fs")), + ) + } else if !os.IsNotExist(err) { + return nil, err } + + options = append(options, fmt.Sprintf("lowerdir=%s", strings.Join(parents, ":"))) return []containerd.Mount{ { Type: "overlay", diff --git a/snapshot/overlay/overlay_test.go b/snapshot/overlay/overlay_test.go index fbd3e5b..3a19d68 100644 --- a/snapshot/overlay/overlay_test.go +++ b/snapshot/overlay/overlay_test.go @@ -2,6 +2,7 @@ package overlay import ( "context" + "fmt" "io/ioutil" "os" "path/filepath" @@ -196,3 +197,49 @@ func TestOverlayOverlayRead(t *testing.T) { return } } + +func TestOverlayView(t *testing.T) { + ctx := context.TODO() + root, err := ioutil.TempDir("", "overlay") + if err != nil { + t.Fatal(err) + } + defer os.RemoveAll(root) + o, err := NewSnapshotter(root) + if err != nil { + t.Fatal(err) + } + key := "/tmp/test" + mounts, err := o.Prepare(ctx, key, "") + if err != nil { + t.Fatal(err) + } + m := mounts[0] + if err := ioutil.WriteFile(filepath.Join(m.Source, "foo"), []byte("hi"), 0660); err != nil { + t.Fatal(err) + } + if err := o.Commit(ctx, "base", key); err != nil { + t.Fatal(err) + } + mounts, err = o.View(ctx, "/tmp/test2", "base") + if err != nil { + t.Fatal(err) + } + if len(mounts) != 1 { + t.Fatalf("should only have 1 mount but received %d", len(mounts)) + } + m = mounts[0] + if m.Type != "overlay" { + t.Errorf("mount type should be overlay but received %q", m.Type) + } + if m.Source != "overlay" { + t.Errorf("mount source should be overlay but received %q", m.Source) + } + if len(m.Options) != 1 { + t.Errorf("expected 1 mount option but got %d", len(m.Options)) + } + expected := fmt.Sprintf("lowerdir=%s", filepath.Join(root, "committed", hash("base"), "fs")) + if m.Options[0] != expected { + t.Errorf("expected option %q but received %q", expected, m.Options[0]) + } +} From 1723444ba2587a878605622f5c01188e578a97e9 Mon Sep 17 00:00:00 2001 From: Derek McGowan Date: Fri, 17 Feb 2017 18:43:10 -0800 Subject: [PATCH 2/4] snapshot/overlay: prevent committing view Signed-off-by: Derek McGowan (github: dmcgowan) --- snapshot/overlay/overlay.go | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/snapshot/overlay/overlay.go b/snapshot/overlay/overlay.go index 69f1c88..588aec2 100644 --- a/snapshot/overlay/overlay.go +++ b/snapshot/overlay/overlay.go @@ -253,6 +253,13 @@ func (a *activeDir) setParent(name string) error { } func (a *activeDir) commit(name string, c *cache) error { + if _, err := os.Stat(filepath.Join(a.path, "fs")); err != nil { + if os.IsNotExist(err) { + return errors.New("cannot commit view") + } + return err + } + // TODO(stevvooe): This doesn't quite meet the current model. The new model // is to copy all of this out and let the transaction continue. We don't // really have tests for it yet, but this will be the spot to fix it. From eeb88554ac9262eea2b96d0071fec00fc7e269fb Mon Sep 17 00:00:00 2001 From: Derek McGowan Date: Mon, 27 Feb 2017 13:21:20 -0800 Subject: [PATCH 3/4] snapshot/overlay: use readonly bindmount for single parent view Signed-off-by: Derek McGowan (github: dmcgowan) --- snapshot/overlay/overlay.go | 11 +++++++++ snapshot/overlay/overlay_test.go | 41 +++++++++++++++++++++++++++++--- 2 files changed, 49 insertions(+), 3 deletions(-) diff --git a/snapshot/overlay/overlay.go b/snapshot/overlay/overlay.go index 588aec2..533824a 100644 --- a/snapshot/overlay/overlay.go +++ b/snapshot/overlay/overlay.go @@ -330,6 +330,17 @@ func (a *activeDir) mounts(c *cache) ([]containerd.Mount, error) { ) } else if !os.IsNotExist(err) { return nil, err + } else if len(parents) == 1 { + return []containerd.Mount{ + { + Source: parents[0], + Type: "bind", + Options: []string{ + "ro", + "rbind", + }, + }, + }, nil } options = append(options, fmt.Sprintf("lowerdir=%s", strings.Join(parents, ":"))) diff --git a/snapshot/overlay/overlay_test.go b/snapshot/overlay/overlay_test.go index 3a19d68..4bd594a 100644 --- a/snapshot/overlay/overlay_test.go +++ b/snapshot/overlay/overlay_test.go @@ -209,7 +209,7 @@ func TestOverlayView(t *testing.T) { if err != nil { t.Fatal(err) } - key := "/tmp/test" + key := "/tmp/base" mounts, err := o.Prepare(ctx, key, "") if err != nil { t.Fatal(err) @@ -221,7 +221,42 @@ func TestOverlayView(t *testing.T) { if err := o.Commit(ctx, "base", key); err != nil { t.Fatal(err) } - mounts, err = o.View(ctx, "/tmp/test2", "base") + + key = "/tmp/top" + _, err = o.Prepare(ctx, key, "base") + if err != nil { + t.Fatal(err) + } + if err := ioutil.WriteFile(filepath.Join(root, "active", hash(key), "fs", "foo"), []byte("hi, again"), 0660); err != nil { + t.Fatal(err) + } + if err := o.Commit(ctx, "top", key); err != nil { + t.Fatal(err) + } + + mounts, err = o.View(ctx, "/tmp/view1", "base") + if err != nil { + t.Fatal(err) + } + if len(mounts) != 1 { + t.Fatalf("should only have 1 mount but received %d", len(mounts)) + } + m = mounts[0] + if m.Type != "bind" { + t.Errorf("mount type should be bind but received %q", m.Type) + } + expected := filepath.Join(root, "committed", hash("base"), "fs") + if m.Source != expected { + t.Errorf("expected source %q but received %q", expected, m.Source) + } + if m.Options[0] != "ro" { + t.Errorf("expected mount option ro but received %q", m.Options[0]) + } + if m.Options[1] != "rbind" { + t.Errorf("expected mount option rbind but received %q", m.Options[1]) + } + + mounts, err = o.View(ctx, "/tmp/view2", "top") if err != nil { t.Fatal(err) } @@ -238,7 +273,7 @@ func TestOverlayView(t *testing.T) { if len(m.Options) != 1 { t.Errorf("expected 1 mount option but got %d", len(m.Options)) } - expected := fmt.Sprintf("lowerdir=%s", filepath.Join(root, "committed", hash("base"), "fs")) + expected = fmt.Sprintf("lowerdir=%s:%s", filepath.Join(root, "committed", hash("top"), "fs"), filepath.Join(root, "committed", hash("base"), "fs")) if m.Options[0] != expected { t.Errorf("expected option %q but received %q", expected, m.Options[0]) } From c48e9a763f67f68d5b8cb86a5d98fca8f581c1ca Mon Sep 17 00:00:00 2001 From: Derek McGowan Date: Mon, 27 Feb 2017 13:39:52 -0800 Subject: [PATCH 4/4] Allow ro mounts without a parent Signed-off-by: Derek McGowan (github: dmcgowan) --- snapshot/overlay/overlay.go | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/snapshot/overlay/overlay.go b/snapshot/overlay/overlay.go index 533824a..d12770e 100644 --- a/snapshot/overlay/overlay.go +++ b/snapshot/overlay/overlay.go @@ -206,7 +206,7 @@ func (o *Snapshotter) newActiveDir(key string, readonly bool) (*activeDir, error } } } else { - if err := os.MkdirAll(path, 0700); err != nil { + if err := os.MkdirAll(filepath.Join(path, "fs"), 0700); err != nil { a.delete() return nil, err } @@ -309,13 +309,21 @@ func (a *activeDir) mounts(c *cache) ([]containerd.Mount, error) { } if len(parents) == 0 { // if we only have one layer/no parents then just return a bind mount as overlay - // will not work, readonly always has parent + // will not work + roFlag := "rw" + if _, err := os.Stat(filepath.Join(a.path, "work")); err != nil { + if !os.IsNotExist(err) { + return nil, err + } + roFlag = "ro" + } + return []containerd.Mount{ { Source: filepath.Join(a.path, "fs"), Type: "bind", Options: []string{ - "rw", + roFlag, "rbind", }, }, @@ -323,7 +331,7 @@ func (a *activeDir) mounts(c *cache) ([]containerd.Mount, error) { } var options []string - if _, err := os.Stat(filepath.Join(a.path, "fs")); err == nil { + if _, err := os.Stat(filepath.Join(a.path, "work")); err == nil { options = append(options, fmt.Sprintf("workdir=%s", filepath.Join(a.path, "work")), fmt.Sprintf("upperdir=%s", filepath.Join(a.path, "fs")),