content: refactor content store for API
After iterating on the GRPC API, the changes required for the actual implementation are now included in the content store. The begin change is the move to a single, atomic `Ingester.Writer` method for locking content ingestion on a key. From this, comes several new interface definitions. The main benefit here is the clarification between `Status` and `Info` that came out of the GPRC API. `Status` tells the status of a write, whereas `Info` is for querying metadata about various blobs. Signed-off-by: Stephen J Day <stephen.day@docker.com>
This commit is contained in:
parent
baaf7543dc
commit
621164bc84
14 changed files with 573 additions and 611 deletions
23
cmd/dist/active.go
vendored
23
cmd/dist/active.go
vendored
|
@ -3,11 +3,9 @@ package main
|
|||
import (
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"text/tabwriter"
|
||||
"time"
|
||||
|
||||
"github.com/docker/containerd/content"
|
||||
units "github.com/docker/go-units"
|
||||
"github.com/urfave/cli"
|
||||
)
|
||||
|
@ -26,24 +24,11 @@ var activeCommand = cli.Command{
|
|||
cli.StringFlag{
|
||||
Name: "root",
|
||||
Usage: "path to content store root",
|
||||
Value: ".content", // TODO(stevvooe): for now, just use the PWD/.content
|
||||
Value: "/tmp/content", // TODO(stevvooe): for now, just use the PWD/.content
|
||||
},
|
||||
},
|
||||
Action: func(context *cli.Context) error {
|
||||
var (
|
||||
// ctx = contextpkg.Background()
|
||||
root = context.String("root")
|
||||
)
|
||||
|
||||
if !filepath.IsAbs(root) {
|
||||
var err error
|
||||
root, err = filepath.Abs(root)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
cs, err := content.Open(root)
|
||||
cs, err := resolveContentStore(context)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
@ -58,8 +43,8 @@ var activeCommand = cli.Command{
|
|||
for _, active := range active {
|
||||
fmt.Fprintf(tw, "%s\t%s\t%s\n",
|
||||
active.Ref,
|
||||
units.HumanSize(float64(active.Size)),
|
||||
units.HumanDuration(time.Since(active.ModTime)))
|
||||
units.HumanSize(float64(active.Offset)),
|
||||
units.HumanDuration(time.Since(active.StartedAt)))
|
||||
}
|
||||
tw.Flush()
|
||||
|
||||
|
|
34
cmd/dist/common.go
vendored
Normal file
34
cmd/dist/common.go
vendored
Normal file
|
@ -0,0 +1,34 @@
|
|||
package main
|
||||
|
||||
import (
|
||||
"net"
|
||||
"path/filepath"
|
||||
"time"
|
||||
|
||||
"github.com/docker/containerd/content"
|
||||
"github.com/urfave/cli"
|
||||
"google.golang.org/grpc"
|
||||
)
|
||||
|
||||
func resolveContentStore(context *cli.Context) (*content.Store, error) {
|
||||
root := context.GlobalString("root")
|
||||
if !filepath.IsAbs(root) {
|
||||
var err error
|
||||
root, err = filepath.Abs(root)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
return content.NewStore(root)
|
||||
}
|
||||
|
||||
func connectGRPC(context *cli.Context) (*grpc.ClientConn, error) {
|
||||
socket := context.GlobalString("socket")
|
||||
return grpc.Dial(socket,
|
||||
grpc.WithBlock(),
|
||||
grpc.WithInsecure(),
|
||||
grpc.WithDialer(func(addr string, timeout time.Duration) (net.Conn, error) {
|
||||
return net.DialTimeout("unix", socket, timeout)
|
||||
}),
|
||||
)
|
||||
}
|
21
cmd/dist/delete.go
vendored
21
cmd/dist/delete.go
vendored
|
@ -3,9 +3,7 @@ package main
|
|||
import (
|
||||
contextpkg "context"
|
||||
"fmt"
|
||||
"path/filepath"
|
||||
|
||||
"github.com/docker/containerd/content"
|
||||
"github.com/docker/containerd/log"
|
||||
digest "github.com/opencontainers/go-digest"
|
||||
"github.com/urfave/cli"
|
||||
|
@ -18,30 +16,15 @@ var deleteCommand = cli.Command{
|
|||
ArgsUsage: "[flags] [<digest>, ...]",
|
||||
Description: `Delete one or more blobs permanently. Successfully deleted
|
||||
blobs are printed to stdout.`,
|
||||
Flags: []cli.Flag{
|
||||
cli.StringFlag{
|
||||
Name: "root",
|
||||
Usage: "path to content store root",
|
||||
Value: ".content", // TODO(stevvooe): for now, just use the PWD/.content
|
||||
},
|
||||
},
|
||||
Flags: []cli.Flag{},
|
||||
Action: func(context *cli.Context) error {
|
||||
var (
|
||||
ctx = contextpkg.Background()
|
||||
root = context.String("root")
|
||||
args = []string(context.Args())
|
||||
exitError error
|
||||
)
|
||||
|
||||
if !filepath.IsAbs(root) {
|
||||
var err error
|
||||
root, err = filepath.Abs(root)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
cs, err := content.Open(root)
|
||||
cs, err := resolveContentStore(context)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
|
39
cmd/dist/get.go
vendored
Normal file
39
cmd/dist/get.go
vendored
Normal file
|
@ -0,0 +1,39 @@
|
|||
package main
|
||||
|
||||
import (
|
||||
"io"
|
||||
"os"
|
||||
|
||||
digest "github.com/opencontainers/go-digest"
|
||||
"github.com/urfave/cli"
|
||||
)
|
||||
|
||||
var getCommand = cli.Command{
|
||||
Name: "get",
|
||||
Usage: "get the data for an object",
|
||||
ArgsUsage: "[flags] [<digest>, ...]",
|
||||
Description: `Display the paths to one or more blobs.
|
||||
|
||||
Output paths can be used to directly access blobs on disk.`,
|
||||
Flags: []cli.Flag{},
|
||||
Action: func(context *cli.Context) error {
|
||||
cs, err := resolveContentStore(context)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
dgst, err := digest.Parse(context.Args().First())
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
rc, err := cs.Open(dgst)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer rc.Close()
|
||||
|
||||
_, err = io.Copy(os.Stdout, rc)
|
||||
return err
|
||||
},
|
||||
}
|
52
cmd/dist/ingest.go
vendored
52
cmd/dist/ingest.go
vendored
|
@ -4,8 +4,6 @@ import (
|
|||
contextpkg "context"
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
|
||||
"github.com/docker/containerd/content"
|
||||
"github.com/opencontainers/go-digest"
|
||||
|
@ -18,17 +16,6 @@ var ingestCommand = cli.Command{
|
|||
ArgsUsage: "[flags] <key>",
|
||||
Description: `Ingest objects into the local content store.`,
|
||||
Flags: []cli.Flag{
|
||||
cli.DurationFlag{
|
||||
Name: "timeout",
|
||||
Usage: "total timeout for fetch",
|
||||
EnvVar: "CONTAINERD_FETCH_TIMEOUT",
|
||||
},
|
||||
cli.StringFlag{
|
||||
Name: "path, p",
|
||||
Usage: "path to content store",
|
||||
Value: ".content", // TODO(stevvooe): for now, just use the PWD/.content
|
||||
EnvVar: "CONTAINERD_DIST_CONTENT_STORE",
|
||||
},
|
||||
cli.Int64Flag{
|
||||
Name: "expected-size",
|
||||
Usage: "validate against provided size",
|
||||
|
@ -40,57 +27,32 @@ var ingestCommand = cli.Command{
|
|||
},
|
||||
Action: func(context *cli.Context) error {
|
||||
var (
|
||||
ctx = contextpkg.Background()
|
||||
timeout = context.Duration("timeout")
|
||||
root = context.String("path")
|
||||
ctx = background
|
||||
cancel func()
|
||||
ref = context.Args().First()
|
||||
expectedSize = context.Int64("expected-size")
|
||||
expectedDigest = digest.Digest(context.String("expected-digest"))
|
||||
)
|
||||
|
||||
if timeout > 0 {
|
||||
var cancel func()
|
||||
ctx, cancel = contextpkg.WithTimeout(ctx, timeout)
|
||||
defer cancel()
|
||||
}
|
||||
ctx, cancel = contextpkg.WithCancel(ctx)
|
||||
defer cancel()
|
||||
|
||||
if err := expectedDigest.Validate(); expectedDigest != "" && err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if !filepath.IsAbs(root) {
|
||||
var err error
|
||||
root, err = filepath.Abs(root)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
cs, err := content.Open(root)
|
||||
cs, err := resolveContentStore(context)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if expectedDigest != "" {
|
||||
if ok, err := cs.Exists(expectedDigest); err != nil {
|
||||
return err
|
||||
} else if ok {
|
||||
fmt.Fprintf(os.Stderr, "content with digest %v already exists\n", expectedDigest)
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
||||
if ref == "" {
|
||||
if expectedDigest == "" {
|
||||
return fmt.Errorf("must specify a transaction reference or expected digest")
|
||||
}
|
||||
|
||||
ref = strings.Replace(expectedDigest.String(), ":", "-", -1)
|
||||
return fmt.Errorf("must specify a transaction reference")
|
||||
}
|
||||
|
||||
// TODO(stevvooe): Allow ingest to be reentrant. Currently, we expect
|
||||
// all data to be written in a single invocation. Allow multiple writes
|
||||
// to the same transaction key followed by a commit.
|
||||
return content.WriteBlob(cs, os.Stdin, ref, expectedSize, expectedDigest)
|
||||
return content.WriteBlob(ctx, cs, os.Stdin, ref, expectedSize, expectedDigest)
|
||||
},
|
||||
}
|
||||
|
|
17
cmd/dist/list.go
vendored
17
cmd/dist/list.go
vendored
|
@ -4,7 +4,6 @@ import (
|
|||
contextpkg "context"
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"text/tabwriter"
|
||||
"time"
|
||||
|
||||
|
@ -22,11 +21,6 @@ var listCommand = cli.Command{
|
|||
ArgsUsage: "[flags] [<prefix>, ...]",
|
||||
Description: `List blobs in the content store.`,
|
||||
Flags: []cli.Flag{
|
||||
cli.StringFlag{
|
||||
Name: "root",
|
||||
Usage: "path to content store root",
|
||||
Value: ".content", // TODO(stevvooe): for now, just use the PWD/.content
|
||||
},
|
||||
cli.BoolFlag{
|
||||
Name: "quiet, q",
|
||||
Usage: "print only the blob digest",
|
||||
|
@ -35,20 +29,11 @@ var listCommand = cli.Command{
|
|||
Action: func(context *cli.Context) error {
|
||||
var (
|
||||
ctx = contextpkg.Background()
|
||||
root = context.String("root")
|
||||
quiet = context.Bool("quiet")
|
||||
args = []string(context.Args())
|
||||
)
|
||||
|
||||
if !filepath.IsAbs(root) {
|
||||
var err error
|
||||
root, err = filepath.Abs(root)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
cs, err := content.Open(root)
|
||||
cs, err := resolveContentStore(context)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
|
27
cmd/dist/main.go
vendored
27
cmd/dist/main.go
vendored
|
@ -1,6 +1,7 @@
|
|||
package main
|
||||
|
||||
import (
|
||||
contextpkg "context"
|
||||
"fmt"
|
||||
"os"
|
||||
|
||||
|
@ -9,6 +10,10 @@ import (
|
|||
"github.com/urfave/cli"
|
||||
)
|
||||
|
||||
var (
|
||||
background = contextpkg.Background()
|
||||
)
|
||||
|
||||
func main() {
|
||||
app := cli.NewApp()
|
||||
app.Name = "dist"
|
||||
|
@ -27,20 +32,38 @@ distribution tool
|
|||
Name: "debug",
|
||||
Usage: "enable debug output in logs",
|
||||
},
|
||||
cli.DurationFlag{
|
||||
Name: "timeout",
|
||||
Usage: "total timeout for fetch",
|
||||
EnvVar: "CONTAINERD_FETCH_TIMEOUT",
|
||||
},
|
||||
cli.StringFlag{
|
||||
Name: "root",
|
||||
Usage: "path to content store root",
|
||||
Value: "/tmp/content", // TODO(stevvooe): for now, just use the PWD/.content
|
||||
},
|
||||
}
|
||||
app.Commands = []cli.Command{
|
||||
fetchCommand,
|
||||
ingestCommand,
|
||||
activeCommand,
|
||||
pathCommand,
|
||||
getCommand,
|
||||
deleteCommand,
|
||||
listCommand,
|
||||
applyCommand,
|
||||
}
|
||||
app.Before = func(context *cli.Context) error {
|
||||
if context.GlobalBool("debug") {
|
||||
var (
|
||||
debug = context.GlobalBool("debug")
|
||||
timeout = context.GlobalDuration("timeout")
|
||||
)
|
||||
if debug {
|
||||
logrus.SetLevel(logrus.DebugLevel)
|
||||
}
|
||||
|
||||
if timeout > 0 {
|
||||
background, _ = contextpkg.WithTimeout(background, timeout)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
if err := app.Run(os.Args); err != nil {
|
||||
|
|
89
cmd/dist/path.go
vendored
89
cmd/dist/path.go
vendored
|
@ -1,89 +0,0 @@
|
|||
package main
|
||||
|
||||
import (
|
||||
contextpkg "context"
|
||||
"fmt"
|
||||
"path/filepath"
|
||||
|
||||
"github.com/docker/containerd/content"
|
||||
"github.com/docker/containerd/log"
|
||||
digest "github.com/opencontainers/go-digest"
|
||||
"github.com/urfave/cli"
|
||||
)
|
||||
|
||||
var pathCommand = cli.Command{
|
||||
Name: "path",
|
||||
Usage: "print the path to one or more blobs",
|
||||
ArgsUsage: "[flags] [<digest>, ...]",
|
||||
Description: `Display the paths to one or more blobs.
|
||||
|
||||
Output paths can be used to directly access blobs on disk.`,
|
||||
Flags: []cli.Flag{
|
||||
cli.StringFlag{
|
||||
Name: "root",
|
||||
Usage: "path to content store root",
|
||||
Value: ".content", // TODO(stevvooe): for now, just use the PWD/.content
|
||||
EnvVar: "CONTAINERD_DIST_CONTENT_STORE",
|
||||
},
|
||||
cli.BoolFlag{
|
||||
Name: "quiet, q",
|
||||
Usage: "elide digests in output",
|
||||
},
|
||||
},
|
||||
Action: func(context *cli.Context) error {
|
||||
var (
|
||||
ctx = contextpkg.Background()
|
||||
root = context.String("root")
|
||||
args = []string(context.Args())
|
||||
quiet = context.Bool("quiet")
|
||||
exitError error
|
||||
)
|
||||
|
||||
if !filepath.IsAbs(root) {
|
||||
var err error
|
||||
root, err = filepath.Abs(root)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
cs, err := content.Open(root)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// TODO(stevvooe): Take the set of paths from stdin.
|
||||
|
||||
if len(args) < 1 {
|
||||
return fmt.Errorf("please specify a blob digest")
|
||||
}
|
||||
|
||||
for _, arg := range args {
|
||||
dgst, err := digest.Parse(arg)
|
||||
if err != nil {
|
||||
log.G(ctx).WithError(err).Errorf("parsing %q as digest failed", arg)
|
||||
if exitError == nil {
|
||||
exitError = err
|
||||
}
|
||||
continue
|
||||
}
|
||||
|
||||
p, err := cs.GetPath(dgst)
|
||||
if err != nil {
|
||||
log.G(ctx).WithError(err).Errorf("getting path for %q failed", dgst)
|
||||
if exitError == nil {
|
||||
exitError = err
|
||||
}
|
||||
continue
|
||||
}
|
||||
|
||||
if !quiet {
|
||||
fmt.Println(dgst, p)
|
||||
} else {
|
||||
fmt.Println(p)
|
||||
}
|
||||
}
|
||||
|
||||
return exitError
|
||||
},
|
||||
}
|
Loading…
Add table
Add a link
Reference in a new issue