content: break up into multiple files
Break up the content store prototype into a few logical files. We have a file for the store, the writer and helpers. Also, the writer has been modified to remove write and exec permissions on blobs in the store. Signed-off-by: Stephen J Day <stephen.day@docker.com>
This commit is contained in:
Stephen J Day
parent
efed5a918e
commit
3469905bbb
4 changed files with 168 additions and 107 deletions
|
|
@ -40,50 +40,6 @@ func OpenContentStore(root string) (*ContentStore, error) {
|
|||
}, nil
|
||||
}
|
||||
|
||||
// OpenBlob opens the blob for reading identified by dgst.
|
||||
//
|
||||
// The opened blob may also implement seek. Callers can detect with io.Seeker.
|
||||
func OpenBlob(cs *ContentStore, dgst digest.Digest) (io.ReadCloser, error) {
|
||||
path, err := cs.GetPath(dgst)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
fp, err := os.Open(path)
|
||||
return fp, err
|
||||
}
|
||||
|
||||
// WriteBlob writes data with the expected digest into the content store. If
|
||||
// expected already exists, the method returns immediately and the reader will
|
||||
// not be consumed.
|
||||
//
|
||||
// This is useful when the digest and size are known beforehand.
|
||||
//
|
||||
// Copy is buffered, so no need to wrap reader in buffered io.
|
||||
func WriteBlob(cs *ContentStore, r io.Reader, size int64, expected digest.Digest) error {
|
||||
cw, err := cs.Begin(expected.Hex())
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
buf := bufPool.Get().([]byte)
|
||||
defer bufPool.Put(buf)
|
||||
|
||||
nn, err := io.CopyBuffer(cw, r, buf)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if nn != size {
|
||||
return errors.Errorf("failed size verification: %v != %v", nn, size)
|
||||
}
|
||||
|
||||
if err := cw.Commit(size, expected); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (cs *ContentStore) GetPath(dgst digest.Digest) (string, error) {
|
||||
p := filepath.Join(cs.root, "blobs", dgst.Algorithm().String(), dgst.Hex())
|
||||
if _, err := os.Stat(p); err != nil {
|
||||
|
|
@ -190,59 +146,3 @@ func (cs *ContentStore) ingestPaths(ref string) (string, string, error) {
|
|||
|
||||
return fp, filepath.Join(fp, "data"), nil
|
||||
}
|
||||
|
||||
// ContentWriter represents a write transaction against the blob store.
|
||||
//
|
||||
//
|
||||
type ContentWriter struct {
|
||||
cs *ContentStore
|
||||
fp *os.File // opened data file
|
||||
path string // path to writer dir
|
||||
offset int64
|
||||
digester digest.Digester
|
||||
}
|
||||
|
||||
func (cw *ContentWriter) Write(p []byte) (n int, err error) {
|
||||
n, err = cw.fp.Write(p)
|
||||
cw.digester.Hash().Write(p[:n])
|
||||
return n, err
|
||||
}
|
||||
|
||||
func (cw *ContentWriter) Commit(size int64, expected digest.Digest) error {
|
||||
if err := cw.fp.Sync(); err != nil {
|
||||
return errors.Wrap(err, "sync failed")
|
||||
}
|
||||
|
||||
fi, err := cw.fp.Stat()
|
||||
if err != nil {
|
||||
return errors.Wrap(err, "stat on data file failed")
|
||||
}
|
||||
|
||||
if size != fi.Size() {
|
||||
return errors.Errorf("failed size validation: %v != %v", fi.Size(), size)
|
||||
}
|
||||
|
||||
dgst := cw.digester.Digest()
|
||||
if expected != dgst {
|
||||
return errors.Errorf("unexpected digest: %v != %v", dgst, expected)
|
||||
}
|
||||
|
||||
apath := filepath.Join(cw.cs.root, "blobs", dgst.Algorithm().String())
|
||||
if err := os.MkdirAll(apath, 0755); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
dpath := filepath.Join(apath, dgst.Hex())
|
||||
|
||||
// clean up!!
|
||||
defer os.RemoveAll(cw.path)
|
||||
return os.Rename(filepath.Join(cw.path, "data"), dpath)
|
||||
}
|
||||
|
||||
// Close the writer, leaving the progress in tact.
|
||||
//
|
||||
// If one needs to resume the transaction, a new writer can be obtained from
|
||||
// `ContentStore.Resume` using the same key.
|
||||
func (cw *ContentWriter) Close() error {
|
||||
return cw.fp.Close()
|
||||
}
|
||||
|
|
|
|||
|
|
@ -4,6 +4,7 @@ import (
|
|||
"bufio"
|
||||
"bytes"
|
||||
"crypto/rand"
|
||||
_ "crypto/sha256" // required for digest package
|
||||
"fmt"
|
||||
"io"
|
||||
"io/ioutil"
|
||||
|
|
@ -73,13 +74,7 @@ func TestContentWriter(t *testing.T) {
|
|||
t.Fatal(err)
|
||||
}
|
||||
|
||||
path, err := cs.GetPath(expected)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if path != filepath.Join(tmpdir, "blobs", expected.Algorithm().String(), expected.Hex()) {
|
||||
t.Fatalf("unxpected path: %q", path)
|
||||
}
|
||||
path := checkBlobPath(t, cs, expected)
|
||||
|
||||
// read the data back, make sure its the same
|
||||
pp, err := ioutil.ReadFile(path)
|
||||
|
|
@ -105,6 +100,27 @@ func checkCopy(t *testing.T, size int64, dst io.Writer, src io.Reader) {
|
|||
}
|
||||
}
|
||||
|
||||
func checkBlobPath(t *testing.T, cs *ContentStore, dgst digest.Digest) string {
|
||||
path, err := cs.GetPath(dgst)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if path != filepath.Join(cs.root, "blobs", dgst.Algorithm().String(), dgst.Hex()) {
|
||||
t.Fatalf("unxpected path: %q", path)
|
||||
}
|
||||
fi, err := os.Stat(path)
|
||||
if err != nil {
|
||||
t.Fatalf("error stating blob path: %v", err)
|
||||
}
|
||||
|
||||
// ensure that only read bits are set.
|
||||
if ((fi.Mode() & os.ModePerm) & 0333) != 0 {
|
||||
t.Fatalf("incorrect permissions: %v", fi.Mode())
|
||||
}
|
||||
|
||||
return path
|
||||
}
|
||||
|
||||
func dumpDir(root string) error {
|
||||
return filepath.Walk(root, func(path string, fi os.FileInfo, err error) error {
|
||||
if err != nil {
|
||||
|
|
|
|||
53
content/helpers.go
Normal file
53
content/helpers.go
Normal file
|
|
@ -0,0 +1,53 @@
|
|||
package content
|
||||
|
||||
import (
|
||||
"io"
|
||||
"os"
|
||||
|
||||
"github.com/docker/distribution/digest"
|
||||
"github.com/pkg/errors"
|
||||
)
|
||||
|
||||
// OpenBlob opens the blob for reading identified by dgst.
|
||||
//
|
||||
// The opened blob may also implement seek. Callers can detect with io.Seeker.
|
||||
func OpenBlob(cs *ContentStore, dgst digest.Digest) (io.ReadCloser, error) {
|
||||
path, err := cs.GetPath(dgst)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
fp, err := os.Open(path)
|
||||
return fp, err
|
||||
}
|
||||
|
||||
// WriteBlob writes data with the expected digest into the content store. If
|
||||
// expected already exists, the method returns immediately and the reader will
|
||||
// not be consumed.
|
||||
//
|
||||
// This is useful when the digest and size are known beforehand.
|
||||
//
|
||||
// Copy is buffered, so no need to wrap reader in buffered io.
|
||||
func WriteBlob(cs *ContentStore, r io.Reader, size int64, expected digest.Digest) error {
|
||||
cw, err := cs.Begin(expected.Hex())
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
buf := bufPool.Get().([]byte)
|
||||
defer bufPool.Put(buf)
|
||||
|
||||
nn, err := io.CopyBuffer(cw, r, buf)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if nn != size {
|
||||
return errors.Errorf("failed size verification: %v != %v", nn, size)
|
||||
}
|
||||
|
||||
if err := cw.Commit(size, expected); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
92
content/writer.go
Normal file
92
content/writer.go
Normal file
|
|
@ -0,0 +1,92 @@
|
|||
package content
|
||||
|
||||
import (
|
||||
"os"
|
||||
"path/filepath"
|
||||
|
||||
"github.com/docker/distribution/digest"
|
||||
"github.com/pkg/errors"
|
||||
)
|
||||
|
||||
// ContentWriter represents a write transaction against the blob store.
|
||||
//
|
||||
//
|
||||
type ContentWriter struct {
|
||||
cs *ContentStore
|
||||
fp *os.File // opened data file
|
||||
path string // path to writer dir
|
||||
offset int64
|
||||
digester digest.Digester
|
||||
}
|
||||
|
||||
// Write p to the transaction.
|
||||
//
|
||||
// Note that writes are unbuffered to the backing file. When writing, it is
|
||||
// recommended to wrap in a bufio.Writer or, preferrably, use io.CopyBuffer.
|
||||
func (cw *ContentWriter) Write(p []byte) (n int, err error) {
|
||||
n, err = cw.fp.Write(p)
|
||||
cw.digester.Hash().Write(p[:n])
|
||||
return n, err
|
||||
}
|
||||
|
||||
func (cw *ContentWriter) Commit(size int64, expected digest.Digest) error {
|
||||
if err := cw.fp.Sync(); err != nil {
|
||||
return errors.Wrap(err, "sync failed")
|
||||
}
|
||||
|
||||
fi, err := cw.fp.Stat()
|
||||
if err != nil {
|
||||
return errors.Wrap(err, "stat on ingest file failed")
|
||||
}
|
||||
|
||||
// change to readonly, more important for read, but provides _some_
|
||||
// protection from this point on. We use the existing perms with a mask
|
||||
// only allowing reads honoring the umask on creation.
|
||||
//
|
||||
// This removes write and exec, only allowing read per the creation umask.
|
||||
if err := cw.fp.Chmod((fi.Mode() & os.ModePerm) &^ 0333); err != nil {
|
||||
return errors.Wrap(err, "failed to change ingest file permissions")
|
||||
}
|
||||
|
||||
if size != fi.Size() {
|
||||
return errors.Errorf("failed size validation: %v != %v", fi.Size(), size)
|
||||
}
|
||||
|
||||
dgst := cw.digester.Digest()
|
||||
if expected != dgst {
|
||||
return errors.Errorf("unexpected digest: %v != %v", dgst, expected)
|
||||
}
|
||||
|
||||
apath := filepath.Join(cw.cs.root, "blobs", dgst.Algorithm().String())
|
||||
if err := os.MkdirAll(apath, 0755); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
var (
|
||||
ingest = filepath.Join(cw.path, "data")
|
||||
target = filepath.Join(apath, dgst.Hex())
|
||||
)
|
||||
|
||||
// clean up!!
|
||||
defer os.RemoveAll(cw.path)
|
||||
if err := os.Rename(ingest, target); err != nil {
|
||||
if os.IsExist(err) {
|
||||
// collision with the target file!
|
||||
return nil
|
||||
}
|
||||
return err
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// Close the writer, flushing any unwritten data and leaving the progress in
|
||||
// tact.
|
||||
//
|
||||
// If one needs to resume the transaction, a new writer can be obtained from
|
||||
// `ContentStore.Resume` using the same key. The write can then be continued
|
||||
// from it was left off.
|
||||
func (cw *ContentWriter) Close() (err error) {
|
||||
cw.fp.Sync()
|
||||
return cw.fp.Close()
|
||||
}
|
||||
Loading…
Reference in a new issue