diff --git a/circle.yml b/circle.yml index 16a6c817..3309f55b 100644 --- a/circle.yml +++ b/circle.yml @@ -21,7 +21,7 @@ test: - test -z $(gofmt -s -l . | tee /dev/stderr) - go vet ./... - test -z $(golint ./... | tee /dev/stderr) - - go test -test.v ./... + - go test -test.v -test.short ./... # Disabling the race detector due to massive memory usage. # - go test -race -test.v ./...: diff --git a/storagedriver/filesystem/driver.go b/storagedriver/filesystem/driver.go index 05ec6175..3e352125 100644 --- a/storagedriver/filesystem/driver.go +++ b/storagedriver/filesystem/driver.go @@ -203,6 +203,10 @@ func (d *Driver) Move(sourcePath string, destPath string) error { return storagedriver.PathNotFoundError{Path: sourcePath} } + if err := os.MkdirAll(path.Dir(dest), 0755); err != nil { + return err + } + err := os.Rename(source, dest) return err } diff --git a/storagedriver/inmemory/driver.go b/storagedriver/inmemory/driver.go index 0b68e021..841ce56c 100644 --- a/storagedriver/inmemory/driver.go +++ b/storagedriver/inmemory/driver.go @@ -5,7 +5,6 @@ import ( "fmt" "io" "io/ioutil" - "strings" "sync" "time" @@ -87,7 +86,7 @@ func (d *Driver) ReadStream(path string, offset int64) (io.ReadCloser, error) { return nil, storagedriver.InvalidOffsetError{Path: path, Offset: offset} } - path = d.normalize(path) + path = normalize(path) found := d.root.find(path) if found.path() != path { @@ -111,7 +110,7 @@ func (d *Driver) WriteStream(path string, offset int64, reader io.Reader) (nn in return 0, storagedriver.InvalidOffsetError{Path: path, Offset: offset} } - normalized := d.normalize(path) + normalized := normalize(path) f, err := d.root.mkfile(normalized) if err != nil { @@ -139,7 +138,7 @@ func (d *Driver) Stat(path string) (storagedriver.FileInfo, error) { d.mutex.RLock() defer d.mutex.RUnlock() - normalized := d.normalize(path) + normalized := normalize(path) found := d.root.find(path) if found.path() != normalized { @@ -162,7 +161,7 @@ func (d *Driver) Stat(path string) (storagedriver.FileInfo, error) { // List returns a list of the objects that are direct descendants of the given // path. func (d *Driver) List(path string) ([]string, error) { - normalized := d.normalize(path) + normalized := normalize(path) found := d.root.find(normalized) @@ -192,7 +191,7 @@ func (d *Driver) Move(sourcePath string, destPath string) error { d.mutex.Lock() defer d.mutex.Unlock() - normalizedSrc, normalizedDst := d.normalize(sourcePath), d.normalize(destPath) + normalizedSrc, normalizedDst := normalize(sourcePath), normalize(destPath) err := d.root.move(normalizedSrc, normalizedDst) switch err { @@ -208,7 +207,7 @@ func (d *Driver) Delete(path string) error { d.mutex.Lock() defer d.mutex.Unlock() - normalized := d.normalize(path) + normalized := normalize(path) err := d.root.delete(normalized) switch err { @@ -218,10 +217,3 @@ func (d *Driver) Delete(path string) error { return err } } - -func (d *Driver) normalize(p string) string { - if !strings.HasPrefix(p, "/") { - p = "/" + p // Ghetto path absolution. - } - return p -} diff --git a/storagedriver/inmemory/mfs.go b/storagedriver/inmemory/mfs.go index 9d75cb0e..6ff35db9 100644 --- a/storagedriver/inmemory/mfs.go +++ b/storagedriver/inmemory/mfs.go @@ -11,7 +11,7 @@ import ( var ( errExists = fmt.Errorf("exists") - errNotExists = fmt.Errorf("exists") + errNotExists = fmt.Errorf("notexists") errIsNotDir = fmt.Errorf("notdir") errIsDir = fmt.Errorf("isdir") ) @@ -139,9 +139,7 @@ func (d *dir) mkfile(p string) (*file, error) { // mkdirs creates any missing directory entries in p and returns the result. func (d *dir) mkdirs(p string) (*dir, error) { - if p == "" { - p = "/" - } + p = normalize(p) n := d.find(p) @@ -210,7 +208,7 @@ func (d *dir) move(src, dst string) error { srcDirname, srcFilename := path.Split(src) sp := d.find(srcDirname) - if sp.path() != srcDirname { + if normalize(srcDirname) != normalize(sp.path()) { return errNotExists } @@ -237,7 +235,7 @@ func (d *dir) delete(p string) error { dirname, filename := path.Split(p) parent := d.find(dirname) - if dirname != parent.path() { + if normalize(dirname) != normalize(parent.path()) { return errNotExists } @@ -328,3 +326,7 @@ func (c *common) path() string { func (c *common) modtime() time.Time { return c.mod } + +func normalize(p string) string { + return "/" + strings.Trim(p, "/") +} diff --git a/storagedriver/s3/s3_test.go b/storagedriver/s3/s3_test.go index f7b4f80e..fd17cd58 100644 --- a/storagedriver/s3/s3_test.go +++ b/storagedriver/s3/s3_test.go @@ -20,32 +20,37 @@ func Test(t *testing.T) { check.TestingT(t) } func init() { accessKey := os.Getenv("AWS_ACCESS_KEY") secretKey := os.Getenv("AWS_SECRET_KEY") - region := os.Getenv("AWS_REGION") bucket := os.Getenv("S3_BUCKET") encrypt := os.Getenv("S3_ENCRYPT") - s3DriverConstructor := func() (storagedriver.StorageDriver, error) { + s3DriverConstructor := func(region aws.Region) (storagedriver.StorageDriver, error) { shouldEncrypt, err := strconv.ParseBool(encrypt) if err != nil { return nil, err } - return New(accessKey, secretKey, aws.GetRegion(region), shouldEncrypt, bucket) + return New(accessKey, secretKey, region, shouldEncrypt, bucket) } // Skip S3 storage driver tests if environment variable parameters are not provided skipCheck := func() string { - if accessKey == "" || secretKey == "" || region == "" || bucket == "" || encrypt == "" { - return "Must set AWS_ACCESS_KEY, AWS_SECRET_KEY, AWS_REGION, S3_BUCKET, and S3_ENCRYPT to run S3 tests" + if accessKey == "" || secretKey == "" || bucket == "" || encrypt == "" { + return "Must set AWS_ACCESS_KEY, AWS_SECRET_KEY, S3_BUCKET, and S3_ENCRYPT to run S3 tests" } return "" } - testsuites.RegisterInProcessSuite(s3DriverConstructor, skipCheck) - testsuites.RegisterIPCSuite(driverName, map[string]string{ - "accesskey": accessKey, - "secretkey": secretKey, - "region": region, - "bucket": bucket, - "encrypt": encrypt, - }, skipCheck) + for _, region := range aws.Regions { + if region == aws.USGovWest { + continue + } + + testsuites.RegisterInProcessSuite(s3DriverConstructor(region), skipCheck) + testsuites.RegisterIPCSuite(driverName, map[string]string{ + "accesskey": accessKey, + "secretkey": secretKey, + "region": region.Name, + "bucket": bucket, + "encrypt": encrypt, + }, skipCheck) + } } diff --git a/storagedriver/testsuites/testsuites.go b/storagedriver/testsuites/testsuites.go index f745781e..3aa8642c 100644 --- a/storagedriver/testsuites/testsuites.go +++ b/storagedriver/testsuites/testsuites.go @@ -2,6 +2,7 @@ package testsuites import ( "bytes" + "crypto/sha1" "io" "io/ioutil" "math/rand" @@ -109,35 +110,54 @@ func (suite *DriverSuite) TearDownSuite(c *check.C) { // TestWriteRead1 tests a simple write-read workflow. func (suite *DriverSuite) TestWriteRead1(c *check.C) { - filename := randomString(32) + filename := randomPath(32) contents := []byte("a") suite.writeReadCompare(c, filename, contents) } // TestWriteRead2 tests a simple write-read workflow with unicode data. func (suite *DriverSuite) TestWriteRead2(c *check.C) { - filename := randomString(32) + filename := randomPath(32) contents := []byte("\xc3\x9f") suite.writeReadCompare(c, filename, contents) } // TestWriteRead3 tests a simple write-read workflow with a small string. func (suite *DriverSuite) TestWriteRead3(c *check.C) { - filename := randomString(32) - contents := []byte(randomString(32)) + filename := randomPath(32) + contents := randomContents(32) suite.writeReadCompare(c, filename, contents) } // TestWriteRead4 tests a simple write-read workflow with 1MB of data. func (suite *DriverSuite) TestWriteRead4(c *check.C) { - filename := randomString(32) - contents := []byte(randomString(1024 * 1024)) + filename := randomPath(32) + contents := randomContents(1024 * 1024) + suite.writeReadCompare(c, filename, contents) +} + +// TestWriteReadNonUTF8 tests that non-utf8 data may be written to the storage +// driver safely. +func (suite *DriverSuite) TestWriteReadNonUTF8(c *check.C) { + filename := randomPath(32) + contents := []byte{0x80, 0x80, 0x80, 0x80} + suite.writeReadCompare(c, filename, contents) +} + +// TestTruncate tests that putting smaller contents than an original file does +// remove the excess contents. +func (suite *DriverSuite) TestTruncate(c *check.C) { + filename := randomPath(32) + contents := randomContents(1024 * 1024) + suite.writeReadCompare(c, filename, contents) + + contents = randomContents(1024) suite.writeReadCompare(c, filename, contents) } // TestReadNonexistent tests reading content from an empty path. func (suite *DriverSuite) TestReadNonexistent(c *check.C) { - filename := randomString(32) + filename := randomPath(32) _, err := suite.StorageDriver.GetContent(filename) c.Assert(err, check.NotNil) c.Assert(err, check.FitsTypeOf, storagedriver.PathNotFoundError{}) @@ -145,7 +165,7 @@ func (suite *DriverSuite) TestReadNonexistent(c *check.C) { // TestWriteReadStreams1 tests a simple write-read streaming workflow. func (suite *DriverSuite) TestWriteReadStreams1(c *check.C) { - filename := randomString(32) + filename := randomPath(32) contents := []byte("a") suite.writeReadCompareStreams(c, filename, contents) } @@ -153,7 +173,7 @@ func (suite *DriverSuite) TestWriteReadStreams1(c *check.C) { // TestWriteReadStreams2 tests a simple write-read streaming workflow with // unicode data. func (suite *DriverSuite) TestWriteReadStreams2(c *check.C) { - filename := randomString(32) + filename := randomPath(32) contents := []byte("\xc3\x9f") suite.writeReadCompareStreams(c, filename, contents) } @@ -161,30 +181,68 @@ func (suite *DriverSuite) TestWriteReadStreams2(c *check.C) { // TestWriteReadStreams3 tests a simple write-read streaming workflow with a // small amount of data. func (suite *DriverSuite) TestWriteReadStreams3(c *check.C) { - filename := randomString(32) - contents := []byte(randomString(32)) + filename := randomPath(32) + contents := randomContents(32) suite.writeReadCompareStreams(c, filename, contents) } // TestWriteReadStreams4 tests a simple write-read streaming workflow with 1MB // of data. func (suite *DriverSuite) TestWriteReadStreams4(c *check.C) { - filename := randomString(32) - contents := []byte(randomString(1024 * 1024)) + filename := randomPath(32) + contents := randomContents(1024 * 1024) suite.writeReadCompareStreams(c, filename, contents) } +// TestWriteReadStreamsNonUTF8 tests that non-utf8 data may be written to the +// storage driver safely. +func (suite *DriverSuite) TestWriteReadStreamsNonUTF8(c *check.C) { + filename := randomPath(32) + contents := []byte{0x80, 0x80, 0x80, 0x80} + suite.writeReadCompareStreams(c, filename, contents) +} + +// TestWriteReadLargeStreams tests that a 5GB file may be written to the storage +// driver safely. +func (suite *DriverSuite) TestWriteReadLargeStreams(c *check.C) { + if testing.Short() { + c.Skip("Skipping test in short mode") + } + + filename := randomPath(32) + defer suite.StorageDriver.Delete(firstPart(filename)) + + checksum := sha1.New() + var offset int64 + var chunkSize int64 = 1024 * 1024 + + for i := 0; i < 5*1024; i++ { + contents := randomContents(chunkSize) + written, err := suite.StorageDriver.WriteStream(filename, offset, io.TeeReader(bytes.NewReader(contents), checksum)) + c.Assert(err, check.IsNil) + c.Assert(written, check.Equals, chunkSize) + offset += chunkSize + } + reader, err := suite.StorageDriver.ReadStream(filename, 0) + c.Assert(err, check.IsNil) + + writtenChecksum := sha1.New() + io.Copy(writtenChecksum, reader) + + c.Assert(writtenChecksum.Sum(nil), check.DeepEquals, checksum.Sum(nil)) +} + // TestReadStreamWithOffset tests that the appropriate data is streamed when // reading with a given offset. func (suite *DriverSuite) TestReadStreamWithOffset(c *check.C) { - filename := randomString(32) - defer suite.StorageDriver.Delete(filename) + filename := randomPath(32) + defer suite.StorageDriver.Delete(firstPart(filename)) chunkSize := int64(32) - contentsChunk1 := []byte(randomString(chunkSize)) - contentsChunk2 := []byte(randomString(chunkSize)) - contentsChunk3 := []byte(randomString(chunkSize)) + contentsChunk1 := randomContents(chunkSize) + contentsChunk2 := randomContents(chunkSize) + contentsChunk3 := randomContents(chunkSize) err := suite.StorageDriver.PutContent(filename, append(append(contentsChunk1, contentsChunk2...), contentsChunk3...)) c.Assert(err, check.IsNil) @@ -256,15 +314,15 @@ func (suite *DriverSuite) TestReadStreamWithOffset(c *check.C) { // TestContinueStreamAppend tests that a stream write can be appended to without // corrupting the data. func (suite *DriverSuite) TestContinueStreamAppend(c *check.C) { - filename := randomString(32) - defer suite.StorageDriver.Delete(filename) + filename := randomPath(32) + defer suite.StorageDriver.Delete(firstPart(filename)) chunkSize := int64(10 * 1024 * 1024) - contentsChunk1 := []byte(randomString(chunkSize)) - contentsChunk2 := []byte(randomString(chunkSize)) - contentsChunk3 := []byte(randomString(chunkSize)) - contentsChunk4 := []byte(randomString(chunkSize)) + contentsChunk1 := randomContents(chunkSize) + contentsChunk2 := randomContents(chunkSize) + contentsChunk3 := randomContents(chunkSize) + contentsChunk4 := randomContents(chunkSize) zeroChunk := make([]byte, int64(chunkSize)) fullContents := append(append(contentsChunk1, contentsChunk2...), contentsChunk3...) @@ -337,7 +395,7 @@ func (suite *DriverSuite) TestContinueStreamAppend(c *check.C) { // TestReadNonexistentStream tests that reading a stream for a nonexistent path // fails. func (suite *DriverSuite) TestReadNonexistentStream(c *check.C) { - filename := randomString(32) + filename := randomPath(32) _, err := suite.StorageDriver.ReadStream(filename, 0) c.Assert(err, check.NotNil) c.Assert(err, check.FitsTypeOf, storagedriver.PathNotFoundError{}) @@ -345,15 +403,15 @@ func (suite *DriverSuite) TestReadNonexistentStream(c *check.C) { // TestList checks the returned list of keys after populating a directory tree. func (suite *DriverSuite) TestList(c *check.C) { - rootDirectory := "/" + randomString(int64(8+rand.Intn(8))) - defer suite.StorageDriver.Delete(rootDirectory) + rootDirectory := "/" + randomFilename(int64(8+rand.Intn(8))) + defer suite.StorageDriver.Delete("/") - parentDirectory := rootDirectory + "/" + randomString(int64(8+rand.Intn(8))) + parentDirectory := rootDirectory + "/" + randomFilename(int64(8+rand.Intn(8))) childFiles := make([]string, 50) for i := 0; i < len(childFiles); i++ { - childFile := parentDirectory + "/" + randomString(int64(8+rand.Intn(8))) + childFile := parentDirectory + "/" + randomFilename(int64(8+rand.Intn(8))) childFiles[i] = childFile - err := suite.StorageDriver.PutContent(childFile, []byte(randomString(32))) + err := suite.StorageDriver.PutContent(childFile, randomContents(32)) c.Assert(err, check.IsNil) } sort.Strings(childFiles) @@ -381,12 +439,12 @@ func (suite *DriverSuite) TestList(c *check.C) { // TestMove checks that a moved object no longer exists at the source path and // does exist at the destination. func (suite *DriverSuite) TestMove(c *check.C) { - contents := []byte(randomString(32)) - sourcePath := randomString(32) - destPath := randomString(32) + contents := randomContents(32) + sourcePath := randomPath(32) + destPath := randomPath(32) - defer suite.StorageDriver.Delete(sourcePath) - defer suite.StorageDriver.Delete(destPath) + defer suite.StorageDriver.Delete(firstPart(sourcePath)) + defer suite.StorageDriver.Delete(firstPart(destPath)) err := suite.StorageDriver.PutContent(sourcePath, contents) c.Assert(err, check.IsNil) @@ -405,8 +463,8 @@ func (suite *DriverSuite) TestMove(c *check.C) { // TestMoveNonexistent checks that moving a nonexistent key fails func (suite *DriverSuite) TestMoveNonexistent(c *check.C) { - sourcePath := randomString(32) - destPath := randomString(32) + sourcePath := randomPath(32) + destPath := randomPath(32) err := suite.StorageDriver.Move(sourcePath, destPath) c.Assert(err, check.NotNil) @@ -416,10 +474,10 @@ func (suite *DriverSuite) TestMoveNonexistent(c *check.C) { // TestDelete checks that the delete operation removes data from the storage // driver func (suite *DriverSuite) TestDelete(c *check.C) { - filename := randomString(32) - contents := []byte(randomString(32)) + filename := randomPath(32) + contents := randomContents(32) - defer suite.StorageDriver.Delete(filename) + defer suite.StorageDriver.Delete(firstPart(filename)) err := suite.StorageDriver.PutContent(filename, contents) c.Assert(err, check.IsNil) @@ -434,7 +492,7 @@ func (suite *DriverSuite) TestDelete(c *check.C) { // TestDeleteNonexistent checks that removing a nonexistent key fails. func (suite *DriverSuite) TestDeleteNonexistent(c *check.C) { - filename := randomString(32) + filename := randomPath(32) err := suite.StorageDriver.Delete(filename) c.Assert(err, check.NotNil) c.Assert(err, check.FitsTypeOf, storagedriver.PathNotFoundError{}) @@ -442,13 +500,13 @@ func (suite *DriverSuite) TestDeleteNonexistent(c *check.C) { // TestDeleteFolder checks that deleting a folder removes all child elements. func (suite *DriverSuite) TestDeleteFolder(c *check.C) { - dirname := randomString(32) - filename1 := randomString(32) - filename2 := randomString(32) - contents := []byte(randomString(32)) + dirname := randomPath(32) + filename1 := randomPath(32) + filename2 := randomPath(32) + filename3 := randomPath(32) + contents := randomContents(32) - defer suite.StorageDriver.Delete(path.Join(dirname, filename1)) - defer suite.StorageDriver.Delete(path.Join(dirname, filename2)) + defer suite.StorageDriver.Delete(firstPart(dirname)) err := suite.StorageDriver.PutContent(path.Join(dirname, filename1), contents) c.Assert(err, check.IsNil) @@ -456,6 +514,22 @@ func (suite *DriverSuite) TestDeleteFolder(c *check.C) { err = suite.StorageDriver.PutContent(path.Join(dirname, filename2), contents) c.Assert(err, check.IsNil) + err = suite.StorageDriver.PutContent(path.Join(dirname, filename3), contents) + c.Assert(err, check.IsNil) + + err = suite.StorageDriver.Delete(path.Join(dirname, filename1)) + c.Assert(err, check.IsNil) + + _, err = suite.StorageDriver.GetContent(path.Join(dirname, filename1)) + c.Assert(err, check.NotNil) + c.Assert(err, check.FitsTypeOf, storagedriver.PathNotFoundError{}) + + _, err = suite.StorageDriver.GetContent(path.Join(dirname, filename2)) + c.Assert(err, check.IsNil) + + _, err = suite.StorageDriver.GetContent(path.Join(dirname, filename3)) + c.Assert(err, check.IsNil) + err = suite.StorageDriver.Delete(dirname) c.Assert(err, check.IsNil) @@ -466,22 +540,28 @@ func (suite *DriverSuite) TestDeleteFolder(c *check.C) { _, err = suite.StorageDriver.GetContent(path.Join(dirname, filename2)) c.Assert(err, check.NotNil) c.Assert(err, check.FitsTypeOf, storagedriver.PathNotFoundError{}) + + _, err = suite.StorageDriver.GetContent(path.Join(dirname, filename3)) + c.Assert(err, check.NotNil) + c.Assert(err, check.FitsTypeOf, storagedriver.PathNotFoundError{}) } // TestStatCall runs verifies the implementation of the storagedriver's Stat call. func (suite *DriverSuite) TestStatCall(c *check.C) { - content := randomString(4096) - dirPath := randomString(32) - fileName := randomString(32) + content := randomContents(4096) + dirPath := randomPath(32) + fileName := randomFilename(32) filePath := path.Join(dirPath, fileName) + defer suite.StorageDriver.Delete(dirPath) + // Call on non-existent file/dir, check error. fi, err := suite.StorageDriver.Stat(filePath) c.Assert(err, check.NotNil) c.Assert(err, check.FitsTypeOf, storagedriver.PathNotFoundError{}) c.Assert(fi, check.IsNil) - err = suite.StorageDriver.PutContent(filePath, []byte(content)) + err = suite.StorageDriver.PutContent(filePath, content) c.Assert(err, check.IsNil) // Call on regular file, check results @@ -555,7 +635,7 @@ func (suite *DriverSuite) testFileStreams(c *check.C, size int64) { tfName := path.Base(tf.Name()) defer suite.StorageDriver.Delete(tfName) - contents := []byte(randomString(size)) + contents := randomContents(size) _, err = tf.Write(contents) c.Assert(err, check.IsNil) @@ -578,7 +658,7 @@ func (suite *DriverSuite) testFileStreams(c *check.C, size int64) { } func (suite *DriverSuite) writeReadCompare(c *check.C, filename string, contents []byte) { - defer suite.StorageDriver.Delete(filename) + defer suite.StorageDriver.Delete(firstPart(filename)) err := suite.StorageDriver.PutContent(filename, contents) c.Assert(err, check.IsNil) @@ -590,7 +670,7 @@ func (suite *DriverSuite) writeReadCompare(c *check.C, filename string, contents } func (suite *DriverSuite) writeReadCompareStreams(c *check.C, filename string, contents []byte) { - defer suite.StorageDriver.Delete(filename) + defer suite.StorageDriver.Delete(firstPart(filename)) nn, err := suite.StorageDriver.WriteStream(filename, 0, bytes.NewReader(contents)) c.Assert(err, check.IsNil) @@ -606,12 +686,55 @@ func (suite *DriverSuite) writeReadCompareStreams(c *check.C, filename string, c c.Assert(readContents, check.DeepEquals, contents) } -var pathChars = []byte("abcdefghijklmnopqrstuvwxyz") +var filenameChars = []byte("abcdefghijklmnopqrstuvwxyz0123456789") -func randomString(length int64) string { +func randomPath(length int64) string { + path := "" + for int64(len(path)) < length { + chunkLength := rand.Int63n(length-int64(len(path))) + 1 + chunk := randomFilename(chunkLength) + path += chunk + if length-int64(len(path)) == 1 { + path += randomFilename(1) + } else if length-int64(len(path)) > 1 { + path += "/" + } + } + return path +} + +func randomFilename(length int64) string { b := make([]byte, length) for i := range b { - b[i] = pathChars[rand.Intn(len(pathChars))] + b[i] = filenameChars[rand.Intn(len(filenameChars))] } return string(b) } + +func randomContents(length int64) []byte { + b := make([]byte, length) + for i := range b { + b[i] = byte(rand.Intn(2 << 8)) + } + return b +} + +func firstPart(filePath string) string { + for { + if filePath[len(filePath)-1] == '/' { + filePath = filePath[:len(filePath)-1] + } + + dir, file := path.Split(filePath) + if dir == "" && file == "" { + return "/" + } + if dir == "" { + return file + } + if file == "" { + return dir + } + filePath = dir + } +}