vbatts/registry
1
0
Fork 0
Code Issues Pull requests Releases Wiki Activity

Adds new storagedriver.FileWriter interface

Browse source 
Updates registry storage code to use this for better resumable writes.
Implements this interface for the following drivers:
 + Inmemory
 + Filesystem
 + S3
 + Azure

Signed-off-by: Brian Bland <brian.bland@docker.com>
This commit is contained in:
Brian Bland 2016-02-08 14:29:21 -08:00
parent 46b2c3fadf
commit ff03381d49
37 changed files with 1429 additions and 3456 deletions
Download patch file Download diff file Expand all files Collapse all files

2
Godeps/Godeps.json generated
View file

@ -43,7 +43,7 @@
},
{
"ImportPath": "github.com/Azure/azure-sdk-for-go/storage",
"Rev": "97d9593768bbbbd316f9c055dfc5f780933cd7fc"
"Rev": "95361a2573b1fa92a00c5fc2707a80308483c6f9"
},
{
"ImportPath": "github.com/Sirupsen/logrus",

164
Godeps/_workspace/src/github.com/Azure/azure-sdk-for-go/storage/blob.go generated vendored
View file

@ -167,8 +167,9 @@ type BlobType string
// Types of page blobs
const (
BlobTypeBlock BlobType = "BlockBlob"
BlobTypePage BlobType = "PageBlob"
BlobTypeBlock BlobType = "BlockBlob"
BlobTypePage BlobType = "PageBlob"
BlobTypeAppend BlobType = "AppendBlob"
)
// PageWriteType defines the type updates that are going to be
@ -330,7 +331,6 @@ func (b BlobStorageClient) createContainer(name string, access ContainerAccessTy
uri := b.client.getEndpoint(blobServiceName, pathForContainer(name), url.Values{"restype": {"container"}})
headers := b.client.getStandardHeaders()
headers["Content-Length"] = "0"
if access != "" {
headers["x-ms-blob-public-access"] = string(access)
}
@ -541,17 +541,102 @@ func (b BlobStorageClient) GetBlobProperties(container, name string) (*BlobPrope
}, nil
}
// SetBlobMetadata replaces the metadata for the specified blob.
//
// Some keys may be converted to Camel-Case before sending. All keys
// are returned in lower case by GetBlobMetadata. HTTP header names
// are case-insensitive so case munging should not matter to other
// applications either.
//
// See https://msdn.microsoft.com/en-us/library/azure/dd179414.aspx
func (b BlobStorageClient) SetBlobMetadata(container, name string, metadata map[string]string) error {
params := url.Values{"comp": {"metadata"}}
uri := b.client.getEndpoint(blobServiceName, pathForBlob(container, name), params)
headers := b.client.getStandardHeaders()
for k, v := range metadata {
headers[userDefinedMetadataHeaderPrefix+k] = v
}
resp, err := b.client.exec("PUT", uri, headers, nil)
if err != nil {
return err
}
defer resp.body.Close()
return checkRespCode(resp.statusCode, []int{http.StatusOK})
}
// GetBlobMetadata returns all user-defined metadata for the specified blob.
//
// All metadata keys will be returned in lower case. (HTTP header
// names are case-insensitive.)
//
// See https://msdn.microsoft.com/en-us/library/azure/dd179414.aspx
func (b BlobStorageClient) GetBlobMetadata(container, name string) (map[string]string, error) {
params := url.Values{"comp": {"metadata"}}
uri := b.client.getEndpoint(blobServiceName, pathForBlob(container, name), params)
headers := b.client.getStandardHeaders()
resp, err := b.client.exec("GET", uri, headers, nil)
if err != nil {
return nil, err
}
defer resp.body.Close()
if err := checkRespCode(resp.statusCode, []int{http.StatusOK}); err != nil {
return nil, err
}
metadata := make(map[string]string)
for k, v := range resp.headers {
// Can't trust CanonicalHeaderKey() to munge case
// reliably. "_" is allowed in identifiers:
// https://msdn.microsoft.com/en-us/library/azure/dd179414.aspx
// https://msdn.microsoft.com/library/aa664670(VS.71).aspx
// http://tools.ietf.org/html/rfc7230#section-3.2
// ...but "_" is considered invalid by
// CanonicalMIMEHeaderKey in
// https://golang.org/src/net/textproto/reader.go?s=14615:14659#L542
// so k can be "X-Ms-Meta-Foo" or "x-ms-meta-foo_bar".
k = strings.ToLower(k)
if len(v) == 0 || !strings.HasPrefix(k, strings.ToLower(userDefinedMetadataHeaderPrefix)) {
continue
}
// metadata["foo"] = content of the last X-Ms-Meta-Foo header
k = k[len(userDefinedMetadataHeaderPrefix):]
metadata[k] = v[len(v)-1]
}
return metadata, nil
}
// CreateBlockBlob initializes an empty block blob with no blocks.
//
// See https://msdn.microsoft.com/en-us/library/azure/dd179451.aspx
func (b BlobStorageClient) CreateBlockBlob(container, name string) error {
return b.CreateBlockBlobFromReader(container, name, 0, nil, nil)
}
// CreateBlockBlobFromReader initializes a block blob using data from
// reader. Size must be the number of bytes read from reader. To
// create an empty blob, use size==0 and reader==nil.
//
// The API rejects requests with size > 64 MiB (but this limit is not
// checked by the SDK). To write a larger blob, use CreateBlockBlob,
// PutBlock, and PutBlockList.
//
// See https://msdn.microsoft.com/en-us/library/azure/dd179451.aspx
func (b BlobStorageClient) CreateBlockBlobFromReader(container, name string, size uint64, blob io.Reader, extraHeaders map[string]string) error {
path := fmt.Sprintf("%s/%s", container, name)
uri := b.client.getEndpoint(blobServiceName, path, url.Values{})
headers := b.client.getStandardHeaders()
headers["x-ms-blob-type"] = string(BlobTypeBlock)
headers["Content-Length"] = fmt.Sprintf("%v", 0)
headers["Content-Length"] = fmt.Sprintf("%d", size)
resp, err := b.client.exec("PUT", uri, headers, nil)
for k, v := range extraHeaders {
headers[k] = v
}
resp, err := b.client.exec("PUT", uri, headers, blob)
if err != nil {
return err
}
@ -562,26 +647,37 @@ func (b BlobStorageClient) CreateBlockBlob(container, name string) error {
// PutBlock saves the given data chunk to the specified block blob with
// given ID.
//
// The API rejects chunks larger than 4 MiB (but this limit is not
// checked by the SDK).
//
// See https://msdn.microsoft.com/en-us/library/azure/dd135726.aspx
func (b BlobStorageClient) PutBlock(container, name, blockID string, chunk []byte) error {
return b.PutBlockWithLength(container, name, blockID, uint64(len(chunk)), bytes.NewReader(chunk))
return b.PutBlockWithLength(container, name, blockID, uint64(len(chunk)), bytes.NewReader(chunk), nil)
}
// PutBlockWithLength saves the given data stream of exactly specified size to
// the block blob with given ID. It is an alternative to PutBlocks where data
// comes as stream but the length is known in advance.
//
// The API rejects requests with size > 4 MiB (but this limit is not
// checked by the SDK).
//
// See https://msdn.microsoft.com/en-us/library/azure/dd135726.aspx
func (b BlobStorageClient) PutBlockWithLength(container, name, blockID string, size uint64, blob io.Reader) error {
func (b BlobStorageClient) PutBlockWithLength(container, name, blockID string, size uint64, blob io.Reader, extraHeaders map[string]string) error {
uri := b.client.getEndpoint(blobServiceName, pathForBlob(container, name), url.Values{"comp": {"block"}, "blockid": {blockID}})
headers := b.client.getStandardHeaders()
headers["x-ms-blob-type"] = string(BlobTypeBlock)
headers["Content-Length"] = fmt.Sprintf("%v", size)
for k, v := range extraHeaders {
headers[k] = v
}
resp, err := b.client.exec("PUT", uri, headers, blob)
if err != nil {
return err
}
defer resp.body.Close()
return checkRespCode(resp.statusCode, []int{http.StatusCreated})
}
@ -628,13 +724,16 @@ func (b BlobStorageClient) GetBlockList(container, name string, blockType BlockL
// be created using this method before writing pages.
//
// See https://msdn.microsoft.com/en-us/library/azure/dd179451.aspx
func (b BlobStorageClient) PutPageBlob(container, name string, size int64) error {
func (b BlobStorageClient) PutPageBlob(container, name string, size int64, extraHeaders map[string]string) error {
path := fmt.Sprintf("%s/%s", container, name)
uri := b.client.getEndpoint(blobServiceName, path, url.Values{})
headers := b.client.getStandardHeaders()
headers["x-ms-blob-type"] = string(BlobTypePage)
headers["x-ms-blob-content-length"] = fmt.Sprintf("%v", size)
headers["Content-Length"] = fmt.Sprintf("%v", 0)
for k, v := range extraHeaders {
headers[k] = v
}
resp, err := b.client.exec("PUT", uri, headers, nil)
if err != nil {
@ -700,6 +799,48 @@ func (b BlobStorageClient) GetPageRanges(container, name string) (GetPageRangesR
return out, err
}
// PutAppendBlob initializes an empty append blob with specified name. An
// append blob must be created using this method before appending blocks.
//
// See https://msdn.microsoft.com/en-us/library/azure/dd179451.aspx
func (b BlobStorageClient) PutAppendBlob(container, name string, extraHeaders map[string]string) error {
path := fmt.Sprintf("%s/%s", container, name)
uri := b.client.getEndpoint(blobServiceName, path, url.Values{})
headers := b.client.getStandardHeaders()
headers["x-ms-blob-type"] = string(BlobTypeAppend)
for k, v := range extraHeaders {
headers[k] = v
}
resp, err := b.client.exec("PUT", uri, headers, nil)
if err != nil {
return err
}
defer resp.body.Close()
return checkRespCode(resp.statusCode, []int{http.StatusCreated})
}
// AppendBlock appends a block to an append blob.
//
// See https://msdn.microsoft.com/en-us/library/azure/mt427365.aspx
func (b BlobStorageClient) AppendBlock(container, name string, chunk []byte) error {
path := fmt.Sprintf("%s/%s", container, name)
uri := b.client.getEndpoint(blobServiceName, path, url.Values{"comp": {"appendblock"}})
headers := b.client.getStandardHeaders()
headers["x-ms-blob-type"] = string(BlobTypeAppend)
headers["Content-Length"] = fmt.Sprintf("%v", len(chunk))
resp, err := b.client.exec("PUT", uri, headers, bytes.NewReader(chunk))
if err != nil {
return err
}
defer resp.body.Close()
return checkRespCode(resp.statusCode, []int{http.StatusCreated})
}
// CopyBlob starts a blob copy operation and waits for the operation to
// complete. sourceBlob parameter must be a canonical URL to the blob (can be
// obtained using GetBlobURL method.) There is no SLA on blob copy and therefore
@ -719,7 +860,6 @@ func (b BlobStorageClient) startBlobCopy(container, name, sourceBlob string) (st
uri := b.client.getEndpoint(blobServiceName, pathForBlob(container, name), url.Values{})
headers := b.client.getStandardHeaders()
headers["Content-Length"] = "0"
headers["x-ms-copy-source"] = sourceBlob
resp, err := b.client.exec("PUT", uri, headers, nil)
@ -850,6 +990,10 @@ func (b BlobStorageClient) GetBlobSASURI(container, name string, expiry time.Tim
func blobSASStringToSign(signedVersion, canonicalizedResource, signedExpiry, signedPermissions string) (string, error) {
var signedStart, signedIdentifier, rscc, rscd, rsce, rscl, rsct string
if signedVersion >= "2015-02-21" {
canonicalizedResource = "/blob" + canonicalizedResource
}
// reference: http://msdn.microsoft.com/en-us/library/azure/dn140255.aspx
if signedVersion >= "2013-08-15" {
return fmt.Sprintf("%s\n%s\n%s\n%s\n%s\n%s\n%s\n%s\n%s\n%s\n%s", signedPermissions, signedStart, signedExpiry, canonicalizedResource, signedIdentifier, signedVersion, rscc, rscd, rsce, rscl, rsct), nil

625
Godeps/_workspace/src/github.com/Azure/azure-sdk-for-go/storage/blob_test.go generated vendored
View file

@ -1,625 +0,0 @@
package storage
import (
"bytes"
"crypto/rand"
"encoding/base64"
"fmt"
"io/ioutil"
"net/http"
"net/url"
"sort"
"sync"
"testing"
"time"
chk "gopkg.in/check.v1"
)
type StorageBlobSuite struct{}
var _ = chk.Suite(&StorageBlobSuite{})
const testContainerPrefix = "zzzztest-"
func getBlobClient(c *chk.C) BlobStorageClient {
return getBasicClient(c).GetBlobService()
}
func (s *StorageBlobSuite) Test_pathForContainer(c *chk.C) {
c.Assert(pathForContainer("foo"), chk.Equals, "/foo")
}
func (s *StorageBlobSuite) Test_pathForBlob(c *chk.C) {
c.Assert(pathForBlob("foo", "blob"), chk.Equals, "/foo/blob")
}
func (s *StorageBlobSuite) Test_blobSASStringToSign(c *chk.C) {
_, err := blobSASStringToSign("2012-02-12", "CS", "SE", "SP")
c.Assert(err, chk.NotNil) // not implemented SAS for versions earlier than 2013-08-15
out, err := blobSASStringToSign("2013-08-15", "CS", "SE", "SP")
c.Assert(err, chk.IsNil)
c.Assert(out, chk.Equals, "SP\n\nSE\nCS\n\n2013-08-15\n\n\n\n\n")
}
func (s *StorageBlobSuite) TestGetBlobSASURI(c *chk.C) {
api, err := NewClient("foo", "YmFy", DefaultBaseURL, "2013-08-15", true)
c.Assert(err, chk.IsNil)
cli := api.GetBlobService()
expiry := time.Time{}
expectedParts := url.URL{
Scheme: "https",
Host: "foo.blob.core.windows.net",
Path: "container/name",
RawQuery: url.Values{
"sv": {"2013-08-15"},
"sig": {"/OXG7rWh08jYwtU03GzJM0DHZtidRGpC6g69rSGm3I0="},
"sr": {"b"},
"sp": {"r"},
"se": {"0001-01-01T00:00:00Z"},
}.Encode()}
u, err := cli.GetBlobSASURI("container", "name", expiry, "r")
c.Assert(err, chk.IsNil)
sasParts, err := url.Parse(u)
c.Assert(err, chk.IsNil)
c.Assert(expectedParts.String(), chk.Equals, sasParts.String())
c.Assert(expectedParts.Query(), chk.DeepEquals, sasParts.Query())
}
func (s *StorageBlobSuite) TestBlobSASURICorrectness(c *chk.C) {
cli := getBlobClient(c)
cnt := randContainer()
blob := randString(20)
body := []byte(randString(100))
expiry := time.Now().UTC().Add(time.Hour)
permissions := "r"
c.Assert(cli.CreateContainer(cnt, ContainerAccessTypePrivate), chk.IsNil)
defer cli.DeleteContainer(cnt)
c.Assert(cli.putSingleBlockBlob(cnt, blob, body), chk.IsNil)
sasURI, err := cli.GetBlobSASURI(cnt, blob, expiry, permissions)
c.Assert(err, chk.IsNil)
resp, err := http.Get(sasURI)
c.Assert(err, chk.IsNil)
blobResp, err := ioutil.ReadAll(resp.Body)
defer resp.Body.Close()
c.Assert(err, chk.IsNil)
c.Assert(resp.StatusCode, chk.Equals, http.StatusOK)
c.Assert(len(blobResp), chk.Equals, len(body))
}
func (s *StorageBlobSuite) TestListContainersPagination(c *chk.C) {
cli := getBlobClient(c)
c.Assert(deleteTestContainers(cli), chk.IsNil)
const n = 5
const pageSize = 2
// Create test containers
created := []string{}
for i := 0; i < n; i++ {
name := randContainer()
c.Assert(cli.CreateContainer(name, ContainerAccessTypePrivate), chk.IsNil)
created = append(created, name)
}
sort.Strings(created)
// Defer test container deletions
defer func() {
var wg sync.WaitGroup
for _, cnt := range created {
wg.Add(1)
go func(name string) {
c.Assert(cli.DeleteContainer(name), chk.IsNil)
wg.Done()
}(cnt)
}
wg.Wait()
}()
// Paginate results
seen := []string{}
marker := ""
for {
resp, err := cli.ListContainers(ListContainersParameters{
Prefix: testContainerPrefix,
MaxResults: pageSize,
Marker: marker})
c.Assert(err, chk.IsNil)
containers := resp.Containers
if len(containers) > pageSize {
c.Fatalf("Got a bigger page. Expected: %d, got: %d", pageSize, len(containers))
}
for _, c := range containers {
seen = append(seen, c.Name)
}
marker = resp.NextMarker
if marker == "" || len(containers) == 0 {
break
}
}
c.Assert(seen, chk.DeepEquals, created)
}
func (s *StorageBlobSuite) TestContainerExists(c *chk.C) {
cnt := randContainer()
cli := getBlobClient(c)
ok, err := cli.ContainerExists(cnt)
c.Assert(err, chk.IsNil)
c.Assert(ok, chk.Equals, false)
c.Assert(cli.CreateContainer(cnt, ContainerAccessTypeBlob), chk.IsNil)
defer cli.DeleteContainer(cnt)
ok, err = cli.ContainerExists(cnt)
c.Assert(err, chk.IsNil)
c.Assert(ok, chk.Equals, true)
}
func (s *StorageBlobSuite) TestCreateDeleteContainer(c *chk.C) {
cnt := randContainer()
cli := getBlobClient(c)
c.Assert(cli.CreateContainer(cnt, ContainerAccessTypePrivate), chk.IsNil)
c.Assert(cli.DeleteContainer(cnt), chk.IsNil)
}
func (s *StorageBlobSuite) TestCreateContainerIfNotExists(c *chk.C) {
cnt := randContainer()
cli := getBlobClient(c)
// First create
ok, err := cli.CreateContainerIfNotExists(cnt, ContainerAccessTypePrivate)
c.Assert(err, chk.IsNil)
c.Assert(ok, chk.Equals, true)
// Second create, should not give errors
ok, err = cli.CreateContainerIfNotExists(cnt, ContainerAccessTypePrivate)
c.Assert(err, chk.IsNil)
defer cli.DeleteContainer(cnt)
c.Assert(ok, chk.Equals, false)
}
func (s *StorageBlobSuite) TestDeleteContainerIfExists(c *chk.C) {
cnt := randContainer()
cli := getBlobClient(c)
// Nonexisting container
c.Assert(cli.DeleteContainer(cnt), chk.NotNil)
ok, err := cli.DeleteContainerIfExists(cnt)
c.Assert(err, chk.IsNil)
c.Assert(ok, chk.Equals, false)
// Existing container
c.Assert(cli.CreateContainer(cnt, ContainerAccessTypePrivate), chk.IsNil)
ok, err = cli.DeleteContainerIfExists(cnt)
c.Assert(err, chk.IsNil)
c.Assert(ok, chk.Equals, true)
}
func (s *StorageBlobSuite) TestBlobExists(c *chk.C) {
cnt := randContainer()
blob := randString(20)
cli := getBlobClient(c)
c.Assert(cli.CreateContainer(cnt, ContainerAccessTypeBlob), chk.IsNil)
defer cli.DeleteContainer(cnt)
c.Assert(cli.putSingleBlockBlob(cnt, blob, []byte("Hello!")), chk.IsNil)
defer cli.DeleteBlob(cnt, blob)
ok, err := cli.BlobExists(cnt, blob+".foo")
c.Assert(err, chk.IsNil)
c.Assert(ok, chk.Equals, false)
ok, err = cli.BlobExists(cnt, blob)
c.Assert(err, chk.IsNil)
c.Assert(ok, chk.Equals, true)
}
func (s *StorageBlobSuite) TestGetBlobURL(c *chk.C) {
api, err := NewBasicClient("foo", "YmFy")
c.Assert(err, chk.IsNil)
cli := api.GetBlobService()
c.Assert(cli.GetBlobURL("c", "nested/blob"), chk.Equals, "https://foo.blob.core.windows.net/c/nested/blob")
c.Assert(cli.GetBlobURL("", "blob"), chk.Equals, "https://foo.blob.core.windows.net/$root/blob")
c.Assert(cli.GetBlobURL("", "nested/blob"), chk.Equals, "https://foo.blob.core.windows.net/$root/nested/blob")
}
func (s *StorageBlobSuite) TestBlobCopy(c *chk.C) {
if testing.Short() {
c.Skip("skipping blob copy in short mode, no SLA on async operation")
}
cli := getBlobClient(c)
cnt := randContainer()
src := randString(20)
dst := randString(20)
body := []byte(randString(1024))
c.Assert(cli.CreateContainer(cnt, ContainerAccessTypePrivate), chk.IsNil)
defer cli.deleteContainer(cnt)
c.Assert(cli.putSingleBlockBlob(cnt, src, body), chk.IsNil)
defer cli.DeleteBlob(cnt, src)
c.Assert(cli.CopyBlob(cnt, dst, cli.GetBlobURL(cnt, src)), chk.IsNil)
defer cli.DeleteBlob(cnt, dst)
blobBody, err := cli.GetBlob(cnt, dst)
c.Assert(err, chk.IsNil)
b, err := ioutil.ReadAll(blobBody)
defer blobBody.Close()
c.Assert(err, chk.IsNil)
c.Assert(b, chk.DeepEquals, body)
}
func (s *StorageBlobSuite) TestDeleteBlobIfExists(c *chk.C) {
cnt := randContainer()
blob := randString(20)
cli := getBlobClient(c)
c.Assert(cli.DeleteBlob(cnt, blob), chk.NotNil)
ok, err := cli.DeleteBlobIfExists(cnt, blob)
c.Assert(err, chk.IsNil)
c.Assert(ok, chk.Equals, false)
}
func (s *StorageBlobSuite) TestGetBlobProperties(c *chk.C) {
cnt := randContainer()
blob := randString(20)
contents := randString(64)
cli := getBlobClient(c)
c.Assert(cli.CreateContainer(cnt, ContainerAccessTypePrivate), chk.IsNil)
defer cli.DeleteContainer(cnt)
// Nonexisting blob
_, err := cli.GetBlobProperties(cnt, blob)
c.Assert(err, chk.NotNil)
// Put the blob
c.Assert(cli.putSingleBlockBlob(cnt, blob, []byte(contents)), chk.IsNil)
// Get blob properties
props, err := cli.GetBlobProperties(cnt, blob)
c.Assert(err, chk.IsNil)
c.Assert(props.ContentLength, chk.Equals, int64(len(contents)))
c.Assert(props.BlobType, chk.Equals, BlobTypeBlock)
}
func (s *StorageBlobSuite) TestListBlobsPagination(c *chk.C) {
cli := getBlobClient(c)
cnt := randContainer()
c.Assert(cli.CreateContainer(cnt, ContainerAccessTypePrivate), chk.IsNil)
defer cli.DeleteContainer(cnt)
blobs := []string{}
const n = 5
const pageSize = 2
for i := 0; i < n; i++ {
name := randString(20)
c.Assert(cli.putSingleBlockBlob(cnt, name, []byte("Hello, world!")), chk.IsNil)
blobs = append(blobs, name)
}
sort.Strings(blobs)
// Paginate
seen := []string{}
marker := ""
for {
resp, err := cli.ListBlobs(cnt, ListBlobsParameters{
MaxResults: pageSize,
Marker: marker})
c.Assert(err, chk.IsNil)
for _, v := range resp.Blobs {
seen = append(seen, v.Name)
}
marker = resp.NextMarker
if marker == "" || len(resp.Blobs) == 0 {
break
}
}
// Compare
c.Assert(seen, chk.DeepEquals, blobs)
}
func (s *StorageBlobSuite) TestPutEmptyBlockBlob(c *chk.C) {
cli := getBlobClient(c)
cnt := randContainer()
c.Assert(cli.CreateContainer(cnt, ContainerAccessTypePrivate), chk.IsNil)
defer cli.deleteContainer(cnt)
blob := randString(20)
c.Assert(cli.putSingleBlockBlob(cnt, blob, []byte{}), chk.IsNil)
props, err := cli.GetBlobProperties(cnt, blob)
c.Assert(err, chk.IsNil)
c.Assert(props.ContentLength, chk.Not(chk.Equals), 0)
}
func (s *StorageBlobSuite) TestGetBlobRange(c *chk.C) {
cnt := randContainer()
blob := randString(20)
body := "0123456789"
cli := getBlobClient(c)
c.Assert(cli.CreateContainer(cnt, ContainerAccessTypeBlob), chk.IsNil)
defer cli.DeleteContainer(cnt)
c.Assert(cli.putSingleBlockBlob(cnt, blob, []byte(body)), chk.IsNil)
defer cli.DeleteBlob(cnt, blob)
// Read 1-3
for _, r := range []struct {
rangeStr string
expected string
}{
{"0-", body},
{"1-3", body[1 : 3+1]},
{"3-", body[3:]},
} {
resp, err := cli.GetBlobRange(cnt, blob, r.rangeStr)
c.Assert(err, chk.IsNil)
blobBody, err := ioutil.ReadAll(resp)
c.Assert(err, chk.IsNil)
str := string(blobBody)
c.Assert(str, chk.Equals, r.expected)
}
}
func (s *StorageBlobSuite) TestPutBlock(c *chk.C) {
cli := getBlobClient(c)
cnt := randContainer()
c.Assert(cli.CreateContainer(cnt, ContainerAccessTypePrivate), chk.IsNil)
defer cli.deleteContainer(cnt)
blob := randString(20)
chunk := []byte(randString(1024))
blockID := base64.StdEncoding.EncodeToString([]byte("foo"))
c.Assert(cli.PutBlock(cnt, blob, blockID, chunk), chk.IsNil)
}
func (s *StorageBlobSuite) TestGetBlockList_PutBlockList(c *chk.C) {
cli := getBlobClient(c)
cnt := randContainer()
c.Assert(cli.CreateContainer(cnt, ContainerAccessTypePrivate), chk.IsNil)
defer cli.deleteContainer(cnt)
blob := randString(20)
chunk := []byte(randString(1024))
blockID := base64.StdEncoding.EncodeToString([]byte("foo"))
// Put one block
c.Assert(cli.PutBlock(cnt, blob, blockID, chunk), chk.IsNil)
defer cli.deleteBlob(cnt, blob)
// Get committed blocks
committed, err := cli.GetBlockList(cnt, blob, BlockListTypeCommitted)
c.Assert(err, chk.IsNil)
if len(committed.CommittedBlocks) > 0 {
c.Fatal("There are committed blocks")
}
// Get uncommitted blocks
uncommitted, err := cli.GetBlockList(cnt, blob, BlockListTypeUncommitted)
c.Assert(err, chk.IsNil)
c.Assert(len(uncommitted.UncommittedBlocks), chk.Equals, 1)
// Commit block list
c.Assert(cli.PutBlockList(cnt, blob, []Block{{blockID, BlockStatusUncommitted}}), chk.IsNil)
// Get all blocks
all, err := cli.GetBlockList(cnt, blob, BlockListTypeAll)
c.Assert(err, chk.IsNil)
c.Assert(len(all.CommittedBlocks), chk.Equals, 1)
c.Assert(len(all.UncommittedBlocks), chk.Equals, 0)
// Verify the block
thatBlock := all.CommittedBlocks[0]
c.Assert(thatBlock.Name, chk.Equals, blockID)
c.Assert(thatBlock.Size, chk.Equals, int64(len(chunk)))
}
func (s *StorageBlobSuite) TestCreateBlockBlob(c *chk.C) {
cli := getBlobClient(c)
cnt := randContainer()
c.Assert(cli.CreateContainer(cnt, ContainerAccessTypePrivate), chk.IsNil)
defer cli.deleteContainer(cnt)
blob := randString(20)
c.Assert(cli.CreateBlockBlob(cnt, blob), chk.IsNil)
// Verify
blocks, err := cli.GetBlockList(cnt, blob, BlockListTypeAll)
c.Assert(err, chk.IsNil)
c.Assert(len(blocks.CommittedBlocks), chk.Equals, 0)
c.Assert(len(blocks.UncommittedBlocks), chk.Equals, 0)
}
func (s *StorageBlobSuite) TestPutPageBlob(c *chk.C) {
cli := getBlobClient(c)
cnt := randContainer()
c.Assert(cli.CreateContainer(cnt, ContainerAccessTypePrivate), chk.IsNil)
defer cli.deleteContainer(cnt)
blob := randString(20)
size := int64(10 * 1024 * 1024)
c.Assert(cli.PutPageBlob(cnt, blob, size), chk.IsNil)
// Verify
props, err := cli.GetBlobProperties(cnt, blob)
c.Assert(err, chk.IsNil)
c.Assert(props.ContentLength, chk.Equals, size)
c.Assert(props.BlobType, chk.Equals, BlobTypePage)
}
func (s *StorageBlobSuite) TestPutPagesUpdate(c *chk.C) {
cli := getBlobClient(c)
cnt := randContainer()
c.Assert(cli.CreateContainer(cnt, ContainerAccessTypePrivate), chk.IsNil)
defer cli.deleteContainer(cnt)
blob := randString(20)
size := int64(10 * 1024 * 1024) // larger than we'll use
c.Assert(cli.PutPageBlob(cnt, blob, size), chk.IsNil)
chunk1 := []byte(randString(1024))
chunk2 := []byte(randString(512))
// Append chunks
c.Assert(cli.PutPage(cnt, blob, 0, int64(len(chunk1)-1), PageWriteTypeUpdate, chunk1), chk.IsNil)
c.Assert(cli.PutPage(cnt, blob, int64(len(chunk1)), int64(len(chunk1)+len(chunk2)-1), PageWriteTypeUpdate, chunk2), chk.IsNil)
// Verify contents
out, err := cli.GetBlobRange(cnt, blob, fmt.Sprintf("%v-%v", 0, len(chunk1)+len(chunk2)-1))
c.Assert(err, chk.IsNil)
defer out.Close()
blobContents, err := ioutil.ReadAll(out)
c.Assert(err, chk.IsNil)
c.Assert(blobContents, chk.DeepEquals, append(chunk1, chunk2...))
out.Close()
// Overwrite first half of chunk1
chunk0 := []byte(randString(512))
c.Assert(cli.PutPage(cnt, blob, 0, int64(len(chunk0)-1), PageWriteTypeUpdate, chunk0), chk.IsNil)
// Verify contents
out, err = cli.GetBlobRange(cnt, blob, fmt.Sprintf("%v-%v", 0, len(chunk1)+len(chunk2)-1))
c.Assert(err, chk.IsNil)
defer out.Close()
blobContents, err = ioutil.ReadAll(out)
c.Assert(err, chk.IsNil)
c.Assert(blobContents, chk.DeepEquals, append(append(chunk0, chunk1[512:]...), chunk2...))
}
func (s *StorageBlobSuite) TestPutPagesClear(c *chk.C) {
cli := getBlobClient(c)
cnt := randContainer()
c.Assert(cli.CreateContainer(cnt, ContainerAccessTypePrivate), chk.IsNil)
defer cli.deleteContainer(cnt)
blob := randString(20)
size := int64(10 * 1024 * 1024) // larger than we'll use
c.Assert(cli.PutPageBlob(cnt, blob, size), chk.IsNil)
// Put 0-2047
chunk := []byte(randString(2048))
c.Assert(cli.PutPage(cnt, blob, 0, 2047, PageWriteTypeUpdate, chunk), chk.IsNil)
// Clear 512-1023
c.Assert(cli.PutPage(cnt, blob, 512, 1023, PageWriteTypeClear, nil), chk.IsNil)
// Verify contents
out, err := cli.GetBlobRange(cnt, blob, "0-2047")
c.Assert(err, chk.IsNil)
contents, err := ioutil.ReadAll(out)
c.Assert(err, chk.IsNil)
defer out.Close()
c.Assert(contents, chk.DeepEquals, append(append(chunk[:512], make([]byte, 512)...), chunk[1024:]...))
}
func (s *StorageBlobSuite) TestGetPageRanges(c *chk.C) {
cli := getBlobClient(c)
cnt := randContainer()
c.Assert(cli.CreateContainer(cnt, ContainerAccessTypePrivate), chk.IsNil)
defer cli.deleteContainer(cnt)
blob := randString(20)
size := int64(10 * 1024 * 1024) // larger than we'll use
c.Assert(cli.PutPageBlob(cnt, blob, size), chk.IsNil)
// Get page ranges on empty blob
out, err := cli.GetPageRanges(cnt, blob)
c.Assert(err, chk.IsNil)
c.Assert(len(out.PageList), chk.Equals, 0)
// Add 0-512 page
c.Assert(cli.PutPage(cnt, blob, 0, 511, PageWriteTypeUpdate, []byte(randString(512))), chk.IsNil)
out, err = cli.GetPageRanges(cnt, blob)
c.Assert(err, chk.IsNil)
c.Assert(len(out.PageList), chk.Equals, 1)
// Add 1024-2048
c.Assert(cli.PutPage(cnt, blob, 1024, 2047, PageWriteTypeUpdate, []byte(randString(1024))), chk.IsNil)
out, err = cli.GetPageRanges(cnt, blob)
c.Assert(err, chk.IsNil)
c.Assert(len(out.PageList), chk.Equals, 2)
}
func deleteTestContainers(cli BlobStorageClient) error {
for {
resp, err := cli.ListContainers(ListContainersParameters{Prefix: testContainerPrefix})
if err != nil {
return err
}
if len(resp.Containers) == 0 {
break
}
for _, c := range resp.Containers {
err = cli.DeleteContainer(c.Name)
if err != nil {
return err
}
}
}
return nil
}
func (b BlobStorageClient) putSingleBlockBlob(container, name string, chunk []byte) error {
if len(chunk) > MaxBlobBlockSize {
return fmt.Errorf("storage: provided chunk (%d bytes) cannot fit into single-block blob (max %d bytes)", len(chunk), MaxBlobBlockSize)
}
uri := b.client.getEndpoint(blobServiceName, pathForBlob(container, name), url.Values{})
headers := b.client.getStandardHeaders()
headers["x-ms-blob-type"] = string(BlobTypeBlock)
headers["Content-Length"] = fmt.Sprintf("%v", len(chunk))
resp, err := b.client.exec("PUT", uri, headers, bytes.NewReader(chunk))
if err != nil {
return err
}
return checkRespCode(resp.statusCode, []int{http.StatusCreated})
}
func randContainer() string {
return testContainerPrefix + randString(32-len(testContainerPrefix))
}
func randString(n int) string {
if n <= 0 {
panic("negative number")
}
const alphanum = "0123456789abcdefghijklmnopqrstuvwxyz"
var bytes = make([]byte, n)
rand.Read(bytes)
for i, b := range bytes {
bytes[i] = alphanum[b%byte(len(alphanum))]
}
return string(bytes)
}

45
Godeps/_workspace/src/github.com/Azure/azure-sdk-for-go/storage/client.go generated vendored
View file

@ -5,6 +5,7 @@ import (
"bytes"
"encoding/base64"
"encoding/xml"
"errors"
"fmt"
"io"
"io/ioutil"
@ -12,6 +13,7 @@ import (
"net/url"
"regexp"
"sort"
"strconv"
"strings"
)
@ -22,13 +24,14 @@ const (
// DefaultAPIVersion is the Azure Storage API version string used when a
// basic client is created.
DefaultAPIVersion = "2014-02-14"
DefaultAPIVersion = "2015-02-21"
defaultUseHTTPS = true
blobServiceName = "blob"
tableServiceName = "table"
queueServiceName = "queue"
fileServiceName = "file"
)
// Client is the object that needs to be constructed to perform
@ -79,6 +82,11 @@ func (e UnexpectedStatusCodeError) Error() string {
return fmt.Sprintf("storage: status code from service response is %s; was expecting %s", got, strings.Join(expected, " or "))
}
// Got is the actual status code returned by Azure.
func (e UnexpectedStatusCodeError) Got() int {
return e.got
}
// NewBasicClient constructs a Client with given storage service name and
// key.
func NewBasicClient(accountName, accountKey string) (Client, error) {
@ -154,6 +162,12 @@ func (c Client) GetQueueService() QueueServiceClient {
return QueueServiceClient{c}
}
// GetFileService returns a FileServiceClient which can operate on the file
// service of the storage account.
func (c Client) GetFileService() FileServiceClient {
return FileServiceClient{c}
}
func (c Client) createAuthorizationHeader(canonicalizedString string) string {
signature := c.computeHmac256(canonicalizedString)
return fmt.Sprintf("%s %s:%s", "SharedKey", c.accountName, signature)
@ -252,18 +266,22 @@ func (c Client) buildCanonicalizedResource(uri string) (string, error) {
}
func (c Client) buildCanonicalizedString(verb string, headers map[string]string, canonicalizedResource string) string {
contentLength := headers["Content-Length"]
if contentLength == "0" {
contentLength = ""
}
canonicalizedString := fmt.Sprintf("%s\n%s\n%s\n%s\n%s\n%s\n%s\n%s\n%s\n%s\n%s\n%s\n%s\n%s",
verb,
headers["Content-Encoding"],
headers["Content-Language"],
headers["Content-Length"],
contentLength,
headers["Content-MD5"],
headers["Content-Type"],
headers["Date"],
headers["If-Modified-Singe"],
headers["If-Modified-Since"],
headers["If-Match"],
headers["If-None-Match"],
headers["If-Unmodified-Singe"],
headers["If-Unmodified-Since"],
headers["Range"],
c.buildCanonicalizedHeader(headers),
canonicalizedResource)
@ -283,6 +301,20 @@ func (c Client) exec(verb, url string, headers map[string]string, body io.Reader
}
req, err := http.NewRequest(verb, url, body)
if err != nil {
return nil, errors.New("azure/storage: error creating request: " + err.Error())
}
if clstr, ok := headers["Content-Length"]; ok {
// content length header is being signed, but completely ignored by golang.
// instead we have to use the ContentLength property on the request struct
// (see https://golang.org/src/net/http/request.go?s=18140:18370#L536 and
// https://golang.org/src/net/http/transfer.go?s=1739:2467#L49)
req.ContentLength, err = strconv.ParseInt(clstr, 10, 64)
if err != nil {
return nil, err
}
}
for k, v := range headers {
req.Header.Add(k, v)
}
@ -293,7 +325,7 @@ func (c Client) exec(verb, url string, headers map[string]string, body io.Reader
}
statusCode := resp.StatusCode
if statusCode >= 400 && statusCode <= 505 {
if statusCode >= 400 && statusCode <= 505 && statusCode != 404 {
var respBody []byte
respBody, err = readResponseBody(resp)
if err != nil {
@ -344,7 +376,8 @@ func serviceErrFromXML(body []byte, statusCode int, requestID string) (AzureStor
}
func (e AzureStorageServiceError) Error() string {
return fmt.Sprintf("storage: service returned error: StatusCode=%d, ErrorCode=%s, ErrorMessage=%s, RequestId=%s", e.StatusCode, e.Code, e.Message, e.RequestID)
return fmt.Sprintf("storage: service returned error: StatusCode=%d, ErrorCode=%s, ErrorMessage=%s, RequestId=%s, QueryParameterName=%s, QueryParameterValue=%s",
e.StatusCode, e.Code, e.Message, e.RequestID, e.QueryParameterName, e.QueryParameterValue)
}
// checkRespCode returns UnexpectedStatusError if the given response code is not

156
Godeps/_workspace/src/github.com/Azure/azure-sdk-for-go/storage/client_test.go generated vendored
View file

@ -1,156 +0,0 @@
package storage
import (
"encoding/base64"
"net/url"
"os"
"testing"
chk "gopkg.in/check.v1"
)
// Hook up gocheck to testing
func Test(t *testing.T) { chk.TestingT(t) }
type StorageClientSuite struct{}
var _ = chk.Suite(&StorageClientSuite{})
// getBasicClient returns a test client from storage credentials in the env
func getBasicClient(c *chk.C) Client {
name := os.Getenv("ACCOUNT_NAME")
if name == "" {
c.Fatal("ACCOUNT_NAME not set, need an empty storage account to test")
}
key := os.Getenv("ACCOUNT_KEY")
if key == "" {
c.Fatal("ACCOUNT_KEY not set")
}
cli, err := NewBasicClient(name, key)
c.Assert(err, chk.IsNil)
return cli
}
func (s *StorageClientSuite) TestGetBaseURL_Basic_Https(c *chk.C) {
cli, err := NewBasicClient("foo", "YmFy")
c.Assert(err, chk.IsNil)
c.Assert(cli.apiVersion, chk.Equals, DefaultAPIVersion)
c.Assert(err, chk.IsNil)
c.Assert(cli.getBaseURL("table"), chk.Equals, "https://foo.table.core.windows.net")
}
func (s *StorageClientSuite) TestGetBaseURL_Custom_NoHttps(c *chk.C) {
apiVersion := "2015-01-01" // a non existing one
cli, err := NewClient("foo", "YmFy", "core.chinacloudapi.cn", apiVersion, false)
c.Assert(err, chk.IsNil)
c.Assert(cli.apiVersion, chk.Equals, apiVersion)
c.Assert(cli.getBaseURL("table"), chk.Equals, "http://foo.table.core.chinacloudapi.cn")
}
func (s *StorageClientSuite) TestGetEndpoint_None(c *chk.C) {
cli, err := NewBasicClient("foo", "YmFy")
c.Assert(err, chk.IsNil)
output := cli.getEndpoint(blobServiceName, "", url.Values{})
c.Assert(output, chk.Equals, "https://foo.blob.core.windows.net/")
}
func (s *StorageClientSuite) TestGetEndpoint_PathOnly(c *chk.C) {
cli, err := NewBasicClient("foo", "YmFy")
c.Assert(err, chk.IsNil)
output := cli.getEndpoint(blobServiceName, "path", url.Values{})
c.Assert(output, chk.Equals, "https://foo.blob.core.windows.net/path")
}
func (s *StorageClientSuite) TestGetEndpoint_ParamsOnly(c *chk.C) {
cli, err := NewBasicClient("foo", "YmFy")
c.Assert(err, chk.IsNil)
params := url.Values{}
params.Set("a", "b")
params.Set("c", "d")
output := cli.getEndpoint(blobServiceName, "", params)
c.Assert(output, chk.Equals, "https://foo.blob.core.windows.net/?a=b&c=d")
}
func (s *StorageClientSuite) TestGetEndpoint_Mixed(c *chk.C) {
cli, err := NewBasicClient("foo", "YmFy")
c.Assert(err, chk.IsNil)
params := url.Values{}
params.Set("a", "b")
params.Set("c", "d")
output := cli.getEndpoint(blobServiceName, "path", params)
c.Assert(output, chk.Equals, "https://foo.blob.core.windows.net/path?a=b&c=d")
}
func (s *StorageClientSuite) Test_getStandardHeaders(c *chk.C) {
cli, err := NewBasicClient("foo", "YmFy")
c.Assert(err, chk.IsNil)
headers := cli.getStandardHeaders()
c.Assert(len(headers), chk.Equals, 2)
c.Assert(headers["x-ms-version"], chk.Equals, cli.apiVersion)
if _, ok := headers["x-ms-date"]; !ok {
c.Fatal("Missing date header")
}
}
func (s *StorageClientSuite) Test_buildCanonicalizedResource(c *chk.C) {
cli, err := NewBasicClient("foo", "YmFy")
c.Assert(err, chk.IsNil)
type test struct{ url, expected string }
tests := []test{
{"https://foo.blob.core.windows.net/path?a=b&c=d", "/foo/path\na:b\nc:d"},
{"https://foo.blob.core.windows.net/?comp=list", "/foo/\ncomp:list"},
{"https://foo.blob.core.windows.net/cnt/blob", "/foo/cnt/blob"},
}
for _, i := range tests {
out, err := cli.buildCanonicalizedResource(i.url)
c.Assert(err, chk.IsNil)
c.Assert(out, chk.Equals, i.expected)
}
}
func (s *StorageClientSuite) Test_buildCanonicalizedHeader(c *chk.C) {
cli, err := NewBasicClient("foo", "YmFy")
c.Assert(err, chk.IsNil)
type test struct {
headers map[string]string
expected string
}
tests := []test{
{map[string]string{}, ""},
{map[string]string{"x-ms-foo": "bar"}, "x-ms-foo:bar"},
{map[string]string{"foo:": "bar"}, ""},
{map[string]string{"foo:": "bar", "x-ms-foo": "bar"}, "x-ms-foo:bar"},
{map[string]string{
"x-ms-version": "9999-99-99",
"x-ms-blob-type": "BlockBlob"}, "x-ms-blob-type:BlockBlob\nx-ms-version:9999-99-99"}}
for _, i := range tests {
c.Assert(cli.buildCanonicalizedHeader(i.headers), chk.Equals, i.expected)
}
}
func (s *StorageClientSuite) TestReturnsStorageServiceError(c *chk.C) {
// attempt to delete a nonexisting container
_, err := getBlobClient(c).deleteContainer(randContainer())
c.Assert(err, chk.NotNil)
v, ok := err.(AzureStorageServiceError)
c.Check(ok, chk.Equals, true)
c.Assert(v.StatusCode, chk.Equals, 404)
c.Assert(v.Code, chk.Equals, "ContainerNotFound")
c.Assert(v.Code, chk.Not(chk.Equals), "")
}
func (s *StorageClientSuite) Test_createAuthorizationHeader(c *chk.C) {
key := base64.StdEncoding.EncodeToString([]byte("bar"))
cli, err := NewBasicClient("foo", key)
c.Assert(err, chk.IsNil)
canonicalizedString := `foobarzoo`
expected := `SharedKey foo:h5U0ATVX6SpbFX1H6GNuxIMeXXCILLoIvhflPtuQZ30=`
c.Assert(cli.createAuthorizationHeader(canonicalizedString), chk.Equals, expected)
}

91
Godeps/_workspace/src/github.com/Azure/azure-sdk-for-go/storage/file.go generated vendored Normal file
View file

@ -0,0 +1,91 @@
package storage
import (
"fmt"
"net/http"
"net/url"
)
// FileServiceClient contains operations for Microsoft Azure File Service.
type FileServiceClient struct {
client Client
}
// pathForFileShare returns the URL path segment for a File Share resource
func pathForFileShare(name string) string {
return fmt.Sprintf("/%s", name)
}
// CreateShare operation creates a new share under the specified account. If the
// share with the same name already exists, the operation fails.
//
// See https://msdn.microsoft.com/en-us/library/azure/dn167008.aspx
func (f FileServiceClient) CreateShare(name string) error {
resp, err := f.createShare(name)
if err != nil {
return err
}
defer resp.body.Close()
return checkRespCode(resp.statusCode, []int{http.StatusCreated})
}
// CreateShareIfNotExists creates a new share under the specified account if
// it does not exist. Returns true if container is newly created or false if
// container already exists.
//
// See https://msdn.microsoft.com/en-us/library/azure/dn167008.aspx
func (f FileServiceClient) CreateShareIfNotExists(name string) (bool, error) {
resp, err := f.createShare(name)
if resp != nil {
defer resp.body.Close()
if resp.statusCode == http.StatusCreated || resp.statusCode == http.StatusConflict {
return resp.statusCode == http.StatusCreated, nil
}
}
return false, err
}
// CreateShare creates a Azure File Share and returns its response
func (f FileServiceClient) createShare(name string) (*storageResponse, error) {
uri := f.client.getEndpoint(fileServiceName, pathForFileShare(name), url.Values{"restype": {"share"}})
headers := f.client.getStandardHeaders()
return f.client.exec("PUT", uri, headers, nil)
}
// DeleteShare operation marks the specified share for deletion. The share
// and any files contained within it are later deleted during garbage
// collection.
//
// See https://msdn.microsoft.com/en-us/library/azure/dn689090.aspx
func (f FileServiceClient) DeleteShare(name string) error {
resp, err := f.deleteShare(name)
if err != nil {
return err
}
defer resp.body.Close()
return checkRespCode(resp.statusCode, []int{http.StatusAccepted})
}
// DeleteShareIfExists operation marks the specified share for deletion if it
// exists. The share and any files contained within it are later deleted during
// garbage collection. Returns true if share existed and deleted with this call,
// false otherwise.
//
// See https://msdn.microsoft.com/en-us/library/azure/dn689090.aspx
func (f FileServiceClient) DeleteShareIfExists(name string) (bool, error) {
resp, err := f.deleteShare(name)
if resp != nil {
defer resp.body.Close()
if resp.statusCode == http.StatusAccepted || resp.statusCode == http.StatusNotFound {
return resp.statusCode == http.StatusAccepted, nil
}
}
return false, err
}
// deleteShare makes the call to Delete Share operation endpoint and returns
// the response
func (f FileServiceClient) deleteShare(name string) (*storageResponse, error) {
uri := f.client.getEndpoint(fileServiceName, pathForFileShare(name), url.Values{"restype": {"share"}})
return f.client.exec("DELETE", uri, f.client.getStandardHeaders(), nil)
}

78
Godeps/_workspace/src/github.com/Azure/azure-sdk-for-go/storage/queue.go generated vendored
View file

@ -6,6 +6,13 @@ import (
"net/http"
"net/url"
"strconv"
"strings"
)
const (
// casing is per Golang's http.Header canonicalizing the header names.
approximateMessagesCountHeader = "X-Ms-Approximate-Messages-Count"
userDefinedMetadataHeaderPrefix = "X-Ms-Meta-"
)
// QueueServiceClient contains operations for Microsoft Azure Queue Storage
@ -111,13 +118,82 @@ type PeekMessageResponse struct {
MessageText string `xml:"MessageText"`
}
// QueueMetadataResponse represents user defined metadata and queue
// properties on a specific queue.
//
// See https://msdn.microsoft.com/en-us/library/azure/dd179384.aspx
type QueueMetadataResponse struct {
ApproximateMessageCount int
UserDefinedMetadata map[string]string
}
// SetMetadata operation sets user-defined metadata on the specified queue.
// Metadata is associated with the queue as name-value pairs.
//
// See https://msdn.microsoft.com/en-us/library/azure/dd179348.aspx
func (c QueueServiceClient) SetMetadata(name string, metadata map[string]string) error {
uri := c.client.getEndpoint(queueServiceName, pathForQueue(name), url.Values{"comp": []string{"metadata"}})
headers := c.client.getStandardHeaders()
for k, v := range metadata {
headers[userDefinedMetadataHeaderPrefix+k] = v
}
resp, err := c.client.exec("PUT", uri, headers, nil)
if err != nil {
return err
}
defer resp.body.Close()
return checkRespCode(resp.statusCode, []int{http.StatusNoContent})
}
// GetMetadata operation retrieves user-defined metadata and queue
// properties on the specified queue. Metadata is associated with
// the queue as name-values pairs.
//
// See https://msdn.microsoft.com/en-us/library/azure/dd179384.aspx
//
// Because the way Golang's http client (and http.Header in particular)
// canonicalize header names, the returned metadata names would always
// be all lower case.
func (c QueueServiceClient) GetMetadata(name string) (QueueMetadataResponse, error) {
qm := QueueMetadataResponse{}
qm.UserDefinedMetadata = make(map[string]string)
uri := c.client.getEndpoint(queueServiceName, pathForQueue(name), url.Values{"comp": []string{"metadata"}})
headers := c.client.getStandardHeaders()
resp, err := c.client.exec("GET", uri, headers, nil)
if err != nil {
return qm, err
}
defer resp.body.Close()
for k, v := range resp.headers {
if len(v) != 1 {
return qm, fmt.Errorf("Unexpected number of values (%d) in response header '%s'", len(v), k)
}
value := v[0]
if k == approximateMessagesCountHeader {
qm.ApproximateMessageCount, err = strconv.Atoi(value)
if err != nil {
return qm, fmt.Errorf("Unexpected value in response header '%s': '%s' ", k, value)
}
} else if strings.HasPrefix(k, userDefinedMetadataHeaderPrefix) {
name := strings.TrimPrefix(k, userDefinedMetadataHeaderPrefix)
qm.UserDefinedMetadata[strings.ToLower(name)] = value
}
}
return qm, checkRespCode(resp.statusCode, []int{http.StatusOK})
}
// CreateQueue operation creates a queue under the given account.
//
// See https://msdn.microsoft.com/en-us/library/azure/dd179342.aspx
func (c QueueServiceClient) CreateQueue(name string) error {
uri := c.client.getEndpoint(queueServiceName, pathForQueue(name), url.Values{})
headers := c.client.getStandardHeaders()
headers["Content-Length"] = "0"
resp, err := c.client.exec("PUT", uri, headers, nil)
if err != nil {
return err

91
Godeps/_workspace/src/github.com/Azure/azure-sdk-for-go/storage/queue_test.go generated vendored
View file

@ -1,91 +0,0 @@
package storage
import (
chk "gopkg.in/check.v1"
)
type StorageQueueSuite struct{}
var _ = chk.Suite(&StorageQueueSuite{})
func getQueueClient(c *chk.C) QueueServiceClient {
return getBasicClient(c).GetQueueService()
}
func (s *StorageQueueSuite) Test_pathForQueue(c *chk.C) {
c.Assert(pathForQueue("q"), chk.Equals, "/q")
}
func (s *StorageQueueSuite) Test_pathForQueueMessages(c *chk.C) {
c.Assert(pathForQueueMessages("q"), chk.Equals, "/q/messages")
}
func (s *StorageQueueSuite) Test_pathForMessage(c *chk.C) {
c.Assert(pathForMessage("q", "m"), chk.Equals, "/q/messages/m")
}
func (s *StorageQueueSuite) TestCreateQueue_DeleteQueue(c *chk.C) {
cli := getQueueClient(c)
name := randString(20)
c.Assert(cli.CreateQueue(name), chk.IsNil)
c.Assert(cli.DeleteQueue(name), chk.IsNil)
}
func (s *StorageQueueSuite) TestQueueExists(c *chk.C) {
cli := getQueueClient(c)
ok, err := cli.QueueExists("nonexistent-queue")
c.Assert(err, chk.IsNil)
c.Assert(ok, chk.Equals, false)
name := randString(20)
c.Assert(cli.CreateQueue(name), chk.IsNil)
defer cli.DeleteQueue(name)
ok, err = cli.QueueExists(name)
c.Assert(err, chk.IsNil)
c.Assert(ok, chk.Equals, true)
}
func (s *StorageQueueSuite) TestPostMessage_PeekMessage_DeleteMessage(c *chk.C) {
q := randString(20)
cli := getQueueClient(c)
c.Assert(cli.CreateQueue(q), chk.IsNil)
defer cli.DeleteQueue(q)
msg := randString(64 * 1024) // exercise max length
c.Assert(cli.PutMessage(q, msg, PutMessageParameters{}), chk.IsNil)
r, err := cli.PeekMessages(q, PeekMessagesParameters{})
c.Assert(err, chk.IsNil)
c.Assert(len(r.QueueMessagesList), chk.Equals, 1)
c.Assert(r.QueueMessagesList[0].MessageText, chk.Equals, msg)
}
func (s *StorageQueueSuite) TestGetMessages(c *chk.C) {
q := randString(20)
cli := getQueueClient(c)
c.Assert(cli.CreateQueue(q), chk.IsNil)
defer cli.DeleteQueue(q)
n := 4
for i := 0; i < n; i++ {
c.Assert(cli.PutMessage(q, randString(10), PutMessageParameters{}), chk.IsNil)
}
r, err := cli.GetMessages(q, GetMessagesParameters{NumOfMessages: n})
c.Assert(err, chk.IsNil)
c.Assert(len(r.QueueMessagesList), chk.Equals, n)
}
func (s *StorageQueueSuite) TestDeleteMessages(c *chk.C) {
q := randString(20)
cli := getQueueClient(c)
c.Assert(cli.CreateQueue(q), chk.IsNil)
defer cli.DeleteQueue(q)
c.Assert(cli.PutMessage(q, "message", PutMessageParameters{}), chk.IsNil)
r, err := cli.GetMessages(q, GetMessagesParameters{VisibilityTimeout: 1})
c.Assert(err, chk.IsNil)
c.Assert(len(r.QueueMessagesList), chk.Equals, 1)
m := r.QueueMessagesList[0]
c.Assert(cli.DeleteMessage(q, m.MessageID, m.PopReceipt), chk.IsNil)
}

69
Godeps/_workspace/src/github.com/Azure/azure-sdk-for-go/storage/util_test.go generated vendored
View file

@ -1,69 +0,0 @@
package storage
import (
"encoding/xml"
"io/ioutil"
"net/url"
"strings"
"time"
chk "gopkg.in/check.v1"
)
func (s *StorageClientSuite) Test_timeRfc1123Formatted(c *chk.C) {
now := time.Now().UTC()
expectedLayout := "Mon, 02 Jan 2006 15:04:05 GMT"
c.Assert(timeRfc1123Formatted(now), chk.Equals, now.Format(expectedLayout))
}
func (s *StorageClientSuite) Test_mergeParams(c *chk.C) {
v1 := url.Values{
"k1": {"v1"},
"k2": {"v2"}}
v2 := url.Values{
"k1": {"v11"},
"k3": {"v3"}}
out := mergeParams(v1, v2)
c.Assert(out.Get("k1"), chk.Equals, "v1")
c.Assert(out.Get("k2"), chk.Equals, "v2")
c.Assert(out.Get("k3"), chk.Equals, "v3")
c.Assert(out["k1"], chk.DeepEquals, []string{"v1", "v11"})
}
func (s *StorageClientSuite) Test_prepareBlockListRequest(c *chk.C) {
empty := []Block{}
expected := `<?xml version="1.0" encoding="utf-8"?><BlockList></BlockList>`
c.Assert(prepareBlockListRequest(empty), chk.DeepEquals, expected)
blocks := []Block{{"foo", BlockStatusLatest}, {"bar", BlockStatusUncommitted}}
expected = `<?xml version="1.0" encoding="utf-8"?><BlockList><Latest>foo</Latest><Uncommitted>bar</Uncommitted></BlockList>`
c.Assert(prepareBlockListRequest(blocks), chk.DeepEquals, expected)
}
func (s *StorageClientSuite) Test_xmlUnmarshal(c *chk.C) {
xml := `<?xml version="1.0" encoding="utf-8"?>
<Blob>
<Name>myblob</Name>
</Blob>`
var blob Blob
body := ioutil.NopCloser(strings.NewReader(xml))
c.Assert(xmlUnmarshal(body, &blob), chk.IsNil)
c.Assert(blob.Name, chk.Equals, "myblob")
}
func (s *StorageClientSuite) Test_xmlMarshal(c *chk.C) {
type t struct {
XMLName xml.Name `xml:"S"`
Name string `xml:"Name"`
}
b := t{Name: "myblob"}
expected := `<S><Name>myblob</Name></S>`
r, i, err := xmlMarshal(b)
c.Assert(err, chk.IsNil)
o, err := ioutil.ReadAll(r)
c.Assert(err, chk.IsNil)
out := string(o)
c.Assert(out, chk.Equals, expected)
c.Assert(i, chk.Equals, len(expected))
}