Update vendored azure-sdk-for-go

Updating to a recent version of Azure Storage SDK to be able to patch some memory leaks through configurable HTTP client changes which were made possible by recent patches to it. Signed-off-by: Ahmet Alp Balkan <ahmetalpbalkan@gmail.com>
2016-10-28 15:46:05 -07:00 · 2016-10-28 15:46:05 -07:00 · 2ab25288a2
commit 2ab25288a2
parent 62e88f0fe7
10 changed files with 1241 additions and 40 deletions
--- a/vendor/github.com/Azure/azure-sdk-for-go/storage/blob.go
+++ b/vendor/github.com/Azure/azure-sdk-for-go/storage/blob.go
@ -55,7 +55,51 @@ type ContainerListResponse struct {
 type Blob struct {
 	Name       string         `xml:"Name"`
 	Properties BlobProperties `xml:"Properties"`
-	// TODO (ahmetalpbalkan) Metadata
+	Metadata   BlobMetadata   `xml:"Metadata"`
+}
+
+// BlobMetadata is a set of custom name/value pairs.
+//
+// See https://msdn.microsoft.com/en-us/library/azure/dd179404.aspx
+type BlobMetadata map[string]string
+
+type blobMetadataEntries struct {
+	Entries []blobMetadataEntry `xml:",any"`
+}
+type blobMetadataEntry struct {
+	XMLName xml.Name
+	Value   string `xml:",chardata"`
+}
+
+// UnmarshalXML converts the xml:Metadata into Metadata map
+func (bm *BlobMetadata) UnmarshalXML(d *xml.Decoder, start xml.StartElement) error {
+	var entries blobMetadataEntries
+	if err := d.DecodeElement(&entries, &start); err != nil {
+		return err
+	}
+	for _, entry := range entries.Entries {
+		if *bm == nil {
+			*bm = make(BlobMetadata)
+		}
+		(*bm)[strings.ToLower(entry.XMLName.Local)] = entry.Value
+	}
+	return nil
+}
+
+// MarshalXML implements the xml.Marshaler interface. It encodes
+// metadata name/value pairs as they would appear in an Azure
+// ListBlobs response.
+func (bm BlobMetadata) MarshalXML(enc *xml.Encoder, start xml.StartElement) error {
+	entries := make([]blobMetadataEntry, 0, len(bm))
+	for k, v := range bm {
+		entries = append(entries, blobMetadataEntry{
+			XMLName: xml.Name{Local: http.CanonicalHeaderKey(k)},
+			Value:   v,
+		})
+	}
+	return enc.EncodeElement(blobMetadataEntries{
+		Entries: entries,
+	}, start)
 }

 // BlobProperties contains various properties of a blob
@ -67,6 +111,8 @@ type BlobProperties struct {
 	ContentLength         int64    `xml:"Content-Length"`
 	ContentType           string   `xml:"Content-Type"`
 	ContentEncoding       string   `xml:"Content-Encoding"`
+	CacheControl          string   `xml:"Cache-Control"`
+	ContentLanguage       string   `xml:"Cache-Language"`
 	BlobType              BlobType `xml:"x-ms-blob-blob-type"`
 	SequenceNumber        int64    `xml:"x-ms-blob-sequence-number"`
 	CopyID                string   `xml:"CopyId"`
@ -75,6 +121,17 @@ type BlobProperties struct {
 	CopyProgress          string   `xml:"CopyProgress"`
 	CopyCompletionTime    string   `xml:"CopyCompletionTime"`
 	CopyStatusDescription string   `xml:"CopyStatusDescription"`
+	LeaseStatus           string   `xml:"LeaseStatus"`
+}
+
+// BlobHeaders contains various properties of a blob and is an entry
+// in SetBlobProperties
+type BlobHeaders struct {
+	ContentMD5      string `header:"x-ms-blob-content-md5"`
+	ContentLanguage string `header:"x-ms-blob-content-language"`
+	ContentEncoding string `header:"x-ms-blob-content-encoding"`
+	ContentType     string `header:"x-ms-blob-content-type"`
+	CacheControl    string `header:"x-ms-blob-cache-control"`
 }

 // BlobListResponse contains the response fields from ListBlobs call.
@ -88,6 +145,16 @@ type BlobListResponse struct {
 	NextMarker string   `xml:"NextMarker"`
 	MaxResults int64    `xml:"MaxResults"`
 	Blobs      []Blob   `xml:"Blobs>Blob"`
+
+	// BlobPrefix is used to traverse blobs as if it were a file system.
+	// It is returned if ListBlobsParameters.Delimiter is specified.
+	// The list here can be thought of as "folders" that may contain
+	// other folders or blobs.
+	BlobPrefixes []string `xml:"Blobs>BlobPrefix>Name"`
+
+	// Delimiter is used to traverse blobs as if it were a file system.
+	// It is returned if ListBlobsParameters.Delimiter is specified.
+	Delimiter string `xml:"Delimiter"`
 }

 // ListContainersParameters defines the set of customizable parameters to make a
@ -189,6 +256,23 @@ const (
 	blobCopyStatusFailed  = "failed"
 )

+// lease constants.
+const (
+	leaseHeaderPrefix = "x-ms-lease-"
+	leaseID           = "x-ms-lease-id"
+	leaseAction       = "x-ms-lease-action"
+	leaseBreakPeriod  = "x-ms-lease-break-period"
+	leaseDuration     = "x-ms-lease-duration"
+	leaseProposedID   = "x-ms-proposed-lease-id"
+	leaseTime         = "x-ms-lease-time"
+
+	acquireLease = "acquire"
+	renewLease   = "renew"
+	changeLease  = "change"
+	releaseLease = "release"
+	breakLease   = "break"
+)
+
 // BlockListType is used to filter out types of blocks in a Get Blocks List call
 // for a block blob.
 //
@ -419,7 +503,6 @@ func (b BlobStorageClient) ListBlobs(container string, params ListBlobsParameter
 func (b BlobStorageClient) BlobExists(container, name string) (bool, error) {
 	verb := "HEAD"
 	uri := b.client.getEndpoint(blobServiceName, pathForBlob(container, name), url.Values{})
-
 	headers := b.client.getStandardHeaders()
 	resp, err := b.client.exec(verb, uri, headers, nil)
 	if resp != nil {
@ -447,7 +530,7 @@ func (b BlobStorageClient) GetBlobURL(container, name string) string {
 //
 // See https://msdn.microsoft.com/en-us/library/azure/dd179440.aspx
 func (b BlobStorageClient) GetBlob(container, name string) (io.ReadCloser, error) {
-	resp, err := b.getBlobRange(container, name, "")
+	resp, err := b.getBlobRange(container, name, "", nil)
 	if err != nil {
 		return nil, err
 	}
@ -462,8 +545,8 @@ func (b BlobStorageClient) GetBlob(container, name string) (io.ReadCloser, error
 // string must be in a format like "0-", "10-100" as defined in HTTP 1.1 spec.
 //
 // See https://msdn.microsoft.com/en-us/library/azure/dd179440.aspx
-func (b BlobStorageClient) GetBlobRange(container, name, bytesRange string) (io.ReadCloser, error) {
-	resp, err := b.getBlobRange(container, name, bytesRange)
+func (b BlobStorageClient) GetBlobRange(container, name, bytesRange string, extraHeaders map[string]string) (io.ReadCloser, error) {
+	resp, err := b.getBlobRange(container, name, bytesRange, extraHeaders)
 	if err != nil {
 		return nil, err
 	}
@ -474,7 +557,7 @@ func (b BlobStorageClient) GetBlobRange(container, name, bytesRange string) (io.
 	return resp.body, nil
 }

-func (b BlobStorageClient) getBlobRange(container, name, bytesRange string) (*storageResponse, error) {
+func (b BlobStorageClient) getBlobRange(container, name, bytesRange string, extraHeaders map[string]string) (*storageResponse, error) {
 	verb := "GET"
 	uri := b.client.getEndpoint(blobServiceName, pathForBlob(container, name), url.Values{})

@ -482,6 +565,11 @@ func (b BlobStorageClient) getBlobRange(container, name, bytesRange string) (*st
 	if bytesRange != "" {
 		headers["Range"] = fmt.Sprintf("bytes=%s", bytesRange)
 	}
+
+	for k, v := range extraHeaders {
+		headers[k] = v
+	}
+
 	resp, err := b.client.exec(verb, uri, headers, nil)
 	if err != nil {
 		return nil, err
@ -489,6 +577,134 @@ func (b BlobStorageClient) getBlobRange(container, name, bytesRange string) (*st
 	return resp, err
 }

+// leasePut is common PUT code for the various aquire/release/break etc functions.
+func (b BlobStorageClient) leaseCommonPut(container string, name string, headers map[string]string, expectedStatus int) (http.Header, error) {
+	params := url.Values{"comp": {"lease"}}
+	uri := b.client.getEndpoint(blobServiceName, pathForBlob(container, name), params)
+
+	resp, err := b.client.exec("PUT", uri, headers, nil)
+	if err != nil {
+		return nil, err
+	}
+	defer resp.body.Close()
+
+	if err := checkRespCode(resp.statusCode, []int{expectedStatus}); err != nil {
+		return nil, err
+	}
+
+	return resp.headers, nil
+}
+
+// AcquireLease creates a lease for a blob as per https://msdn.microsoft.com/en-us/library/azure/ee691972.aspx
+// returns leaseID acquired
+func (b BlobStorageClient) AcquireLease(container string, name string, leaseTimeInSeconds int, proposedLeaseID string) (returnedLeaseID string, err error) {
+	headers := b.client.getStandardHeaders()
+	headers[leaseAction] = acquireLease
+	headers[leaseProposedID] = proposedLeaseID
+	headers[leaseDuration] = strconv.Itoa(leaseTimeInSeconds)
+
+	respHeaders, err := b.leaseCommonPut(container, name, headers, http.StatusCreated)
+	if err != nil {
+		return "", err
+	}
+
+	returnedLeaseID = respHeaders.Get(http.CanonicalHeaderKey(leaseID))
+
+	if returnedLeaseID != "" {
+		return returnedLeaseID, nil
+	}
+
+	// what should we return in case of HTTP 201 but no lease ID?
+	// or it just cant happen? (brave words)
+	return "", errors.New("LeaseID not returned")
+}
+
+// BreakLease breaks the lease for a blob as per https://msdn.microsoft.com/en-us/library/azure/ee691972.aspx
+// Returns the timeout remaining in the lease in seconds
+func (b BlobStorageClient) BreakLease(container string, name string) (breakTimeout int, err error) {
+	headers := b.client.getStandardHeaders()
+	headers[leaseAction] = breakLease
+	return b.breakLeaseCommon(container, name, headers)
+}
+
+// BreakLeaseWithBreakPeriod breaks the lease for a blob as per https://msdn.microsoft.com/en-us/library/azure/ee691972.aspx
+// breakPeriodInSeconds is used to determine how long until new lease can be created.
+// Returns the timeout remaining in the lease in seconds
+func (b BlobStorageClient) BreakLeaseWithBreakPeriod(container string, name string, breakPeriodInSeconds int) (breakTimeout int, err error) {
+	headers := b.client.getStandardHeaders()
+	headers[leaseAction] = breakLease
+	headers[leaseBreakPeriod] = strconv.Itoa(breakPeriodInSeconds)
+	return b.breakLeaseCommon(container, name, headers)
+}
+
+// breakLeaseCommon is common code for both version of BreakLease (with and without break period)
+func (b BlobStorageClient) breakLeaseCommon(container string, name string, headers map[string]string) (breakTimeout int, err error) {
+
+	respHeaders, err := b.leaseCommonPut(container, name, headers, http.StatusAccepted)
+	if err != nil {
+		return 0, err
+	}
+
+	breakTimeoutStr := respHeaders.Get(http.CanonicalHeaderKey(leaseTime))
+	if breakTimeoutStr != "" {
+		breakTimeout, err = strconv.Atoi(breakTimeoutStr)
+		if err != nil {
+			return 0, err
+		}
+	}
+
+	return breakTimeout, nil
+}
+
+// ChangeLease changes a lease ID for a blob as per https://msdn.microsoft.com/en-us/library/azure/ee691972.aspx
+// Returns the new LeaseID acquired
+func (b BlobStorageClient) ChangeLease(container string, name string, currentLeaseID string, proposedLeaseID string) (newLeaseID string, err error) {
+	headers := b.client.getStandardHeaders()
+	headers[leaseAction] = changeLease
+	headers[leaseID] = currentLeaseID
+	headers[leaseProposedID] = proposedLeaseID
+
+	respHeaders, err := b.leaseCommonPut(container, name, headers, http.StatusOK)
+	if err != nil {
+		return "", err
+	}
+
+	newLeaseID = respHeaders.Get(http.CanonicalHeaderKey(leaseID))
+	if newLeaseID != "" {
+		return newLeaseID, nil
+	}
+
+	return "", errors.New("LeaseID not returned")
+}
+
+// ReleaseLease releases the lease for a blob as per https://msdn.microsoft.com/en-us/library/azure/ee691972.aspx
+func (b BlobStorageClient) ReleaseLease(container string, name string, currentLeaseID string) error {
+	headers := b.client.getStandardHeaders()
+	headers[leaseAction] = releaseLease
+	headers[leaseID] = currentLeaseID
+
+	_, err := b.leaseCommonPut(container, name, headers, http.StatusOK)
+	if err != nil {
+		return err
+	}
+
+	return nil
+}
+
+// RenewLease renews the lease for a blob as per https://msdn.microsoft.com/en-us/library/azure/ee691972.aspx
+func (b BlobStorageClient) RenewLease(container string, name string, currentLeaseID string) error {
+	headers := b.client.getStandardHeaders()
+	headers[leaseAction] = renewLease
+	headers[leaseID] = currentLeaseID
+
+	_, err := b.leaseCommonPut(container, name, headers, http.StatusOK)
+	if err != nil {
+		return err
+	}
+
+	return nil
+}
+
 // GetBlobProperties provides various information about the specified
 // blob. See https://msdn.microsoft.com/en-us/library/azure/dd179394.aspx
 func (b BlobStorageClient) GetBlobProperties(container, name string) (*BlobProperties, error) {
@ -530,6 +746,9 @@ func (b BlobStorageClient) GetBlobProperties(container, name string) (*BlobPrope
 		ContentMD5:            resp.headers.Get("Content-MD5"),
 		ContentLength:         contentLength,
 		ContentEncoding:       resp.headers.Get("Content-Encoding"),
+		ContentType:           resp.headers.Get("Content-Type"),
+		CacheControl:          resp.headers.Get("Cache-Control"),
+		ContentLanguage:       resp.headers.Get("Content-Language"),
 		SequenceNumber:        sequenceNum,
 		CopyCompletionTime:    resp.headers.Get("x-ms-copy-completion-time"),
 		CopyStatusDescription: resp.headers.Get("x-ms-copy-status-description"),
@ -538,9 +757,38 @@ func (b BlobStorageClient) GetBlobProperties(container, name string) (*BlobPrope
 		CopySource:            resp.headers.Get("x-ms-copy-source"),
 		CopyStatus:            resp.headers.Get("x-ms-copy-status"),
 		BlobType:              BlobType(resp.headers.Get("x-ms-blob-type")),
+		LeaseStatus:           resp.headers.Get("x-ms-lease-status"),
 	}, nil
 }

+// SetBlobProperties replaces the BlobHeaders for the specified blob.
+//
+// Some keys may be converted to Camel-Case before sending. All keys
+// are returned in lower case by GetBlobProperties. HTTP header names
+// are case-insensitive so case munging should not matter to other
+// applications either.
+//
+// See https://msdn.microsoft.com/en-us/library/azure/ee691966.aspx
+func (b BlobStorageClient) SetBlobProperties(container, name string, blobHeaders BlobHeaders) error {
+	params := url.Values{"comp": {"properties"}}
+	uri := b.client.getEndpoint(blobServiceName, pathForBlob(container, name), params)
+	headers := b.client.getStandardHeaders()
+
+	extraHeaders := headersFromStruct(blobHeaders)
+
+	for k, v := range extraHeaders {
+		headers[k] = v
+	}
+
+	resp, err := b.client.exec("PUT", uri, headers, nil)
+	if err != nil {
+		return err
+	}
+	defer resp.body.Close()
+
+	return checkRespCode(resp.statusCode, []int{http.StatusOK})
+}
+
 // SetBlobMetadata replaces the metadata for the specified blob.
 //
 // Some keys may be converted to Camel-Case before sending. All keys
@ -549,7 +797,7 @@ func (b BlobStorageClient) GetBlobProperties(container, name string) (*BlobPrope
 // applications either.
 //
 // See https://msdn.microsoft.com/en-us/library/azure/dd179414.aspx
-func (b BlobStorageClient) SetBlobMetadata(container, name string, metadata map[string]string) error {
+func (b BlobStorageClient) SetBlobMetadata(container, name string, metadata map[string]string, extraHeaders map[string]string) error {
 	params := url.Values{"comp": {"metadata"}}
 	uri := b.client.getEndpoint(blobServiceName, pathForBlob(container, name), params)
 	headers := b.client.getStandardHeaders()
@ -557,6 +805,10 @@ func (b BlobStorageClient) SetBlobMetadata(container, name string, metadata map[
 		headers[userDefinedMetadataHeaderPrefix+k] = v
 	}

+	for k, v := range extraHeaders {
+		headers[k] = v
+	}
+
 	resp, err := b.client.exec("PUT", uri, headers, nil)
 	if err != nil {
 		return err
@ -749,14 +1001,16 @@ func (b BlobStorageClient) PutPageBlob(container, name string, size int64, extra
 // with 512-byte boundaries and chunk must be of size multiplies by 512.
 //
 // See https://msdn.microsoft.com/en-us/library/ee691975.aspx
-func (b BlobStorageClient) PutPage(container, name string, startByte, endByte int64, writeType PageWriteType, chunk []byte) error {
+func (b BlobStorageClient) PutPage(container, name string, startByte, endByte int64, writeType PageWriteType, chunk []byte, extraHeaders map[string]string) error {
 	path := fmt.Sprintf("%s/%s", container, name)
 	uri := b.client.getEndpoint(blobServiceName, path, url.Values{"comp": {"page"}})
 	headers := b.client.getStandardHeaders()
 	headers["x-ms-blob-type"] = string(BlobTypePage)
 	headers["x-ms-page-write"] = string(writeType)
 	headers["x-ms-range"] = fmt.Sprintf("bytes=%v-%v", startByte, endByte)
-
+	for k, v := range extraHeaders {
+		headers[k] = v
+	}
 	var contentLength int64
 	var data io.Reader
 	if writeType == PageWriteTypeClear {
@ -825,13 +1079,17 @@ func (b BlobStorageClient) PutAppendBlob(container, name string, extraHeaders ma
 // AppendBlock appends a block to an append blob.
 //
 // See https://msdn.microsoft.com/en-us/library/azure/mt427365.aspx
-func (b BlobStorageClient) AppendBlock(container, name string, chunk []byte) error {
+func (b BlobStorageClient) AppendBlock(container, name string, chunk []byte, extraHeaders map[string]string) error {
 	path := fmt.Sprintf("%s/%s", container, name)
 	uri := b.client.getEndpoint(blobServiceName, path, url.Values{"comp": {"appendblock"}})
 	headers := b.client.getStandardHeaders()
 	headers["x-ms-blob-type"] = string(BlobTypeAppend)
 	headers["Content-Length"] = fmt.Sprintf("%v", len(chunk))

+	for k, v := range extraHeaders {
+		headers[k] = v
+	}
+
 	resp, err := b.client.exec("PUT", uri, headers, bytes.NewReader(chunk))
 	if err != nil {
 		return err
@ -908,8 +1166,8 @@ func (b BlobStorageClient) waitForBlobCopy(container, name, copyID string) error
 // DeleteBlob deletes the given blob from the specified container.
 // If the blob does not exists at the time of the Delete Blob operation, it
 // returns error. See https://msdn.microsoft.com/en-us/library/azure/dd179413.aspx
-func (b BlobStorageClient) DeleteBlob(container, name string) error {
-	resp, err := b.deleteBlob(container, name)
+func (b BlobStorageClient) DeleteBlob(container, name string, extraHeaders map[string]string) error {
+	resp, err := b.deleteBlob(container, name, extraHeaders)
 	if err != nil {
 		return err
 	}
@ -921,19 +1179,24 @@ func (b BlobStorageClient) DeleteBlob(container, name string) error {
 // blob is deleted with this call, returns true. Otherwise returns false.
 //
 // See https://msdn.microsoft.com/en-us/library/azure/dd179413.aspx
-func (b BlobStorageClient) DeleteBlobIfExists(container, name string) (bool, error) {
-	resp, err := b.deleteBlob(container, name)
-	if resp != nil && (resp.statusCode == http.StatusAccepted || resp.statusCode == http.StatusNotFound) {
-		return resp.statusCode == http.StatusAccepted, nil
+func (b BlobStorageClient) DeleteBlobIfExists(container, name string, extraHeaders map[string]string) (bool, error) {
+	resp, err := b.deleteBlob(container, name, extraHeaders)
+	if resp != nil {
+		defer resp.body.Close()
+		if resp.statusCode == http.StatusAccepted || resp.statusCode == http.StatusNotFound {
+			return resp.statusCode == http.StatusAccepted, nil
+		}
 	}
-	defer resp.body.Close()
 	return false, err
 }

-func (b BlobStorageClient) deleteBlob(container, name string) (*storageResponse, error) {
+func (b BlobStorageClient) deleteBlob(container, name string, extraHeaders map[string]string) (*storageResponse, error) {
 	verb := "DELETE"
 	uri := b.client.getEndpoint(blobServiceName, pathForBlob(container, name), url.Values{})
 	headers := b.client.getStandardHeaders()
+	for k, v := range extraHeaders {
+		headers[k] = v
+	}

 	return b.client.exec(verb, uri, headers, nil)
 }
@ -959,10 +1222,25 @@ func (b BlobStorageClient) GetBlobSASURI(container, name string, expiry time.Tim
 		blobURL           = b.GetBlobURL(container, name)
 	)
 	canonicalizedResource, err := b.client.buildCanonicalizedResource(blobURL)
+
 	if err != nil {
 		return "", err
 	}
-	signedExpiry := expiry.Format(time.RFC3339)
+
+	// "The canonicalizedresouce portion of the string is a canonical path to the signed resource.
+	// It must include the service name (blob, table, queue or file) for version 2015-02-21 or
+	// later, the storage account name, and the resource name, and must be URL-decoded.
+	// -- https://msdn.microsoft.com/en-us/library/azure/dn140255.aspx
+
+	// We need to replace + with %2b first to avoid being treated as a space (which is correct for query strings, but not the path component).
+	canonicalizedResource = strings.Replace(canonicalizedResource, "+", "%2b", -1)
+
+	canonicalizedResource, err = url.QueryUnescape(canonicalizedResource)
+	if err != nil {
+		return "", err
+	}
+
+	signedExpiry := expiry.UTC().Format(time.RFC3339)
 	signedResource := "b"

 	stringToSign, err := blobSASStringToSign(b.client.apiVersion, canonicalizedResource, signedExpiry, signedPermissions)