From daa22cacba823c8ad35d522252d8d1783823049f Mon Sep 17 00:00:00 2001 From: Ahmet Alp Balkan Date: Thu, 11 Jun 2015 15:30:18 -0700 Subject: [PATCH] storage/driver/azure: Update vendored Azure SDK This change refreshes the updated version of Azure SDK for Go that has the latest changes. I manually vendored the new SDK (github.com/Azure/azure-sdk-for-go) and I removed `management/` `core/` packages manually simply because they're not used here and they have a fork of `net/http` and `crypto/tls` for a particular reason. It was introducing a 44k SLOC change otherwise... This also undoes the `include_azure` flag (actually Steven removed the driver from imports but forgot to add the build flag apparently, so the flag wasn't really including azure. :smile: ). This also must be obsolete now. Fixes #620, #175. Signed-off-by: Ahmet Alp Balkan --- Dockerfile | 2 +- Godeps/Godeps.json | 5 +- .../Azure/azure-sdk-for-go/.gitignore | 29 + .../Azure/azure-sdk-for-go/.travis.yml | 19 + .../github.com/Azure/azure-sdk-for-go/LICENSE | 202 +++ .../Azure/azure-sdk-for-go/README.md | 88 ++ .../azure-sdk-for-go/storage/blob.go | 406 +++--- .../azure-sdk-for-go/storage/blob_test.go | 625 +++++++++ .../azure-sdk-for-go/storage/client.go | 136 +- .../azure-sdk-for-go/storage/client_test.go | 156 +++ .../Azure/azure-sdk-for-go/storage/queue.go | 230 ++++ .../azure-sdk-for-go/storage/queue_test.go | 91 ++ .../azure-sdk-for-go/storage/util.go | 16 +- .../azure-sdk-for-go/storage/util_test.go | 69 + .../azure-sdk-for-go/storage/blob_test.go | 1123 ----------------- .../azure-sdk-for-go/storage/client_test.go | 203 --- .../azure-sdk-for-go/storage/util_test.go | 80 -- cmd/registry/main.go | 1 + docs/building.md | 5 - docs/storage-drivers/azure.md | 6 - registry/storage/driver/azure/azure.go | 23 +- registry/storage/driver/azure/blockblob.go | 2 +- .../storage/driver/azure/blockblob_test.go | 8 +- registry/storage/driver/azure/blockid.go | 2 +- registry/storage/driver/azure/blockid_test.go | 2 +- registry/storage/driver/azure/randomwriter.go | 14 +- .../storage/driver/azure/randomwriter_test.go | 2 +- 27 files changed, 1835 insertions(+), 1710 deletions(-) create mode 100644 Godeps/_workspace/src/github.com/Azure/azure-sdk-for-go/.gitignore create mode 100644 Godeps/_workspace/src/github.com/Azure/azure-sdk-for-go/.travis.yml create mode 100644 Godeps/_workspace/src/github.com/Azure/azure-sdk-for-go/LICENSE create mode 100644 Godeps/_workspace/src/github.com/Azure/azure-sdk-for-go/README.md rename Godeps/_workspace/src/github.com/{MSOpenTech => Azure}/azure-sdk-for-go/storage/blob.go (69%) create mode 100644 Godeps/_workspace/src/github.com/Azure/azure-sdk-for-go/storage/blob_test.go rename Godeps/_workspace/src/github.com/{MSOpenTech => Azure}/azure-sdk-for-go/storage/client.go (59%) create mode 100644 Godeps/_workspace/src/github.com/Azure/azure-sdk-for-go/storage/client_test.go create mode 100644 Godeps/_workspace/src/github.com/Azure/azure-sdk-for-go/storage/queue.go create mode 100644 Godeps/_workspace/src/github.com/Azure/azure-sdk-for-go/storage/queue_test.go rename Godeps/_workspace/src/github.com/{MSOpenTech => Azure}/azure-sdk-for-go/storage/util.go (73%) create mode 100644 Godeps/_workspace/src/github.com/Azure/azure-sdk-for-go/storage/util_test.go delete mode 100644 Godeps/_workspace/src/github.com/MSOpenTech/azure-sdk-for-go/storage/blob_test.go delete mode 100644 Godeps/_workspace/src/github.com/MSOpenTech/azure-sdk-for-go/storage/client_test.go delete mode 100644 Godeps/_workspace/src/github.com/MSOpenTech/azure-sdk-for-go/storage/util_test.go diff --git a/Dockerfile b/Dockerfile index c1bf8bd5f..266c78634 100644 --- a/Dockerfile +++ b/Dockerfile @@ -6,7 +6,7 @@ RUN apt-get update && \ ENV DISTRIBUTION_DIR /go/src/github.com/docker/distribution ENV GOPATH $DISTRIBUTION_DIR/Godeps/_workspace:$GOPATH -ENV DOCKER_BUILDTAGS include_rados include_azure +ENV DOCKER_BUILDTAGS include_rados WORKDIR $DISTRIBUTION_DIR COPY . $DISTRIBUTION_DIR diff --git a/Godeps/Godeps.json b/Godeps/Godeps.json index 1e8d5d01a..f0b9d21a0 100644 --- a/Godeps/Godeps.json +++ b/Godeps/Godeps.json @@ -18,9 +18,8 @@ "Rev": "cc210f45dcb9889c2769a274522be2bf70edfb99" }, { - "ImportPath": "github.com/MSOpenTech/azure-sdk-for-go/storage", - "Comment": "v1.2-43-gd90753b", - "Rev": "d90753bcad2ed782fcead7392d1e831df29aa2bb" + "ImportPath": "github.com/Azure/azure-sdk-for-go/storage", + "Rev": "97d9593768bbbbd316f9c055dfc5f780933cd7fc" }, { "ImportPath": "github.com/Sirupsen/logrus", diff --git a/Godeps/_workspace/src/github.com/Azure/azure-sdk-for-go/.gitignore b/Godeps/_workspace/src/github.com/Azure/azure-sdk-for-go/.gitignore new file mode 100644 index 000000000..c4c1f537b --- /dev/null +++ b/Godeps/_workspace/src/github.com/Azure/azure-sdk-for-go/.gitignore @@ -0,0 +1,29 @@ +# Compiled Object files, Static and Dynamic libs (Shared Objects) +*.o +*.a +*.so + +# Folders +_obj +_test + +# Architecture specific extensions/prefixes +*.[568vq] +[568vq].out + +*.cgo1.go +*.cgo2.c +_cgo_defun.c +_cgo_gotypes.go +_cgo_export.* + +_testmain.go + +*.exe +*.test +*.prof + +# Editor swap files +*.swp +*~ +.DS_Store diff --git a/Godeps/_workspace/src/github.com/Azure/azure-sdk-for-go/.travis.yml b/Godeps/_workspace/src/github.com/Azure/azure-sdk-for-go/.travis.yml new file mode 100644 index 000000000..e6fabccbc --- /dev/null +++ b/Godeps/_workspace/src/github.com/Azure/azure-sdk-for-go/.travis.yml @@ -0,0 +1,19 @@ +sudo: false + +language: go + +before_script: + - go get -u golang.org/x/tools/cmd/vet + - go get -u github.com/golang/lint/golint + +go: tip +script: + - test -z "$(gofmt -s -l -w management | tee /dev/stderr)" + - test -z "$(gofmt -s -l -w storage | tee /dev/stderr)" + - go build -v ./... + - go test -v ./storage/... -check.v + - test -z "$(golint ./storage/... | tee /dev/stderr)" + - go vet ./storage/... + - go test -v ./management/... + - test -z "$(golint ./management/... | grep -v 'should have comment' | grep -v 'stutters' | tee /dev/stderr)" + - go vet ./management/... diff --git a/Godeps/_workspace/src/github.com/Azure/azure-sdk-for-go/LICENSE b/Godeps/_workspace/src/github.com/Azure/azure-sdk-for-go/LICENSE new file mode 100644 index 000000000..d64569567 --- /dev/null +++ b/Godeps/_workspace/src/github.com/Azure/azure-sdk-for-go/LICENSE @@ -0,0 +1,202 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/Godeps/_workspace/src/github.com/Azure/azure-sdk-for-go/README.md b/Godeps/_workspace/src/github.com/Azure/azure-sdk-for-go/README.md new file mode 100644 index 000000000..13d548570 --- /dev/null +++ b/Godeps/_workspace/src/github.com/Azure/azure-sdk-for-go/README.md @@ -0,0 +1,88 @@ +# Microsoft Azure SDK for Go + +This project provides various Go packages to perform operations +on Microsoft Azure REST APIs. + +[![GoDoc](https://godoc.org/github.com/Azure/azure-sdk-for-go?status.svg)](https://godoc.org/github.com/Azure/azure-sdk-for-go) [![Build Status](https://travis-ci.org/Azure/azure-sdk-for-go.svg?branch=master)](https://travis-ci.org/Azure/azure-sdk-for-go) + +See list of implemented API clients [here](http://godoc.org/github.com/Azure/azure-sdk-for-go). + +> **NOTE:** This repository is under heavy ongoing development and +is likely to break over time. We currently do not have any releases +yet. If you are planning to use the repository, please consider vendoring +the packages in your project and update them when a stable tag is out. + +# Installation + + go get -d github.com/Azure/azure-sdk-for-go/management + +# Usage + +Read Godoc of the repository at: http://godoc.org/github.com/Azure/azure-sdk-for-go/ + +The client currently supports authentication to the Service Management +API with certificates or Azure `.publishSettings` file. You can +download the `.publishSettings` file for your subscriptions +[here](https://manage.windowsazure.com/publishsettings). + +### Example: Creating a Linux Virtual Machine + +```go +package main + +import ( + "encoding/base64" + "fmt" + + "github.com/Azure/azure-sdk-for-go/management" + "github.com/Azure/azure-sdk-for-go/management/hostedservice" + "github.com/Azure/azure-sdk-for-go/management/virtualmachine" + "github.com/Azure/azure-sdk-for-go/management/vmutils" +) + +func main() { + dnsName := "test-vm-from-go" + storageAccount := "mystorageaccount" + location := "West US" + vmSize := "Small" + vmImage := "b39f27a8b8c64d52b05eac6a62ebad85__Ubuntu-14_04-LTS-amd64-server-20140724-en-us-30GB" + userName := "testuser" + userPassword := "Test123" + + client, err := management.ClientFromPublishSettingsFile("path/to/downloaded.publishsettings", "") + if err != nil { + panic(err) + } + + // create hosted service + if err := hostedservice.NewClient(client).CreateHostedService(hostedservice.CreateHostedServiceParameters{ + ServiceName: dnsName, + Location: location, + Label: base64.StdEncoding.EncodeToString([]byte(dnsName))}); err != nil { + panic(err) + } + + // create virtual machine + role := vmutils.NewVMConfiguration(dnsName, vmSize) + vmutils.ConfigureDeploymentFromPlatformImage( + &role, + vmImage, + fmt.Sprintf("http://%s.blob.core.windows.net/sdktest/%s.vhd", storageAccount, dnsName), + "") + vmutils.ConfigureForLinux(&role, dnsName, userName, userPassword) + vmutils.ConfigureWithPublicSSH(&role) + + operationID, err := virtualmachine.NewClient(client). + CreateDeployment(role, dnsName, virtualmachine.CreateDeploymentOptions{}) + if err != nil { + panic(err) + } + if err := client.WaitForOperation(operationID, nil); err != nil { + panic(err) + } +} +``` + +# License + +This project is published under [Apache 2.0 License](LICENSE). diff --git a/Godeps/_workspace/src/github.com/MSOpenTech/azure-sdk-for-go/storage/blob.go b/Godeps/_workspace/src/github.com/Azure/azure-sdk-for-go/storage/blob.go similarity index 69% rename from Godeps/_workspace/src/github.com/MSOpenTech/azure-sdk-for-go/storage/blob.go rename to Godeps/_workspace/src/github.com/Azure/azure-sdk-for-go/storage/blob.go index f451e51fe..a6e2eb8ab 100644 --- a/Godeps/_workspace/src/github.com/MSOpenTech/azure-sdk-for-go/storage/blob.go +++ b/Godeps/_workspace/src/github.com/Azure/azure-sdk-for-go/storage/blob.go @@ -2,7 +2,6 @@ package storage import ( "bytes" - "encoding/base64" "encoding/xml" "errors" "fmt" @@ -14,8 +13,10 @@ import ( "time" ) +// BlobStorageClient contains operations for Microsoft Azure Blob Storage +// Service. type BlobStorageClient struct { - client StorageClient + client Client } // A Container is an entry in ContainerListResponse. @@ -25,8 +26,8 @@ type Container struct { // TODO (ahmetalpbalkan) Metadata } -// ContainerProperties contains various properties of a -// container returned from various endpoints like ListContainers. +// ContainerProperties contains various properties of a container returned from +// various endpoints like ListContainers. type ContainerProperties struct { LastModified string `xml:"Last-Modified"` Etag string `xml:"Etag"` @@ -37,7 +38,9 @@ type ContainerProperties struct { } // ContainerListResponse contains the response fields from -// ListContainers call. https://msdn.microsoft.com/en-us/library/azure/dd179352.aspx +// ListContainers call. +// +// See https://msdn.microsoft.com/en-us/library/azure/dd179352.aspx type ContainerListResponse struct { XMLName xml.Name `xml:"EnumerationResults"` Xmlns string `xml:"xmlns,attr"` @@ -66,7 +69,7 @@ type BlobProperties struct { ContentEncoding string `xml:"Content-Encoding"` BlobType BlobType `xml:"x-ms-blob-blob-type"` SequenceNumber int64 `xml:"x-ms-blob-sequence-number"` - CopyId string `xml:"CopyId"` + CopyID string `xml:"CopyId"` CopyStatus string `xml:"CopyStatus"` CopySource string `xml:"CopySource"` CopyProgress string `xml:"CopyProgress"` @@ -74,8 +77,9 @@ type BlobProperties struct { CopyStatusDescription string `xml:"CopyStatusDescription"` } -// BlobListResponse contains the response fields from -// ListBlobs call. https://msdn.microsoft.com/en-us/library/azure/dd135734.aspx +// BlobListResponse contains the response fields from ListBlobs call. +// +// See https://msdn.microsoft.com/en-us/library/azure/dd135734.aspx type BlobListResponse struct { XMLName xml.Name `xml:"EnumerationResults"` Xmlns string `xml:"xmlns,attr"` @@ -86,8 +90,10 @@ type BlobListResponse struct { Blobs []Blob `xml:"Blobs>Blob"` } -// ListContainersParameters defines the set of customizable -// parameters to make a List Containers call. https://msdn.microsoft.com/en-us/library/azure/dd179352.aspx +// ListContainersParameters defines the set of customizable parameters to make a +// List Containers call. +// +// See https://msdn.microsoft.com/en-us/library/azure/dd179352.aspx type ListContainersParameters struct { Prefix string Marker string @@ -119,7 +125,9 @@ func (p ListContainersParameters) getParameters() url.Values { } // ListBlobsParameters defines the set of customizable -// parameters to make a List Blobs call. https://msdn.microsoft.com/en-us/library/azure/dd135734.aspx +// parameters to make a List Blobs call. +// +// See https://msdn.microsoft.com/en-us/library/azure/dd135734.aspx type ListBlobsParameters struct { Prefix string Delimiter string @@ -157,6 +165,7 @@ func (p ListBlobsParameters) getParameters() url.Values { // BlobType defines the type of the Azure Blob. type BlobType string +// Types of page blobs const ( BlobTypeBlock BlobType = "BlockBlob" BlobTypePage BlobType = "PageBlob" @@ -166,6 +175,7 @@ const ( // done on the page blob. type PageWriteType string +// Types of operations on page blobs const ( PageWriteTypeUpdate PageWriteType = "update" PageWriteTypeClear PageWriteType = "clear" @@ -178,29 +188,35 @@ const ( blobCopyStatusFailed = "failed" ) -// BlockListType is used to filter out types of blocks -// in a Get Blocks List call for a block blob. See -// https://msdn.microsoft.com/en-us/library/azure/dd179400.aspx -// for all block types. +// BlockListType is used to filter out types of blocks in a Get Blocks List call +// for a block blob. +// +// See https://msdn.microsoft.com/en-us/library/azure/dd179400.aspx for all +// block types. type BlockListType string +// Filters for listing blocks in block blobs const ( BlockListTypeAll BlockListType = "all" BlockListTypeCommitted BlockListType = "committed" BlockListTypeUncommitted BlockListType = "uncommitted" ) -// ContainerAccessType defines the access level to the container -// from a public request. See https://msdn.microsoft.com/en-us/library/azure/dd179468.aspx -// and "x-ms-blob-public-access" header. +// ContainerAccessType defines the access level to the container from a public +// request. +// +// See https://msdn.microsoft.com/en-us/library/azure/dd179468.aspx and "x-ms- +// blob-public-access" header. type ContainerAccessType string +// Access options for containers const ( ContainerAccessTypePrivate ContainerAccessType = "" ContainerAccessTypeBlob ContainerAccessType = "blob" ContainerAccessTypeContainer ContainerAccessType = "container" ) +// Maximum sizes (per REST API) for various concepts const ( MaxBlobBlockSize = 4 * 1024 * 1024 MaxBlobPageSize = 4 * 1024 * 1024 @@ -210,6 +226,7 @@ const ( // be in. type BlockStatus string +// List of statuses that can be used to refer to a block in a block list const ( BlockStatusUncommitted BlockStatus = "Uncommitted" BlockStatusCommitted BlockStatus = "Committed" @@ -219,12 +236,13 @@ const ( // Block is used to create Block entities for Put Block List // call. type Block struct { - Id string + ID string Status BlockStatus } -// BlockListResponse contains the response fields from -// Get Block List call. https://msdn.microsoft.com/en-us/library/azure/dd179400.aspx +// BlockListResponse contains the response fields from Get Block List call. +// +// See https://msdn.microsoft.com/en-us/library/azure/dd179400.aspx type BlockListResponse struct { XMLName xml.Name `xml:"BlockList"` CommittedBlocks []BlockResponse `xml:"CommittedBlocks>Block"` @@ -239,31 +257,32 @@ type BlockResponse struct { } // GetPageRangesResponse contains the reponse fields from -// Get Page Ranges call. https://msdn.microsoft.com/en-us/library/azure/ee691973.aspx +// Get Page Ranges call. +// +// See https://msdn.microsoft.com/en-us/library/azure/ee691973.aspx type GetPageRangesResponse struct { XMLName xml.Name `xml:"PageList"` PageList []PageRange `xml:"PageRange"` } // PageRange contains information about a page of a page blob from -// Get Pages Range call. https://msdn.microsoft.com/en-us/library/azure/ee691973.aspx +// Get Pages Range call. +// +// See https://msdn.microsoft.com/en-us/library/azure/ee691973.aspx type PageRange struct { Start int64 `xml:"Start"` End int64 `xml:"End"` } var ( - ErrNotCreated = errors.New("storage: operation has returned a successful error code other than 201 Created.") - ErrNotAccepted = errors.New("storage: operation has returned a successful error code other than 202 Accepted.") - errBlobCopyAborted = errors.New("storage: blob copy is aborted") - errBlobCopyIdMismatch = errors.New("storage: blob copy id is a mismatch") + errBlobCopyIDMismatch = errors.New("storage: blob copy id is a mismatch") ) -const errUnexpectedStatus = "storage: was expecting status code: %d, got: %d" - // ListContainers returns the list of containers in a storage account along with -// pagination token and other response details. See https://msdn.microsoft.com/en-us/library/azure/dd179352.aspx +// pagination token and other response details. +// +// See https://msdn.microsoft.com/en-us/library/azure/dd179352.aspx func (b BlobStorageClient) ListContainers(params ListContainersParameters) (ContainerListResponse, error) { q := mergeParams(params.getParameters(), url.Values{"comp": {"list"}}) uri := b.client.getEndpoint(blobServiceName, "", q) @@ -274,31 +293,34 @@ func (b BlobStorageClient) ListContainers(params ListContainersParameters) (Cont if err != nil { return out, err } + defer resp.body.Close() err = xmlUnmarshal(resp.body, &out) return out, err } // CreateContainer creates a blob container within the storage account -// with given name and access level. See https://msdn.microsoft.com/en-us/library/azure/dd179468.aspx -// Returns error if container already exists. +// with given name and access level. Returns error if container already exists. +// +// See https://msdn.microsoft.com/en-us/library/azure/dd179468.aspx func (b BlobStorageClient) CreateContainer(name string, access ContainerAccessType) error { resp, err := b.createContainer(name, access) if err != nil { return err } - if resp.statusCode != http.StatusCreated { - return ErrNotCreated - } - return nil + defer resp.body.Close() + return checkRespCode(resp.statusCode, []int{http.StatusCreated}) } // CreateContainerIfNotExists creates a blob container if it does not exist. Returns // true if container is newly created or false if container already exists. func (b BlobStorageClient) CreateContainerIfNotExists(name string, access ContainerAccessType) (bool, error) { resp, err := b.createContainer(name, access) - if resp != nil && (resp.statusCode == http.StatusCreated || resp.statusCode == http.StatusConflict) { - return resp.statusCode == http.StatusCreated, nil + if resp != nil { + defer resp.body.Close() + if resp.statusCode == http.StatusCreated || resp.statusCode == http.StatusConflict { + return resp.statusCode == http.StatusCreated, nil + } } return false, err } @@ -323,34 +345,41 @@ func (b BlobStorageClient) ContainerExists(name string) (bool, error) { headers := b.client.getStandardHeaders() resp, err := b.client.exec(verb, uri, headers, nil) - if resp != nil && (resp.statusCode == http.StatusOK || resp.statusCode == http.StatusNotFound) { - return resp.statusCode == http.StatusOK, nil + if resp != nil { + defer resp.body.Close() + if resp.statusCode == http.StatusOK || resp.statusCode == http.StatusNotFound { + return resp.statusCode == http.StatusOK, nil + } } return false, err } // DeleteContainer deletes the container with given name on the storage -// account. See https://msdn.microsoft.com/en-us/library/azure/dd179408.aspx -// If the container does not exist returns error. +// account. If the container does not exist returns error. +// +// See https://msdn.microsoft.com/en-us/library/azure/dd179408.aspx func (b BlobStorageClient) DeleteContainer(name string) error { resp, err := b.deleteContainer(name) if err != nil { return err } - if resp.statusCode != http.StatusAccepted { - return ErrNotAccepted - } - return nil + defer resp.body.Close() + return checkRespCode(resp.statusCode, []int{http.StatusAccepted}) } -// DeleteContainer deletes the container with given name on the storage -// account if it exists. See https://msdn.microsoft.com/en-us/library/azure/dd179408.aspx -// Returns true if container is deleted with this call, or false -// if the container did not exist at the time of the Delete Container operation. +// DeleteContainerIfExists deletes the container with given name on the storage +// account if it exists. Returns true if container is deleted with this call, or +// false if the container did not exist at the time of the Delete Container +// operation. +// +// See https://msdn.microsoft.com/en-us/library/azure/dd179408.aspx func (b BlobStorageClient) DeleteContainerIfExists(name string) (bool, error) { resp, err := b.deleteContainer(name) - if resp != nil && (resp.statusCode == http.StatusAccepted || resp.statusCode == http.StatusNotFound) { - return resp.statusCode == http.StatusAccepted, nil + if resp != nil { + defer resp.body.Close() + if resp.statusCode == http.StatusAccepted || resp.statusCode == http.StatusNotFound { + return resp.statusCode == http.StatusAccepted, nil + } } return false, err } @@ -365,6 +394,7 @@ func (b BlobStorageClient) deleteContainer(name string) (*storageResponse, error // ListBlobs returns an object that contains list of blobs in the container, // pagination token and other information in the response of List Blobs call. +// // See https://msdn.microsoft.com/en-us/library/azure/dd135734.aspx func (b BlobStorageClient) ListBlobs(container string, params ListBlobsParameters) (BlobListResponse, error) { q := mergeParams(params.getParameters(), url.Values{ @@ -378,60 +408,68 @@ func (b BlobStorageClient) ListBlobs(container string, params ListBlobsParameter if err != nil { return out, err } + defer resp.body.Close() err = xmlUnmarshal(resp.body, &out) return out, err } -// BlobExists returns true if a blob with given name exists on the -// specified container of the storage account. +// BlobExists returns true if a blob with given name exists on the specified +// container of the storage account. func (b BlobStorageClient) BlobExists(container, name string) (bool, error) { verb := "HEAD" uri := b.client.getEndpoint(blobServiceName, pathForBlob(container, name), url.Values{}) headers := b.client.getStandardHeaders() resp, err := b.client.exec(verb, uri, headers, nil) - if resp != nil && (resp.statusCode == http.StatusOK || resp.statusCode == http.StatusNotFound) { - return resp.statusCode == http.StatusOK, nil + if resp != nil { + defer resp.body.Close() + if resp.statusCode == http.StatusOK || resp.statusCode == http.StatusNotFound { + return resp.statusCode == http.StatusOK, nil + } } return false, err } -// GetBlobUrl gets the canonical URL to the blob with the specified -// name in the specified container. This method does not create a -// publicly accessible URL if the blob or container is private and this -// method does not check if the blob exists. -func (b BlobStorageClient) GetBlobUrl(container, name string) string { +// GetBlobURL gets the canonical URL to the blob with the specified name in the +// specified container. This method does not create a publicly accessible URL if +// the blob or container is private and this method does not check if the blob +// exists. +func (b BlobStorageClient) GetBlobURL(container, name string) string { if container == "" { container = "$root" } return b.client.getEndpoint(blobServiceName, pathForBlob(container, name), url.Values{}) } -// GetBlob downloads a blob to a stream. See https://msdn.microsoft.com/en-us/library/azure/dd179440.aspx +// GetBlob returns a stream to read the blob. Caller must call Close() the +// reader to close on the underlying connection. +// +// See https://msdn.microsoft.com/en-us/library/azure/dd179440.aspx func (b BlobStorageClient) GetBlob(container, name string) (io.ReadCloser, error) { resp, err := b.getBlobRange(container, name, "") if err != nil { return nil, err } - if resp.statusCode != http.StatusOK { - return nil, fmt.Errorf(errUnexpectedStatus, http.StatusOK, resp.statusCode) + if err := checkRespCode(resp.statusCode, []int{http.StatusOK}); err != nil { + return nil, err } return resp.body, nil } -// GetBlobRange reads the specified range of a blob to a stream. -// The bytesRange string must be in a format like "0-", "10-100" -// as defined in HTTP 1.1 spec. See https://msdn.microsoft.com/en-us/library/azure/dd179440.aspx +// GetBlobRange reads the specified range of a blob to a stream. The bytesRange +// string must be in a format like "0-", "10-100" as defined in HTTP 1.1 spec. +// +// See https://msdn.microsoft.com/en-us/library/azure/dd179440.aspx func (b BlobStorageClient) GetBlobRange(container, name, bytesRange string) (io.ReadCloser, error) { resp, err := b.getBlobRange(container, name, bytesRange) if err != nil { return nil, err } - if resp.statusCode != http.StatusPartialContent { - return nil, fmt.Errorf(errUnexpectedStatus, http.StatusPartialContent, resp.statusCode) + if err := checkRespCode(resp.statusCode, []int{http.StatusPartialContent}); err != nil { + return nil, err } return resp.body, nil } @@ -462,9 +500,10 @@ func (b BlobStorageClient) GetBlobProperties(container, name string) (*BlobPrope if err != nil { return nil, err } + defer resp.body.Close() - if resp.statusCode != http.StatusOK { - return nil, fmt.Errorf(errUnexpectedStatus, http.StatusOK, resp.statusCode) + if err := checkRespCode(resp.statusCode, []int{http.StatusOK}); err != nil { + return nil, err } var contentLength int64 @@ -494,7 +533,7 @@ func (b BlobStorageClient) GetBlobProperties(container, name string) (*BlobPrope SequenceNumber: sequenceNum, CopyCompletionTime: resp.headers.Get("x-ms-copy-completion-time"), CopyStatusDescription: resp.headers.Get("x-ms-copy-status-description"), - CopyId: resp.headers.Get("x-ms-copy-id"), + CopyID: resp.headers.Get("x-ms-copy-id"), CopyProgress: resp.headers.Get("x-ms-copy-progress"), CopySource: resp.headers.Get("x-ms-copy-source"), CopyStatus: resp.headers.Get("x-ms-copy-status"), @@ -503,6 +542,7 @@ func (b BlobStorageClient) GetBlobProperties(container, name string) (*BlobPrope } // CreateBlockBlob initializes an empty block blob with no blocks. +// // See https://msdn.microsoft.com/en-us/library/azure/dd179451.aspx func (b BlobStorageClient) CreateBlockBlob(container, name string) error { path := fmt.Sprintf("%s/%s", container, name) @@ -515,96 +555,25 @@ func (b BlobStorageClient) CreateBlockBlob(container, name string) error { if err != nil { return err } - if resp.statusCode != http.StatusCreated { - return ErrNotCreated - } - return nil -} - -// PutBlockBlob uploads given stream into a block blob by splitting -// data stream into chunks and uploading as blocks. Commits the block -// list at the end. This is a helper method built on top of PutBlock -// and PutBlockList methods with sequential block ID counting logic. -func (b BlobStorageClient) PutBlockBlob(container, name string, blob io.Reader) error { // TODO (ahmetalpbalkan) consider ReadCloser and closing - return b.putBlockBlob(container, name, blob, MaxBlobBlockSize) -} - -func (b BlobStorageClient) putBlockBlob(container, name string, blob io.Reader, chunkSize int) error { - if chunkSize <= 0 || chunkSize > MaxBlobBlockSize { - chunkSize = MaxBlobBlockSize - } - - chunk := make([]byte, chunkSize) - n, err := blob.Read(chunk) - if err != nil && err != io.EOF { - return err - } - - if err == io.EOF { - // Fits into one block - return b.putSingleBlockBlob(container, name, chunk[:n]) - } else { - // Does not fit into one block. Upload block by block then commit the block list - blockList := []Block{} - - // Put blocks - for blockNum := 0; ; blockNum++ { - id := base64.StdEncoding.EncodeToString([]byte(fmt.Sprintf("%011d", blockNum))) - data := chunk[:n] - err = b.PutBlock(container, name, id, data) - if err != nil { - return err - } - blockList = append(blockList, Block{id, BlockStatusLatest}) - - // Read next block - n, err = blob.Read(chunk) - if err != nil && err != io.EOF { - return err - } - if err == io.EOF { - break - } - } - - // Commit block list - return b.PutBlockList(container, name, blockList) - } -} - -func (b BlobStorageClient) putSingleBlockBlob(container, name string, chunk []byte) error { - if len(chunk) > MaxBlobBlockSize { - return fmt.Errorf("storage: provided chunk (%d bytes) cannot fit into single-block blob (max %d bytes)", len(chunk), MaxBlobBlockSize) - } - - uri := b.client.getEndpoint(blobServiceName, pathForBlob(container, name), url.Values{}) - headers := b.client.getStandardHeaders() - headers["x-ms-blob-type"] = string(BlobTypeBlock) - headers["Content-Length"] = fmt.Sprintf("%v", len(chunk)) - - resp, err := b.client.exec("PUT", uri, headers, bytes.NewReader(chunk)) - if err != nil { - return err - } - if resp.statusCode != http.StatusCreated { - return ErrNotCreated - } - - return nil + defer resp.body.Close() + return checkRespCode(resp.statusCode, []int{http.StatusCreated}) } // PutBlock saves the given data chunk to the specified block blob with -// given ID. See https://msdn.microsoft.com/en-us/library/azure/dd135726.aspx -func (b BlobStorageClient) PutBlock(container, name, blockId string, chunk []byte) error { - return b.PutBlockWithLength(container, name, blockId, uint64(len(chunk)), bytes.NewReader(chunk)) +// given ID. +// +// See https://msdn.microsoft.com/en-us/library/azure/dd135726.aspx +func (b BlobStorageClient) PutBlock(container, name, blockID string, chunk []byte) error { + return b.PutBlockWithLength(container, name, blockID, uint64(len(chunk)), bytes.NewReader(chunk)) } -// PutBlockWithLength saves the given data stream of exactly specified size to the block blob -// with given ID. See https://msdn.microsoft.com/en-us/library/azure/dd135726.aspx -// It is an alternative to PutBlocks where data comes as stream but the length is -// known in advance. -func (b BlobStorageClient) PutBlockWithLength(container, name, blockId string, size uint64, blob io.Reader) error { - uri := b.client.getEndpoint(blobServiceName, pathForBlob(container, name), url.Values{"comp": {"block"}, "blockid": {blockId}}) +// PutBlockWithLength saves the given data stream of exactly specified size to +// the block blob with given ID. It is an alternative to PutBlocks where data +// comes as stream but the length is known in advance. +// +// See https://msdn.microsoft.com/en-us/library/azure/dd135726.aspx +func (b BlobStorageClient) PutBlockWithLength(container, name, blockID string, size uint64, blob io.Reader) error { + uri := b.client.getEndpoint(blobServiceName, pathForBlob(container, name), url.Values{"comp": {"block"}, "blockid": {blockID}}) headers := b.client.getStandardHeaders() headers["x-ms-blob-type"] = string(BlobTypeBlock) headers["Content-Length"] = fmt.Sprintf("%v", size) @@ -613,34 +582,31 @@ func (b BlobStorageClient) PutBlockWithLength(container, name, blockId string, s if err != nil { return err } - if resp.statusCode != http.StatusCreated { - return ErrNotCreated - } - - return nil + defer resp.body.Close() + return checkRespCode(resp.statusCode, []int{http.StatusCreated}) } -// PutBlockList saves list of blocks to the specified block blob. See -// https://msdn.microsoft.com/en-us/library/azure/dd179467.aspx +// PutBlockList saves list of blocks to the specified block blob. +// +// See https://msdn.microsoft.com/en-us/library/azure/dd179467.aspx func (b BlobStorageClient) PutBlockList(container, name string, blocks []Block) error { - blockListXml := prepareBlockListRequest(blocks) + blockListXML := prepareBlockListRequest(blocks) uri := b.client.getEndpoint(blobServiceName, pathForBlob(container, name), url.Values{"comp": {"blocklist"}}) headers := b.client.getStandardHeaders() - headers["Content-Length"] = fmt.Sprintf("%v", len(blockListXml)) + headers["Content-Length"] = fmt.Sprintf("%v", len(blockListXML)) - resp, err := b.client.exec("PUT", uri, headers, strings.NewReader(blockListXml)) + resp, err := b.client.exec("PUT", uri, headers, strings.NewReader(blockListXML)) if err != nil { return err } - if resp.statusCode != http.StatusCreated { - return ErrNotCreated - } - return nil + defer resp.body.Close() + return checkRespCode(resp.statusCode, []int{http.StatusCreated}) } -// GetBlockList retrieves list of blocks in the specified block blob. See -// https://msdn.microsoft.com/en-us/library/azure/dd179400.aspx +// GetBlockList retrieves list of blocks in the specified block blob. +// +// See https://msdn.microsoft.com/en-us/library/azure/dd179400.aspx func (b BlobStorageClient) GetBlockList(container, name string, blockType BlockListType) (BlockListResponse, error) { params := url.Values{"comp": {"blocklist"}, "blocklisttype": {string(blockType)}} uri := b.client.getEndpoint(blobServiceName, pathForBlob(container, name), params) @@ -651,6 +617,7 @@ func (b BlobStorageClient) GetBlockList(container, name string, blockType BlockL if err != nil { return out, err } + defer resp.body.Close() err = xmlUnmarshal(resp.body, &out) return out, err @@ -659,6 +626,7 @@ func (b BlobStorageClient) GetBlockList(container, name string, blockType BlockL // PutPageBlob initializes an empty page blob with specified name and maximum // size in bytes (size must be aligned to a 512-byte boundary). A page blob must // be created using this method before writing pages. +// // See https://msdn.microsoft.com/en-us/library/azure/dd179451.aspx func (b BlobStorageClient) PutPageBlob(container, name string, size int64) error { path := fmt.Sprintf("%s/%s", container, name) @@ -672,15 +640,15 @@ func (b BlobStorageClient) PutPageBlob(container, name string, size int64) error if err != nil { return err } - if resp.statusCode != http.StatusCreated { - return ErrNotCreated - } - return nil + defer resp.body.Close() + + return checkRespCode(resp.statusCode, []int{http.StatusCreated}) } // PutPage writes a range of pages to a page blob or clears the given range. // In case of 'clear' writes, given chunk is discarded. Ranges must be aligned // with 512-byte boundaries and chunk must be of size multiplies by 512. +// // See https://msdn.microsoft.com/en-us/library/ee691975.aspx func (b BlobStorageClient) PutPage(container, name string, startByte, endByte int64, writeType PageWriteType, chunk []byte) error { path := fmt.Sprintf("%s/%s", container, name) @@ -705,13 +673,13 @@ func (b BlobStorageClient) PutPage(container, name string, startByte, endByte in if err != nil { return err } - if resp.statusCode != http.StatusCreated { - return ErrNotCreated - } - return nil + defer resp.body.Close() + + return checkRespCode(resp.statusCode, []int{http.StatusCreated}) } // GetPageRanges returns the list of valid page ranges for a page blob. +// // See https://msdn.microsoft.com/en-us/library/azure/ee691973.aspx func (b BlobStorageClient) GetPageRanges(container, name string) (GetPageRangesResponse, error) { path := fmt.Sprintf("%s/%s", container, name) @@ -723,26 +691,28 @@ func (b BlobStorageClient) GetPageRanges(container, name string) (GetPageRangesR if err != nil { return out, err } + defer resp.body.Close() - if resp.statusCode != http.StatusOK { - return out, fmt.Errorf(errUnexpectedStatus, http.StatusOK, resp.statusCode) + if err := checkRespCode(resp.statusCode, []int{http.StatusOK}); err != nil { + return out, err } - err = xmlUnmarshal(resp.body, &out) return out, err } -// CopyBlob starts a blob copy operation and waits for the operation to complete. -// sourceBlob parameter must be a canonical URL to the blob (can be obtained using -// GetBlobURL method.) There is no SLA on blob copy and therefore this helper -// method works faster on smaller files. See https://msdn.microsoft.com/en-us/library/azure/dd894037.aspx +// CopyBlob starts a blob copy operation and waits for the operation to +// complete. sourceBlob parameter must be a canonical URL to the blob (can be +// obtained using GetBlobURL method.) There is no SLA on blob copy and therefore +// this helper method works faster on smaller files. +// +// See https://msdn.microsoft.com/en-us/library/azure/dd894037.aspx func (b BlobStorageClient) CopyBlob(container, name, sourceBlob string) error { - copyId, err := b.startBlobCopy(container, name, sourceBlob) + copyID, err := b.startBlobCopy(container, name, sourceBlob) if err != nil { return err } - return b.waitForBlobCopy(container, name, copyId) + return b.waitForBlobCopy(container, name, copyID) } func (b BlobStorageClient) startBlobCopy(container, name, sourceBlob string) (string, error) { @@ -756,26 +726,28 @@ func (b BlobStorageClient) startBlobCopy(container, name, sourceBlob string) (st if err != nil { return "", err } - if resp.statusCode != http.StatusAccepted && resp.statusCode != http.StatusCreated { - return "", fmt.Errorf(errUnexpectedStatus, []int{http.StatusAccepted, http.StatusCreated}, resp.statusCode) + defer resp.body.Close() + + if err := checkRespCode(resp.statusCode, []int{http.StatusAccepted, http.StatusCreated}); err != nil { + return "", err } - copyId := resp.headers.Get("x-ms-copy-id") - if copyId == "" { + copyID := resp.headers.Get("x-ms-copy-id") + if copyID == "" { return "", errors.New("Got empty copy id header") } - return copyId, nil + return copyID, nil } -func (b BlobStorageClient) waitForBlobCopy(container, name, copyId string) error { +func (b BlobStorageClient) waitForBlobCopy(container, name, copyID string) error { for { props, err := b.GetBlobProperties(container, name) if err != nil { return err } - if props.CopyId != copyId { - return errBlobCopyIdMismatch + if props.CopyID != copyID { + return errBlobCopyIDMismatch } switch props.CopyStatus { @@ -786,7 +758,7 @@ func (b BlobStorageClient) waitForBlobCopy(container, name, copyId string) error case blobCopyStatusAborted: return errBlobCopyAborted case blobCopyStatusFailed: - return fmt.Errorf("storage: blob copy failed. Id=%s Description=%s", props.CopyId, props.CopyStatusDescription) + return fmt.Errorf("storage: blob copy failed. Id=%s Description=%s", props.CopyID, props.CopyStatusDescription) default: return fmt.Errorf("storage: unhandled blob copy status: '%s'", props.CopyStatus) } @@ -801,20 +773,20 @@ func (b BlobStorageClient) DeleteBlob(container, name string) error { if err != nil { return err } - if resp.statusCode != http.StatusAccepted { - return ErrNotAccepted - } - return nil + defer resp.body.Close() + return checkRespCode(resp.statusCode, []int{http.StatusAccepted}) } -// DeleteBlobIfExists deletes the given blob from the specified container -// If the blob is deleted with this call, returns true. Otherwise returns -// false. See https://msdn.microsoft.com/en-us/library/azure/dd179413.aspx +// DeleteBlobIfExists deletes the given blob from the specified container If the +// blob is deleted with this call, returns true. Otherwise returns false. +// +// See https://msdn.microsoft.com/en-us/library/azure/dd179413.aspx func (b BlobStorageClient) DeleteBlobIfExists(container, name string) (bool, error) { resp, err := b.deleteBlob(container, name) if resp != nil && (resp.statusCode == http.StatusAccepted || resp.statusCode == http.StatusNotFound) { return resp.statusCode == http.StatusAccepted, nil } + defer resp.body.Close() return false, err } @@ -831,19 +803,22 @@ func pathForContainer(name string) string { return fmt.Sprintf("/%s", name) } -// helper method to construct the path to a blob given its container and blob name +// helper method to construct the path to a blob given its container and blob +// name func pathForBlob(container, name string) string { return fmt.Sprintf("/%s/%s", container, name) } -// GetBlobSASURI creates an URL to the specified blob which contains the Shared Access Signature -// with specified permissions and expiration time. See https://msdn.microsoft.com/en-us/library/azure/ee395415.aspx +// GetBlobSASURI creates an URL to the specified blob which contains the Shared +// Access Signature with specified permissions and expiration time. +// +// See https://msdn.microsoft.com/en-us/library/azure/ee395415.aspx func (b BlobStorageClient) GetBlobSASURI(container, name string, expiry time.Time, permissions string) (string, error) { var ( signedPermissions = permissions - blobUrl = b.GetBlobUrl(container, name) + blobURL = b.GetBlobURL(container, name) ) - canonicalizedResource, err := b.client.buildCanonicalizedResource(blobUrl) + canonicalizedResource, err := b.client.buildCanonicalizedResource(blobURL) if err != nil { return "", err } @@ -864,12 +839,12 @@ func (b BlobStorageClient) GetBlobSASURI(container, name string, expiry time.Tim "sig": {sig}, } - sasUrl, err := url.Parse(blobUrl) + sasURL, err := url.Parse(blobURL) if err != nil { return "", err } - sasUrl.RawQuery = sasParams.Encode() - return sasUrl.String(), nil + sasURL.RawQuery = sasParams.Encode() + return sasURL.String(), nil } func blobSASStringToSign(signedVersion, canonicalizedResource, signedExpiry, signedPermissions string) (string, error) { @@ -878,7 +853,6 @@ func blobSASStringToSign(signedVersion, canonicalizedResource, signedExpiry, sig // reference: http://msdn.microsoft.com/en-us/library/azure/dn140255.aspx if signedVersion >= "2013-08-15" { return fmt.Sprintf("%s\n%s\n%s\n%s\n%s\n%s\n%s\n%s\n%s\n%s\n%s", signedPermissions, signedStart, signedExpiry, canonicalizedResource, signedIdentifier, signedVersion, rscc, rscd, rsce, rscl, rsct), nil - } else { - return "", errors.New("storage: not implemented SAS for versions earlier than 2013-08-15") } + return "", errors.New("storage: not implemented SAS for versions earlier than 2013-08-15") } diff --git a/Godeps/_workspace/src/github.com/Azure/azure-sdk-for-go/storage/blob_test.go b/Godeps/_workspace/src/github.com/Azure/azure-sdk-for-go/storage/blob_test.go new file mode 100644 index 000000000..14a2f6b2c --- /dev/null +++ b/Godeps/_workspace/src/github.com/Azure/azure-sdk-for-go/storage/blob_test.go @@ -0,0 +1,625 @@ +package storage + +import ( + "bytes" + "crypto/rand" + "encoding/base64" + "fmt" + "io/ioutil" + "net/http" + "net/url" + "sort" + "sync" + "testing" + "time" + + chk "gopkg.in/check.v1" +) + +type StorageBlobSuite struct{} + +var _ = chk.Suite(&StorageBlobSuite{}) + +const testContainerPrefix = "zzzztest-" + +func getBlobClient(c *chk.C) BlobStorageClient { + return getBasicClient(c).GetBlobService() +} + +func (s *StorageBlobSuite) Test_pathForContainer(c *chk.C) { + c.Assert(pathForContainer("foo"), chk.Equals, "/foo") +} + +func (s *StorageBlobSuite) Test_pathForBlob(c *chk.C) { + c.Assert(pathForBlob("foo", "blob"), chk.Equals, "/foo/blob") +} + +func (s *StorageBlobSuite) Test_blobSASStringToSign(c *chk.C) { + _, err := blobSASStringToSign("2012-02-12", "CS", "SE", "SP") + c.Assert(err, chk.NotNil) // not implemented SAS for versions earlier than 2013-08-15 + + out, err := blobSASStringToSign("2013-08-15", "CS", "SE", "SP") + c.Assert(err, chk.IsNil) + c.Assert(out, chk.Equals, "SP\n\nSE\nCS\n\n2013-08-15\n\n\n\n\n") +} + +func (s *StorageBlobSuite) TestGetBlobSASURI(c *chk.C) { + api, err := NewClient("foo", "YmFy", DefaultBaseURL, "2013-08-15", true) + c.Assert(err, chk.IsNil) + cli := api.GetBlobService() + expiry := time.Time{} + + expectedParts := url.URL{ + Scheme: "https", + Host: "foo.blob.core.windows.net", + Path: "container/name", + RawQuery: url.Values{ + "sv": {"2013-08-15"}, + "sig": {"/OXG7rWh08jYwtU03GzJM0DHZtidRGpC6g69rSGm3I0="}, + "sr": {"b"}, + "sp": {"r"}, + "se": {"0001-01-01T00:00:00Z"}, + }.Encode()} + + u, err := cli.GetBlobSASURI("container", "name", expiry, "r") + c.Assert(err, chk.IsNil) + sasParts, err := url.Parse(u) + c.Assert(err, chk.IsNil) + c.Assert(expectedParts.String(), chk.Equals, sasParts.String()) + c.Assert(expectedParts.Query(), chk.DeepEquals, sasParts.Query()) +} + +func (s *StorageBlobSuite) TestBlobSASURICorrectness(c *chk.C) { + cli := getBlobClient(c) + cnt := randContainer() + blob := randString(20) + body := []byte(randString(100)) + expiry := time.Now().UTC().Add(time.Hour) + permissions := "r" + + c.Assert(cli.CreateContainer(cnt, ContainerAccessTypePrivate), chk.IsNil) + defer cli.DeleteContainer(cnt) + + c.Assert(cli.putSingleBlockBlob(cnt, blob, body), chk.IsNil) + + sasURI, err := cli.GetBlobSASURI(cnt, blob, expiry, permissions) + c.Assert(err, chk.IsNil) + + resp, err := http.Get(sasURI) + c.Assert(err, chk.IsNil) + + blobResp, err := ioutil.ReadAll(resp.Body) + defer resp.Body.Close() + c.Assert(err, chk.IsNil) + + c.Assert(resp.StatusCode, chk.Equals, http.StatusOK) + c.Assert(len(blobResp), chk.Equals, len(body)) +} + +func (s *StorageBlobSuite) TestListContainersPagination(c *chk.C) { + cli := getBlobClient(c) + c.Assert(deleteTestContainers(cli), chk.IsNil) + + const n = 5 + const pageSize = 2 + + // Create test containers + created := []string{} + for i := 0; i < n; i++ { + name := randContainer() + c.Assert(cli.CreateContainer(name, ContainerAccessTypePrivate), chk.IsNil) + created = append(created, name) + } + sort.Strings(created) + + // Defer test container deletions + defer func() { + var wg sync.WaitGroup + for _, cnt := range created { + wg.Add(1) + go func(name string) { + c.Assert(cli.DeleteContainer(name), chk.IsNil) + wg.Done() + }(cnt) + } + wg.Wait() + }() + + // Paginate results + seen := []string{} + marker := "" + for { + resp, err := cli.ListContainers(ListContainersParameters{ + Prefix: testContainerPrefix, + MaxResults: pageSize, + Marker: marker}) + c.Assert(err, chk.IsNil) + + containers := resp.Containers + if len(containers) > pageSize { + c.Fatalf("Got a bigger page. Expected: %d, got: %d", pageSize, len(containers)) + } + + for _, c := range containers { + seen = append(seen, c.Name) + } + + marker = resp.NextMarker + if marker == "" || len(containers) == 0 { + break + } + } + + c.Assert(seen, chk.DeepEquals, created) +} + +func (s *StorageBlobSuite) TestContainerExists(c *chk.C) { + cnt := randContainer() + cli := getBlobClient(c) + ok, err := cli.ContainerExists(cnt) + c.Assert(err, chk.IsNil) + c.Assert(ok, chk.Equals, false) + + c.Assert(cli.CreateContainer(cnt, ContainerAccessTypeBlob), chk.IsNil) + defer cli.DeleteContainer(cnt) + + ok, err = cli.ContainerExists(cnt) + c.Assert(err, chk.IsNil) + c.Assert(ok, chk.Equals, true) +} + +func (s *StorageBlobSuite) TestCreateDeleteContainer(c *chk.C) { + cnt := randContainer() + cli := getBlobClient(c) + c.Assert(cli.CreateContainer(cnt, ContainerAccessTypePrivate), chk.IsNil) + c.Assert(cli.DeleteContainer(cnt), chk.IsNil) +} + +func (s *StorageBlobSuite) TestCreateContainerIfNotExists(c *chk.C) { + cnt := randContainer() + cli := getBlobClient(c) + + // First create + ok, err := cli.CreateContainerIfNotExists(cnt, ContainerAccessTypePrivate) + c.Assert(err, chk.IsNil) + c.Assert(ok, chk.Equals, true) + + // Second create, should not give errors + ok, err = cli.CreateContainerIfNotExists(cnt, ContainerAccessTypePrivate) + c.Assert(err, chk.IsNil) + defer cli.DeleteContainer(cnt) + c.Assert(ok, chk.Equals, false) +} + +func (s *StorageBlobSuite) TestDeleteContainerIfExists(c *chk.C) { + cnt := randContainer() + cli := getBlobClient(c) + + // Nonexisting container + c.Assert(cli.DeleteContainer(cnt), chk.NotNil) + + ok, err := cli.DeleteContainerIfExists(cnt) + c.Assert(err, chk.IsNil) + c.Assert(ok, chk.Equals, false) + + // Existing container + c.Assert(cli.CreateContainer(cnt, ContainerAccessTypePrivate), chk.IsNil) + ok, err = cli.DeleteContainerIfExists(cnt) + c.Assert(err, chk.IsNil) + c.Assert(ok, chk.Equals, true) +} + +func (s *StorageBlobSuite) TestBlobExists(c *chk.C) { + cnt := randContainer() + blob := randString(20) + cli := getBlobClient(c) + + c.Assert(cli.CreateContainer(cnt, ContainerAccessTypeBlob), chk.IsNil) + defer cli.DeleteContainer(cnt) + c.Assert(cli.putSingleBlockBlob(cnt, blob, []byte("Hello!")), chk.IsNil) + defer cli.DeleteBlob(cnt, blob) + + ok, err := cli.BlobExists(cnt, blob+".foo") + c.Assert(err, chk.IsNil) + c.Assert(ok, chk.Equals, false) + + ok, err = cli.BlobExists(cnt, blob) + c.Assert(err, chk.IsNil) + c.Assert(ok, chk.Equals, true) +} + +func (s *StorageBlobSuite) TestGetBlobURL(c *chk.C) { + api, err := NewBasicClient("foo", "YmFy") + c.Assert(err, chk.IsNil) + cli := api.GetBlobService() + + c.Assert(cli.GetBlobURL("c", "nested/blob"), chk.Equals, "https://foo.blob.core.windows.net/c/nested/blob") + c.Assert(cli.GetBlobURL("", "blob"), chk.Equals, "https://foo.blob.core.windows.net/$root/blob") + c.Assert(cli.GetBlobURL("", "nested/blob"), chk.Equals, "https://foo.blob.core.windows.net/$root/nested/blob") +} + +func (s *StorageBlobSuite) TestBlobCopy(c *chk.C) { + if testing.Short() { + c.Skip("skipping blob copy in short mode, no SLA on async operation") + } + + cli := getBlobClient(c) + cnt := randContainer() + src := randString(20) + dst := randString(20) + body := []byte(randString(1024)) + + c.Assert(cli.CreateContainer(cnt, ContainerAccessTypePrivate), chk.IsNil) + defer cli.deleteContainer(cnt) + + c.Assert(cli.putSingleBlockBlob(cnt, src, body), chk.IsNil) + defer cli.DeleteBlob(cnt, src) + + c.Assert(cli.CopyBlob(cnt, dst, cli.GetBlobURL(cnt, src)), chk.IsNil) + defer cli.DeleteBlob(cnt, dst) + + blobBody, err := cli.GetBlob(cnt, dst) + c.Assert(err, chk.IsNil) + + b, err := ioutil.ReadAll(blobBody) + defer blobBody.Close() + c.Assert(err, chk.IsNil) + c.Assert(b, chk.DeepEquals, body) +} + +func (s *StorageBlobSuite) TestDeleteBlobIfExists(c *chk.C) { + cnt := randContainer() + blob := randString(20) + + cli := getBlobClient(c) + c.Assert(cli.DeleteBlob(cnt, blob), chk.NotNil) + + ok, err := cli.DeleteBlobIfExists(cnt, blob) + c.Assert(err, chk.IsNil) + c.Assert(ok, chk.Equals, false) +} + +func (s *StorageBlobSuite) TestGetBlobProperties(c *chk.C) { + cnt := randContainer() + blob := randString(20) + contents := randString(64) + + cli := getBlobClient(c) + c.Assert(cli.CreateContainer(cnt, ContainerAccessTypePrivate), chk.IsNil) + defer cli.DeleteContainer(cnt) + + // Nonexisting blob + _, err := cli.GetBlobProperties(cnt, blob) + c.Assert(err, chk.NotNil) + + // Put the blob + c.Assert(cli.putSingleBlockBlob(cnt, blob, []byte(contents)), chk.IsNil) + + // Get blob properties + props, err := cli.GetBlobProperties(cnt, blob) + c.Assert(err, chk.IsNil) + + c.Assert(props.ContentLength, chk.Equals, int64(len(contents))) + c.Assert(props.BlobType, chk.Equals, BlobTypeBlock) +} + +func (s *StorageBlobSuite) TestListBlobsPagination(c *chk.C) { + cli := getBlobClient(c) + cnt := randContainer() + + c.Assert(cli.CreateContainer(cnt, ContainerAccessTypePrivate), chk.IsNil) + defer cli.DeleteContainer(cnt) + + blobs := []string{} + const n = 5 + const pageSize = 2 + for i := 0; i < n; i++ { + name := randString(20) + c.Assert(cli.putSingleBlockBlob(cnt, name, []byte("Hello, world!")), chk.IsNil) + blobs = append(blobs, name) + } + sort.Strings(blobs) + + // Paginate + seen := []string{} + marker := "" + for { + resp, err := cli.ListBlobs(cnt, ListBlobsParameters{ + MaxResults: pageSize, + Marker: marker}) + c.Assert(err, chk.IsNil) + + for _, v := range resp.Blobs { + seen = append(seen, v.Name) + } + + marker = resp.NextMarker + if marker == "" || len(resp.Blobs) == 0 { + break + } + } + + // Compare + c.Assert(seen, chk.DeepEquals, blobs) +} + +func (s *StorageBlobSuite) TestPutEmptyBlockBlob(c *chk.C) { + cli := getBlobClient(c) + cnt := randContainer() + + c.Assert(cli.CreateContainer(cnt, ContainerAccessTypePrivate), chk.IsNil) + defer cli.deleteContainer(cnt) + + blob := randString(20) + c.Assert(cli.putSingleBlockBlob(cnt, blob, []byte{}), chk.IsNil) + + props, err := cli.GetBlobProperties(cnt, blob) + c.Assert(err, chk.IsNil) + c.Assert(props.ContentLength, chk.Not(chk.Equals), 0) +} + +func (s *StorageBlobSuite) TestGetBlobRange(c *chk.C) { + cnt := randContainer() + blob := randString(20) + body := "0123456789" + + cli := getBlobClient(c) + c.Assert(cli.CreateContainer(cnt, ContainerAccessTypeBlob), chk.IsNil) + defer cli.DeleteContainer(cnt) + + c.Assert(cli.putSingleBlockBlob(cnt, blob, []byte(body)), chk.IsNil) + defer cli.DeleteBlob(cnt, blob) + + // Read 1-3 + for _, r := range []struct { + rangeStr string + expected string + }{ + {"0-", body}, + {"1-3", body[1 : 3+1]}, + {"3-", body[3:]}, + } { + resp, err := cli.GetBlobRange(cnt, blob, r.rangeStr) + c.Assert(err, chk.IsNil) + blobBody, err := ioutil.ReadAll(resp) + c.Assert(err, chk.IsNil) + + str := string(blobBody) + c.Assert(str, chk.Equals, r.expected) + } +} + +func (s *StorageBlobSuite) TestPutBlock(c *chk.C) { + cli := getBlobClient(c) + cnt := randContainer() + c.Assert(cli.CreateContainer(cnt, ContainerAccessTypePrivate), chk.IsNil) + defer cli.deleteContainer(cnt) + + blob := randString(20) + chunk := []byte(randString(1024)) + blockID := base64.StdEncoding.EncodeToString([]byte("foo")) + c.Assert(cli.PutBlock(cnt, blob, blockID, chunk), chk.IsNil) +} + +func (s *StorageBlobSuite) TestGetBlockList_PutBlockList(c *chk.C) { + cli := getBlobClient(c) + cnt := randContainer() + c.Assert(cli.CreateContainer(cnt, ContainerAccessTypePrivate), chk.IsNil) + defer cli.deleteContainer(cnt) + + blob := randString(20) + chunk := []byte(randString(1024)) + blockID := base64.StdEncoding.EncodeToString([]byte("foo")) + + // Put one block + c.Assert(cli.PutBlock(cnt, blob, blockID, chunk), chk.IsNil) + defer cli.deleteBlob(cnt, blob) + + // Get committed blocks + committed, err := cli.GetBlockList(cnt, blob, BlockListTypeCommitted) + c.Assert(err, chk.IsNil) + + if len(committed.CommittedBlocks) > 0 { + c.Fatal("There are committed blocks") + } + + // Get uncommitted blocks + uncommitted, err := cli.GetBlockList(cnt, blob, BlockListTypeUncommitted) + c.Assert(err, chk.IsNil) + + c.Assert(len(uncommitted.UncommittedBlocks), chk.Equals, 1) + // Commit block list + c.Assert(cli.PutBlockList(cnt, blob, []Block{{blockID, BlockStatusUncommitted}}), chk.IsNil) + + // Get all blocks + all, err := cli.GetBlockList(cnt, blob, BlockListTypeAll) + c.Assert(err, chk.IsNil) + c.Assert(len(all.CommittedBlocks), chk.Equals, 1) + c.Assert(len(all.UncommittedBlocks), chk.Equals, 0) + + // Verify the block + thatBlock := all.CommittedBlocks[0] + c.Assert(thatBlock.Name, chk.Equals, blockID) + c.Assert(thatBlock.Size, chk.Equals, int64(len(chunk))) +} + +func (s *StorageBlobSuite) TestCreateBlockBlob(c *chk.C) { + cli := getBlobClient(c) + cnt := randContainer() + c.Assert(cli.CreateContainer(cnt, ContainerAccessTypePrivate), chk.IsNil) + defer cli.deleteContainer(cnt) + + blob := randString(20) + c.Assert(cli.CreateBlockBlob(cnt, blob), chk.IsNil) + + // Verify + blocks, err := cli.GetBlockList(cnt, blob, BlockListTypeAll) + c.Assert(err, chk.IsNil) + c.Assert(len(blocks.CommittedBlocks), chk.Equals, 0) + c.Assert(len(blocks.UncommittedBlocks), chk.Equals, 0) +} + +func (s *StorageBlobSuite) TestPutPageBlob(c *chk.C) { + cli := getBlobClient(c) + cnt := randContainer() + c.Assert(cli.CreateContainer(cnt, ContainerAccessTypePrivate), chk.IsNil) + defer cli.deleteContainer(cnt) + + blob := randString(20) + size := int64(10 * 1024 * 1024) + c.Assert(cli.PutPageBlob(cnt, blob, size), chk.IsNil) + + // Verify + props, err := cli.GetBlobProperties(cnt, blob) + c.Assert(err, chk.IsNil) + c.Assert(props.ContentLength, chk.Equals, size) + c.Assert(props.BlobType, chk.Equals, BlobTypePage) +} + +func (s *StorageBlobSuite) TestPutPagesUpdate(c *chk.C) { + cli := getBlobClient(c) + cnt := randContainer() + c.Assert(cli.CreateContainer(cnt, ContainerAccessTypePrivate), chk.IsNil) + defer cli.deleteContainer(cnt) + + blob := randString(20) + size := int64(10 * 1024 * 1024) // larger than we'll use + c.Assert(cli.PutPageBlob(cnt, blob, size), chk.IsNil) + + chunk1 := []byte(randString(1024)) + chunk2 := []byte(randString(512)) + + // Append chunks + c.Assert(cli.PutPage(cnt, blob, 0, int64(len(chunk1)-1), PageWriteTypeUpdate, chunk1), chk.IsNil) + c.Assert(cli.PutPage(cnt, blob, int64(len(chunk1)), int64(len(chunk1)+len(chunk2)-1), PageWriteTypeUpdate, chunk2), chk.IsNil) + + // Verify contents + out, err := cli.GetBlobRange(cnt, blob, fmt.Sprintf("%v-%v", 0, len(chunk1)+len(chunk2)-1)) + c.Assert(err, chk.IsNil) + defer out.Close() + blobContents, err := ioutil.ReadAll(out) + c.Assert(err, chk.IsNil) + c.Assert(blobContents, chk.DeepEquals, append(chunk1, chunk2...)) + out.Close() + + // Overwrite first half of chunk1 + chunk0 := []byte(randString(512)) + c.Assert(cli.PutPage(cnt, blob, 0, int64(len(chunk0)-1), PageWriteTypeUpdate, chunk0), chk.IsNil) + + // Verify contents + out, err = cli.GetBlobRange(cnt, blob, fmt.Sprintf("%v-%v", 0, len(chunk1)+len(chunk2)-1)) + c.Assert(err, chk.IsNil) + defer out.Close() + blobContents, err = ioutil.ReadAll(out) + c.Assert(err, chk.IsNil) + c.Assert(blobContents, chk.DeepEquals, append(append(chunk0, chunk1[512:]...), chunk2...)) +} + +func (s *StorageBlobSuite) TestPutPagesClear(c *chk.C) { + cli := getBlobClient(c) + cnt := randContainer() + c.Assert(cli.CreateContainer(cnt, ContainerAccessTypePrivate), chk.IsNil) + defer cli.deleteContainer(cnt) + + blob := randString(20) + size := int64(10 * 1024 * 1024) // larger than we'll use + c.Assert(cli.PutPageBlob(cnt, blob, size), chk.IsNil) + + // Put 0-2047 + chunk := []byte(randString(2048)) + c.Assert(cli.PutPage(cnt, blob, 0, 2047, PageWriteTypeUpdate, chunk), chk.IsNil) + + // Clear 512-1023 + c.Assert(cli.PutPage(cnt, blob, 512, 1023, PageWriteTypeClear, nil), chk.IsNil) + + // Verify contents + out, err := cli.GetBlobRange(cnt, blob, "0-2047") + c.Assert(err, chk.IsNil) + contents, err := ioutil.ReadAll(out) + c.Assert(err, chk.IsNil) + defer out.Close() + c.Assert(contents, chk.DeepEquals, append(append(chunk[:512], make([]byte, 512)...), chunk[1024:]...)) +} + +func (s *StorageBlobSuite) TestGetPageRanges(c *chk.C) { + cli := getBlobClient(c) + cnt := randContainer() + c.Assert(cli.CreateContainer(cnt, ContainerAccessTypePrivate), chk.IsNil) + defer cli.deleteContainer(cnt) + + blob := randString(20) + size := int64(10 * 1024 * 1024) // larger than we'll use + c.Assert(cli.PutPageBlob(cnt, blob, size), chk.IsNil) + + // Get page ranges on empty blob + out, err := cli.GetPageRanges(cnt, blob) + c.Assert(err, chk.IsNil) + c.Assert(len(out.PageList), chk.Equals, 0) + + // Add 0-512 page + c.Assert(cli.PutPage(cnt, blob, 0, 511, PageWriteTypeUpdate, []byte(randString(512))), chk.IsNil) + + out, err = cli.GetPageRanges(cnt, blob) + c.Assert(err, chk.IsNil) + c.Assert(len(out.PageList), chk.Equals, 1) + + // Add 1024-2048 + c.Assert(cli.PutPage(cnt, blob, 1024, 2047, PageWriteTypeUpdate, []byte(randString(1024))), chk.IsNil) + + out, err = cli.GetPageRanges(cnt, blob) + c.Assert(err, chk.IsNil) + c.Assert(len(out.PageList), chk.Equals, 2) +} + +func deleteTestContainers(cli BlobStorageClient) error { + for { + resp, err := cli.ListContainers(ListContainersParameters{Prefix: testContainerPrefix}) + if err != nil { + return err + } + if len(resp.Containers) == 0 { + break + } + for _, c := range resp.Containers { + err = cli.DeleteContainer(c.Name) + if err != nil { + return err + } + } + } + return nil +} + +func (b BlobStorageClient) putSingleBlockBlob(container, name string, chunk []byte) error { + if len(chunk) > MaxBlobBlockSize { + return fmt.Errorf("storage: provided chunk (%d bytes) cannot fit into single-block blob (max %d bytes)", len(chunk), MaxBlobBlockSize) + } + + uri := b.client.getEndpoint(blobServiceName, pathForBlob(container, name), url.Values{}) + headers := b.client.getStandardHeaders() + headers["x-ms-blob-type"] = string(BlobTypeBlock) + headers["Content-Length"] = fmt.Sprintf("%v", len(chunk)) + + resp, err := b.client.exec("PUT", uri, headers, bytes.NewReader(chunk)) + if err != nil { + return err + } + return checkRespCode(resp.statusCode, []int{http.StatusCreated}) +} + +func randContainer() string { + return testContainerPrefix + randString(32-len(testContainerPrefix)) +} + +func randString(n int) string { + if n <= 0 { + panic("negative number") + } + const alphanum = "0123456789abcdefghijklmnopqrstuvwxyz" + var bytes = make([]byte, n) + rand.Read(bytes) + for i, b := range bytes { + bytes[i] = alphanum[b%byte(len(alphanum))] + } + return string(bytes) +} diff --git a/Godeps/_workspace/src/github.com/MSOpenTech/azure-sdk-for-go/storage/client.go b/Godeps/_workspace/src/github.com/Azure/azure-sdk-for-go/storage/client.go similarity index 59% rename from Godeps/_workspace/src/github.com/MSOpenTech/azure-sdk-for-go/storage/client.go rename to Godeps/_workspace/src/github.com/Azure/azure-sdk-for-go/storage/client.go index 2928dbab3..6c171050a 100644 --- a/Godeps/_workspace/src/github.com/MSOpenTech/azure-sdk-for-go/storage/client.go +++ b/Godeps/_workspace/src/github.com/Azure/azure-sdk-for-go/storage/client.go @@ -1,3 +1,4 @@ +// Package storage provides clients for Microsoft Azure Storage Services. package storage import ( @@ -15,22 +16,28 @@ import ( ) const ( - DefaultBaseUrl = "core.windows.net" - DefaultApiVersion = "2014-02-14" - defaultUseHttps = true + // DefaultBaseURL is the domain name used for storage requests when a + // default client is created. + DefaultBaseURL = "core.windows.net" + + // DefaultAPIVersion is the Azure Storage API version string used when a + // basic client is created. + DefaultAPIVersion = "2014-02-14" + + defaultUseHTTPS = true blobServiceName = "blob" tableServiceName = "table" queueServiceName = "queue" ) -// StorageClient is the object that needs to be constructed -// to perform operations on the storage account. -type StorageClient struct { +// Client is the object that needs to be constructed to perform +// operations on the storage account. +type Client struct { accountName string accountKey []byte - useHttps bool - baseUrl string + useHTTPS bool + baseURL string apiVersion string } @@ -40,10 +47,10 @@ type storageResponse struct { body io.ReadCloser } -// StorageServiceError contains fields of the error response from +// AzureStorageServiceError contains fields of the error response from // Azure Storage Service REST API. See https://msdn.microsoft.com/en-us/library/azure/dd179382.aspx // Some fields might be specific to certain calls. -type StorageServiceError struct { +type AzureStorageServiceError struct { Code string `xml:"Code"` Message string `xml:"Message"` AuthenticationErrorDetail string `xml:"AuthenticationErrorDetail"` @@ -51,25 +58,43 @@ type StorageServiceError struct { QueryParameterValue string `xml:"QueryParameterValue"` Reason string `xml:"Reason"` StatusCode int - RequestId string + RequestID string } -// NewBasicClient constructs a StorageClient with given storage service name -// and key. -func NewBasicClient(accountName, accountKey string) (StorageClient, error) { - return NewClient(accountName, accountKey, DefaultBaseUrl, DefaultApiVersion, defaultUseHttps) +// UnexpectedStatusCodeError is returned when a storage service responds with neither an error +// nor with an HTTP status code indicating success. +type UnexpectedStatusCodeError struct { + allowed []int + got int } -// NewClient constructs a StorageClient. This should be used if the caller -// wants to specify whether to use HTTPS, a specific REST API version or a -// custom storage endpoint than Azure Public Cloud. -func NewClient(accountName, accountKey, blobServiceBaseUrl, apiVersion string, useHttps bool) (StorageClient, error) { - var c StorageClient +func (e UnexpectedStatusCodeError) Error() string { + s := func(i int) string { return fmt.Sprintf("%d %s", i, http.StatusText(i)) } + + got := s(e.got) + expected := []string{} + for _, v := range e.allowed { + expected = append(expected, s(v)) + } + return fmt.Sprintf("storage: status code from service response is %s; was expecting %s", got, strings.Join(expected, " or ")) +} + +// NewBasicClient constructs a Client with given storage service name and +// key. +func NewBasicClient(accountName, accountKey string) (Client, error) { + return NewClient(accountName, accountKey, DefaultBaseURL, DefaultAPIVersion, defaultUseHTTPS) +} + +// NewClient constructs a Client. This should be used if the caller wants +// to specify whether to use HTTPS, a specific REST API version or a custom +// storage endpoint than Azure Public Cloud. +func NewClient(accountName, accountKey, blobServiceBaseURL, apiVersion string, useHTTPS bool) (Client, error) { + var c Client if accountName == "" { return c, fmt.Errorf("azure: account name required") } else if accountKey == "" { return c, fmt.Errorf("azure: account key required") - } else if blobServiceBaseUrl == "" { + } else if blobServiceBaseURL == "" { return c, fmt.Errorf("azure: base storage service url required") } @@ -78,22 +103,22 @@ func NewClient(accountName, accountKey, blobServiceBaseUrl, apiVersion string, u return c, err } - return StorageClient{ + return Client{ accountName: accountName, accountKey: key, - useHttps: useHttps, - baseUrl: blobServiceBaseUrl, + useHTTPS: useHTTPS, + baseURL: blobServiceBaseURL, apiVersion: apiVersion, }, nil } -func (c StorageClient) getBaseUrl(service string) string { +func (c Client) getBaseURL(service string) string { scheme := "http" - if c.useHttps { + if c.useHTTPS { scheme = "https" } - host := fmt.Sprintf("%s.%s.%s", c.accountName, service, c.baseUrl) + host := fmt.Sprintf("%s.%s.%s", c.accountName, service, c.baseURL) u := &url.URL{ Scheme: scheme, @@ -101,8 +126,8 @@ func (c StorageClient) getBaseUrl(service string) string { return u.String() } -func (c StorageClient) getEndpoint(service, path string, params url.Values) string { - u, err := url.Parse(c.getBaseUrl(service)) +func (c Client) getEndpoint(service, path string, params url.Values) string { + u, err := url.Parse(c.getBaseURL(service)) if err != nil { // really should not be happening panic(err) @@ -117,18 +142,24 @@ func (c StorageClient) getEndpoint(service, path string, params url.Values) stri return u.String() } -// GetBlobService returns a BlobStorageClient which can operate on the -// blob service of the storage account. -func (c StorageClient) GetBlobService() *BlobStorageClient { - return &BlobStorageClient{c} +// GetBlobService returns a BlobStorageClient which can operate on the blob +// service of the storage account. +func (c Client) GetBlobService() BlobStorageClient { + return BlobStorageClient{c} } -func (c StorageClient) createAuthorizationHeader(canonicalizedString string) string { +// GetQueueService returns a QueueServiceClient which can operate on the queue +// service of the storage account. +func (c Client) GetQueueService() QueueServiceClient { + return QueueServiceClient{c} +} + +func (c Client) createAuthorizationHeader(canonicalizedString string) string { signature := c.computeHmac256(canonicalizedString) return fmt.Sprintf("%s %s:%s", "SharedKey", c.accountName, signature) } -func (c StorageClient) getAuthorizationHeader(verb, url string, headers map[string]string) (string, error) { +func (c Client) getAuthorizationHeader(verb, url string, headers map[string]string) (string, error) { canonicalizedResource, err := c.buildCanonicalizedResource(url) if err != nil { return "", err @@ -138,14 +169,14 @@ func (c StorageClient) getAuthorizationHeader(verb, url string, headers map[stri return c.createAuthorizationHeader(canonicalizedString), nil } -func (c StorageClient) getStandardHeaders() map[string]string { +func (c Client) getStandardHeaders() map[string]string { return map[string]string{ "x-ms-version": c.apiVersion, "x-ms-date": currentTimeRfc1123Formatted(), } } -func (c StorageClient) buildCanonicalizedHeader(headers map[string]string) string { +func (c Client) buildCanonicalizedHeader(headers map[string]string) string { cm := make(map[string]string) for k, v := range headers { @@ -179,7 +210,7 @@ func (c StorageClient) buildCanonicalizedHeader(headers map[string]string) strin return ch } -func (c StorageClient) buildCanonicalizedResource(uri string) (string, error) { +func (c Client) buildCanonicalizedResource(uri string) (string, error) { errMsg := "buildCanonicalizedResource error: %s" u, err := url.Parse(uri) if err != nil { @@ -220,7 +251,7 @@ func (c StorageClient) buildCanonicalizedResource(uri string) (string, error) { return cr, nil } -func (c StorageClient) buildCanonicalizedString(verb string, headers map[string]string, canonicalizedResource string) string { +func (c Client) buildCanonicalizedString(verb string, headers map[string]string, canonicalizedResource string) string { canonicalizedString := fmt.Sprintf("%s\n%s\n%s\n%s\n%s\n%s\n%s\n%s\n%s\n%s\n%s\n%s\n%s\n%s", verb, headers["Content-Encoding"], @@ -240,7 +271,7 @@ func (c StorageClient) buildCanonicalizedString(verb string, headers map[string] return canonicalizedString } -func (c StorageClient) exec(verb, url string, headers map[string]string, body io.Reader) (*storageResponse, error) { +func (c Client) exec(verb, url string, headers map[string]string, body io.Reader) (*storageResponse, error) { authHeader, err := c.getAuthorizationHeader(verb, url, headers) if err != nil { return nil, err @@ -271,10 +302,10 @@ func (c StorageClient) exec(verb, url string, headers map[string]string, body io if len(respBody) == 0 { // no error in response body - err = fmt.Errorf("storage: service returned without a response body (%s).", resp.Status) + err = fmt.Errorf("storage: service returned without a response body (%s)", resp.Status) } else { // response contains storage service error object, unmarshal - storageErr, errIn := serviceErrFromXml(respBody, resp.StatusCode, resp.Header.Get("x-ms-request-id")) + storageErr, errIn := serviceErrFromXML(respBody, resp.StatusCode, resp.Header.Get("x-ms-request-id")) if err != nil { // error unmarshaling the error response err = errIn } @@ -302,16 +333,27 @@ func readResponseBody(resp *http.Response) ([]byte, error) { return out, err } -func serviceErrFromXml(body []byte, statusCode int, requestId string) (StorageServiceError, error) { - var storageErr StorageServiceError +func serviceErrFromXML(body []byte, statusCode int, requestID string) (AzureStorageServiceError, error) { + var storageErr AzureStorageServiceError if err := xml.Unmarshal(body, &storageErr); err != nil { return storageErr, err } storageErr.StatusCode = statusCode - storageErr.RequestId = requestId + storageErr.RequestID = requestID return storageErr, nil } -func (e StorageServiceError) Error() string { - return fmt.Sprintf("storage: remote server returned error. StatusCode=%d, ErrorCode=%s, ErrorMessage=%s, RequestId=%s", e.StatusCode, e.Code, e.Message, e.RequestId) +func (e AzureStorageServiceError) Error() string { + return fmt.Sprintf("storage: service returned error: StatusCode=%d, ErrorCode=%s, ErrorMessage=%s, RequestId=%s", e.StatusCode, e.Code, e.Message, e.RequestID) +} + +// checkRespCode returns UnexpectedStatusError if the given response code is not +// one of the allowed status codes; otherwise nil. +func checkRespCode(respCode int, allowed []int) error { + for _, v := range allowed { + if respCode == v { + return nil + } + } + return UnexpectedStatusCodeError{allowed, respCode} } diff --git a/Godeps/_workspace/src/github.com/Azure/azure-sdk-for-go/storage/client_test.go b/Godeps/_workspace/src/github.com/Azure/azure-sdk-for-go/storage/client_test.go new file mode 100644 index 000000000..5bc521100 --- /dev/null +++ b/Godeps/_workspace/src/github.com/Azure/azure-sdk-for-go/storage/client_test.go @@ -0,0 +1,156 @@ +package storage + +import ( + "encoding/base64" + "net/url" + "os" + "testing" + + chk "gopkg.in/check.v1" +) + +// Hook up gocheck to testing +func Test(t *testing.T) { chk.TestingT(t) } + +type StorageClientSuite struct{} + +var _ = chk.Suite(&StorageClientSuite{}) + +// getBasicClient returns a test client from storage credentials in the env +func getBasicClient(c *chk.C) Client { + name := os.Getenv("ACCOUNT_NAME") + if name == "" { + c.Fatal("ACCOUNT_NAME not set, need an empty storage account to test") + } + key := os.Getenv("ACCOUNT_KEY") + if key == "" { + c.Fatal("ACCOUNT_KEY not set") + } + cli, err := NewBasicClient(name, key) + c.Assert(err, chk.IsNil) + return cli +} + +func (s *StorageClientSuite) TestGetBaseURL_Basic_Https(c *chk.C) { + cli, err := NewBasicClient("foo", "YmFy") + c.Assert(err, chk.IsNil) + c.Assert(cli.apiVersion, chk.Equals, DefaultAPIVersion) + c.Assert(err, chk.IsNil) + c.Assert(cli.getBaseURL("table"), chk.Equals, "https://foo.table.core.windows.net") +} + +func (s *StorageClientSuite) TestGetBaseURL_Custom_NoHttps(c *chk.C) { + apiVersion := "2015-01-01" // a non existing one + cli, err := NewClient("foo", "YmFy", "core.chinacloudapi.cn", apiVersion, false) + c.Assert(err, chk.IsNil) + c.Assert(cli.apiVersion, chk.Equals, apiVersion) + c.Assert(cli.getBaseURL("table"), chk.Equals, "http://foo.table.core.chinacloudapi.cn") +} + +func (s *StorageClientSuite) TestGetEndpoint_None(c *chk.C) { + cli, err := NewBasicClient("foo", "YmFy") + c.Assert(err, chk.IsNil) + output := cli.getEndpoint(blobServiceName, "", url.Values{}) + c.Assert(output, chk.Equals, "https://foo.blob.core.windows.net/") +} + +func (s *StorageClientSuite) TestGetEndpoint_PathOnly(c *chk.C) { + cli, err := NewBasicClient("foo", "YmFy") + c.Assert(err, chk.IsNil) + output := cli.getEndpoint(blobServiceName, "path", url.Values{}) + c.Assert(output, chk.Equals, "https://foo.blob.core.windows.net/path") +} + +func (s *StorageClientSuite) TestGetEndpoint_ParamsOnly(c *chk.C) { + cli, err := NewBasicClient("foo", "YmFy") + c.Assert(err, chk.IsNil) + params := url.Values{} + params.Set("a", "b") + params.Set("c", "d") + output := cli.getEndpoint(blobServiceName, "", params) + c.Assert(output, chk.Equals, "https://foo.blob.core.windows.net/?a=b&c=d") +} + +func (s *StorageClientSuite) TestGetEndpoint_Mixed(c *chk.C) { + cli, err := NewBasicClient("foo", "YmFy") + c.Assert(err, chk.IsNil) + params := url.Values{} + params.Set("a", "b") + params.Set("c", "d") + output := cli.getEndpoint(blobServiceName, "path", params) + c.Assert(output, chk.Equals, "https://foo.blob.core.windows.net/path?a=b&c=d") +} + +func (s *StorageClientSuite) Test_getStandardHeaders(c *chk.C) { + cli, err := NewBasicClient("foo", "YmFy") + c.Assert(err, chk.IsNil) + + headers := cli.getStandardHeaders() + c.Assert(len(headers), chk.Equals, 2) + c.Assert(headers["x-ms-version"], chk.Equals, cli.apiVersion) + if _, ok := headers["x-ms-date"]; !ok { + c.Fatal("Missing date header") + } +} + +func (s *StorageClientSuite) Test_buildCanonicalizedResource(c *chk.C) { + cli, err := NewBasicClient("foo", "YmFy") + c.Assert(err, chk.IsNil) + + type test struct{ url, expected string } + tests := []test{ + {"https://foo.blob.core.windows.net/path?a=b&c=d", "/foo/path\na:b\nc:d"}, + {"https://foo.blob.core.windows.net/?comp=list", "/foo/\ncomp:list"}, + {"https://foo.blob.core.windows.net/cnt/blob", "/foo/cnt/blob"}, + } + + for _, i := range tests { + out, err := cli.buildCanonicalizedResource(i.url) + c.Assert(err, chk.IsNil) + c.Assert(out, chk.Equals, i.expected) + } +} + +func (s *StorageClientSuite) Test_buildCanonicalizedHeader(c *chk.C) { + cli, err := NewBasicClient("foo", "YmFy") + c.Assert(err, chk.IsNil) + + type test struct { + headers map[string]string + expected string + } + tests := []test{ + {map[string]string{}, ""}, + {map[string]string{"x-ms-foo": "bar"}, "x-ms-foo:bar"}, + {map[string]string{"foo:": "bar"}, ""}, + {map[string]string{"foo:": "bar", "x-ms-foo": "bar"}, "x-ms-foo:bar"}, + {map[string]string{ + "x-ms-version": "9999-99-99", + "x-ms-blob-type": "BlockBlob"}, "x-ms-blob-type:BlockBlob\nx-ms-version:9999-99-99"}} + + for _, i := range tests { + c.Assert(cli.buildCanonicalizedHeader(i.headers), chk.Equals, i.expected) + } +} + +func (s *StorageClientSuite) TestReturnsStorageServiceError(c *chk.C) { + // attempt to delete a nonexisting container + _, err := getBlobClient(c).deleteContainer(randContainer()) + c.Assert(err, chk.NotNil) + + v, ok := err.(AzureStorageServiceError) + c.Check(ok, chk.Equals, true) + c.Assert(v.StatusCode, chk.Equals, 404) + c.Assert(v.Code, chk.Equals, "ContainerNotFound") + c.Assert(v.Code, chk.Not(chk.Equals), "") +} + +func (s *StorageClientSuite) Test_createAuthorizationHeader(c *chk.C) { + key := base64.StdEncoding.EncodeToString([]byte("bar")) + cli, err := NewBasicClient("foo", key) + c.Assert(err, chk.IsNil) + + canonicalizedString := `foobarzoo` + expected := `SharedKey foo:h5U0ATVX6SpbFX1H6GNuxIMeXXCILLoIvhflPtuQZ30=` + c.Assert(cli.createAuthorizationHeader(canonicalizedString), chk.Equals, expected) +} diff --git a/Godeps/_workspace/src/github.com/Azure/azure-sdk-for-go/storage/queue.go b/Godeps/_workspace/src/github.com/Azure/azure-sdk-for-go/storage/queue.go new file mode 100644 index 000000000..fa017f4c8 --- /dev/null +++ b/Godeps/_workspace/src/github.com/Azure/azure-sdk-for-go/storage/queue.go @@ -0,0 +1,230 @@ +package storage + +import ( + "encoding/xml" + "fmt" + "net/http" + "net/url" + "strconv" +) + +// QueueServiceClient contains operations for Microsoft Azure Queue Storage +// Service. +type QueueServiceClient struct { + client Client +} + +func pathForQueue(queue string) string { return fmt.Sprintf("/%s", queue) } +func pathForQueueMessages(queue string) string { return fmt.Sprintf("/%s/messages", queue) } +func pathForMessage(queue, name string) string { return fmt.Sprintf("/%s/messages/%s", queue, name) } + +type putMessageRequest struct { + XMLName xml.Name `xml:"QueueMessage"` + MessageText string `xml:"MessageText"` +} + +// PutMessageParameters is the set of options can be specified for Put Messsage +// operation. A zero struct does not use any preferences for the request. +type PutMessageParameters struct { + VisibilityTimeout int + MessageTTL int +} + +func (p PutMessageParameters) getParameters() url.Values { + out := url.Values{} + if p.VisibilityTimeout != 0 { + out.Set("visibilitytimeout", strconv.Itoa(p.VisibilityTimeout)) + } + if p.MessageTTL != 0 { + out.Set("messagettl", strconv.Itoa(p.MessageTTL)) + } + return out +} + +// GetMessagesParameters is the set of options can be specified for Get +// Messsages operation. A zero struct does not use any preferences for the +// request. +type GetMessagesParameters struct { + NumOfMessages int + VisibilityTimeout int +} + +func (p GetMessagesParameters) getParameters() url.Values { + out := url.Values{} + if p.NumOfMessages != 0 { + out.Set("numofmessages", strconv.Itoa(p.NumOfMessages)) + } + if p.VisibilityTimeout != 0 { + out.Set("visibilitytimeout", strconv.Itoa(p.VisibilityTimeout)) + } + return out +} + +// PeekMessagesParameters is the set of options can be specified for Peek +// Messsage operation. A zero struct does not use any preferences for the +// request. +type PeekMessagesParameters struct { + NumOfMessages int +} + +func (p PeekMessagesParameters) getParameters() url.Values { + out := url.Values{"peekonly": {"true"}} // Required for peek operation + if p.NumOfMessages != 0 { + out.Set("numofmessages", strconv.Itoa(p.NumOfMessages)) + } + return out +} + +// GetMessagesResponse represents a response returned from Get Messages +// operation. +type GetMessagesResponse struct { + XMLName xml.Name `xml:"QueueMessagesList"` + QueueMessagesList []GetMessageResponse `xml:"QueueMessage"` +} + +// GetMessageResponse represents a QueueMessage object returned from Get +// Messages operation response. +type GetMessageResponse struct { + MessageID string `xml:"MessageId"` + InsertionTime string `xml:"InsertionTime"` + ExpirationTime string `xml:"ExpirationTime"` + PopReceipt string `xml:"PopReceipt"` + TimeNextVisible string `xml:"TimeNextVisible"` + DequeueCount int `xml:"DequeueCount"` + MessageText string `xml:"MessageText"` +} + +// PeekMessagesResponse represents a response returned from Get Messages +// operation. +type PeekMessagesResponse struct { + XMLName xml.Name `xml:"QueueMessagesList"` + QueueMessagesList []PeekMessageResponse `xml:"QueueMessage"` +} + +// PeekMessageResponse represents a QueueMessage object returned from Peek +// Messages operation response. +type PeekMessageResponse struct { + MessageID string `xml:"MessageId"` + InsertionTime string `xml:"InsertionTime"` + ExpirationTime string `xml:"ExpirationTime"` + DequeueCount int `xml:"DequeueCount"` + MessageText string `xml:"MessageText"` +} + +// CreateQueue operation creates a queue under the given account. +// +// See https://msdn.microsoft.com/en-us/library/azure/dd179342.aspx +func (c QueueServiceClient) CreateQueue(name string) error { + uri := c.client.getEndpoint(queueServiceName, pathForQueue(name), url.Values{}) + headers := c.client.getStandardHeaders() + headers["Content-Length"] = "0" + resp, err := c.client.exec("PUT", uri, headers, nil) + if err != nil { + return err + } + defer resp.body.Close() + return checkRespCode(resp.statusCode, []int{http.StatusCreated}) +} + +// DeleteQueue operation permanently deletes the specified queue. +// +// See https://msdn.microsoft.com/en-us/library/azure/dd179436.aspx +func (c QueueServiceClient) DeleteQueue(name string) error { + uri := c.client.getEndpoint(queueServiceName, pathForQueue(name), url.Values{}) + resp, err := c.client.exec("DELETE", uri, c.client.getStandardHeaders(), nil) + if err != nil { + return err + } + defer resp.body.Close() + return checkRespCode(resp.statusCode, []int{http.StatusNoContent}) +} + +// QueueExists returns true if a queue with given name exists. +func (c QueueServiceClient) QueueExists(name string) (bool, error) { + uri := c.client.getEndpoint(queueServiceName, pathForQueue(name), url.Values{"comp": {"metadata"}}) + resp, err := c.client.exec("GET", uri, c.client.getStandardHeaders(), nil) + if resp != nil && (resp.statusCode == http.StatusOK || resp.statusCode == http.StatusNotFound) { + return resp.statusCode == http.StatusOK, nil + } + + return false, err +} + +// PutMessage operation adds a new message to the back of the message queue. +// +// See https://msdn.microsoft.com/en-us/library/azure/dd179346.aspx +func (c QueueServiceClient) PutMessage(queue string, message string, params PutMessageParameters) error { + uri := c.client.getEndpoint(queueServiceName, pathForQueueMessages(queue), params.getParameters()) + req := putMessageRequest{MessageText: message} + body, nn, err := xmlMarshal(req) + if err != nil { + return err + } + headers := c.client.getStandardHeaders() + headers["Content-Length"] = strconv.Itoa(nn) + resp, err := c.client.exec("POST", uri, headers, body) + if err != nil { + return err + } + defer resp.body.Close() + return checkRespCode(resp.statusCode, []int{http.StatusCreated}) +} + +// ClearMessages operation deletes all messages from the specified queue. +// +// See https://msdn.microsoft.com/en-us/library/azure/dd179454.aspx +func (c QueueServiceClient) ClearMessages(queue string) error { + uri := c.client.getEndpoint(queueServiceName, pathForQueueMessages(queue), url.Values{}) + resp, err := c.client.exec("DELETE", uri, c.client.getStandardHeaders(), nil) + if err != nil { + return err + } + defer resp.body.Close() + return checkRespCode(resp.statusCode, []int{http.StatusNoContent}) +} + +// GetMessages operation retrieves one or more messages from the front of the +// queue. +// +// See https://msdn.microsoft.com/en-us/library/azure/dd179474.aspx +func (c QueueServiceClient) GetMessages(queue string, params GetMessagesParameters) (GetMessagesResponse, error) { + var r GetMessagesResponse + uri := c.client.getEndpoint(queueServiceName, pathForQueueMessages(queue), params.getParameters()) + resp, err := c.client.exec("GET", uri, c.client.getStandardHeaders(), nil) + if err != nil { + return r, err + } + defer resp.body.Close() + err = xmlUnmarshal(resp.body, &r) + return r, err +} + +// PeekMessages retrieves one or more messages from the front of the queue, but +// does not alter the visibility of the message. +// +// See https://msdn.microsoft.com/en-us/library/azure/dd179472.aspx +func (c QueueServiceClient) PeekMessages(queue string, params PeekMessagesParameters) (PeekMessagesResponse, error) { + var r PeekMessagesResponse + uri := c.client.getEndpoint(queueServiceName, pathForQueueMessages(queue), params.getParameters()) + resp, err := c.client.exec("GET", uri, c.client.getStandardHeaders(), nil) + if err != nil { + return r, err + } + defer resp.body.Close() + err = xmlUnmarshal(resp.body, &r) + return r, err +} + +// DeleteMessage operation deletes the specified message. +// +// See https://msdn.microsoft.com/en-us/library/azure/dd179347.aspx +func (c QueueServiceClient) DeleteMessage(queue, messageID, popReceipt string) error { + uri := c.client.getEndpoint(queueServiceName, pathForMessage(queue, messageID), url.Values{ + "popreceipt": {popReceipt}}) + resp, err := c.client.exec("DELETE", uri, c.client.getStandardHeaders(), nil) + if err != nil { + return err + } + defer resp.body.Close() + return checkRespCode(resp.statusCode, []int{http.StatusNoContent}) +} diff --git a/Godeps/_workspace/src/github.com/Azure/azure-sdk-for-go/storage/queue_test.go b/Godeps/_workspace/src/github.com/Azure/azure-sdk-for-go/storage/queue_test.go new file mode 100644 index 000000000..5c7bad93c --- /dev/null +++ b/Godeps/_workspace/src/github.com/Azure/azure-sdk-for-go/storage/queue_test.go @@ -0,0 +1,91 @@ +package storage + +import ( + chk "gopkg.in/check.v1" +) + +type StorageQueueSuite struct{} + +var _ = chk.Suite(&StorageQueueSuite{}) + +func getQueueClient(c *chk.C) QueueServiceClient { + return getBasicClient(c).GetQueueService() +} + +func (s *StorageQueueSuite) Test_pathForQueue(c *chk.C) { + c.Assert(pathForQueue("q"), chk.Equals, "/q") +} + +func (s *StorageQueueSuite) Test_pathForQueueMessages(c *chk.C) { + c.Assert(pathForQueueMessages("q"), chk.Equals, "/q/messages") +} + +func (s *StorageQueueSuite) Test_pathForMessage(c *chk.C) { + c.Assert(pathForMessage("q", "m"), chk.Equals, "/q/messages/m") +} + +func (s *StorageQueueSuite) TestCreateQueue_DeleteQueue(c *chk.C) { + cli := getQueueClient(c) + name := randString(20) + c.Assert(cli.CreateQueue(name), chk.IsNil) + c.Assert(cli.DeleteQueue(name), chk.IsNil) +} + +func (s *StorageQueueSuite) TestQueueExists(c *chk.C) { + cli := getQueueClient(c) + ok, err := cli.QueueExists("nonexistent-queue") + c.Assert(err, chk.IsNil) + c.Assert(ok, chk.Equals, false) + + name := randString(20) + c.Assert(cli.CreateQueue(name), chk.IsNil) + defer cli.DeleteQueue(name) + + ok, err = cli.QueueExists(name) + c.Assert(err, chk.IsNil) + c.Assert(ok, chk.Equals, true) +} + +func (s *StorageQueueSuite) TestPostMessage_PeekMessage_DeleteMessage(c *chk.C) { + q := randString(20) + cli := getQueueClient(c) + c.Assert(cli.CreateQueue(q), chk.IsNil) + defer cli.DeleteQueue(q) + + msg := randString(64 * 1024) // exercise max length + c.Assert(cli.PutMessage(q, msg, PutMessageParameters{}), chk.IsNil) + r, err := cli.PeekMessages(q, PeekMessagesParameters{}) + c.Assert(err, chk.IsNil) + c.Assert(len(r.QueueMessagesList), chk.Equals, 1) + c.Assert(r.QueueMessagesList[0].MessageText, chk.Equals, msg) +} + +func (s *StorageQueueSuite) TestGetMessages(c *chk.C) { + q := randString(20) + cli := getQueueClient(c) + c.Assert(cli.CreateQueue(q), chk.IsNil) + defer cli.DeleteQueue(q) + + n := 4 + for i := 0; i < n; i++ { + c.Assert(cli.PutMessage(q, randString(10), PutMessageParameters{}), chk.IsNil) + } + + r, err := cli.GetMessages(q, GetMessagesParameters{NumOfMessages: n}) + c.Assert(err, chk.IsNil) + c.Assert(len(r.QueueMessagesList), chk.Equals, n) +} + +func (s *StorageQueueSuite) TestDeleteMessages(c *chk.C) { + q := randString(20) + cli := getQueueClient(c) + c.Assert(cli.CreateQueue(q), chk.IsNil) + defer cli.DeleteQueue(q) + + c.Assert(cli.PutMessage(q, "message", PutMessageParameters{}), chk.IsNil) + r, err := cli.GetMessages(q, GetMessagesParameters{VisibilityTimeout: 1}) + c.Assert(err, chk.IsNil) + c.Assert(len(r.QueueMessagesList), chk.Equals, 1) + m := r.QueueMessagesList[0] + c.Assert(cli.DeleteMessage(q, m.MessageID, m.PopReceipt), chk.IsNil) +} diff --git a/Godeps/_workspace/src/github.com/MSOpenTech/azure-sdk-for-go/storage/util.go b/Godeps/_workspace/src/github.com/Azure/azure-sdk-for-go/storage/util.go similarity index 73% rename from Godeps/_workspace/src/github.com/MSOpenTech/azure-sdk-for-go/storage/util.go rename to Godeps/_workspace/src/github.com/Azure/azure-sdk-for-go/storage/util.go index 8a0f7b945..33155af7f 100644 --- a/Godeps/_workspace/src/github.com/MSOpenTech/azure-sdk-for-go/storage/util.go +++ b/Godeps/_workspace/src/github.com/Azure/azure-sdk-for-go/storage/util.go @@ -1,6 +1,7 @@ package storage import ( + "bytes" "crypto/hmac" "crypto/sha256" "encoding/base64" @@ -13,7 +14,7 @@ import ( "time" ) -func (c StorageClient) computeHmac256(message string) string { +func (c Client) computeHmac256(message string) string { h := hmac.New(sha256.New, c.accountKey) h.Write([]byte(message)) return base64.StdEncoding.EncodeToString(h.Sum(nil)) @@ -47,17 +48,24 @@ func mergeParams(v1, v2 url.Values) url.Values { func prepareBlockListRequest(blocks []Block) string { s := `` for _, v := range blocks { - s += fmt.Sprintf("<%s>%s", v.Status, v.Id, v.Status) + s += fmt.Sprintf("<%s>%s", v.Status, v.ID, v.Status) } s += `` return s } -func xmlUnmarshal(body io.ReadCloser, v interface{}) error { +func xmlUnmarshal(body io.Reader, v interface{}) error { data, err := ioutil.ReadAll(body) if err != nil { return err } - defer body.Close() return xml.Unmarshal(data, v) } + +func xmlMarshal(v interface{}) (io.Reader, int, error) { + b, err := xml.Marshal(v) + if err != nil { + return nil, 0, err + } + return bytes.NewReader(b), len(b), nil +} diff --git a/Godeps/_workspace/src/github.com/Azure/azure-sdk-for-go/storage/util_test.go b/Godeps/_workspace/src/github.com/Azure/azure-sdk-for-go/storage/util_test.go new file mode 100644 index 000000000..9bf82dcc3 --- /dev/null +++ b/Godeps/_workspace/src/github.com/Azure/azure-sdk-for-go/storage/util_test.go @@ -0,0 +1,69 @@ +package storage + +import ( + "encoding/xml" + "io/ioutil" + "net/url" + "strings" + "time" + + chk "gopkg.in/check.v1" +) + +func (s *StorageClientSuite) Test_timeRfc1123Formatted(c *chk.C) { + now := time.Now().UTC() + expectedLayout := "Mon, 02 Jan 2006 15:04:05 GMT" + c.Assert(timeRfc1123Formatted(now), chk.Equals, now.Format(expectedLayout)) +} + +func (s *StorageClientSuite) Test_mergeParams(c *chk.C) { + v1 := url.Values{ + "k1": {"v1"}, + "k2": {"v2"}} + v2 := url.Values{ + "k1": {"v11"}, + "k3": {"v3"}} + out := mergeParams(v1, v2) + c.Assert(out.Get("k1"), chk.Equals, "v1") + c.Assert(out.Get("k2"), chk.Equals, "v2") + c.Assert(out.Get("k3"), chk.Equals, "v3") + c.Assert(out["k1"], chk.DeepEquals, []string{"v1", "v11"}) +} + +func (s *StorageClientSuite) Test_prepareBlockListRequest(c *chk.C) { + empty := []Block{} + expected := `` + c.Assert(prepareBlockListRequest(empty), chk.DeepEquals, expected) + + blocks := []Block{{"foo", BlockStatusLatest}, {"bar", BlockStatusUncommitted}} + expected = `foobar` + c.Assert(prepareBlockListRequest(blocks), chk.DeepEquals, expected) +} + +func (s *StorageClientSuite) Test_xmlUnmarshal(c *chk.C) { + xml := ` + + myblob + ` + var blob Blob + body := ioutil.NopCloser(strings.NewReader(xml)) + c.Assert(xmlUnmarshal(body, &blob), chk.IsNil) + c.Assert(blob.Name, chk.Equals, "myblob") +} + +func (s *StorageClientSuite) Test_xmlMarshal(c *chk.C) { + type t struct { + XMLName xml.Name `xml:"S"` + Name string `xml:"Name"` + } + + b := t{Name: "myblob"} + expected := `myblob` + r, i, err := xmlMarshal(b) + c.Assert(err, chk.IsNil) + o, err := ioutil.ReadAll(r) + c.Assert(err, chk.IsNil) + out := string(o) + c.Assert(out, chk.Equals, expected) + c.Assert(i, chk.Equals, len(expected)) +} diff --git a/Godeps/_workspace/src/github.com/MSOpenTech/azure-sdk-for-go/storage/blob_test.go b/Godeps/_workspace/src/github.com/MSOpenTech/azure-sdk-for-go/storage/blob_test.go deleted file mode 100644 index 33e3d173d..000000000 --- a/Godeps/_workspace/src/github.com/MSOpenTech/azure-sdk-for-go/storage/blob_test.go +++ /dev/null @@ -1,1123 +0,0 @@ -package storage - -import ( - "bytes" - "crypto/rand" - "encoding/base64" - "errors" - "fmt" - "io/ioutil" - "net/http" - "net/url" - "os" - "reflect" - "sort" - "strings" - "sync" - "testing" - "time" -) - -const testContainerPrefix = "zzzztest-" - -func Test_pathForContainer(t *testing.T) { - out := pathForContainer("foo") - if expected := "/foo"; out != expected { - t.Errorf("Wrong pathForContainer. Expected: '%s', got: '%s'", expected, out) - } -} - -func Test_pathForBlob(t *testing.T) { - out := pathForBlob("foo", "blob") - if expected := "/foo/blob"; out != expected { - t.Errorf("Wrong pathForBlob. Expected: '%s', got: '%s'", expected, out) - } -} - -func Test_blobSASStringToSign(t *testing.T) { - _, err := blobSASStringToSign("2012-02-12", "CS", "SE", "SP") - if err == nil { - t.Fatal("Expected error, got nil") - } - - out, err := blobSASStringToSign("2013-08-15", "CS", "SE", "SP") - if err != nil { - t.Fatal(err) - } - if expected := "SP\n\nSE\nCS\n\n2013-08-15\n\n\n\n\n"; out != expected { - t.Errorf("Wrong stringToSign. Expected: '%s', got: '%s'", expected, out) - } -} - -func TestGetBlobSASURI(t *testing.T) { - api, err := NewClient("foo", "YmFy", DefaultBaseUrl, "2013-08-15", true) - if err != nil { - t.Fatal(err) - } - cli := api.GetBlobService() - expiry := time.Time{} - - expectedParts := url.URL{ - Scheme: "https", - Host: "foo.blob.core.windows.net", - Path: "container/name", - RawQuery: url.Values{ - "sv": {"2013-08-15"}, - "sig": {"/OXG7rWh08jYwtU03GzJM0DHZtidRGpC6g69rSGm3I0="}, - "sr": {"b"}, - "sp": {"r"}, - "se": {"0001-01-01T00:00:00Z"}, - }.Encode()} - - u, err := cli.GetBlobSASURI("container", "name", expiry, "r") - if err != nil { - t.Fatal(err) - } - sasParts, err := url.Parse(u) - if err != nil { - t.Fatal(err) - } - - expectedQuery := expectedParts.Query() - sasQuery := sasParts.Query() - - expectedParts.RawQuery = "" // reset - sasParts.RawQuery = "" - - if expectedParts.String() != sasParts.String() { - t.Fatalf("Base URL wrong for SAS. Expected: '%s', got: '%s'", expectedParts, sasParts) - } - - if len(expectedQuery) != len(sasQuery) { - t.Fatalf("Query string wrong for SAS URL. Expected: '%d keys', got: '%d keys'", len(expectedQuery), len(sasQuery)) - } - - for k, v := range expectedQuery { - out, ok := sasQuery[k] - if !ok { - t.Fatalf("Query parameter '%s' not found in generated SAS query. Expected: '%s'", k, v) - } - if !reflect.DeepEqual(v, out) { - t.Fatalf("Wrong value for query parameter '%s'. Expected: '%s', got: '%s'", k, v, out) - } - } -} - -func TestBlobSASURICorrectness(t *testing.T) { - cli, err := getBlobClient() - if err != nil { - t.Fatal(err) - } - cnt := randContainer() - blob := randString(20) - body := []byte(randString(100)) - expiry := time.Now().UTC().Add(time.Hour) - permissions := "r" - - err = cli.CreateContainer(cnt, ContainerAccessTypePrivate) - if err != nil { - t.Fatal(err) - } - defer cli.DeleteContainer(cnt) - - err = cli.PutBlockBlob(cnt, blob, bytes.NewReader(body)) - if err != nil { - t.Fatal(err) - } - - sasUri, err := cli.GetBlobSASURI(cnt, blob, expiry, permissions) - if err != nil { - t.Fatal(err) - } - - resp, err := http.Get(sasUri) - if err != nil { - t.Logf("SAS URI: %s", sasUri) - t.Fatal(err) - } - - blobResp, err := ioutil.ReadAll(resp.Body) - defer resp.Body.Close() - if err != nil { - t.Fatal(err) - } - - if resp.StatusCode != http.StatusOK { - t.Fatalf("Non-ok status code: %s", resp.Status) - } - - if len(blobResp) != len(body) { - t.Fatalf("Wrong blob size on SAS URI. Expected: %d, Got: %d", len(body), len(blobResp)) - } -} - -func TestListContainersPagination(t *testing.T) { - cli, err := getBlobClient() - if err != nil { - t.Fatal(err) - } - - err = deleteTestContainers(cli) - if err != nil { - t.Fatal(err) - } - - const n = 5 - const pageSize = 2 - - // Create test containers - created := []string{} - for i := 0; i < n; i++ { - name := randContainer() - err := cli.CreateContainer(name, ContainerAccessTypePrivate) - if err != nil { - t.Fatalf("Error creating test container: %s", err) - } - created = append(created, name) - } - sort.Strings(created) - - // Defer test container deletions - defer func() { - var wg sync.WaitGroup - for _, cnt := range created { - wg.Add(1) - go func(name string) { - err := cli.DeleteContainer(name) - if err != nil { - t.Logf("Error while deleting test container: %s", err) - } - wg.Done() - }(cnt) - } - wg.Wait() - }() - - // Paginate results - seen := []string{} - marker := "" - for { - resp, err := cli.ListContainers(ListContainersParameters{ - Prefix: testContainerPrefix, - MaxResults: pageSize, - Marker: marker}) - - if err != nil { - t.Fatal(err) - } - - containers := resp.Containers - - if len(containers) > pageSize { - t.Fatalf("Got a bigger page. Expected: %d, got: %d", pageSize, len(containers)) - } - - for _, c := range containers { - seen = append(seen, c.Name) - } - - marker = resp.NextMarker - if marker == "" || len(containers) == 0 { - break - } - } - - // Compare - if !reflect.DeepEqual(created, seen) { - t.Fatalf("Wrong pagination results:\nExpected:\t\t%v\nGot:\t\t%v", created, seen) - } -} - -func TestContainerExists(t *testing.T) { - cnt := randContainer() - - cli, err := getBlobClient() - if err != nil { - t.Fatal(err) - } - - ok, err := cli.ContainerExists(cnt) - if err != nil { - t.Fatal(err) - } - if ok { - t.Fatalf("Non-existing container returned as existing: %s", cnt) - } - - err = cli.CreateContainer(cnt, ContainerAccessTypeBlob) - if err != nil { - t.Fatal(err) - } - defer cli.DeleteContainer(cnt) - - ok, err = cli.ContainerExists(cnt) - if err != nil { - t.Fatal(err) - } - if !ok { - t.Fatalf("Existing container returned as non-existing: %s", cnt) - } -} - -func TestCreateDeleteContainer(t *testing.T) { - cnt := randContainer() - - cli, err := getBlobClient() - if err != nil { - t.Fatal(err) - } - - err = cli.CreateContainer(cnt, ContainerAccessTypePrivate) - if err != nil { - t.Fatal(err) - } - defer cli.DeleteContainer(cnt) - - err = cli.DeleteContainer(cnt) - if err != nil { - t.Fatal(err) - } -} - -func TestCreateContainerIfNotExists(t *testing.T) { - cnt := randContainer() - - cli, err := getBlobClient() - if err != nil { - t.Fatal(err) - } - - // First create - ok, err := cli.CreateContainerIfNotExists(cnt, ContainerAccessTypePrivate) - if err != nil { - t.Fatal(err) - } - if expected := true; ok != expected { - t.Fatalf("Wrong creation status. Expected: %v; Got: %v", expected, ok) - } - - // Second create, should not give errors - ok, err = cli.CreateContainerIfNotExists(cnt, ContainerAccessTypePrivate) - if err != nil { - t.Fatal(err) - } - if expected := false; ok != expected { - t.Fatalf("Wrong creation status. Expected: %v; Got: %v", expected, ok) - } - - defer cli.DeleteContainer(cnt) -} - -func TestDeleteContainerIfExists(t *testing.T) { - cnt := randContainer() - - cli, err := getBlobClient() - if err != nil { - t.Fatal(err) - } - - // Nonexisting container - err = cli.DeleteContainer(cnt) - if err == nil { - t.Fatal("Expected error, got nil") - } - - ok, err := cli.DeleteContainerIfExists(cnt) - if err != nil { - t.Fatalf("Not supposed to return error, got: %s", err) - } - if expected := false; ok != expected { - t.Fatalf("Wrong deletion status. Expected: %v; Got: %v", expected, ok) - } - - // Existing container - err = cli.CreateContainer(cnt, ContainerAccessTypePrivate) - if err != nil { - t.Fatal(err) - } - ok, err = cli.DeleteContainerIfExists(cnt) - if err != nil { - t.Fatalf("Not supposed to return error, got: %s", err) - } - if expected := true; ok != expected { - t.Fatalf("Wrong deletion status. Expected: %v; Got: %v", expected, ok) - } -} - -func TestBlobExists(t *testing.T) { - cnt := randContainer() - blob := randString(20) - - cli, err := getBlobClient() - if err != nil { - t.Fatal(err) - } - - err = cli.CreateContainer(cnt, ContainerAccessTypeBlob) - if err != nil { - t.Fatal(err) - } - defer cli.DeleteContainer(cnt) - err = cli.PutBlockBlob(cnt, blob, strings.NewReader("Hello!")) - if err != nil { - t.Fatal(err) - } - defer cli.DeleteBlob(cnt, blob) - - ok, err := cli.BlobExists(cnt, blob+".foo") - if err != nil { - t.Fatal(err) - } - if ok { - t.Errorf("Non-existing blob returned as existing: %s/%s", cnt, blob) - } - - ok, err = cli.BlobExists(cnt, blob) - if err != nil { - t.Fatal(err) - } - if !ok { - t.Errorf("Existing blob returned as non-existing: %s/%s", cnt, blob) - } -} - -func TestGetBlobUrl(t *testing.T) { - api, err := NewBasicClient("foo", "YmFy") - if err != nil { - t.Fatal(err) - } - cli := api.GetBlobService() - - out := cli.GetBlobUrl("c", "nested/blob") - if expected := "https://foo.blob.core.windows.net/c/nested/blob"; out != expected { - t.Fatalf("Wrong blob URL. Expected: '%s', got:'%s'", expected, out) - } - - out = cli.GetBlobUrl("", "blob") - if expected := "https://foo.blob.core.windows.net/$root/blob"; out != expected { - t.Fatalf("Wrong blob URL. Expected: '%s', got:'%s'", expected, out) - } - - out = cli.GetBlobUrl("", "nested/blob") - if expected := "https://foo.blob.core.windows.net/$root/nested/blob"; out != expected { - t.Fatalf("Wrong blob URL. Expected: '%s', got:'%s'", expected, out) - } -} - -func TestBlobCopy(t *testing.T) { - if testing.Short() { - t.Skip("skipping blob copy in short mode, no SLA on async operation") - } - - cli, err := getBlobClient() - if err != nil { - t.Fatal(err) - } - - cnt := randContainer() - src := randString(20) - dst := randString(20) - body := []byte(randString(1024)) - - err = cli.CreateContainer(cnt, ContainerAccessTypePrivate) - if err != nil { - t.Fatal(err) - } - defer cli.deleteContainer(cnt) - - err = cli.PutBlockBlob(cnt, src, bytes.NewReader(body)) - if err != nil { - t.Fatal(err) - } - defer cli.DeleteBlob(cnt, src) - - err = cli.CopyBlob(cnt, dst, cli.GetBlobUrl(cnt, src)) - if err != nil { - t.Fatal(err) - } - defer cli.DeleteBlob(cnt, dst) - - blobBody, err := cli.GetBlob(cnt, dst) - if err != nil { - t.Fatal(err) - } - - b, err := ioutil.ReadAll(blobBody) - defer blobBody.Close() - if err != nil { - t.Fatal(err) - } - - if !reflect.DeepEqual(body, b) { - t.Fatalf("Copied blob is wrong. Expected: %d bytes, got: %d bytes\n%s\n%s", len(body), len(b), body, b) - } -} - -func TestDeleteBlobIfExists(t *testing.T) { - cnt := randContainer() - blob := randString(20) - - cli, err := getBlobClient() - if err != nil { - t.Fatal(err) - } - - err = cli.DeleteBlob(cnt, blob) - if err == nil { - t.Fatal("Nonexisting blob did not return error") - } - - ok, err := cli.DeleteBlobIfExists(cnt, blob) - if err != nil { - t.Fatalf("Not supposed to return error: %s", err) - } - if expected := false; ok != expected { - t.Fatalf("Wrong deletion status. Expected: %v; Got: %v", expected, ok) - } -} - -func TestGetBlobProperties(t *testing.T) { - cnt := randContainer() - blob := randString(20) - contents := randString(64) - - cli, err := getBlobClient() - if err != nil { - t.Fatal(err) - } - - err = cli.CreateContainer(cnt, ContainerAccessTypePrivate) - if err != nil { - t.Fatal(err) - } - defer cli.DeleteContainer(cnt) - - // Nonexisting blob - _, err = cli.GetBlobProperties(cnt, blob) - if err == nil { - t.Fatal("Did not return error for non-existing blob") - } - - // Put the blob - err = cli.PutBlockBlob(cnt, blob, strings.NewReader(contents)) - if err != nil { - t.Fatal(err) - } - - // Get blob properties - props, err := cli.GetBlobProperties(cnt, blob) - if err != nil { - t.Fatal(err) - } - - if props.ContentLength != int64(len(contents)) { - t.Fatalf("Got wrong Content-Length: '%d', expected: %d", props.ContentLength, len(contents)) - } - if props.BlobType != BlobTypeBlock { - t.Fatalf("Got wrong BlobType. Expected:'%s', got:'%s'", BlobTypeBlock, props.BlobType) - } -} - -func TestListBlobsPagination(t *testing.T) { - cli, err := getBlobClient() - if err != nil { - t.Fatal(err) - } - - cnt := randContainer() - err = cli.CreateContainer(cnt, ContainerAccessTypePrivate) - if err != nil { - t.Fatal(err) - } - defer cli.DeleteContainer(cnt) - - blobs := []string{} - const n = 5 - const pageSize = 2 - for i := 0; i < n; i++ { - name := randString(20) - err := cli.PutBlockBlob(cnt, name, strings.NewReader("Hello, world!")) - if err != nil { - t.Fatal(err) - } - blobs = append(blobs, name) - } - sort.Strings(blobs) - - // Paginate - seen := []string{} - marker := "" - for { - resp, err := cli.ListBlobs(cnt, ListBlobsParameters{ - MaxResults: pageSize, - Marker: marker}) - if err != nil { - t.Fatal(err) - } - - for _, v := range resp.Blobs { - seen = append(seen, v.Name) - } - - marker = resp.NextMarker - if marker == "" || len(resp.Blobs) == 0 { - break - } - } - - // Compare - if !reflect.DeepEqual(blobs, seen) { - t.Fatalf("Got wrong list of blobs. Expected: %s, Got: %s", blobs, seen) - } - - err = cli.DeleteContainer(cnt) - if err != nil { - t.Fatal(err) - } -} - -func TestPutEmptyBlockBlob(t *testing.T) { - cli, err := getBlobClient() - if err != nil { - t.Fatal(err) - } - - cnt := randContainer() - if err := cli.CreateContainer(cnt, ContainerAccessTypePrivate); err != nil { - t.Fatal(err) - } - defer cli.deleteContainer(cnt) - - blob := randString(20) - err = cli.PutBlockBlob(cnt, blob, bytes.NewReader([]byte{})) - if err != nil { - t.Fatal(err) - } - - props, err := cli.GetBlobProperties(cnt, blob) - if err != nil { - t.Fatal(err) - } - if props.ContentLength != 0 { - t.Fatalf("Wrong content length for empty blob: %d", props.ContentLength) - } -} - -func TestPutSingleBlockBlob(t *testing.T) { - cnt := randContainer() - blob := randString(20) - body := []byte(randString(1024)) - - cli, err := getBlobClient() - if err != nil { - t.Fatal(err) - } - - err = cli.CreateContainer(cnt, ContainerAccessTypeBlob) - if err != nil { - t.Fatal(err) - } - defer cli.DeleteContainer(cnt) - - err = cli.PutBlockBlob(cnt, blob, bytes.NewReader(body)) - if err != nil { - t.Fatal(err) - } - defer cli.DeleteBlob(cnt, blob) - - resp, err := cli.GetBlob(cnt, blob) - if err != nil { - t.Fatal(err) - } - - // Verify contents - respBody, err := ioutil.ReadAll(resp) - defer resp.Close() - if err != nil { - t.Fatal(err) - } - - if !reflect.DeepEqual(body, respBody) { - t.Fatalf("Wrong blob contents.\nExpected: %d bytes, Got: %d byes", len(body), len(respBody)) - } - - // Verify block list - blocks, err := cli.GetBlockList(cnt, blob, BlockListTypeAll) - if err != nil { - t.Fatal(err) - } - if expected := 1; len(blocks.CommittedBlocks) != expected { - t.Fatalf("Wrong committed block count. Expected: %d, Got: %d", expected, len(blocks.CommittedBlocks)) - } - if expected := 0; len(blocks.UncommittedBlocks) != expected { - t.Fatalf("Wrong unccommitted block count. Expected: %d, Got: %d", expected, len(blocks.UncommittedBlocks)) - } - thatBlock := blocks.CommittedBlocks[0] - if expected := base64.StdEncoding.EncodeToString([]byte(fmt.Sprintf("%011d", 0))); thatBlock.Name != expected { - t.Fatalf("Wrong block name. Expected: %s, Got: %s", expected, thatBlock.Name) - } -} - -func TestGetBlobRange(t *testing.T) { - cnt := randContainer() - blob := randString(20) - body := "0123456789" - - cli, err := getBlobClient() - if err != nil { - t.Fatal(err) - } - - err = cli.CreateContainer(cnt, ContainerAccessTypeBlob) - if err != nil { - t.Fatal(err) - } - defer cli.DeleteContainer(cnt) - - err = cli.PutBlockBlob(cnt, blob, strings.NewReader(body)) - if err != nil { - t.Fatal(err) - } - defer cli.DeleteBlob(cnt, blob) - - // Read 1-3 - for _, r := range []struct { - rangeStr string - expected string - }{ - {"0-", body}, - {"1-3", body[1 : 3+1]}, - {"3-", body[3:]}, - } { - resp, err := cli.GetBlobRange(cnt, blob, r.rangeStr) - if err != nil { - t.Fatal(err) - } - blobBody, err := ioutil.ReadAll(resp) - if err != nil { - t.Fatal(err) - } - str := string(blobBody) - if str != r.expected { - t.Fatalf("Got wrong range. Expected: '%s'; Got:'%s'", r.expected, str) - } - } -} - -func TestPutBlock(t *testing.T) { - cli, err := getBlobClient() - if err != nil { - t.Fatal(err) - } - - cnt := randContainer() - if err := cli.CreateContainer(cnt, ContainerAccessTypePrivate); err != nil { - t.Fatal(err) - } - defer cli.deleteContainer(cnt) - - blob := randString(20) - chunk := []byte(randString(1024)) - blockId := base64.StdEncoding.EncodeToString([]byte("foo")) - err = cli.PutBlock(cnt, blob, blockId, chunk) - if err != nil { - t.Fatal(err) - } -} - -func TestPutMultiBlockBlob(t *testing.T) { - var ( - cnt = randContainer() - blob = randString(20) - blockSize = 32 * 1024 // 32 KB - body = []byte(randString(blockSize*2 + blockSize/2)) // 3 blocks - ) - - cli, err := getBlobClient() - if err != nil { - t.Fatal(err) - } - - err = cli.CreateContainer(cnt, ContainerAccessTypeBlob) - if err != nil { - t.Fatal(err) - } - defer cli.DeleteContainer(cnt) - - err = cli.putBlockBlob(cnt, blob, bytes.NewReader(body), blockSize) - if err != nil { - t.Fatal(err) - } - defer cli.DeleteBlob(cnt, blob) - - resp, err := cli.GetBlob(cnt, blob) - if err != nil { - t.Fatal(err) - } - - // Verify contents - respBody, err := ioutil.ReadAll(resp) - defer resp.Close() - if err != nil { - t.Fatal(err) - } - if !reflect.DeepEqual(body, respBody) { - t.Fatalf("Wrong blob contents.\nExpected: %d bytes, Got: %d byes", len(body), len(respBody)) - } - - err = cli.DeleteBlob(cnt, blob) - if err != nil { - t.Fatal(err) - } - - err = cli.DeleteContainer(cnt) - if err != nil { - t.Fatal(err) - } -} - -func TestGetBlockList_PutBlockList(t *testing.T) { - cli, err := getBlobClient() - if err != nil { - t.Fatal(err) - } - - cnt := randContainer() - if err := cli.CreateContainer(cnt, ContainerAccessTypePrivate); err != nil { - t.Fatal(err) - } - defer cli.deleteContainer(cnt) - - blob := randString(20) - chunk := []byte(randString(1024)) - blockId := base64.StdEncoding.EncodeToString([]byte("foo")) - - // Put one block - err = cli.PutBlock(cnt, blob, blockId, chunk) - if err != nil { - t.Fatal(err) - } - defer cli.deleteBlob(cnt, blob) - - // Get committed blocks - committed, err := cli.GetBlockList(cnt, blob, BlockListTypeCommitted) - if err != nil { - t.Fatal(err) - } - - if len(committed.CommittedBlocks) > 0 { - t.Fatal("There are committed blocks") - } - - // Get uncommitted blocks - uncommitted, err := cli.GetBlockList(cnt, blob, BlockListTypeUncommitted) - if err != nil { - t.Fatal(err) - } - - if expected := 1; len(uncommitted.UncommittedBlocks) != expected { - t.Fatalf("Uncommitted blocks wrong. Expected: %d, got: %d", expected, len(uncommitted.UncommittedBlocks)) - } - - // Commit block list - err = cli.PutBlockList(cnt, blob, []Block{{blockId, BlockStatusUncommitted}}) - if err != nil { - t.Fatal(err) - } - - // Get all blocks - all, err := cli.GetBlockList(cnt, blob, BlockListTypeAll) - if err != nil { - t.Fatal(err) - } - - if expected := 1; len(all.CommittedBlocks) != expected { - t.Fatalf("Uncommitted blocks wrong. Expected: %d, got: %d", expected, len(uncommitted.CommittedBlocks)) - } - if expected := 0; len(all.UncommittedBlocks) != expected { - t.Fatalf("Uncommitted blocks wrong. Expected: %d, got: %d", expected, len(uncommitted.UncommittedBlocks)) - } - - // Verify the block - thatBlock := all.CommittedBlocks[0] - if expected := blockId; expected != thatBlock.Name { - t.Fatalf("Wrong block name. Expected: %s, got: %s", expected, thatBlock.Name) - } - if expected := int64(len(chunk)); expected != thatBlock.Size { - t.Fatalf("Wrong block name. Expected: %d, got: %d", expected, thatBlock.Size) - } -} - -func TestCreateBlockBlob(t *testing.T) { - cli, err := getBlobClient() - if err != nil { - t.Fatal(err) - } - - cnt := randContainer() - if err := cli.CreateContainer(cnt, ContainerAccessTypePrivate); err != nil { - t.Fatal(err) - } - defer cli.deleteContainer(cnt) - - blob := randString(20) - if err := cli.CreateBlockBlob(cnt, blob); err != nil { - t.Fatal(err) - } - - // Verify - blocks, err := cli.GetBlockList(cnt, blob, BlockListTypeAll) - if err != nil { - t.Fatal(err) - } - if expected, got := 0, len(blocks.CommittedBlocks); expected != got { - t.Fatalf("Got wrong committed block count. Expected: %v, Got:%v ", expected, got) - } - if expected, got := 0, len(blocks.UncommittedBlocks); expected != got { - t.Fatalf("Got wrong uncommitted block count. Expected: %v, Got:%v ", expected, got) - } -} - -func TestPutPageBlob(t *testing.T) { - cli, err := getBlobClient() - if err != nil { - t.Fatal(err) - } - - cnt := randContainer() - if err := cli.CreateContainer(cnt, ContainerAccessTypePrivate); err != nil { - t.Fatal(err) - } - defer cli.deleteContainer(cnt) - - blob := randString(20) - size := int64(10 * 1024 * 1024) - if err := cli.PutPageBlob(cnt, blob, size); err != nil { - t.Fatal(err) - } - - // Verify - props, err := cli.GetBlobProperties(cnt, blob) - if err != nil { - t.Fatal(err) - } - if expected := size; expected != props.ContentLength { - t.Fatalf("Got wrong Content-Length. Expected: %v, Got:%v ", expected, props.ContentLength) - } - if expected := BlobTypePage; expected != props.BlobType { - t.Fatalf("Got wrong x-ms-blob-type. Expected: %v, Got:%v ", expected, props.BlobType) - } -} - -func TestPutPagesUpdate(t *testing.T) { - cli, err := getBlobClient() - if err != nil { - t.Fatal(err) - } - - cnt := randContainer() - if err := cli.CreateContainer(cnt, ContainerAccessTypePrivate); err != nil { - t.Fatal(err) - } - defer cli.deleteContainer(cnt) - - blob := randString(20) - size := int64(10 * 1024 * 1024) // larger than we'll use - if err := cli.PutPageBlob(cnt, blob, size); err != nil { - t.Fatal(err) - } - - chunk1 := []byte(randString(1024)) - chunk2 := []byte(randString(512)) - // Append chunks - if err := cli.PutPage(cnt, blob, 0, int64(len(chunk1)-1), PageWriteTypeUpdate, chunk1); err != nil { - t.Fatal(err) - } - if err := cli.PutPage(cnt, blob, int64(len(chunk1)), int64(len(chunk1)+len(chunk2)-1), PageWriteTypeUpdate, chunk2); err != nil { - t.Fatal(err) - } - - // Verify contents - out, err := cli.GetBlobRange(cnt, blob, fmt.Sprintf("%v-%v", 0, len(chunk1)+len(chunk2))) - if err != nil { - t.Fatal(err) - } - blobContents, err := ioutil.ReadAll(out) - defer out.Close() - if err != nil { - t.Fatal(err) - } - if expected := append(chunk1, chunk2...); reflect.DeepEqual(blobContents, expected) { - t.Fatalf("Got wrong blob.\nGot:%d bytes, Expected:%d bytes", len(blobContents), len(expected)) - } - out.Close() - - // Overwrite first half of chunk1 - chunk0 := []byte(randString(512)) - if err := cli.PutPage(cnt, blob, 0, int64(len(chunk0)-1), PageWriteTypeUpdate, chunk0); err != nil { - t.Fatal(err) - } - - // Verify contents - out, err = cli.GetBlobRange(cnt, blob, fmt.Sprintf("%v-%v", 0, len(chunk1)+len(chunk2))) - if err != nil { - t.Fatal(err) - } - blobContents, err = ioutil.ReadAll(out) - defer out.Close() - if err != nil { - t.Fatal(err) - } - if expected := append(append(chunk0, chunk1[512:]...), chunk2...); reflect.DeepEqual(blobContents, expected) { - t.Fatalf("Got wrong blob.\nGot:%d bytes, Expected:%d bytes", len(blobContents), len(expected)) - } -} - -func TestPutPagesClear(t *testing.T) { - cli, err := getBlobClient() - if err != nil { - t.Fatal(err) - } - - cnt := randContainer() - if err := cli.CreateContainer(cnt, ContainerAccessTypePrivate); err != nil { - t.Fatal(err) - } - defer cli.deleteContainer(cnt) - - blob := randString(20) - size := int64(10 * 1024 * 1024) // larger than we'll use - - if err := cli.PutPageBlob(cnt, blob, size); err != nil { - t.Fatal(err) - } - - // Put 0-2047 - chunk := []byte(randString(2048)) - if err := cli.PutPage(cnt, blob, 0, 2047, PageWriteTypeUpdate, chunk); err != nil { - t.Fatal(err) - } - - // Clear 512-1023 - if err := cli.PutPage(cnt, blob, 512, 1023, PageWriteTypeClear, nil); err != nil { - t.Fatal(err) - } - - // Get blob contents - if out, err := cli.GetBlobRange(cnt, blob, "0-2048"); err != nil { - t.Fatal(err) - } else { - contents, err := ioutil.ReadAll(out) - defer out.Close() - if err != nil { - t.Fatal(err) - } - - if expected := append(append(chunk[:512], make([]byte, 512)...), chunk[1024:]...); reflect.DeepEqual(contents, expected) { - t.Fatalf("Cleared blob is not the same. Expected: (%d) %v; got: (%d) %v", len(expected), expected, len(contents), contents) - } - } -} - -func TestGetPageRanges(t *testing.T) { - cli, err := getBlobClient() - if err != nil { - t.Fatal(err) - } - - cnt := randContainer() - if err := cli.CreateContainer(cnt, ContainerAccessTypePrivate); err != nil { - t.Fatal(err) - } - defer cli.deleteContainer(cnt) - - blob := randString(20) - size := int64(10 * 1024 * 1024) // larger than we'll use - - if err := cli.PutPageBlob(cnt, blob, size); err != nil { - t.Fatal(err) - } - - // Get page ranges on empty blob - if out, err := cli.GetPageRanges(cnt, blob); err != nil { - t.Fatal(err) - } else if len(out.PageList) != 0 { - t.Fatal("Blob has pages") - } - - // Add 0-512 page - err = cli.PutPage(cnt, blob, 0, 511, PageWriteTypeUpdate, []byte(randString(512))) - if err != nil { - t.Fatal(err) - } - - if out, err := cli.GetPageRanges(cnt, blob); err != nil { - t.Fatal(err) - } else if expected := 1; len(out.PageList) != expected { - t.Fatalf("Expected %d pages, got: %d -- %v", expected, len(out.PageList), out.PageList) - } - - // Add 1024-2048 - err = cli.PutPage(cnt, blob, 1024, 2047, PageWriteTypeUpdate, []byte(randString(1024))) - if err != nil { - t.Fatal(err) - } - - if out, err := cli.GetPageRanges(cnt, blob); err != nil { - t.Fatal(err) - } else if expected := 2; len(out.PageList) != expected { - t.Fatalf("Expected %d pages, got: %d -- %v", expected, len(out.PageList), out.PageList) - } -} - -func deleteTestContainers(cli *BlobStorageClient) error { - for { - resp, err := cli.ListContainers(ListContainersParameters{Prefix: testContainerPrefix}) - if err != nil { - return err - } - if len(resp.Containers) == 0 { - break - } - for _, c := range resp.Containers { - err = cli.DeleteContainer(c.Name) - if err != nil { - return err - } - } - } - return nil -} - -func getBlobClient() (*BlobStorageClient, error) { - name := os.Getenv("ACCOUNT_NAME") - if name == "" { - return nil, errors.New("ACCOUNT_NAME not set, need an empty storage account to test") - } - key := os.Getenv("ACCOUNT_KEY") - if key == "" { - return nil, errors.New("ACCOUNT_KEY not set") - } - cli, err := NewBasicClient(name, key) - if err != nil { - return nil, err - } - return cli.GetBlobService(), nil -} - -func randContainer() string { - return testContainerPrefix + randString(32-len(testContainerPrefix)) -} - -func randString(n int) string { - if n <= 0 { - panic("negative number") - } - const alphanum = "0123456789abcdefghijklmnopqrstuvwxyz" - var bytes = make([]byte, n) - rand.Read(bytes) - for i, b := range bytes { - bytes[i] = alphanum[b%byte(len(alphanum))] - } - return string(bytes) -} diff --git a/Godeps/_workspace/src/github.com/MSOpenTech/azure-sdk-for-go/storage/client_test.go b/Godeps/_workspace/src/github.com/MSOpenTech/azure-sdk-for-go/storage/client_test.go deleted file mode 100644 index 844962a60..000000000 --- a/Godeps/_workspace/src/github.com/MSOpenTech/azure-sdk-for-go/storage/client_test.go +++ /dev/null @@ -1,203 +0,0 @@ -package storage - -import ( - "encoding/base64" - "net/url" - "testing" -) - -func TestGetBaseUrl_Basic_Https(t *testing.T) { - cli, err := NewBasicClient("foo", "YmFy") - if err != nil { - t.Fatal(err) - } - - if cli.apiVersion != DefaultApiVersion { - t.Fatalf("Wrong api version. Expected: '%s', got: '%s'", DefaultApiVersion, cli.apiVersion) - } - - if err != nil { - t.Fatal(err) - } - output := cli.getBaseUrl("table") - - if expected := "https://foo.table.core.windows.net"; output != expected { - t.Fatalf("Wrong base url. Expected: '%s', got: '%s'", expected, output) - } -} - -func TestGetBaseUrl_Custom_NoHttps(t *testing.T) { - apiVersion := DefaultApiVersion - cli, err := NewClient("foo", "YmFy", "core.chinacloudapi.cn", apiVersion, false) - if err != nil { - t.Fatal(err) - } - - if cli.apiVersion != apiVersion { - t.Fatalf("Wrong api version. Expected: '%s', got: '%s'", apiVersion, cli.apiVersion) - } - - output := cli.getBaseUrl("table") - - if expected := "http://foo.table.core.chinacloudapi.cn"; output != expected { - t.Fatalf("Wrong base url. Expected: '%s', got: '%s'", expected, output) - } -} - -func TestGetEndpoint_None(t *testing.T) { - cli, err := NewBasicClient("foo", "YmFy") - if err != nil { - t.Fatal(err) - } - output := cli.getEndpoint(blobServiceName, "", url.Values{}) - - if expected := "https://foo.blob.core.windows.net/"; output != expected { - t.Fatalf("Wrong endpoint url. Expected: '%s', got: '%s'", expected, output) - } -} - -func TestGetEndpoint_PathOnly(t *testing.T) { - cli, err := NewBasicClient("foo", "YmFy") - if err != nil { - t.Fatal(err) - } - output := cli.getEndpoint(blobServiceName, "path", url.Values{}) - - if expected := "https://foo.blob.core.windows.net/path"; output != expected { - t.Fatalf("Wrong endpoint url. Expected: '%s', got: '%s'", expected, output) - } -} - -func TestGetEndpoint_ParamsOnly(t *testing.T) { - cli, err := NewBasicClient("foo", "YmFy") - if err != nil { - t.Fatal(err) - } - params := url.Values{} - params.Set("a", "b") - params.Set("c", "d") - output := cli.getEndpoint(blobServiceName, "", params) - - if expected := "https://foo.blob.core.windows.net/?a=b&c=d"; output != expected { - t.Fatalf("Wrong endpoint url. Expected: '%s', got: '%s'", expected, output) - } -} - -func TestGetEndpoint_Mixed(t *testing.T) { - cli, err := NewBasicClient("foo", "YmFy") - if err != nil { - t.Fatal(err) - } - params := url.Values{} - params.Set("a", "b") - params.Set("c", "d") - output := cli.getEndpoint(blobServiceName, "path", params) - - if expected := "https://foo.blob.core.windows.net/path?a=b&c=d"; output != expected { - t.Fatalf("Wrong endpoint url. Expected: '%s', got: '%s'", expected, output) - } -} - -func Test_getStandardHeaders(t *testing.T) { - cli, err := NewBasicClient("foo", "YmFy") - if err != nil { - t.Fatal(err) - } - - headers := cli.getStandardHeaders() - if len(headers) != 2 { - t.Fatal("Wrong standard header count") - } - if v, ok := headers["x-ms-version"]; !ok || v != cli.apiVersion { - t.Fatal("Wrong version header") - } - if _, ok := headers["x-ms-date"]; !ok { - t.Fatal("Missing date header") - } -} - -func Test_buildCanonicalizedResource(t *testing.T) { - cli, err := NewBasicClient("foo", "YmFy") - if err != nil { - t.Fatal(err) - } - - type test struct{ url, expected string } - tests := []test{ - {"https://foo.blob.core.windows.net/path?a=b&c=d", "/foo/path\na:b\nc:d"}, - {"https://foo.blob.core.windows.net/?comp=list", "/foo/\ncomp:list"}, - {"https://foo.blob.core.windows.net/cnt/blob", "/foo/cnt/blob"}, - } - - for _, i := range tests { - if out, err := cli.buildCanonicalizedResource(i.url); err != nil { - t.Fatal(err) - } else if out != i.expected { - t.Fatalf("Wrong canonicalized resource. Expected:\n'%s', Got:\n'%s'", i.expected, out) - } - } -} - -func Test_buildCanonicalizedHeader(t *testing.T) { - cli, err := NewBasicClient("foo", "YmFy") - if err != nil { - t.Fatal(err) - } - - type test struct { - headers map[string]string - expected string - } - tests := []test{ - {map[string]string{}, ""}, - {map[string]string{"x-ms-foo": "bar"}, "x-ms-foo:bar"}, - {map[string]string{"foo:": "bar"}, ""}, - {map[string]string{"foo:": "bar", "x-ms-foo": "bar"}, "x-ms-foo:bar"}, - {map[string]string{ - "x-ms-version": "9999-99-99", - "x-ms-blob-type": "BlockBlob"}, "x-ms-blob-type:BlockBlob\nx-ms-version:9999-99-99"}} - - for _, i := range tests { - if out := cli.buildCanonicalizedHeader(i.headers); out != i.expected { - t.Fatalf("Wrong canonicalized resource. Expected:\n'%s', Got:\n'%s'", i.expected, out) - } - } -} - -func TestReturnsStorageServiceError(t *testing.T) { - cli, err := getBlobClient() - if err != nil { - t.Fatal(err) - } - - // attempt to delete a nonexisting container - _, err = cli.deleteContainer(randContainer()) - if err == nil { - t.Fatal("Service has not returned an error") - } - - if v, ok := err.(StorageServiceError); !ok { - t.Fatal("Cannot assert to specific error") - } else if v.StatusCode != 404 { - t.Fatalf("Expected status:%d, got: %d", 404, v.StatusCode) - } else if v.Code != "ContainerNotFound" { - t.Fatalf("Expected code: %s, got: %s", "ContainerNotFound", v.Code) - } else if v.RequestId == "" { - t.Fatalf("RequestId does not exist") - } -} - -func Test_createAuthorizationHeader(t *testing.T) { - key := base64.StdEncoding.EncodeToString([]byte("bar")) - cli, err := NewBasicClient("foo", key) - if err != nil { - t.Fatal(err) - } - - canonicalizedString := `foobarzoo` - expected := `SharedKey foo:h5U0ATVX6SpbFX1H6GNuxIMeXXCILLoIvhflPtuQZ30=` - - if out := cli.createAuthorizationHeader(canonicalizedString); out != expected { - t.Fatalf("Wrong authorization header. Expected: '%s', Got:'%s'", expected, out) - } -} diff --git a/Godeps/_workspace/src/github.com/MSOpenTech/azure-sdk-for-go/storage/util_test.go b/Godeps/_workspace/src/github.com/MSOpenTech/azure-sdk-for-go/storage/util_test.go deleted file mode 100644 index d1b2c7949..000000000 --- a/Godeps/_workspace/src/github.com/MSOpenTech/azure-sdk-for-go/storage/util_test.go +++ /dev/null @@ -1,80 +0,0 @@ -package storage - -import ( - "io/ioutil" - "net/url" - "reflect" - "strings" - "testing" - "time" -) - -func Test_timeRfc1123Formatted(t *testing.T) { - now := time.Now().UTC() - - expectedLayout := "Mon, 02 Jan 2006 15:04:05 GMT" - expected := now.Format(expectedLayout) - - if output := timeRfc1123Formatted(now); output != expected { - t.Errorf("Expected: %s, got: %s", expected, output) - } -} - -func Test_mergeParams(t *testing.T) { - v1 := url.Values{ - "k1": {"v1"}, - "k2": {"v2"}} - v2 := url.Values{ - "k1": {"v11"}, - "k3": {"v3"}} - - out := mergeParams(v1, v2) - if v := out.Get("k1"); v != "v1" { - t.Errorf("Wrong value for k1: %s", v) - } - - if v := out.Get("k2"); v != "v2" { - t.Errorf("Wrong value for k2: %s", v) - } - - if v := out.Get("k3"); v != "v3" { - t.Errorf("Wrong value for k3: %s", v) - } - - if v := out["k1"]; !reflect.DeepEqual(v, []string{"v1", "v11"}) { - t.Errorf("Wrong multi-value for k1: %s", v) - } -} - -func Test_prepareBlockListRequest(t *testing.T) { - empty := []Block{} - expected := `` - if out := prepareBlockListRequest(empty); expected != out { - t.Errorf("Wrong block list. Expected: '%s', got: '%s'", expected, out) - } - - blocks := []Block{{"foo", BlockStatusLatest}, {"bar", BlockStatusUncommitted}} - expected = `foobar` - if out := prepareBlockListRequest(blocks); expected != out { - t.Errorf("Wrong block list. Expected: '%s', got: '%s'", expected, out) - } -} - -func Test_xmlUnmarshal(t *testing.T) { - xml := ` - - myblob - ` - - body := ioutil.NopCloser(strings.NewReader(xml)) - - var blob Blob - err := xmlUnmarshal(body, &blob) - if err != nil { - t.Fatal(err) - } - - if blob.Name != "myblob" { - t.Fatal("Got wrong value") - } -} diff --git a/cmd/registry/main.go b/cmd/registry/main.go index 49132bf1b..df1359172 100644 --- a/cmd/registry/main.go +++ b/cmd/registry/main.go @@ -22,6 +22,7 @@ import ( _ "github.com/docker/distribution/registry/auth/token" "github.com/docker/distribution/registry/handlers" "github.com/docker/distribution/registry/listener" + _ "github.com/docker/distribution/registry/storage/driver/azure" _ "github.com/docker/distribution/registry/storage/driver/filesystem" _ "github.com/docker/distribution/registry/storage/driver/inmemory" _ "github.com/docker/distribution/registry/storage/driver/middleware/cloudfront" diff --git a/docs/building.md b/docs/building.md index cd534c715..5c40e0eb7 100644 --- a/docs/building.md +++ b/docs/building.md @@ -139,8 +139,3 @@ To enable the [Ceph RADOS storage driver](storage-drivers/rados.md) ```sh export DOCKER_BUILDTAGS='include_rados' ``` - -To enable the [Azure storage driver](storage-drivers/azure.md), use the -`include_azure` build tag. - - diff --git a/docs/storage-drivers/azure.md b/docs/storage-drivers/azure.md index 01cba5c37..f0c16f04e 100644 --- a/docs/storage-drivers/azure.md +++ b/docs/storage-drivers/azure.md @@ -17,11 +17,5 @@ The following parameters must be used to authenticate and configure the storage * `container`: Name of the root storage container in which all registry data will be stored. Must comply the storage container name [requirements][create-container-api]. * `realm`: (optional) Domain name suffix for the Storage Service API endpoint. Defaults to `core.windows.net`. For example realm for "Azure in China" would be `core.chinacloudapi.cn` and realm for "Azure Government" would be `core.usgovcloudapi.net`. -## Developing - -To include this driver when building Docker Distribution, use the build tag -`include_azure`. Please see the [building documentation][building] for details. - [azure-blob-storage]: http://azure.microsoft.com/en-us/services/storage/ [create-container-api]: https://msdn.microsoft.com/en-us/library/azure/dd179468.aspx -[building]: https://github.com/docker/distribution/blob/master/docs/building.md#optional-build-tags diff --git a/registry/storage/driver/azure/azure.go b/registry/storage/driver/azure/azure.go index d21a8259b..cbb959812 100644 --- a/registry/storage/driver/azure/azure.go +++ b/registry/storage/driver/azure/azure.go @@ -16,7 +16,7 @@ import ( "github.com/docker/distribution/registry/storage/driver/base" "github.com/docker/distribution/registry/storage/driver/factory" - azure "github.com/MSOpenTech/azure-sdk-for-go/storage" + azure "github.com/Azure/azure-sdk-for-go/storage" ) const driverName = "azure" @@ -68,7 +68,7 @@ func FromParameters(parameters map[string]interface{}) (*Driver, error) { realm, ok := parameters[paramRealm] if !ok || fmt.Sprint(realm) == "" { - realm = azure.DefaultBaseUrl + realm = azure.DefaultBaseURL } return New(fmt.Sprint(accountName), fmt.Sprint(accountKey), fmt.Sprint(container), fmt.Sprint(realm)) @@ -76,7 +76,7 @@ func FromParameters(parameters map[string]interface{}) (*Driver, error) { // New constructs a new Driver with the given Azure Storage Account credentials func New(accountName, accountKey, container, realm string) (*Driver, error) { - api, err := azure.NewClient(accountName, accountKey, realm, azure.DefaultApiVersion, true) + api, err := azure.NewClient(accountName, accountKey, realm, azure.DefaultAPIVersion, true) if err != nil { return nil, err } @@ -89,7 +89,7 @@ func New(accountName, accountKey, container, realm string) (*Driver, error) { } d := &driver{ - client: *blobClient, + client: blobClient, container: container} return &Driver{baseEmbed: baseEmbed{Base: base.Base{StorageDriver: d}}}, nil } @@ -114,7 +114,16 @@ func (d *driver) GetContent(ctx context.Context, path string) ([]byte, error) { // PutContent stores the []byte content at a location designated by "path". func (d *driver) PutContent(ctx context.Context, path string, contents []byte) error { - return d.client.PutBlockBlob(d.container, path, ioutil.NopCloser(bytes.NewReader(contents))) + if _, err := d.client.DeleteBlobIfExists(d.container, path); err != nil { + return err + } + if err := d.client.CreateBlockBlob(d.container, path); err != nil { + return err + } + bs := newAzureBlockStorage(d.client) + bw := newRandomBlobWriter(&bs, azure.MaxBlobBlockSize) + _, err := bw.WriteBlobAt(d.container, path, 0, bytes.NewReader(contents)) + return err } // ReadStream retrieves an io.ReadCloser for the content stored at "path" with a @@ -233,7 +242,7 @@ func (d *driver) List(ctx context.Context, path string) ([]string, error) { // Move moves an object stored at sourcePath to destPath, removing the original // object. func (d *driver) Move(ctx context.Context, sourcePath string, destPath string) error { - sourceBlobURL := d.client.GetBlobUrl(d.container, sourcePath) + sourceBlobURL := d.client.GetBlobURL(d.container, sourcePath) err := d.client.CopyBlob(d.container, destPath, sourceBlobURL) if err != nil { if is404(err) { @@ -352,6 +361,6 @@ func (d *driver) listBlobs(container, virtPath string) ([]string, error) { } func is404(err error) bool { - e, ok := err.(azure.StorageServiceError) + e, ok := err.(azure.AzureStorageServiceError) return ok && e.StatusCode == http.StatusNotFound } diff --git a/registry/storage/driver/azure/blockblob.go b/registry/storage/driver/azure/blockblob.go index 10b2bf216..1c1df899c 100644 --- a/registry/storage/driver/azure/blockblob.go +++ b/registry/storage/driver/azure/blockblob.go @@ -4,7 +4,7 @@ import ( "fmt" "io" - azure "github.com/MSOpenTech/azure-sdk-for-go/storage" + azure "github.com/Azure/azure-sdk-for-go/storage" ) // azureBlockStorage is adaptor between azure.BlobStorageClient and diff --git a/registry/storage/driver/azure/blockblob_test.go b/registry/storage/driver/azure/blockblob_test.go index c29b4742c..7ce471957 100644 --- a/registry/storage/driver/azure/blockblob_test.go +++ b/registry/storage/driver/azure/blockblob_test.go @@ -6,7 +6,7 @@ import ( "io" "io/ioutil" - azure "github.com/MSOpenTech/azure-sdk-for-go/storage" + azure "github.com/Azure/azure-sdk-for-go/storage" ) type StorageSimulator struct { @@ -122,12 +122,12 @@ func (s *StorageSimulator) PutBlockList(container, blob string, blocks []azure.B var blockIDs []string for _, v := range blocks { - bl, ok := bb.blocks[v.Id] + bl, ok := bb.blocks[v.ID] if !ok { // check if block ID exists - return fmt.Errorf("Block id '%s' not found", v.Id) + return fmt.Errorf("Block id '%s' not found", v.ID) } bl.committed = true - blockIDs = append(blockIDs, v.Id) + blockIDs = append(blockIDs, v.ID) } // Mark all other blocks uncommitted diff --git a/registry/storage/driver/azure/blockid.go b/registry/storage/driver/azure/blockid.go index f6bda6a86..776c7cd59 100644 --- a/registry/storage/driver/azure/blockid.go +++ b/registry/storage/driver/azure/blockid.go @@ -7,7 +7,7 @@ import ( "sync" "time" - azure "github.com/MSOpenTech/azure-sdk-for-go/storage" + azure "github.com/Azure/azure-sdk-for-go/storage" ) type blockIDGenerator struct { diff --git a/registry/storage/driver/azure/blockid_test.go b/registry/storage/driver/azure/blockid_test.go index 6569e15d7..aab70202a 100644 --- a/registry/storage/driver/azure/blockid_test.go +++ b/registry/storage/driver/azure/blockid_test.go @@ -4,7 +4,7 @@ import ( "math" "testing" - azure "github.com/MSOpenTech/azure-sdk-for-go/storage" + azure "github.com/Azure/azure-sdk-for-go/storage" ) func Test_blockIdGenerator(t *testing.T) { diff --git a/registry/storage/driver/azure/randomwriter.go b/registry/storage/driver/azure/randomwriter.go index b570d5593..f18692d0b 100644 --- a/registry/storage/driver/azure/randomwriter.go +++ b/registry/storage/driver/azure/randomwriter.go @@ -5,7 +5,7 @@ import ( "io" "io/ioutil" - azure "github.com/MSOpenTech/azure-sdk-for-go/storage" + azure "github.com/Azure/azure-sdk-for-go/storage" ) // blockStorage is the interface required from a block storage service @@ -75,7 +75,7 @@ func (r *randomBlobWriter) WriteBlobAt(container, blob string, offset int64, chu // Use existing block list var existingBlocks []azure.Block for _, v := range blocks.CommittedBlocks { - existingBlocks = append(existingBlocks, azure.Block{Id: v.Name, Status: azure.BlockStatusCommitted}) + existingBlocks = append(existingBlocks, azure.Block{ID: v.Name, Status: azure.BlockStatusCommitted}) } blockList = append(existingBlocks, blockList...) } @@ -111,7 +111,7 @@ func (r *randomBlobWriter) writeChunkToBlocks(container, blob string, chunk io.R if err := r.bs.PutBlock(container, blob, blockID, data); err != nil { return newBlocks, nn, err } - newBlocks = append(newBlocks, azure.Block{Id: blockID, Status: azure.BlockStatusUncommitted}) + newBlocks = append(newBlocks, azure.Block{ID: blockID, Status: azure.BlockStatusUncommitted}) } return newBlocks, nn, nil } @@ -131,7 +131,7 @@ func (r *randomBlobWriter) blocksLeftSide(container, blob string, writeOffset in for _, v := range bx.CommittedBlocks { blkSize := int64(v.Size) if o >= blkSize { // use existing block - left = append(left, azure.Block{Id: v.Name, Status: azure.BlockStatusCommitted}) + left = append(left, azure.Block{ID: v.Name, Status: azure.BlockStatusCommitted}) o -= blkSize elapsed += blkSize } else if o > 0 { // current block needs to be splitted @@ -150,7 +150,7 @@ func (r *randomBlobWriter) blocksLeftSide(container, blob string, writeOffset in if err = r.bs.PutBlock(container, blob, newBlockID, data); err != nil { return left, err } - left = append(left, azure.Block{Id: newBlockID, Status: azure.BlockStatusUncommitted}) + left = append(left, azure.Block{ID: newBlockID, Status: azure.BlockStatusUncommitted}) break } } @@ -177,7 +177,7 @@ func (r *randomBlobWriter) blocksRightSide(container, blob string, writeOffset i ) if bs > re { // take the block as is - right = append(right, azure.Block{Id: v.Name, Status: azure.BlockStatusCommitted}) + right = append(right, azure.Block{ID: v.Name, Status: azure.BlockStatusCommitted}) } else if be > re { // current block needs to be splitted part, err := r.bs.GetSectionReader(container, blob, re+1, be-(re+1)+1) if err != nil { @@ -192,7 +192,7 @@ func (r *randomBlobWriter) blocksRightSide(container, blob string, writeOffset i if err = r.bs.PutBlock(container, blob, newBlockID, data); err != nil { return right, err } - right = append(right, azure.Block{Id: newBlockID, Status: azure.BlockStatusUncommitted}) + right = append(right, azure.Block{ID: newBlockID, Status: azure.BlockStatusUncommitted}) } elapsed += int64(v.Size) } diff --git a/registry/storage/driver/azure/randomwriter_test.go b/registry/storage/driver/azure/randomwriter_test.go index 2c7480dbf..32c2509e4 100644 --- a/registry/storage/driver/azure/randomwriter_test.go +++ b/registry/storage/driver/azure/randomwriter_test.go @@ -9,7 +9,7 @@ import ( "strings" "testing" - azure "github.com/MSOpenTech/azure-sdk-for-go/storage" + azure "github.com/Azure/azure-sdk-for-go/storage" ) func TestRandomWriter_writeChunkToBlocks(t *testing.T) {