Merge pull request #713 from BrianBland/ng-client

Adds a low level registry http client interface and implementation

client/client.go

@@ -0,0 +1,506 @@
+package client
+
+import (
+	"bytes"
+	"encoding/json"
+	"fmt"
+	"io"
+	"net/http"
+	"net/url"
+	"regexp"
+	"strconv"
+
+	"github.com/docker/docker-registry"
+)
+
+// Client implements the client interface to the registry http api
+type Client interface {
+	// GetImageManifest returns an image manifest for the image at the given
+	// name, tag pair
+	GetImageManifest(name, tag string) (*registry.ImageManifest, error)
+
+	// PutImageManifest uploads an image manifest for the image at the given
+	// name, tag pair
+	PutImageManifest(name, tag string, imageManifest *registry.ImageManifest) error
+
+	// DeleteImage removes the image at the given name, tag pair
+	DeleteImage(name, tag string) error
+
+	// ListImageTags returns a list of all image tags with the given repository
+	// name
+	ListImageTags(name string) ([]string, error)
+
+	// GetImageLayer returns the image layer at the given name, tarsum pair in
+	// the form of an io.ReadCloser with the length of this layer
+	// A nonzero byteOffset can be provided to receive a partial layer beginning
+	// at the given offset
+	GetImageLayer(name, tarsum string, byteOffset int) (io.ReadCloser, int, error)
+
+	// InitiateLayerUpload starts an image upload for the given name, tarsum
+	// pair and returns a unique location url to use for other layer upload
+	// methods
+	// Returns a *registry.LayerAlreadyExistsError if the layer already exists
+	// on the registry
+	InitiateLayerUpload(name, tarsum string) (string, error)
+
+	// GetLayerUploadStatus returns the byte offset and length of the layer at
+	// the given upload location
+	GetLayerUploadStatus(location string) (int, int, error)
+
+	// UploadLayer uploads a full image layer to the registry
+	UploadLayer(location string, layer io.ReadCloser, length int, checksum *registry.Checksum) error
+
+	// UploadLayerChunk uploads a layer chunk with a given length and startByte
+	// to the registry
+	// FinishChunkedLayerUpload must be called to finalize this upload
+	UploadLayerChunk(location string, layerChunk io.ReadCloser, length, startByte int) error
+
+	// FinishChunkedLayerUpload completes a chunked layer upload at a given
+	// location
+	FinishChunkedLayerUpload(location string, length int, checksum *registry.Checksum) error
+
+	// CancelLayerUpload deletes all content at the unfinished layer upload
+	// location and invalidates any future calls to this layer upload
+	CancelLayerUpload(location string) error
+}
+
+// New returns a new Client which operates against a registry with the
+// given base endpoint
+// This endpoint should not include /v2/ or any part of the url after this
+func New(endpoint string) Client {
+	return &clientImpl{endpoint}
+}
+
+// clientImpl is the default implementation of the Client interface
+type clientImpl struct {
+	Endpoint string
+}
+
+// TODO(bbland): use consistent route generation between server and client
+
+func (r *clientImpl) GetImageManifest(name, tag string) (*registry.ImageManifest, error) {
+	response, err := http.Get(r.imageManifestUrl(name, tag))
+	if err != nil {
+		return nil, err
+	}
+	defer response.Body.Close()
+
+	// TODO(bbland): handle other status codes, like 5xx errors
+	switch {
+	case response.StatusCode == http.StatusOK:
+		break
+	case response.StatusCode == http.StatusNotFound:
+		return nil, &registry.ImageManifestNotFoundError{name, tag}
+	case response.StatusCode >= 400 && response.StatusCode < 500:
+		errors := new(registry.Errors)
+		decoder := json.NewDecoder(response.Body)
+		err = decoder.Decode(&errors)
+		if err != nil {
+			return nil, err
+		}
+		return nil, errors
+	default:
+		return nil, &registry.UnexpectedHttpStatusError{response.Status}
+	}
+
+	decoder := json.NewDecoder(response.Body)
+
+	manifest := new(registry.ImageManifest)
+	err = decoder.Decode(manifest)
+	if err != nil {
+		return nil, err
+	}
+	return manifest, nil
+}
+
+func (r *clientImpl) PutImageManifest(name, tag string, manifest *registry.ImageManifest) error {
+	manifestBytes, err := json.Marshal(manifest)
+	if err != nil {
+		return err
+	}
+
+	putRequest, err := http.NewRequest("PUT",
+		r.imageManifestUrl(name, tag), bytes.NewReader(manifestBytes))
+	if err != nil {
+		return err
+	}
+
+	response, err := http.DefaultClient.Do(putRequest)
+	if err != nil {
+		return err
+	}
+	defer response.Body.Close()
+
+	// TODO(bbland): handle other status codes, like 5xx errors
+	switch {
+	case response.StatusCode == http.StatusOK:
+		return nil
+	case response.StatusCode >= 400 && response.StatusCode < 500:
+		errors := new(registry.Errors)
+		decoder := json.NewDecoder(response.Body)
+		err = decoder.Decode(&errors)
+		if err != nil {
+			return err
+		}
+		return errors
+	default:
+		return &registry.UnexpectedHttpStatusError{response.Status}
+	}
+}
+
+func (r *clientImpl) DeleteImage(name, tag string) error {
+	deleteRequest, err := http.NewRequest("DELETE",
+		r.imageManifestUrl(name, tag), nil)
+	if err != nil {
+		return err
+	}
+
+	response, err := http.DefaultClient.Do(deleteRequest)
+	if err != nil {
+		return err
+	}
+	defer response.Body.Close()
+
+	// TODO(bbland): handle other status codes, like 5xx errors
+	switch {
+	case response.StatusCode == http.StatusNoContent:
+		break
+	case response.StatusCode == http.StatusNotFound:
+		return &registry.ImageManifestNotFoundError{name, tag}
+	case response.StatusCode >= 400 && response.StatusCode < 500:
+		errors := new(registry.Errors)
+		decoder := json.NewDecoder(response.Body)
+		err = decoder.Decode(&errors)
+		if err != nil {
+			return err
+		}
+		return errors
+	default:
+		return &registry.UnexpectedHttpStatusError{response.Status}
+	}
+
+	return nil
+}
+
+func (r *clientImpl) ListImageTags(name string) ([]string, error) {
+	response, err := http.Get(fmt.Sprintf("%s/v2/%s/tags", r.Endpoint, name))
+	if err != nil {
+		return nil, err
+	}
+	defer response.Body.Close()
+
+	// TODO(bbland): handle other status codes, like 5xx errors
+	switch {
+	case response.StatusCode == http.StatusOK:
+		break
+	case response.StatusCode == http.StatusNotFound:
+		return nil, &registry.RepositoryNotFoundError{name}
+	case response.StatusCode >= 400 && response.StatusCode < 500:
+		errors := new(registry.Errors)
+		decoder := json.NewDecoder(response.Body)
+		err = decoder.Decode(&errors)
+		if err != nil {
+			return nil, err
+		}
+		return nil, errors
+	default:
+		return nil, &registry.UnexpectedHttpStatusError{response.Status}
+	}
+
+	tags := struct {
+		Tags []string `json:"tags"`
+	}{}
+
+	decoder := json.NewDecoder(response.Body)
+	err = decoder.Decode(&tags)
+	if err != nil {
+		return nil, err
+	}
+
+	return tags.Tags, nil
+}
+
+func (r *clientImpl) GetImageLayer(name, tarsum string, byteOffset int) (io.ReadCloser, int, error) {
+	getRequest, err := http.NewRequest("GET",
+		fmt.Sprintf("%s/v2/%s/layer/%s", r.Endpoint, name, tarsum), nil)
+	if err != nil {
+		return nil, 0, err
+	}
+
+	getRequest.Header.Add("Range", fmt.Sprintf("%d-", byteOffset))
+	response, err := http.DefaultClient.Do(getRequest)
+	if err != nil {
+		return nil, 0, err
+	}
+
+	if response.StatusCode == http.StatusNotFound {
+		return nil, 0, &registry.LayerNotFoundError{name, tarsum}
+	}
+	// TODO(bbland): handle other status codes, like 5xx errors
+	switch {
+	case response.StatusCode == http.StatusOK:
+		lengthHeader := response.Header.Get("Content-Length")
+		length, err := strconv.ParseInt(lengthHeader, 10, 0)
+		if err != nil {
+			return nil, 0, err
+		}
+		return response.Body, int(length), nil
+	case response.StatusCode == http.StatusNotFound:
+		response.Body.Close()
+		return nil, 0, &registry.LayerNotFoundError{name, tarsum}
+	case response.StatusCode >= 400 && response.StatusCode < 500:
+		errors := new(registry.Errors)
+		decoder := json.NewDecoder(response.Body)
+		err = decoder.Decode(&errors)
+		if err != nil {
+			return nil, 0, err
+		}
+		return nil, 0, errors
+	default:
+		response.Body.Close()
+		return nil, 0, &registry.UnexpectedHttpStatusError{response.Status}
+	}
+}
+
+func (r *clientImpl) InitiateLayerUpload(name, tarsum string) (string, error) {
+	postRequest, err := http.NewRequest("POST",
+		fmt.Sprintf("%s/v2/%s/layer/%s/upload", r.Endpoint, name, tarsum), nil)
+	if err != nil {
+		return "", err
+	}
+
+	response, err := http.DefaultClient.Do(postRequest)
+	if err != nil {
+		return "", err
+	}
+	defer response.Body.Close()
+
+	// TODO(bbland): handle other status codes, like 5xx errors
+	switch {
+	case response.StatusCode == http.StatusAccepted:
+		return response.Header.Get("Location"), nil
+	case response.StatusCode == http.StatusNotModified:
+		return "", &registry.LayerAlreadyExistsError{name, tarsum}
+	case response.StatusCode >= 400 && response.StatusCode < 500:
+		errors := new(registry.Errors)
+		decoder := json.NewDecoder(response.Body)
+		err = decoder.Decode(&errors)
+		if err != nil {
+			return "", err
+		}
+		return "", errors
+	default:
+		return "", &registry.UnexpectedHttpStatusError{response.Status}
+	}
+}
+
+func (r *clientImpl) GetLayerUploadStatus(location string) (int, int, error) {
+	response, err := http.Get(fmt.Sprintf("%s%s", r.Endpoint, location))
+	if err != nil {
+		return 0, 0, err
+	}
+	defer response.Body.Close()
+
+	// TODO(bbland): handle other status codes, like 5xx errors
+	switch {
+	case response.StatusCode == http.StatusNoContent:
+		return parseRangeHeader(response.Header.Get("Range"))
+	case response.StatusCode == http.StatusNotFound:
+		return 0, 0, &registry.LayerUploadNotFoundError{location}
+	case response.StatusCode >= 400 && response.StatusCode < 500:
+		errors := new(registry.Errors)
+		decoder := json.NewDecoder(response.Body)
+		err = decoder.Decode(&errors)
+		if err != nil {
+			return 0, 0, err
+		}
+		return 0, 0, errors
+	default:
+		return 0, 0, &registry.UnexpectedHttpStatusError{response.Status}
+	}
+}
+
+func (r *clientImpl) UploadLayer(location string, layer io.ReadCloser, length int, checksum *registry.Checksum) error {
+	defer layer.Close()
+
+	putRequest, err := http.NewRequest("PUT",
+		fmt.Sprintf("%s%s", r.Endpoint, location), layer)
+	if err != nil {
+		return err
+	}
+
+	queryValues := new(url.Values)
+	queryValues.Set("length", fmt.Sprint(length))
+	queryValues.Set(checksum.HashAlgorithm, checksum.Sum)
+	putRequest.URL.RawQuery = queryValues.Encode()
+
+	putRequest.Header.Set("Content-Type", "application/octet-stream")
+	putRequest.Header.Set("Content-Length", fmt.Sprint(length))
+
+	response, err := http.DefaultClient.Do(putRequest)
+	if err != nil {
+		return err
+	}
+	defer response.Body.Close()
+
+	// TODO(bbland): handle other status codes, like 5xx errors
+	switch {
+	case response.StatusCode == http.StatusCreated:
+		return nil
+	case response.StatusCode == http.StatusNotFound:
+		return &registry.LayerUploadNotFoundError{location}
+	case response.StatusCode >= 400 && response.StatusCode < 500:
+		errors := new(registry.Errors)
+		decoder := json.NewDecoder(response.Body)
+		err = decoder.Decode(&errors)
+		if err != nil {
+			return err
+		}
+		return errors
+	default:
+		return &registry.UnexpectedHttpStatusError{response.Status}
+	}
+}
+
+func (r *clientImpl) UploadLayerChunk(location string, layerChunk io.ReadCloser, length, startByte int) error {
+	defer layerChunk.Close()
+
+	putRequest, err := http.NewRequest("PUT",
+		fmt.Sprintf("%s%s", r.Endpoint, location), layerChunk)
+	if err != nil {
+		return err
+	}
+
+	endByte := startByte + length
+
+	putRequest.Header.Set("Content-Type", "application/octet-stream")
+	putRequest.Header.Set("Content-Length", fmt.Sprint(length))
+	putRequest.Header.Set("Content-Range",
+		fmt.Sprintf("%d-%d/%d", startByte, endByte, endByte))
+
+	response, err := http.DefaultClient.Do(putRequest)
+	if err != nil {
+		return err
+	}
+	defer response.Body.Close()
+
+	// TODO(bbland): handle other status codes, like 5xx errors
+	switch {
+	case response.StatusCode == http.StatusAccepted:
+		return nil
+	case response.StatusCode == http.StatusRequestedRangeNotSatisfiable:
+		lastValidRange, layerSize, err := parseRangeHeader(response.Header.Get("Range"))
+		if err != nil {
+			return err
+		}
+		return &registry.LayerUploadInvalidRangeError{location, lastValidRange, layerSize}
+	case response.StatusCode == http.StatusNotFound:
+		return &registry.LayerUploadNotFoundError{location}
+	case response.StatusCode >= 400 && response.StatusCode < 500:
+		errors := new(registry.Errors)
+		decoder := json.NewDecoder(response.Body)
+		err = decoder.Decode(&errors)
+		if err != nil {
+			return err
+		}
+		return errors
+	default:
+		return &registry.UnexpectedHttpStatusError{response.Status}
+	}
+}
+
+func (r *clientImpl) FinishChunkedLayerUpload(location string, length int, checksum *registry.Checksum) error {
+	putRequest, err := http.NewRequest("PUT",
+		fmt.Sprintf("%s%s", r.Endpoint, location), nil)
+	if err != nil {
+		return err
+	}
+
+	queryValues := new(url.Values)
+	queryValues.Set("length", fmt.Sprint(length))
+	queryValues.Set(checksum.HashAlgorithm, checksum.Sum)
+	putRequest.URL.RawQuery = queryValues.Encode()
+
+	putRequest.Header.Set("Content-Type", "application/octet-stream")
+	putRequest.Header.Set("Content-Length", "0")
+	putRequest.Header.Set("Content-Range",
+		fmt.Sprintf("%d-%d/%d", length, length, length))
+
+	response, err := http.DefaultClient.Do(putRequest)
+	if err != nil {
+		return err
+	}
+	defer response.Body.Close()
+
+	// TODO(bbland): handle other status codes, like 5xx errors
+	switch {
+	case response.StatusCode == http.StatusCreated:
+		return nil
+	case response.StatusCode == http.StatusNotFound:
+		return &registry.LayerUploadNotFoundError{location}
+	case response.StatusCode >= 400 && response.StatusCode < 500:
+		errors := new(registry.Errors)
+		decoder := json.NewDecoder(response.Body)
+		err = decoder.Decode(&errors)
+		if err != nil {
+			return err
+		}
+		return errors
+	default:
+		return &registry.UnexpectedHttpStatusError{response.Status}
+	}
+}
+
+func (r *clientImpl) CancelLayerUpload(location string) error {
+	deleteRequest, err := http.NewRequest("DELETE",
+		fmt.Sprintf("%s%s", r.Endpoint, location), nil)
+	if err != nil {
+		return err
+	}
+
+	response, err := http.DefaultClient.Do(deleteRequest)
+	if err != nil {
+		return err
+	}
+	defer response.Body.Close()
+
+	// TODO(bbland): handle other status codes, like 5xx errors
+	switch {
+	case response.StatusCode == http.StatusNoContent:
+		return nil
+	case response.StatusCode == http.StatusNotFound:
+		return &registry.LayerUploadNotFoundError{location}
+	case response.StatusCode >= 400 && response.StatusCode < 500:
+		errors := new(registry.Errors)
+		decoder := json.NewDecoder(response.Body)
+		err = decoder.Decode(&errors)
+		if err != nil {
+			return err
+		}
+		return errors
+	default:
+		return &registry.UnexpectedHttpStatusError{response.Status}
+	}
+}
+
+// imageManifestUrl is a helper method for returning the full url to an image
+// manifest
+func (r *clientImpl) imageManifestUrl(name, tag string) string {
+	return fmt.Sprintf("%s/v2/%s/image/%s", r.Endpoint, name, tag)
+}
+
+// parseRangeHeader parses out the offset and length from a returned Range
+// header
+func parseRangeHeader(byteRangeHeader string) (int, int, error) {
+	r := regexp.MustCompile("bytes=0-(\\d+)/(\\d+)")
+	submatches := r.FindStringSubmatch(byteRangeHeader)
+	offset, err := strconv.ParseInt(submatches[1], 10, 0)
+	if err != nil {
+		return 0, 0, err
+	}
+	length, err := strconv.ParseInt(submatches[2], 10, 0)
+	if err != nil {
+		return 0, 0, err
+	}
+	return int(offset), int(length), nil
+}

errors.go

@@ -162,16 +162,97 @@ func (errs *Errors) Error() string {
 	}
 }
 
-// detailUnknownLayer provides detail for unknown layer errors, returned by
+// DetailUnknownLayer provides detail for unknown layer errors, returned by
 // image manifest push for layers that are not yet transferred. This intended
 // to only be used on the backend to return detail for this specific error.
 type DetailUnknownLayer struct {
 
 	// Unknown should contain the contents of a layer descriptor, which is a
-	// single json object with the key "blobSum" currently.
+	// single FSLayer currently.
-	Unknown struct {
+	Unknown FSLayer `json:"unknown"`
-
+}
-		// BlobSum contains the uniquely identifying tarsum of the layer.
+
-		BlobSum string `json:"blobSum"`
+// RepositoryNotFoundError is returned when making an operation against a
-	} `json:"unknown"`
+// repository that does not exist in the registry
+type RepositoryNotFoundError struct {
+	Name string
+}
+
+func (e *RepositoryNotFoundError) Error() string {
+	return fmt.Sprintf("No repository found with Name: %s", e.Name)
+}
+
+// ImageManifestNotFoundError is returned when making an operation against a
+// given image manifest that does not exist in the registry
+type ImageManifestNotFoundError struct {
+	Name string
+	Tag  string
+}
+
+func (e *ImageManifestNotFoundError) Error() string {
+	return fmt.Sprintf("No manifest found with Name: %s, Tag: %s",
+		e.Name, e.Tag)
+}
+
+// LayerAlreadyExistsError is returned when attempting to create a new layer
+// that already exists in the registry
+type LayerAlreadyExistsError struct {
+	Name   string
+	TarSum string
+}
+
+func (e *LayerAlreadyExistsError) Error() string {
+	return fmt.Sprintf("Layer already found with Name: %s, TarSum: %s",
+		e.Name, e.TarSum)
+}
+
+// LayerNotFoundError is returned when making an operation against a given image
+// layer that does not exist in the registry
+type LayerNotFoundError struct {
+	Name   string
+	TarSum string
+}
+
+func (e *LayerNotFoundError) Error() string {
+	return fmt.Sprintf("No layer found with Name: %s, TarSum: %s",
+		e.Name, e.TarSum)
+}
+
+// LayerUploadNotFoundError is returned when making a layer upload operation
+// against an invalid layer upload location url
+// This may be the result of using a cancelled, completed, or stale upload
+// locationn
+type LayerUploadNotFoundError struct {
+	Location string
+}
+
+func (e *LayerUploadNotFoundError) Error() string {
+	return fmt.Sprintf("No layer found upload found at Location: %s",
+		e.Location)
+}
+
+// LayerUploadInvalidRangeError is returned when attempting to upload an image
+// layer chunk that is out of order
+// This provides the known LayerSize and LastValidRange which can be used to
+// resume the upload
+type LayerUploadInvalidRangeError struct {
+	Location       string
+	LastValidRange int
+	LayerSize      int
+}
+
+func (e *LayerUploadInvalidRangeError) Error() string {
+	return fmt.Sprintf(
+		"Invalid range provided for upload at Location: %s. Last Valid Range: %d, Layer Size: %d",
+		e.Location, e.LastValidRange, e.LayerSize)
+}
+
+// UnexpectedHttpStatusError is returned when an unexpected http status is
+// returned when making a registry api call
+type UnexpectedHttpStatusError struct {
+	Status string
+}
+
+func (e *UnexpectedHttpStatusError) Error() string {
+	return fmt.Sprintf("Received unexpected http status: %s", e.Status)
 }

images.go

@@ -1,11 +1,75 @@
 package registry
 
 import (
+	"encoding/json"
 	"net/http"
 
 	"github.com/gorilla/handlers"
 )
 
+// ImageManifest defines the structure of an image manifest
+type ImageManifest struct {
+	// Name is the name of the image's repository
+	Name string `json:"name"`
+
+	// Tag is the tag of the image specified by this manifest
+	Tag string `json:"tag"`
+
+	// Architecture is the host architecture on which this image is intended to
+	// run
+	Architecture string `json:"architecture"`
+
+	// FSLayers is a list of filesystem layer blobSums contained in this image
+	FSLayers []FSLayer `json:"fsLayers"`
+
+	// History is a list of unstructured historical data for v1 compatibility
+	History []ManifestHistory `json:"history"`
+
+	// SchemaVersion is the image manifest schema that this image follows
+	SchemaVersion int `json:"schemaVersion"`
+
+	// Raw is the byte representation of the ImageManifest, used for signature
+	// verification
+	Raw []byte `json:"-"`
+}
+
+// imageManifest is used to avoid recursion in unmarshaling
+type imageManifest ImageManifest
+
+func (m *ImageManifest) UnmarshalJSON(b []byte) error {
+	var manifest imageManifest
+	err := json.Unmarshal(b, &manifest)
+	if err != nil {
+		return err
+	}
+
+	*m = ImageManifest(manifest)
+	m.Raw = b
+	return nil
+}
+
+// FSLayer is a container struct for BlobSums defined in an image manifest
+type FSLayer struct {
+	// BlobSum is the tarsum of the referenced filesystem image layer
+	BlobSum string `json:"blobSum"`
+}
+
+// ManifestHistory stores unstructured v1 compatibility information
+type ManifestHistory struct {
+	// V1Compatibility is the raw v1 compatibility information
+	V1Compatibility string `json:"v1Compatibility"`
+}
+
+// Checksum is a container struct for an image checksum
+type Checksum struct {
+	// HashAlgorithm is the algorithm used to compute the checksum
+	// Supported values: md5, sha1, sha256, sha512
+	HashAlgorithm string
+
+	// Sum is the actual checksum value for the given HashAlgorithm
+	Sum string
+}
+
 // imageManifestDispatcher takes the request context and builds the
 // appropriate handler for handling image manifest requests.
 func imageManifestDispatcher(ctx *Context, r *http.Request) http.Handler {