diff --git a/docs/spec/api.md b/docs/spec/api.md index cc447fba..6cc2345c 100644 --- a/docs/spec/api.md +++ b/docs/spec/api.md @@ -132,6 +132,9 @@ specification to correspond with the versions enumerated here.
  • Deleting a manifest by tag has been deprecated.
  • Specified `Docker-Content-Digest` header for appropriate entities.
  • Added error code for unsupported operations.
    • +
  • Added capability of doing streaming upload to PATCH blob upload.
    • +
  • Updated PUT blob upload to no longer take final chunk, now requires entire data or no data.
    • +
  • Removed 416 return code from PUT blob upload.
    • @@ -2189,6 +2192,158 @@ The error codes that may be included in the response body are enumerated below: Upload a chunk of data for the specified upload. +##### Stream upload + +``` +PATCH /v2//blobs/uploads/ +Host: +Authorization: +Content-Type: application/octet-stream + + +``` + +Upload a stream of data to upload without completing the upload. + + +The following parameters should be specified on the request: + +|Name|Kind|Description| +|----|----|-----------| +|`Host`|header|Standard HTTP Host Header. Should be set to the registry host.| +|`Authorization`|header|An RFC7235 compliant authorization header.| +|`name`|path|Name of the target repository.| +|`uuid`|path|A uuid identifying the upload. This field can accept characters that match `[a-zA-Z0-9-_.=]+`.| + + + + +###### On Success: Data Accepted + +``` +204 No Content +Location: /v2//blobs/uploads/ +Range: 0- +Content-Length: 0 +Docker-Upload-UUID: +``` + +The stream of data has been accepted and the current progress is available in the range header. The updated upload location is available in the `Location` header. + +The following headers will be returned with the response: + +|Name|Description| +|----|-----------| +|`Location`|The location of the upload. Clients should assume this changes after each request. Clients should use the contents verbatim to complete the upload, adding parameters where required.| +|`Range`|Range indicating the current progress of the upload.| +|`Content-Length`|The `Content-Length` header must be zero and the body must be empty.| +|`Docker-Upload-UUID`|Identifies the docker upload uuid for the current request.| + + + + +###### On Failure: Bad Request + +``` +400 Bad Request +Content-Type: application/json; charset=utf-8 + +{ + "errors:" [ + { + "code": , + "message": "", + "detail": ... + }, + ... + ] +} +``` + +There was an error processing the upload and it must be restarted. + + + +The error codes that may be included in the response body are enumerated below: + +|Code|Message|Description| +-------|----|------|------------ +| `DIGEST_INVALID` | provided digest did not match uploaded content | When a blob is uploaded, the registry will check that the content matches the digest provided by the client. The error may include a detail structure with the key "digest", including the invalid digest string. This error may also be returned when a manifest includes an invalid layer digest. | +| `NAME_INVALID` | invalid repository name | Invalid repository name encountered either during manifest validation or any API operation. | +| `BLOB_UPLOAD_INVALID` | blob upload invalid | The blob upload encountered an error and can no longer proceed. | + + + +###### On Failure: Unauthorized + +``` +401 Unauthorized +WWW-Authenticate: realm="", ..." +Content-Length: +Content-Type: application/json; charset=utf-8 + +{ + "errors:" [ + { + "code": "UNAUTHORIZED", + "message": "access to the requested resource is not authorized", + "detail": ... + }, + ... + ] +} +``` + +The client does not have access to push to the repository. + +The following headers will be returned on the response: + +|Name|Description| +|----|-----------| +|`WWW-Authenticate`|An RFC7235 compliant authentication challenge header.| +|`Content-Length`|Length of the JSON error response body.| + + + +The error codes that may be included in the response body are enumerated below: + +|Code|Message|Description| +-------|----|------|------------ +| `UNAUTHORIZED` | access to the requested resource is not authorized | The access controller denied access for the operation on a resource. Often this will be accompanied by a 401 Unauthorized response status. | + + + +###### On Failure: Not Found + +``` +404 Not Found +Content-Type: application/json; charset=utf-8 + +{ + "errors:" [ + { + "code": , + "message": "", + "detail": ... + }, + ... + ] +} +``` + +The upload is unknown to the registry. The upload must be restarted. + + + +The error codes that may be included in the response body are enumerated below: + +|Code|Message|Description| +-------|----|------|------------ +| `BLOB_UPLOAD_UNKNOWN` | blob upload unknown to registry | If a blob upload has been cancelled or was never started, this error code may be returned. | + + + +##### Chunked upload ``` PATCH /v2//blobs/uploads/ @@ -2201,7 +2356,7 @@ Content-Type: application/octet-stream ``` -Upload a chunk of data to specified upload without completing the upload. +Upload a chunk of data to specified upload without completing the upload. The data will be uploaded to the specified Content Range. The following parameters should be specified on the request: @@ -2364,14 +2519,13 @@ Complete the upload specified by `uuid`, optionally appending the body as the fi PUT /v2//blobs/uploads/?digest= Host: Authorization: -Content-Range: - -Content-Length: +Content-Length: Content-Type: application/octet-stream - + ``` -Complete the upload, providing the _final_ chunk of data, if necessary. This method may take a body with all the data. If the `Content-Range` header is specified, it may include the final chunk. A request without a body will just complete the upload with previously uploaded content. +Complete the upload, providing all the data in the body, if necessary. A request without a body will just complete the upload with previously uploaded content. The following parameters should be specified on the request: @@ -2380,8 +2534,7 @@ The following parameters should be specified on the request: |----|----|-----------| |`Host`|header|Standard HTTP Host Header. Should be set to the registry host.| |`Authorization`|header|An RFC7235 compliant authorization header.| -|`Content-Range`|header|Range of bytes identifying the block of content represented by the body. Start must the end offset retrieved via status check plus one. Note that this is a non-standard use of the `Content-Range` header. May be omitted if no data is provided.| -|`Content-Length`|header|Length of the chunk being uploaded, corresponding to the length of the request body. May be zero if no data is provided.| +|`Content-Length`|header|Length of the data being uploaded, corresponding to the length of the request body. May be zero if no data is provided.| |`name`|path|Name of the target repository.| |`uuid`|path|A uuid identifying the upload. This field can accept characters that match `[a-zA-Z0-9-_.=]+`.| |`digest`|query|Digest of uploaded blob.| @@ -2514,25 +2667,6 @@ The error codes that may be included in the response body are enumerated below: -###### On Failure: Requested Range Not Satisfiable - -``` -416 Requested Range Not Satisfiable -Location: /v2//blobs/uploads/ -Range: 0- -``` - -The `Content-Range` specification cannot be accepted, either because it does not overlap with the current progress or it is invalid. The contents of the `Range` header may be used to resolve the condition. - -The following headers will be returned on the response: - -|Name|Description| -|----|-----------| -|`Location`|The location of the upload. Clients should assume this changes after each request. Clients should use the contents verbatim to complete the upload, adding parameters where required.| -|`Range`|Range indicating the current progress of the upload.| - - - #### DELETE Blob Upload diff --git a/docs/spec/api.md.tmpl b/docs/spec/api.md.tmpl index 6f89c7d5..b61cca76 100644 --- a/docs/spec/api.md.tmpl +++ b/docs/spec/api.md.tmpl @@ -132,6 +132,9 @@ specification to correspond with the versions enumerated here.
  • Deleting a manifest by tag has been deprecated.
  • Specified `Docker-Content-Digest` header for appropriate entities.
  • Added error code for unsupported operations.
    • +
  • Added capability of doing streaming upload to PATCH blob upload.
    • +
  • Updated PUT blob upload to no longer take final chunk, now requires entire data or no data.
    • +
  • Removed 416 return code from PUT blob upload.
    • diff --git a/docs/spec/implementations.md b/docs/spec/implementations.md new file mode 100644 index 00000000..5cec148f --- /dev/null +++ b/docs/spec/implementations.md @@ -0,0 +1,26 @@ +# Distribution API Implementations + +This is a list of known implementations of the Distribution API spec. + +## [Docker Distribution Registry](https://github.com/docker/distribution) + +Docker distribution is the reference implementation of the distribution API +specification. It aims to fully implement the entire specification. + +### Releases +#### 2.0.1 (_in development_) +Implements API 2.0.1 + +_Known Issues_ + - No resumable push support + - Content ranges ignored + - Blob upload status will always return a starting range of 0 + +#### 2.0.0 +Implements API 2.0.0 + +_Known Issues_ + - No resumable push support + - No PATCH implementation for blob upload + - Content ranges ignored + diff --git a/registry/api/v2/descriptors.go b/registry/api/v2/descriptors.go index 0baa5ee7..d7c4a880 100644 --- a/registry/api/v2/descriptors.go +++ b/registry/api/v2/descriptors.go @@ -1055,7 +1055,74 @@ var routeDescriptors = []RouteDescriptor{ Description: "Upload a chunk of data for the specified upload.", Requests: []RequestDescriptor{ { - Description: "Upload a chunk of data to specified upload without completing the upload.", + Name: "Stream upload", + Description: "Upload a stream of data to upload without completing the upload.", + PathParameters: []ParameterDescriptor{ + nameParameterDescriptor, + uuidParameterDescriptor, + }, + Headers: []ParameterDescriptor{ + hostHeader, + authHeader, + }, + Body: BodyDescriptor{ + ContentType: "application/octet-stream", + Format: "", + }, + Successes: []ResponseDescriptor{ + { + Name: "Data Accepted", + Description: "The stream of data has been accepted and the current progress is available in the range header. The updated upload location is available in the `Location` header.", + StatusCode: http.StatusNoContent, + Headers: []ParameterDescriptor{ + { + Name: "Location", + Type: "url", + Format: "/v2//blobs/uploads/", + Description: "The location of the upload. Clients should assume this changes after each request. Clients should use the contents verbatim to complete the upload, adding parameters where required.", + }, + { + Name: "Range", + Type: "header", + Format: "0-", + Description: "Range indicating the current progress of the upload.", + }, + contentLengthZeroHeader, + dockerUploadUUIDHeader, + }, + }, + }, + Failures: []ResponseDescriptor{ + { + Description: "There was an error processing the upload and it must be restarted.", + StatusCode: http.StatusBadRequest, + ErrorCodes: []ErrorCode{ + ErrorCodeDigestInvalid, + ErrorCodeNameInvalid, + ErrorCodeBlobUploadInvalid, + }, + Body: BodyDescriptor{ + ContentType: "application/json; charset=utf-8", + Format: errorsBody, + }, + }, + unauthorizedResponsePush, + { + Description: "The upload is unknown to the registry. The upload must be restarted.", + StatusCode: http.StatusNotFound, + ErrorCodes: []ErrorCode{ + ErrorCodeBlobUploadUnknown, + }, + Body: BodyDescriptor{ + ContentType: "application/json; charset=utf-8", + Format: errorsBody, + }, + }, + }, + }, + { + Name: "Chunked upload", + Description: "Upload a chunk of data to specified upload without completing the upload. The data will be uploaded to the specified Content Range.", PathParameters: []ParameterDescriptor{ nameParameterDescriptor, uuidParameterDescriptor, @@ -1143,26 +1210,15 @@ var routeDescriptors = []RouteDescriptor{ Description: "Complete the upload specified by `uuid`, optionally appending the body as the final chunk.", Requests: []RequestDescriptor{ { - // TODO(stevvooe): Break this down into three separate requests: - // 1. Complete an upload where all data has already been sent. - // 2. Complete an upload where the entire body is in the PUT. - // 3. Complete an upload where the final, partial chunk is the body. - - Description: "Complete the upload, providing the _final_ chunk of data, if necessary. This method may take a body with all the data. If the `Content-Range` header is specified, it may include the final chunk. A request without a body will just complete the upload with previously uploaded content.", + Description: "Complete the upload, providing all the data in the body, if necessary. A request without a body will just complete the upload with previously uploaded content.", Headers: []ParameterDescriptor{ hostHeader, authHeader, - { - Name: "Content-Range", - Type: "header", - Format: "-", - Description: "Range of bytes identifying the block of content represented by the body. Start must the end offset retrieved via status check plus one. Note that this is a non-standard use of the `Content-Range` header. May be omitted if no data is provided.", - }, { Name: "Content-Length", Type: "integer", - Format: "", - Description: "Length of the chunk being uploaded, corresponding to the length of the request body. May be zero if no data is provided.", + Format: "", + Description: "Length of the data being uploaded, corresponding to the length of the request body. May be zero if no data is provided.", }, }, PathParameters: []ParameterDescriptor{ @@ -1181,7 +1237,7 @@ var routeDescriptors = []RouteDescriptor{ }, Body: BodyDescriptor{ ContentType: "application/octet-stream", - Format: "", + Format: "", }, Successes: []ResponseDescriptor{ { @@ -1232,24 +1288,6 @@ var routeDescriptors = []RouteDescriptor{ Format: errorsBody, }, }, - { - Description: "The `Content-Range` specification cannot be accepted, either because it does not overlap with the current progress or it is invalid. The contents of the `Range` header may be used to resolve the condition.", - StatusCode: http.StatusRequestedRangeNotSatisfiable, - Headers: []ParameterDescriptor{ - { - Name: "Location", - Type: "url", - Format: "/v2//blobs/uploads/", - Description: "The location of the upload. Clients should assume this changes after each request. Clients should use the contents verbatim to complete the upload, adding parameters where required.", - }, - { - Name: "Range", - Type: "header", - Format: "0-", - Description: "Range indicating the current progress of the upload.", - }, - }, - }, }, }, }, diff --git a/registry/handlers/api_test.go b/registry/handlers/api_test.go index 3dd7e6ec..1e31477f 100644 --- a/registry/handlers/api_test.go +++ b/registry/handlers/api_test.go @@ -209,6 +209,13 @@ func TestLayerAPI(t *testing.T) { uploadURLBase, uploadUUID = startPushLayer(t, env.builder, imageName) pushLayer(t, env.builder, imageName, layerDigest, uploadURLBase, layerFile) + // ------------------------------------------ + // Now, push just a chunk + layerFile.Seek(0, 0) + + uploadURLBase, uploadUUID = startPushLayer(t, env.builder, imageName) + uploadURLBase, dgst := pushChunk(t, env.builder, imageName, uploadURLBase, layerFile, layerLength) + finishUpload(t, env.builder, imageName, uploadURLBase, dgst) // ------------------------ // Use a head request to see if the layer exists. resp, err = http.Head(layerURL) @@ -616,6 +623,75 @@ func pushLayer(t *testing.T, ub *v2.URLBuilder, name string, dgst digest.Digest, return resp.Header.Get("Location") } +func finishUpload(t *testing.T, ub *v2.URLBuilder, name string, uploadURLBase string, dgst digest.Digest) string { + resp, err := doPushLayer(t, ub, name, dgst, uploadURLBase, nil) + if err != nil { + t.Fatalf("unexpected error doing push layer request: %v", err) + } + defer resp.Body.Close() + + checkResponse(t, "putting monolithic chunk", resp, http.StatusCreated) + + expectedLayerURL, err := ub.BuildBlobURL(name, dgst) + if err != nil { + t.Fatalf("error building expected layer url: %v", err) + } + + checkHeaders(t, resp, http.Header{ + "Location": []string{expectedLayerURL}, + "Content-Length": []string{"0"}, + "Docker-Content-Digest": []string{dgst.String()}, + }) + + return resp.Header.Get("Location") +} + +func doPushChunk(t *testing.T, uploadURLBase string, body io.Reader) (*http.Response, digest.Digest, error) { + u, err := url.Parse(uploadURLBase) + if err != nil { + t.Fatalf("unexpected error parsing pushLayer url: %v", err) + } + + u.RawQuery = url.Values{ + "_state": u.Query()["_state"], + }.Encode() + + uploadURL := u.String() + + digester := digest.NewCanonicalDigester() + + req, err := http.NewRequest("PATCH", uploadURL, io.TeeReader(body, digester)) + if err != nil { + t.Fatalf("unexpected error creating new request: %v", err) + } + req.Header.Set("Content-Type", "application/octet-stream") + + resp, err := http.DefaultClient.Do(req) + + return resp, digester.Digest(), err +} + +func pushChunk(t *testing.T, ub *v2.URLBuilder, name string, uploadURLBase string, body io.Reader, length int64) (string, digest.Digest) { + resp, dgst, err := doPushChunk(t, uploadURLBase, body) + if err != nil { + t.Fatalf("unexpected error doing push layer request: %v", err) + } + defer resp.Body.Close() + + checkResponse(t, "putting chunk", resp, http.StatusAccepted) + + if err != nil { + t.Fatalf("error generating sha256 digest of body") + } + + checkHeaders(t, resp, http.Header{ + "Range": []string{fmt.Sprintf("0-%d", length-1)}, + "Content-Length": []string{"0"}, + }) + + return resp.Header.Get("Location"), dgst +} + func checkResponse(t *testing.T, msg string, resp *http.Response, expectedStatus int) { if resp.StatusCode != expectedStatus { t.Logf("unexpected status %s: %v != %v", msg, resp.StatusCode, expectedStatus) diff --git a/registry/handlers/layerupload.go b/registry/handlers/layerupload.go index 5cfa4554..1591d98d 100644 --- a/registry/handlers/layerupload.go +++ b/registry/handlers/layerupload.go @@ -23,11 +23,10 @@ func layerUploadDispatcher(ctx *Context, r *http.Request) http.Handler { } handler := http.Handler(handlers.MethodHandler{ - "POST": http.HandlerFunc(luh.StartLayerUpload), - "GET": http.HandlerFunc(luh.GetUploadStatus), - "HEAD": http.HandlerFunc(luh.GetUploadStatus), - // TODO(stevvooe): Must implement patch support. - // "PATCH": http.HandlerFunc(luh.PutLayerChunk), + "POST": http.HandlerFunc(luh.StartLayerUpload), + "GET": http.HandlerFunc(luh.GetUploadStatus), + "HEAD": http.HandlerFunc(luh.GetUploadStatus), + "PATCH": http.HandlerFunc(luh.PatchLayerData), "PUT": http.HandlerFunc(luh.PutLayerUploadComplete), "DELETE": http.HandlerFunc(luh.CancelLayerUpload), }) @@ -133,7 +132,7 @@ func (luh *layerUploadHandler) StartLayerUpload(w http.ResponseWriter, r *http.R luh.Upload = upload defer luh.Upload.Close() - if err := luh.layerUploadResponse(w, r); err != nil { + if err := luh.layerUploadResponse(w, r, true); err != nil { w.WriteHeader(http.StatusInternalServerError) // Error conditions here? luh.Errors.Push(v2.ErrorCodeUnknown, err) return @@ -151,7 +150,10 @@ func (luh *layerUploadHandler) GetUploadStatus(w http.ResponseWriter, r *http.Re return } - if err := luh.layerUploadResponse(w, r); err != nil { + // TODO(dmcgowan): Set last argument to false in layerUploadResponse when + // resumable upload is supported. This will enable returning a non-zero + // range for clients to begin uploading at an offset. + if err := luh.layerUploadResponse(w, r, true); err != nil { w.WriteHeader(http.StatusInternalServerError) // Error conditions here? luh.Errors.Push(v2.ErrorCodeUnknown, err) return @@ -161,11 +163,45 @@ func (luh *layerUploadHandler) GetUploadStatus(w http.ResponseWriter, r *http.Re w.WriteHeader(http.StatusNoContent) } -// PutLayerUploadComplete takes the final request of a layer upload. The final -// chunk may include all the layer data, the final chunk of layer data or no -// layer data. Any data provided is received and verified. If successful, the -// layer is linked into the blob store and 201 Created is returned with the -// canonical url of the layer. +// PatchLayerData writes data to an upload. +func (luh *layerUploadHandler) PatchLayerData(w http.ResponseWriter, r *http.Request) { + if luh.Upload == nil { + w.WriteHeader(http.StatusNotFound) + luh.Errors.Push(v2.ErrorCodeBlobUploadUnknown) + return + } + + ct := r.Header.Get("Content-Type") + if ct != "" && ct != "application/octet-stream" { + w.WriteHeader(http.StatusBadRequest) + // TODO(dmcgowan): encode error + return + } + + // TODO(dmcgowan): support Content-Range header to seek and write range + + // Copy the data + if _, err := io.Copy(luh.Upload, r.Body); err != nil { + ctxu.GetLogger(luh).Errorf("unknown error copying into upload: %v", err) + w.WriteHeader(http.StatusInternalServerError) + luh.Errors.Push(v2.ErrorCodeUnknown, err) + return + } + + if err := luh.layerUploadResponse(w, r, false); err != nil { + w.WriteHeader(http.StatusInternalServerError) // Error conditions here? + luh.Errors.Push(v2.ErrorCodeUnknown, err) + return + } + + w.WriteHeader(http.StatusAccepted) +} + +// PutLayerUploadComplete takes the final request of a layer upload. The +// request may include all the layer data or no layer data. Any data +// provided is received and verified. If successful, the layer is linked +// into the blob store and 201 Created is returned with the canonical +// url of the layer. func (luh *layerUploadHandler) PutLayerUploadComplete(w http.ResponseWriter, r *http.Request) { if luh.Upload == nil { w.WriteHeader(http.StatusNotFound) @@ -190,14 +226,11 @@ func (luh *layerUploadHandler) PutLayerUploadComplete(w http.ResponseWriter, r * return } - // TODO(stevvooe): Check the incoming range header here, per the - // specification. LayerUpload should be seeked (sought?) to that position. - // TODO(stevvooe): Consider checking the error on this copy. // Theoretically, problems should be detected during verification but we // may miss a root cause. - // Read in the final chunk, if any. + // Read in the data, if any. if _, err := io.Copy(luh.Upload, r.Body); err != nil { ctxu.GetLogger(luh).Errorf("unknown error copying into upload: %v", err) w.WriteHeader(http.StatusInternalServerError) @@ -260,13 +293,19 @@ func (luh *layerUploadHandler) CancelLayerUpload(w http.ResponseWriter, r *http. // layerUploadResponse provides a standard request for uploading layers and // chunk responses. This sets the correct headers but the response status is -// left to the caller. -func (luh *layerUploadHandler) layerUploadResponse(w http.ResponseWriter, r *http.Request) error { +// left to the caller. The fresh argument is used to ensure that new layer +// uploads always start at a 0 offset. This allows disabling resumable push +// by always returning a 0 offset on check status. +func (luh *layerUploadHandler) layerUploadResponse(w http.ResponseWriter, r *http.Request, fresh bool) error { - offset, err := luh.Upload.Seek(0, os.SEEK_CUR) - if err != nil { - ctxu.GetLogger(luh).Errorf("unable get current offset of layer upload: %v", err) - return err + var offset int64 + if !fresh { + var err error + offset, err = luh.Upload.Seek(0, os.SEEK_CUR) + if err != nil { + ctxu.GetLogger(luh).Errorf("unable get current offset of layer upload: %v", err) + return err + } } // TODO(stevvooe): Need a better way to manage the upload state automatically. @@ -291,10 +330,15 @@ func (luh *layerUploadHandler) layerUploadResponse(w http.ResponseWriter, r *htt return err } + endRange := offset + if endRange > 0 { + endRange = endRange - 1 + } + w.Header().Set("Docker-Upload-UUID", luh.UUID) w.Header().Set("Location", uploadURL) w.Header().Set("Content-Length", "0") - w.Header().Set("Range", fmt.Sprintf("0-%d", luh.State.Offset)) + w.Header().Set("Range", fmt.Sprintf("0-%d", endRange)) return nil }