From d7a2b14489cacf30b996700a7e0c8f555f0804e0 Mon Sep 17 00:00:00 2001 From: wang yan Date: Wed, 19 Aug 2020 17:07:38 +0800 Subject: [PATCH] add content range handling in patch blob Fixes #3141 1, return 416 for Out-of-order blob upload 2, return 400 for content length and content size mismatch Signed-off-by: wang yan --- registry/api/v2/errors.go | 11 ++++++++ registry/handlers/api_test.go | 48 ++++++++++++++++++++++++++++----- registry/handlers/blobupload.go | 25 ++++++++++++++++- registry/handlers/helpers.go | 20 ++++++++++++++ 4 files changed, 96 insertions(+), 8 deletions(-) diff --git a/registry/api/v2/errors.go b/registry/api/v2/errors.go index c413efbb..cf5c818e 100644 --- a/registry/api/v2/errors.go +++ b/registry/api/v2/errors.go @@ -32,6 +32,17 @@ var ( HTTPStatusCode: http.StatusBadRequest, }) + // ErrorCodeRangeInvalid is returned when uploading a blob if the provided + // content range is invalid. + ErrorCodeRangeInvalid = errcode.Register(errGroup, errcode.ErrorDescriptor{ + Value: "RANGE_INVALID", + Message: "invalid content range", + Description: `When a layer is uploaded, the provided range is checked + against the uploaded chunk. This error is returned if the range is + out of order.`, + HTTPStatusCode: http.StatusRequestedRangeNotSatisfiable, + }) + // ErrorCodeNameInvalid is returned when the name in the manifest does not // match the provided name. ErrorCodeNameInvalid = errcode.Register(errGroup, errcode.ErrorDescriptor{ diff --git a/registry/handlers/api_test.go b/registry/handlers/api_test.go index f53ba7c8..e0c30b36 100644 --- a/registry/handlers/api_test.go +++ b/registry/handlers/api_test.go @@ -533,6 +533,32 @@ func testBlobAPI(t *testing.T, env *testEnv, args blobArgs) *testEnv { uploadURLBase, dgst := pushChunk(t, env.builder, imageName, uploadURLBase, layerFile, layerLength) finishUpload(t, env.builder, imageName, uploadURLBase, dgst) + // ----------------------------------------- + // Do layer push with invalid content range + layerFile.Seek(0, io.SeekStart) + uploadURLBase, _ = startPushLayer(t, env, imageName) + sizeInvalid := chunkOptions{ + contentRange: "0-20", + } + resp, err = doPushChunk(t, uploadURLBase, layerFile, sizeInvalid) + if err != nil { + t.Fatalf("unexpected error doing push layer request: %v", err) + } + defer resp.Body.Close() + checkResponse(t, "putting size invalid chunk", resp, http.StatusBadRequest) + + layerFile.Seek(0, io.SeekStart) + uploadURLBase, _ = startPushLayer(t, env, imageName) + outOfOrder := chunkOptions{ + contentRange: "3-22", + } + resp, err = doPushChunk(t, uploadURLBase, layerFile, outOfOrder) + if err != nil { + t.Fatalf("unexpected error doing push layer request: %v", err) + } + defer resp.Body.Close() + checkResponse(t, "putting range out of order chunk", resp, http.StatusRequestedRangeNotSatisfiable) + // ------------------------ // Use a head request to see if the layer exists. resp, err = http.Head(layerURL) @@ -2242,7 +2268,12 @@ func finishUpload(t *testing.T, ub *v2.URLBuilder, name reference.Named, uploadU return resp.Header.Get("Location") } -func doPushChunk(t *testing.T, uploadURLBase string, body io.Reader) (*http.Response, digest.Digest, error) { +type chunkOptions struct { + // Content-Range header to set when pushing chunks + contentRange string +} + +func doPushChunk(t *testing.T, uploadURLBase string, body io.Reader, options chunkOptions) (*http.Response, error) { u, err := url.Parse(uploadURLBase) if err != nil { t.Fatalf("unexpected error parsing pushLayer url: %v", err) @@ -2254,21 +2285,24 @@ func doPushChunk(t *testing.T, uploadURLBase string, body io.Reader) (*http.Resp uploadURL := u.String() - digester := digest.Canonical.Digester() - - req, err := http.NewRequest("PATCH", uploadURL, io.TeeReader(body, digester.Hash())) + req, err := http.NewRequest("PATCH", uploadURL, body) if err != nil { t.Fatalf("unexpected error creating new request: %v", err) } req.Header.Set("Content-Type", "application/octet-stream") + if options.contentRange != "" { + req.Header.Set("Content-Range", options.contentRange) + } resp, err := http.DefaultClient.Do(req) - return resp, digester.Digest(), err + return resp, err } func pushChunk(t *testing.T, ub *v2.URLBuilder, name reference.Named, uploadURLBase string, body io.Reader, length int64) (string, digest.Digest) { - resp, dgst, err := doPushChunk(t, uploadURLBase, body) + digester := digest.Canonical.Digester() + + resp, err := doPushChunk(t, uploadURLBase, io.TeeReader(body, digester.Hash()), chunkOptions{}) if err != nil { t.Fatalf("unexpected error doing push layer request: %v", err) } @@ -2285,7 +2319,7 @@ func pushChunk(t *testing.T, ub *v2.URLBuilder, name reference.Named, uploadURLB "Content-Length": []string{"0"}, }) - return resp.Header.Get("Location"), dgst + return resp.Header.Get("Location"), digester.Digest() } func checkResponse(t *testing.T, msg string, resp *http.Response, expectedStatus int) { diff --git a/registry/handlers/blobupload.go b/registry/handlers/blobupload.go index edb9c11d..d712fe00 100644 --- a/registry/handlers/blobupload.go +++ b/registry/handlers/blobupload.go @@ -4,6 +4,7 @@ import ( "fmt" "net/http" "net/url" + "strconv" "github.com/distribution/distribution/v3" dcontext "github.com/distribution/distribution/v3/context" @@ -133,7 +134,29 @@ func (buh *blobUploadHandler) PatchBlobData(w http.ResponseWriter, r *http.Reque return } - // TODO(dmcgowan): support Content-Range header to seek and write range + cr := r.Header.Get("Content-Range") + cl := r.Header.Get("Content-Length") + if cr != "" && cl != "" { + start, end, err := parseContentRange(cr) + if err != nil { + buh.Errors = append(buh.Errors, errcode.ErrorCodeUnknown.WithDetail(err.Error())) + return + } + if start > end || start != buh.Upload.Size() { + buh.Errors = append(buh.Errors, v2.ErrorCodeRangeInvalid) + return + } + + clInt, err := strconv.ParseInt(cl, 10, 64) + if err != nil { + buh.Errors = append(buh.Errors, errcode.ErrorCodeUnknown.WithDetail(err.Error())) + return + } + if clInt != (end-start)+1 { + buh.Errors = append(buh.Errors, v2.ErrorCodeSizeInvalid) + return + } + } if err := copyFullPayload(buh, w, r, buh.Upload, -1, "blob PATCH"); err != nil { buh.Errors = append(buh.Errors, errcode.ErrorCodeUnknown.WithDetail(err.Error())) diff --git a/registry/handlers/helpers.go b/registry/handlers/helpers.go index 2e89c65b..69324df8 100644 --- a/registry/handlers/helpers.go +++ b/registry/handlers/helpers.go @@ -3,8 +3,11 @@ package handlers import ( "context" "errors" + "fmt" "io" "net/http" + "strconv" + "strings" dcontext "github.com/distribution/distribution/v3/context" ) @@ -64,3 +67,20 @@ func copyFullPayload(ctx context.Context, responseWriter http.ResponseWriter, r return nil } + +func parseContentRange(cr string) (int64, int64, error) { + ranges := strings.Split(cr, "-") + if len(ranges) != 2 { + return -1, -1, fmt.Errorf("invalid content range format, %s", cr) + } + start, err := strconv.ParseInt(ranges[0], 10, 64) + if err != nil { + return -1, -1, err + } + end, err := strconv.ParseInt(ranges[1], 10, 64) + if err != nil { + return -1, -1, err + } + + return start, end, nil +}