From b7f7b0a2c992558c22d23254390f8a4223160541 Mon Sep 17 00:00:00 2001 From: Derek McGowan Date: Wed, 1 Oct 2014 18:26:06 -0700 Subject: [PATCH] Add provenance pull flow for official images Add support for pulling signed images from a version 2 registry. Only official images within the library namespace will be pull from the new registry and check the build signature. Signed-off-by: Derek McGowan (github: dmcgowan) --- docs/registry.go | 1 + docs/registry_mock_test.go | 6 + docs/session.go | 12 +- docs/session_v2.go | 386 +++++++++++++++++++++++++++++++++++++ docs/types.go | 12 +- 5 files changed, 409 insertions(+), 8 deletions(-) create mode 100644 docs/session_v2.go diff --git a/docs/registry.go b/docs/registry.go index 203dfa6f..fd74b751 100644 --- a/docs/registry.go +++ b/docs/registry.go @@ -20,6 +20,7 @@ import ( var ( ErrAlreadyExists = errors.New("Image already exists") ErrInvalidRepositoryName = errors.New("Invalid repository name (ex: \"registry.domain.tld/myrepos\")") + ErrDoesNotExist = errors.New("Image does not exist") errLoginRequired = errors.New("Authentication is required.") validHex = regexp.MustCompile(`^([a-f0-9]{64})$`) validNamespace = regexp.MustCompile(`^([a-z0-9_]{4,30})$`) diff --git a/docs/registry_mock_test.go b/docs/registry_mock_test.go index 8851dcbe..379dc78f 100644 --- a/docs/registry_mock_test.go +++ b/docs/registry_mock_test.go @@ -83,6 +83,8 @@ var ( func init() { r := mux.NewRouter() + + // /v1/ r.HandleFunc("/v1/_ping", handlerGetPing).Methods("GET") r.HandleFunc("/v1/images/{image_id:[^/]+}/{action:json|layer|ancestry}", handlerGetImage).Methods("GET") r.HandleFunc("/v1/images/{image_id:[^/]+}/{action:json|layer|checksum}", handlerPutImage).Methods("PUT") @@ -93,6 +95,10 @@ func init() { r.HandleFunc("/v1/repositories/{repository:.+}{action:/images|/}", handlerImages).Methods("GET", "PUT", "DELETE") r.HandleFunc("/v1/repositories/{repository:.+}/auth", handlerAuth).Methods("PUT") r.HandleFunc("/v1/search", handlerSearch).Methods("GET") + + // /v2/ + r.HandleFunc("/v2/version", handlerGetPing).Methods("GET") + testHttpServer = httptest.NewServer(handlerAccessLog(r)) } diff --git a/docs/session.go b/docs/session.go index 48626308..5067b8d5 100644 --- a/docs/session.go +++ b/docs/session.go @@ -47,7 +47,7 @@ func NewSession(authConfig *AuthConfig, factory *utils.HTTPRequestFactory, endpo // If we're working with a standalone private registry over HTTPS, send Basic Auth headers // alongside our requests. - if r.indexEndpoint.String() != IndexServerAddress() && r.indexEndpoint.URL.Scheme == "https" { + if r.indexEndpoint.VersionString(1) != IndexServerAddress() && r.indexEndpoint.URL.Scheme == "https" { info, err := r.indexEndpoint.Ping() if err != nil { return nil, err @@ -261,7 +261,7 @@ func buildEndpointsList(headers []string, indexEp string) ([]string, error) { } func (r *Session) GetRepositoryData(remote string) (*RepositoryData, error) { - repositoryTarget := fmt.Sprintf("%srepositories/%s/images", r.indexEndpoint.String(), remote) + repositoryTarget := fmt.Sprintf("%srepositories/%s/images", r.indexEndpoint.VersionString(1), remote) log.Debugf("[registry] Calling GET %s", repositoryTarget) @@ -295,7 +295,7 @@ func (r *Session) GetRepositoryData(remote string) (*RepositoryData, error) { var endpoints []string if res.Header.Get("X-Docker-Endpoints") != "" { - endpoints, err = buildEndpointsList(res.Header["X-Docker-Endpoints"], r.indexEndpoint.String()) + endpoints, err = buildEndpointsList(res.Header["X-Docker-Endpoints"], r.indexEndpoint.VersionString(1)) if err != nil { return nil, err } @@ -488,7 +488,7 @@ func (r *Session) PushImageJSONIndex(remote string, imgList []*ImgData, validate if validate { suffix = "images" } - u := fmt.Sprintf("%srepositories/%s/%s", r.indexEndpoint.String(), remote, suffix) + u := fmt.Sprintf("%srepositories/%s/%s", r.indexEndpoint.VersionString(1), remote, suffix) log.Debugf("[registry] PUT %s", u) log.Debugf("Image list pushed to index:\n%s", imgListJSON) req, err := r.reqFactory.NewRequest("PUT", u, bytes.NewReader(imgListJSON)) @@ -546,7 +546,7 @@ func (r *Session) PushImageJSONIndex(remote string, imgList []*ImgData, validate } if res.Header.Get("X-Docker-Endpoints") != "" { - endpoints, err = buildEndpointsList(res.Header["X-Docker-Endpoints"], r.indexEndpoint.String()) + endpoints, err = buildEndpointsList(res.Header["X-Docker-Endpoints"], r.indexEndpoint.VersionString(1)) if err != nil { return nil, err } @@ -572,7 +572,7 @@ func (r *Session) PushImageJSONIndex(remote string, imgList []*ImgData, validate func (r *Session) SearchRepositories(term string) (*SearchResults, error) { log.Debugf("Index server: %s", r.indexEndpoint) - u := r.indexEndpoint.String() + "search?q=" + url.QueryEscape(term) + u := r.indexEndpoint.VersionString(1) + "search?q=" + url.QueryEscape(term) req, err := r.reqFactory.NewRequest("GET", u, nil) if err != nil { return nil, err diff --git a/docs/session_v2.go b/docs/session_v2.go new file mode 100644 index 00000000..2e0de49b --- /dev/null +++ b/docs/session_v2.go @@ -0,0 +1,386 @@ +package registry + +import ( + "encoding/json" + "fmt" + "io" + "io/ioutil" + "net/url" + "strconv" + + "github.com/docker/docker/pkg/log" + "github.com/docker/docker/utils" + "github.com/gorilla/mux" +) + +func newV2RegistryRouter() *mux.Router { + router := mux.NewRouter() + + v2Router := router.PathPrefix("/v2/").Subrouter() + + // Version Info + v2Router.Path("/version").Name("version") + + // Image Manifests + v2Router.Path("/manifest/{imagename:[a-z0-9-._/]+}/{tagname:[a-zA-Z0-9-._]+}").Name("manifests") + + // List Image Tags + v2Router.Path("/tags/{imagename:[a-z0-9-._/]+}").Name("tags") + + // Download a blob + v2Router.Path("/blob/{imagename:[a-z0-9-._/]+}/{sumtype:[a-z0-9_+-]+}/{sum:[a-fA-F0-9]{4,}}").Name("downloadBlob") + + // Upload a blob + v2Router.Path("/blob/{imagename:[a-z0-9-._/]+}/{sumtype:[a-z0-9_+-]+}").Name("uploadBlob") + + // Mounting a blob in an image + v2Router.Path("/mountblob/{imagename:[a-z0-9-._/]+}/{sumtype:[a-z0-9_+-]+}/{sum:[a-fA-F0-9]{4,}}").Name("mountBlob") + + return router +} + +// APIVersion2 /v2/ +var v2HTTPRoutes = newV2RegistryRouter() + +func getV2URL(e *Endpoint, routeName string, vars map[string]string) (*url.URL, error) { + route := v2HTTPRoutes.Get(routeName) + if route == nil { + return nil, fmt.Errorf("unknown regisry v2 route name: %q", routeName) + } + + varReplace := make([]string, 0, len(vars)*2) + for key, val := range vars { + varReplace = append(varReplace, key, val) + } + + routePath, err := route.URLPath(varReplace...) + if err != nil { + return nil, fmt.Errorf("unable to make registry route %q with vars %v: %s", routeName, vars, err) + } + + return &url.URL{ + Scheme: e.URL.Scheme, + Host: e.URL.Host, + Path: routePath.Path, + }, nil +} + +// V2 Provenance POC + +func (r *Session) GetV2Version(token []string) (*RegistryInfo, error) { + routeURL, err := getV2URL(r.indexEndpoint, "version", nil) + if err != nil { + return nil, err + } + + method := "GET" + log.Debugf("[registry] Calling %q %s", method, routeURL.String()) + + req, err := r.reqFactory.NewRequest(method, routeURL.String(), nil) + if err != nil { + return nil, err + } + setTokenAuth(req, token) + res, _, err := r.doRequest(req) + if err != nil { + return nil, err + } + defer res.Body.Close() + if res.StatusCode != 200 { + return nil, utils.NewHTTPRequestError(fmt.Sprintf("Server error: %d fetching Version", res.StatusCode), res) + } + + decoder := json.NewDecoder(res.Body) + versionInfo := new(RegistryInfo) + + err = decoder.Decode(versionInfo) + if err != nil { + return nil, fmt.Errorf("unable to decode GetV2Version JSON response: %s", err) + } + + return versionInfo, nil +} + +// +// 1) Check if TarSum of each layer exists /v2/ +// 1.a) if 200, continue +// 1.b) if 300, then push the +// 1.c) if anything else, err +// 2) PUT the created/signed manifest +// +func (r *Session) GetV2ImageManifest(imageName, tagName string, token []string) ([]byte, error) { + vars := map[string]string{ + "imagename": imageName, + "tagname": tagName, + } + + routeURL, err := getV2URL(r.indexEndpoint, "manifests", vars) + if err != nil { + return nil, err + } + + method := "GET" + log.Debugf("[registry] Calling %q %s", method, routeURL.String()) + + req, err := r.reqFactory.NewRequest(method, routeURL.String(), nil) + if err != nil { + return nil, err + } + setTokenAuth(req, token) + res, _, err := r.doRequest(req) + if err != nil { + return nil, err + } + defer res.Body.Close() + if res.StatusCode != 200 { + if res.StatusCode == 401 { + return nil, errLoginRequired + } else if res.StatusCode == 404 { + return nil, ErrDoesNotExist + } + return nil, utils.NewHTTPRequestError(fmt.Sprintf("Server error: %d trying to fetch for %s:%s", res.StatusCode, imageName, tagName), res) + } + + buf, err := ioutil.ReadAll(res.Body) + if err != nil { + return nil, fmt.Errorf("Error while reading the http response: %s", err) + } + return buf, nil +} + +// - Succeeded to mount for this image scope +// - Failed with no error (So continue to Push the Blob) +// - Failed with error +func (r *Session) PostV2ImageMountBlob(imageName, sumType, sum string, token []string) (bool, error) { + vars := map[string]string{ + "imagename": imageName, + "sumtype": sumType, + "sum": sum, + } + + routeURL, err := getV2URL(r.indexEndpoint, "mountBlob", vars) + if err != nil { + return false, err + } + + method := "POST" + log.Debugf("[registry] Calling %q %s", method, routeURL.String()) + + req, err := r.reqFactory.NewRequest(method, routeURL.String(), nil) + if err != nil { + return false, err + } + setTokenAuth(req, token) + res, _, err := r.doRequest(req) + if err != nil { + return false, err + } + res.Body.Close() // close early, since we're not needing a body on this call .. yet? + switch res.StatusCode { + case 200: + // return something indicating no push needed + return true, nil + case 300: + // return something indicating blob push needed + return false, nil + } + return false, fmt.Errorf("Failed to mount %q - %s:%s : %d", imageName, sumType, sum, res.StatusCode) +} + +func (r *Session) GetV2ImageBlob(imageName, sumType, sum string, blobWrtr io.Writer, token []string) error { + vars := map[string]string{ + "imagename": imageName, + "sumtype": sumType, + "sum": sum, + } + + routeURL, err := getV2URL(r.indexEndpoint, "downloadBlob", vars) + if err != nil { + return err + } + + method := "GET" + log.Debugf("[registry] Calling %q %s", method, routeURL.String()) + req, err := r.reqFactory.NewRequest(method, routeURL.String(), nil) + if err != nil { + return err + } + setTokenAuth(req, token) + res, _, err := r.doRequest(req) + if err != nil { + return err + } + defer res.Body.Close() + if res.StatusCode != 200 { + if res.StatusCode == 401 { + return errLoginRequired + } + return utils.NewHTTPRequestError(fmt.Sprintf("Server error: %d trying to pull %s blob", res.StatusCode, imageName), res) + } + + _, err = io.Copy(blobWrtr, res.Body) + return err +} + +func (r *Session) GetV2ImageBlobReader(imageName, sumType, sum string, token []string) (io.ReadCloser, int64, error) { + vars := map[string]string{ + "imagename": imageName, + "sumtype": sumType, + "sum": sum, + } + + routeURL, err := getV2URL(r.indexEndpoint, "downloadBlob", vars) + if err != nil { + return nil, 0, err + } + + method := "GET" + log.Debugf("[registry] Calling %q %s", method, routeURL.String()) + req, err := r.reqFactory.NewRequest(method, routeURL.String(), nil) + if err != nil { + return nil, 0, err + } + setTokenAuth(req, token) + res, _, err := r.doRequest(req) + if err != nil { + return nil, 0, err + } + if res.StatusCode != 200 { + if res.StatusCode == 401 { + return nil, 0, errLoginRequired + } + return nil, 0, utils.NewHTTPRequestError(fmt.Sprintf("Server error: %d trying to pull %s blob", res.StatusCode, imageName), res) + } + lenStr := res.Header.Get("Content-Length") + l, err := strconv.ParseInt(lenStr, 10, 64) + if err != nil { + return nil, 0, err + } + + return res.Body, l, err +} + +// Push the image to the server for storage. +// 'layer' is an uncompressed reader of the blob to be pushed. +// The server will generate it's own checksum calculation. +func (r *Session) PutV2ImageBlob(imageName, sumType string, blobRdr io.Reader, token []string) (serverChecksum string, err error) { + vars := map[string]string{ + "imagename": imageName, + "sumtype": sumType, + } + + routeURL, err := getV2URL(r.indexEndpoint, "uploadBlob", vars) + if err != nil { + return "", err + } + + method := "PUT" + log.Debugf("[registry] Calling %q %s", method, routeURL.String()) + req, err := r.reqFactory.NewRequest(method, routeURL.String(), blobRdr) + if err != nil { + return "", err + } + setTokenAuth(req, token) + res, _, err := r.doRequest(req) + if err != nil { + return "", err + } + defer res.Body.Close() + if res.StatusCode != 201 { + if res.StatusCode == 401 { + return "", errLoginRequired + } + return "", utils.NewHTTPRequestError(fmt.Sprintf("Server error: %d trying to push %s blob", res.StatusCode, imageName), res) + } + + type sumReturn struct { + Checksum string `json:"checksum"` + } + + decoder := json.NewDecoder(res.Body) + var sumInfo sumReturn + + err = decoder.Decode(&sumInfo) + if err != nil { + return "", fmt.Errorf("unable to decode PutV2ImageBlob JSON response: %s", err) + } + + // XXX this is a json struct from the registry, with its checksum + return sumInfo.Checksum, nil +} + +// Finally Push the (signed) manifest of the blobs we've just pushed +func (r *Session) PutV2ImageManifest(imageName, tagName string, manifestRdr io.Reader, token []string) error { + vars := map[string]string{ + "imagename": imageName, + "tagname": tagName, + } + + routeURL, err := getV2URL(r.indexEndpoint, "manifests", vars) + if err != nil { + return err + } + + method := "PUT" + log.Debugf("[registry] Calling %q %s", method, routeURL.String()) + req, err := r.reqFactory.NewRequest(method, routeURL.String(), manifestRdr) + if err != nil { + return err + } + setTokenAuth(req, token) + res, _, err := r.doRequest(req) + if err != nil { + return err + } + res.Body.Close() + if res.StatusCode != 201 { + if res.StatusCode == 401 { + return errLoginRequired + } + return utils.NewHTTPRequestError(fmt.Sprintf("Server error: %d trying to push %s:%s manifest", res.StatusCode, imageName, tagName), res) + } + + return nil +} + +// Given a repository name, returns a json array of string tags +func (r *Session) GetV2RemoteTags(imageName string, token []string) ([]string, error) { + vars := map[string]string{ + "imagename": imageName, + } + + routeURL, err := getV2URL(r.indexEndpoint, "tags", vars) + if err != nil { + return nil, err + } + + method := "GET" + log.Debugf("[registry] Calling %q %s", method, routeURL.String()) + + req, err := r.reqFactory.NewRequest(method, routeURL.String(), nil) + if err != nil { + return nil, err + } + setTokenAuth(req, token) + res, _, err := r.doRequest(req) + if err != nil { + return nil, err + } + defer res.Body.Close() + if res.StatusCode != 200 { + if res.StatusCode == 401 { + return nil, errLoginRequired + } else if res.StatusCode == 404 { + return nil, ErrDoesNotExist + } + return nil, utils.NewHTTPRequestError(fmt.Sprintf("Server error: %d trying to fetch for %s", res.StatusCode, imageName), res) + } + + decoder := json.NewDecoder(res.Body) + var tags []string + err = decoder.Decode(&tags) + if err != nil { + return nil, fmt.Errorf("Error while decoding the http response: %s", err) + } + return tags, nil +} diff --git a/docs/types.go b/docs/types.go index 3db236da..2ba5af0d 100644 --- a/docs/types.go +++ b/docs/types.go @@ -32,6 +32,15 @@ type RegistryInfo struct { Standalone bool `json:"standalone"` } +type ManifestData struct { + Name string `json:"name"` + Tag string `json:"tag"` + Architecture string `json:"architecture"` + BlobSums []string `json:"blobSums"` + History []string `json:"history"` + SchemaVersion int `json:"schemaVersion"` +} + type APIVersion int func (av APIVersion) String() string { @@ -45,7 +54,6 @@ var apiVersions = map[APIVersion]string{ } const ( - _ = iota - APIVersion1 = iota + APIVersion1 = iota + 1 APIVersion2 )