diff --git a/docs/auth.go b/docs/auth.go index 906a37dd..7c0709a4 100644 --- a/docs/auth.go +++ b/docs/auth.go @@ -14,13 +14,16 @@ import ( "github.com/docker/docker/utils" ) -// Where we store the config file -const CONFIGFILE = ".dockercfg" +const ( + // Where we store the config file + CONFIGFILE = ".dockercfg" -// Only used for user auth + account creation -const INDEXSERVER = "https://index.docker.io/v1/" + // Only used for user auth + account creation + INDEXSERVER = "https://index.docker.io/v1/" + REGISTRYSERVER = "https://registry-1.docker.io/v1/" -//const INDEXSERVER = "https://registry-stage.hub.docker.com/v1/" + // INDEXSERVER = "https://registry-stage.hub.docker.com/v1/" +) var ( ErrConfigFileMissing = errors.New("The Auth config file is missing") diff --git a/docs/endpoint.go b/docs/endpoint.go new file mode 100644 index 00000000..5313a807 --- /dev/null +++ b/docs/endpoint.go @@ -0,0 +1,129 @@ +package registry + +import ( + "encoding/json" + "errors" + "fmt" + "io/ioutil" + "net/http" + "net/url" + "strings" + + "github.com/docker/docker/pkg/log" +) + +// scans string for api version in the URL path. returns the trimmed hostname, if version found, string and API version. +func scanForApiVersion(hostname string) (string, APIVersion) { + var ( + chunks []string + apiVersionStr string + ) + if strings.HasSuffix(hostname, "/") { + chunks = strings.Split(hostname[:len(hostname)-1], "/") + apiVersionStr = chunks[len(chunks)-1] + } else { + chunks = strings.Split(hostname, "/") + apiVersionStr = chunks[len(chunks)-1] + } + for k, v := range apiVersions { + if apiVersionStr == v { + hostname = strings.Join(chunks[:len(chunks)-1], "/") + return hostname, k + } + } + return hostname, DefaultAPIVersion +} + +func NewEndpoint(hostname string) (*Endpoint, error) { + var ( + endpoint Endpoint + trimmedHostname string + err error + ) + if !strings.HasPrefix(hostname, "http") { + hostname = "https://" + hostname + } + trimmedHostname, endpoint.Version = scanForApiVersion(hostname) + endpoint.URL, err = url.Parse(trimmedHostname) + if err != nil { + return nil, err + } + + endpoint.URL.Scheme = "https" + if _, err := endpoint.Ping(); err != nil { + log.Debugf("Registry %s does not work (%s), falling back to http", endpoint, err) + // TODO: Check if http fallback is enabled + endpoint.URL.Scheme = "http" + if _, err = endpoint.Ping(); err != nil { + return nil, errors.New("Invalid Registry endpoint: " + err.Error()) + } + } + + return &endpoint, nil +} + +type Endpoint struct { + URL *url.URL + Version APIVersion +} + +// Get the formated URL for the root of this registry Endpoint +func (e Endpoint) String() string { + return fmt.Sprintf("%s/v%d/", e.URL.String(), e.Version) +} + +func (e Endpoint) VersionString(version APIVersion) string { + return fmt.Sprintf("%s/v%d/", e.URL.String(), version) +} + +func (e Endpoint) Ping() (RegistryInfo, error) { + if e.String() == IndexServerAddress() { + // Skip the check, we now this one is valid + // (and we never want to fallback to http in case of error) + return RegistryInfo{Standalone: false}, nil + } + + req, err := http.NewRequest("GET", e.String()+"_ping", nil) + if err != nil { + return RegistryInfo{Standalone: false}, err + } + + resp, _, err := doRequest(req, nil, ConnectTimeout) + if err != nil { + return RegistryInfo{Standalone: false}, err + } + + defer resp.Body.Close() + + jsonString, err := ioutil.ReadAll(resp.Body) + if err != nil { + return RegistryInfo{Standalone: false}, fmt.Errorf("Error while reading the http response: %s", err) + } + + // If the header is absent, we assume true for compatibility with earlier + // versions of the registry. default to true + info := RegistryInfo{ + Standalone: true, + } + if err := json.Unmarshal(jsonString, &info); err != nil { + log.Debugf("Error unmarshalling the _ping RegistryInfo: %s", err) + // don't stop here. Just assume sane defaults + } + if hdr := resp.Header.Get("X-Docker-Registry-Version"); hdr != "" { + log.Debugf("Registry version header: '%s'", hdr) + info.Version = hdr + } + log.Debugf("RegistryInfo.Version: %q", info.Version) + + standalone := resp.Header.Get("X-Docker-Registry-Standalone") + log.Debugf("Registry standalone header: '%s'", standalone) + // Accepted values are "true" (case-insensitive) and "1". + if strings.EqualFold(standalone, "true") || standalone == "1" { + info.Standalone = true + } else if len(standalone) > 0 { + // there is a header set, and it is not "true" or "1", so assume fails + info.Standalone = false + } + log.Debugf("RegistryInfo.Standalone: %t", info.Standalone) + return info, nil +} diff --git a/docs/registry.go b/docs/registry.go index 9c76aca9..fd74b751 100644 --- a/docs/registry.go +++ b/docs/registry.go @@ -3,7 +3,6 @@ package registry import ( "crypto/tls" "crypto/x509" - "encoding/json" "errors" "fmt" "io/ioutil" @@ -15,14 +14,17 @@ import ( "strings" "time" - "github.com/docker/docker/pkg/log" "github.com/docker/docker/utils" ) var ( ErrAlreadyExists = errors.New("Image already exists") ErrInvalidRepositoryName = errors.New("Invalid repository name (ex: \"registry.domain.tld/myrepos\")") + ErrDoesNotExist = errors.New("Image does not exist") errLoginRequired = errors.New("Authentication is required.") + validHex = regexp.MustCompile(`^([a-f0-9]{64})$`) + validNamespace = regexp.MustCompile(`^([a-z0-9_]{4,30})$`) + validRepo = regexp.MustCompile(`^([a-z0-9-_.]+)$`) ) type TimeoutType uint32 @@ -105,22 +107,20 @@ func doRequest(req *http.Request, jar http.CookieJar, timeout TimeoutType) (*htt data, err := ioutil.ReadFile(path.Join(hostDir, f.Name())) if err != nil { return nil, nil, err - } else { - pool.AppendCertsFromPEM(data) } + pool.AppendCertsFromPEM(data) } if strings.HasSuffix(f.Name(), ".cert") { certName := f.Name() keyName := certName[:len(certName)-5] + ".key" if !hasFile(fs, keyName) { return nil, nil, fmt.Errorf("Missing key %s for certificate %s", keyName, certName) - } else { - cert, err := tls.LoadX509KeyPair(path.Join(hostDir, certName), path.Join(hostDir, keyName)) - if err != nil { - return nil, nil, err - } - certs = append(certs, &cert) } + cert, err := tls.LoadX509KeyPair(path.Join(hostDir, certName), path.Join(hostDir, keyName)) + if err != nil { + return nil, nil, err + } + certs = append(certs, &cert) } if strings.HasSuffix(f.Name(), ".key") { keyName := f.Name() @@ -138,77 +138,19 @@ func doRequest(req *http.Request, jar http.CookieJar, timeout TimeoutType) (*htt return nil, nil, err } return res, client, nil - } else { - for i, cert := range certs { - client := newClient(jar, pool, cert, timeout) - res, err := client.Do(req) - if i == len(certs)-1 { - // If this is the last cert, always return the result - return res, client, err - } else { - // Otherwise, continue to next cert if 403 or 5xx - if err == nil && res.StatusCode != 403 && !(res.StatusCode >= 500 && res.StatusCode < 600) { - return res, client, err - } - } + } + for i, cert := range certs { + client := newClient(jar, pool, cert, timeout) + res, err := client.Do(req) + // If this is the last cert, otherwise, continue to next cert if 403 or 5xx + if i == len(certs)-1 || err == nil && res.StatusCode != 403 && res.StatusCode < 500 { + return res, client, err } } return nil, nil, nil } -func pingRegistryEndpoint(endpoint string) (RegistryInfo, error) { - if endpoint == IndexServerAddress() { - // Skip the check, we now this one is valid - // (and we never want to fallback to http in case of error) - return RegistryInfo{Standalone: false}, nil - } - - req, err := http.NewRequest("GET", endpoint+"_ping", nil) - if err != nil { - return RegistryInfo{Standalone: false}, err - } - - resp, _, err := doRequest(req, nil, ConnectTimeout) - if err != nil { - return RegistryInfo{Standalone: false}, err - } - - defer resp.Body.Close() - - jsonString, err := ioutil.ReadAll(resp.Body) - if err != nil { - return RegistryInfo{Standalone: false}, fmt.Errorf("Error while reading the http response: %s", err) - } - - // If the header is absent, we assume true for compatibility with earlier - // versions of the registry. default to true - info := RegistryInfo{ - Standalone: true, - } - if err := json.Unmarshal(jsonString, &info); err != nil { - log.Debugf("Error unmarshalling the _ping RegistryInfo: %s", err) - // don't stop here. Just assume sane defaults - } - if hdr := resp.Header.Get("X-Docker-Registry-Version"); hdr != "" { - log.Debugf("Registry version header: '%s'", hdr) - info.Version = hdr - } - log.Debugf("RegistryInfo.Version: %q", info.Version) - - standalone := resp.Header.Get("X-Docker-Registry-Standalone") - log.Debugf("Registry standalone header: '%s'", standalone) - // Accepted values are "true" (case-insensitive) and "1". - if strings.EqualFold(standalone, "true") || standalone == "1" { - info.Standalone = true - } else if len(standalone) > 0 { - // there is a header set, and it is not "true" or "1", so assume fails - info.Standalone = false - } - log.Debugf("RegistryInfo.Standalone: %q", info.Standalone) - return info, nil -} - func validateRepositoryName(repositoryName string) error { var ( namespace string @@ -218,15 +160,17 @@ func validateRepositoryName(repositoryName string) error { if len(nameParts) < 2 { namespace = "library" name = nameParts[0] + + if validHex.MatchString(name) { + return fmt.Errorf("Invalid repository name (%s), cannot specify 64-byte hexadecimal strings", name) + } } else { namespace = nameParts[0] name = nameParts[1] } - validNamespace := regexp.MustCompile(`^([a-z0-9_]{4,30})$`) if !validNamespace.MatchString(namespace) { return fmt.Errorf("Invalid namespace name (%s), only [a-z0-9_] are allowed, size between 4 and 30", namespace) } - validRepo := regexp.MustCompile(`^([a-z0-9-_.]+)$`) if !validRepo.MatchString(name) { return fmt.Errorf("Invalid repository name (%s), only [a-z0-9-_.] are allowed", name) } @@ -258,33 +202,6 @@ func ResolveRepositoryName(reposName string) (string, string, error) { return hostname, reposName, nil } -// this method expands the registry name as used in the prefix of a repo -// to a full url. if it already is a url, there will be no change. -// The registry is pinged to test if it http or https -func ExpandAndVerifyRegistryUrl(hostname string) (string, error) { - if strings.HasPrefix(hostname, "http:") || strings.HasPrefix(hostname, "https:") { - // if there is no slash after https:// (8 characters) then we have no path in the url - if strings.LastIndex(hostname, "/") < 9 { - // there is no path given. Expand with default path - hostname = hostname + "/v1/" - } - if _, err := pingRegistryEndpoint(hostname); err != nil { - return "", errors.New("Invalid Registry endpoint: " + err.Error()) - } - return hostname, nil - } - endpoint := fmt.Sprintf("https://%s/v1/", hostname) - if _, err := pingRegistryEndpoint(endpoint); err != nil { - log.Debugf("Registry %s does not work (%s), falling back to http", endpoint, err) - endpoint = fmt.Sprintf("http://%s/v1/", hostname) - if _, err = pingRegistryEndpoint(endpoint); err != nil { - //TODO: triggering highland build can be done there without "failing" - return "", errors.New("Invalid Registry endpoint: " + err.Error()) - } - } - return endpoint, nil -} - func trustedLocation(req *http.Request) bool { var ( trusteds = []string{"docker.com", "docker.io"} @@ -306,12 +223,12 @@ func AddRequiredHeadersToRedirectedRequests(req *http.Request, via []*http.Reque if via != nil && via[0] != nil { if trustedLocation(req) && trustedLocation(via[0]) { req.Header = via[0].Header - } else { - for k, v := range via[0].Header { - if k != "Authorization" { - for _, vv := range v { - req.Header.Add(k, vv) - } + return nil + } + for k, v := range via[0].Header { + if k != "Authorization" { + for _, vv := range v { + req.Header.Add(k, vv) } } } diff --git a/docs/registry_mock_test.go b/docs/registry_mock_test.go index 2b4cd9de..379dc78f 100644 --- a/docs/registry_mock_test.go +++ b/docs/registry_mock_test.go @@ -83,6 +83,8 @@ var ( func init() { r := mux.NewRouter() + + // /v1/ r.HandleFunc("/v1/_ping", handlerGetPing).Methods("GET") r.HandleFunc("/v1/images/{image_id:[^/]+}/{action:json|layer|ancestry}", handlerGetImage).Methods("GET") r.HandleFunc("/v1/images/{image_id:[^/]+}/{action:json|layer|checksum}", handlerPutImage).Methods("PUT") @@ -93,6 +95,10 @@ func init() { r.HandleFunc("/v1/repositories/{repository:.+}{action:/images|/}", handlerImages).Methods("GET", "PUT", "DELETE") r.HandleFunc("/v1/repositories/{repository:.+}/auth", handlerAuth).Methods("PUT") r.HandleFunc("/v1/search", handlerSearch).Methods("GET") + + // /v2/ + r.HandleFunc("/v2/version", handlerGetPing).Methods("GET") + testHttpServer = httptest.NewServer(handlerAccessLog(r)) } @@ -236,6 +242,7 @@ func handlerGetDeleteTags(w http.ResponseWriter, r *http.Request) { tags, exists := testRepositories[repositoryName] if !exists { apiError(w, "Repository not found", 404) + return } if r.Method == "DELETE" { delete(testRepositories, repositoryName) @@ -255,10 +262,12 @@ func handlerGetTag(w http.ResponseWriter, r *http.Request) { tags, exists := testRepositories[repositoryName] if !exists { apiError(w, "Repository not found", 404) + return } tag, exists := tags[tagName] if !exists { apiError(w, "Tag not found", 404) + return } writeResponse(w, tag, 200) } diff --git a/docs/registry_test.go b/docs/registry_test.go index 303879e8..ab417812 100644 --- a/docs/registry_test.go +++ b/docs/registry_test.go @@ -18,7 +18,11 @@ var ( func spawnTestRegistrySession(t *testing.T) *Session { authConfig := &AuthConfig{} - r, err := NewSession(authConfig, utils.NewHTTPRequestFactory(), makeURL("/v1/"), true) + endpoint, err := NewEndpoint(makeURL("/v1/")) + if err != nil { + t.Fatal(err) + } + r, err := NewSession(authConfig, utils.NewHTTPRequestFactory(), endpoint, true) if err != nil { t.Fatal(err) } @@ -26,7 +30,11 @@ func spawnTestRegistrySession(t *testing.T) *Session { } func TestPingRegistryEndpoint(t *testing.T) { - regInfo, err := pingRegistryEndpoint(makeURL("/v1/")) + ep, err := NewEndpoint(makeURL("/v1/")) + if err != nil { + t.Fatal(err) + } + regInfo, err := ep.Ping() if err != nil { t.Fatal(err) } @@ -197,7 +205,7 @@ func TestPushImageJSONIndex(t *testing.T) { if repoData == nil { t.Fatal("Expected RepositoryData object") } - repoData, err = r.PushImageJSONIndex("foo42/bar", imgData, true, []string{r.indexEndpoint}) + repoData, err = r.PushImageJSONIndex("foo42/bar", imgData, true, []string{r.indexEndpoint.String()}) if err != nil { t.Fatal(err) } @@ -224,6 +232,10 @@ func TestValidRepositoryName(t *testing.T) { if err := validateRepositoryName("docker/docker"); err != nil { t.Fatal(err) } + // Support 64-byte non-hexadecimal names (hexadecimal names are forbidden) + if err := validateRepositoryName("thisisthesongthatneverendsitgoesonandonandonthisisthesongthatnev"); err != nil { + t.Fatal(err) + } if err := validateRepositoryName("docker/Docker"); err == nil { t.Log("Repository name should be invalid") t.Fail() @@ -232,17 +244,21 @@ func TestValidRepositoryName(t *testing.T) { t.Log("Repository name should be invalid") t.Fail() } + if err := validateRepositoryName("1a3f5e7d9c1b3a5f7e9d1c3b5a7f9e1d3c5b7a9f1e3d5d7c9b1a3f5e7d9c1b3a"); err == nil { + t.Log("Repository name should be invalid, 64-byte hexadecimal names forbidden") + t.Fail() + } } func TestTrustedLocation(t *testing.T) { - for _, url := range []string{"http://example.com", "https://example.com:7777", "http://docker.io", "http://test.docker.io", "https://fakedocker.com"} { + for _, url := range []string{"http://example.com", "https://example.com:7777", "http://docker.io", "http://test.docker.com", "https://fakedocker.com"} { req, _ := http.NewRequest("GET", url, nil) if trustedLocation(req) == true { t.Fatalf("'%s' shouldn't be detected as a trusted location", url) } } - for _, url := range []string{"https://docker.io", "https://test.docker.io:80"} { + for _, url := range []string{"https://docker.io", "https://test.docker.com:80"} { req, _ := http.NewRequest("GET", url, nil) if trustedLocation(req) == false { t.Fatalf("'%s' should be detected as a trusted location", url) diff --git a/docs/service.go b/docs/service.go index 0e6f1bda..f7b35300 100644 --- a/docs/service.go +++ b/docs/service.go @@ -40,11 +40,14 @@ func (s *Service) Auth(job *engine.Job) engine.Status { job.GetenvJson("authConfig", authConfig) // TODO: this is only done here because auth and registry need to be merged into one pkg if addr := authConfig.ServerAddress; addr != "" && addr != IndexServerAddress() { - addr, err = ExpandAndVerifyRegistryUrl(addr) + endpoint, err := NewEndpoint(addr) if err != nil { return job.Error(err) } - authConfig.ServerAddress = addr + if _, err := endpoint.Ping(); err != nil { + return job.Error(err) + } + authConfig.ServerAddress = endpoint.String() } status, err := Login(authConfig, HTTPRequestFactory(nil)) if err != nil { @@ -86,11 +89,11 @@ func (s *Service) Search(job *engine.Job) engine.Status { if err != nil { return job.Error(err) } - hostname, err = ExpandAndVerifyRegistryUrl(hostname) + endpoint, err := NewEndpoint(hostname) if err != nil { return job.Error(err) } - r, err := NewSession(authConfig, HTTPRequestFactory(metaHeaders), hostname, true) + r, err := NewSession(authConfig, HTTPRequestFactory(metaHeaders), endpoint, true) if err != nil { return job.Error(err) } diff --git a/docs/session.go b/docs/session.go index 82b931f2..5067b8d5 100644 --- a/docs/session.go +++ b/docs/session.go @@ -25,15 +25,15 @@ import ( type Session struct { authConfig *AuthConfig reqFactory *utils.HTTPRequestFactory - indexEndpoint string + indexEndpoint *Endpoint jar *cookiejar.Jar timeout TimeoutType } -func NewSession(authConfig *AuthConfig, factory *utils.HTTPRequestFactory, indexEndpoint string, timeout bool) (r *Session, err error) { +func NewSession(authConfig *AuthConfig, factory *utils.HTTPRequestFactory, endpoint *Endpoint, timeout bool) (r *Session, err error) { r = &Session{ authConfig: authConfig, - indexEndpoint: indexEndpoint, + indexEndpoint: endpoint, } if timeout { @@ -47,13 +47,13 @@ func NewSession(authConfig *AuthConfig, factory *utils.HTTPRequestFactory, index // If we're working with a standalone private registry over HTTPS, send Basic Auth headers // alongside our requests. - if indexEndpoint != IndexServerAddress() && strings.HasPrefix(indexEndpoint, "https://") { - info, err := pingRegistryEndpoint(indexEndpoint) + if r.indexEndpoint.VersionString(1) != IndexServerAddress() && r.indexEndpoint.URL.Scheme == "https" { + info, err := r.indexEndpoint.Ping() if err != nil { return nil, err } if info.Standalone { - log.Debugf("Endpoint %s is eligible for private registry registry. Enabling decorator.", indexEndpoint) + log.Debugf("Endpoint %s is eligible for private registry registry. Enabling decorator.", r.indexEndpoint.String()) dec := utils.NewHTTPAuthDecorator(authConfig.Username, authConfig.Password) factory.AddDecorator(dec) } @@ -153,10 +153,11 @@ func (r *Session) GetRemoteImageJSON(imgID, registry string, token []string) ([] func (r *Session) GetRemoteImageLayer(imgID, registry string, token []string, imgSize int64) (io.ReadCloser, error) { var ( - retries = 5 - client *http.Client - res *http.Response - imageURL = fmt.Sprintf("%simages/%s/layer", registry, imgID) + retries = 5 + statusCode = 0 + client *http.Client + res *http.Response + imageURL = fmt.Sprintf("%simages/%s/layer", registry, imgID) ) req, err := r.reqFactory.NewRequest("GET", imageURL, nil) @@ -165,12 +166,19 @@ func (r *Session) GetRemoteImageLayer(imgID, registry string, token []string, im } setTokenAuth(req, token) for i := 1; i <= retries; i++ { + statusCode = 0 res, client, err = r.doRequest(req) if err != nil { - res.Body.Close() + log.Debugf("Error contacting registry: %s", err) + if res != nil { + if res.Body != nil { + res.Body.Close() + } + statusCode = res.StatusCode + } if i == retries { return nil, fmt.Errorf("Server error: Status %d while fetching image layer (%s)", - res.StatusCode, imgID) + statusCode, imgID) } time.Sleep(time.Duration(i) * 5 * time.Second) continue @@ -253,8 +261,7 @@ func buildEndpointsList(headers []string, indexEp string) ([]string, error) { } func (r *Session) GetRepositoryData(remote string) (*RepositoryData, error) { - indexEp := r.indexEndpoint - repositoryTarget := fmt.Sprintf("%srepositories/%s/images", indexEp, remote) + repositoryTarget := fmt.Sprintf("%srepositories/%s/images", r.indexEndpoint.VersionString(1), remote) log.Debugf("[registry] Calling GET %s", repositoryTarget) @@ -288,17 +295,13 @@ func (r *Session) GetRepositoryData(remote string) (*RepositoryData, error) { var endpoints []string if res.Header.Get("X-Docker-Endpoints") != "" { - endpoints, err = buildEndpointsList(res.Header["X-Docker-Endpoints"], indexEp) + endpoints, err = buildEndpointsList(res.Header["X-Docker-Endpoints"], r.indexEndpoint.VersionString(1)) if err != nil { return nil, err } } else { // Assume the endpoint is on the same host - u, err := url.Parse(indexEp) - if err != nil { - return nil, err - } - endpoints = append(endpoints, fmt.Sprintf("%s://%s/v1/", u.Scheme, req.URL.Host)) + endpoints = append(endpoints, fmt.Sprintf("%s://%s/v1/", r.indexEndpoint.URL.Scheme, req.URL.Host)) } checksumsJSON, err := ioutil.ReadAll(res.Body) @@ -399,7 +402,10 @@ func (r *Session) PushImageLayerRegistry(imgID string, layer io.Reader, registry log.Debugf("[registry] Calling PUT %s", registry+"images/"+imgID+"/layer") - tarsumLayer := &tarsum.TarSum{Reader: layer} + tarsumLayer, err := tarsum.NewTarSum(layer, false, tarsum.Version0) + if err != nil { + return "", "", err + } h := sha256.New() h.Write(jsonRaw) h.Write([]byte{'\n'}) @@ -463,7 +469,6 @@ func (r *Session) PushRegistryTag(remote, revision, tag, registry string, token func (r *Session) PushImageJSONIndex(remote string, imgList []*ImgData, validate bool, regs []string) (*RepositoryData, error) { cleanImgList := []*ImgData{} - indexEp := r.indexEndpoint if validate { for _, elem := range imgList { @@ -483,7 +488,7 @@ func (r *Session) PushImageJSONIndex(remote string, imgList []*ImgData, validate if validate { suffix = "images" } - u := fmt.Sprintf("%srepositories/%s/%s", indexEp, remote, suffix) + u := fmt.Sprintf("%srepositories/%s/%s", r.indexEndpoint.VersionString(1), remote, suffix) log.Debugf("[registry] PUT %s", u) log.Debugf("Image list pushed to index:\n%s", imgListJSON) req, err := r.reqFactory.NewRequest("PUT", u, bytes.NewReader(imgListJSON)) @@ -541,7 +546,7 @@ func (r *Session) PushImageJSONIndex(remote string, imgList []*ImgData, validate } if res.Header.Get("X-Docker-Endpoints") != "" { - endpoints, err = buildEndpointsList(res.Header["X-Docker-Endpoints"], indexEp) + endpoints, err = buildEndpointsList(res.Header["X-Docker-Endpoints"], r.indexEndpoint.VersionString(1)) if err != nil { return nil, err } @@ -567,7 +572,7 @@ func (r *Session) PushImageJSONIndex(remote string, imgList []*ImgData, validate func (r *Session) SearchRepositories(term string) (*SearchResults, error) { log.Debugf("Index server: %s", r.indexEndpoint) - u := r.indexEndpoint + "search?q=" + url.QueryEscape(term) + u := r.indexEndpoint.VersionString(1) + "search?q=" + url.QueryEscape(term) req, err := r.reqFactory.NewRequest("GET", u, nil) if err != nil { return nil, err diff --git a/docs/session_v2.go b/docs/session_v2.go new file mode 100644 index 00000000..c0bc19b3 --- /dev/null +++ b/docs/session_v2.go @@ -0,0 +1,390 @@ +package registry + +import ( + "encoding/json" + "fmt" + "io" + "io/ioutil" + "net/url" + "strconv" + + "github.com/docker/docker/pkg/log" + "github.com/docker/docker/utils" + "github.com/gorilla/mux" +) + +func newV2RegistryRouter() *mux.Router { + router := mux.NewRouter() + + v2Router := router.PathPrefix("/v2/").Subrouter() + + // Version Info + v2Router.Path("/version").Name("version") + + // Image Manifests + v2Router.Path("/manifest/{imagename:[a-z0-9-._/]+}/{tagname:[a-zA-Z0-9-._]+}").Name("manifests") + + // List Image Tags + v2Router.Path("/tags/{imagename:[a-z0-9-._/]+}").Name("tags") + + // Download a blob + v2Router.Path("/blob/{imagename:[a-z0-9-._/]+}/{sumtype:[a-z0-9._+-]+}/{sum:[a-fA-F0-9]{4,}}").Name("downloadBlob") + + // Upload a blob + v2Router.Path("/blob/{imagename:[a-z0-9-._/]+}/{sumtype:[a-z0-9._+-]+}").Name("uploadBlob") + + // Mounting a blob in an image + v2Router.Path("/mountblob/{imagename:[a-z0-9-._/]+}/{sumtype:[a-z0-9._+-]+}/{sum:[a-fA-F0-9]{4,}}").Name("mountBlob") + + return router +} + +// APIVersion2 /v2/ +var v2HTTPRoutes = newV2RegistryRouter() + +func getV2URL(e *Endpoint, routeName string, vars map[string]string) (*url.URL, error) { + route := v2HTTPRoutes.Get(routeName) + if route == nil { + return nil, fmt.Errorf("unknown regisry v2 route name: %q", routeName) + } + + varReplace := make([]string, 0, len(vars)*2) + for key, val := range vars { + varReplace = append(varReplace, key, val) + } + + routePath, err := route.URLPath(varReplace...) + if err != nil { + return nil, fmt.Errorf("unable to make registry route %q with vars %v: %s", routeName, vars, err) + } + u, err := url.Parse(REGISTRYSERVER) + if err != nil { + return nil, fmt.Errorf("invalid registry url: %s", err) + } + + return &url.URL{ + Scheme: u.Scheme, + Host: u.Host, + Path: routePath.Path, + }, nil +} + +// V2 Provenance POC + +func (r *Session) GetV2Version(token []string) (*RegistryInfo, error) { + routeURL, err := getV2URL(r.indexEndpoint, "version", nil) + if err != nil { + return nil, err + } + + method := "GET" + log.Debugf("[registry] Calling %q %s", method, routeURL.String()) + + req, err := r.reqFactory.NewRequest(method, routeURL.String(), nil) + if err != nil { + return nil, err + } + setTokenAuth(req, token) + res, _, err := r.doRequest(req) + if err != nil { + return nil, err + } + defer res.Body.Close() + if res.StatusCode != 200 { + return nil, utils.NewHTTPRequestError(fmt.Sprintf("Server error: %d fetching Version", res.StatusCode), res) + } + + decoder := json.NewDecoder(res.Body) + versionInfo := new(RegistryInfo) + + err = decoder.Decode(versionInfo) + if err != nil { + return nil, fmt.Errorf("unable to decode GetV2Version JSON response: %s", err) + } + + return versionInfo, nil +} + +// +// 1) Check if TarSum of each layer exists /v2/ +// 1.a) if 200, continue +// 1.b) if 300, then push the +// 1.c) if anything else, err +// 2) PUT the created/signed manifest +// +func (r *Session) GetV2ImageManifest(imageName, tagName string, token []string) ([]byte, error) { + vars := map[string]string{ + "imagename": imageName, + "tagname": tagName, + } + + routeURL, err := getV2URL(r.indexEndpoint, "manifests", vars) + if err != nil { + return nil, err + } + + method := "GET" + log.Debugf("[registry] Calling %q %s", method, routeURL.String()) + + req, err := r.reqFactory.NewRequest(method, routeURL.String(), nil) + if err != nil { + return nil, err + } + setTokenAuth(req, token) + res, _, err := r.doRequest(req) + if err != nil { + return nil, err + } + defer res.Body.Close() + if res.StatusCode != 200 { + if res.StatusCode == 401 { + return nil, errLoginRequired + } else if res.StatusCode == 404 { + return nil, ErrDoesNotExist + } + return nil, utils.NewHTTPRequestError(fmt.Sprintf("Server error: %d trying to fetch for %s:%s", res.StatusCode, imageName, tagName), res) + } + + buf, err := ioutil.ReadAll(res.Body) + if err != nil { + return nil, fmt.Errorf("Error while reading the http response: %s", err) + } + return buf, nil +} + +// - Succeeded to mount for this image scope +// - Failed with no error (So continue to Push the Blob) +// - Failed with error +func (r *Session) PostV2ImageMountBlob(imageName, sumType, sum string, token []string) (bool, error) { + vars := map[string]string{ + "imagename": imageName, + "sumtype": sumType, + "sum": sum, + } + + routeURL, err := getV2URL(r.indexEndpoint, "mountBlob", vars) + if err != nil { + return false, err + } + + method := "POST" + log.Debugf("[registry] Calling %q %s", method, routeURL.String()) + + req, err := r.reqFactory.NewRequest(method, routeURL.String(), nil) + if err != nil { + return false, err + } + setTokenAuth(req, token) + res, _, err := r.doRequest(req) + if err != nil { + return false, err + } + res.Body.Close() // close early, since we're not needing a body on this call .. yet? + switch res.StatusCode { + case 200: + // return something indicating no push needed + return true, nil + case 300: + // return something indicating blob push needed + return false, nil + } + return false, fmt.Errorf("Failed to mount %q - %s:%s : %d", imageName, sumType, sum, res.StatusCode) +} + +func (r *Session) GetV2ImageBlob(imageName, sumType, sum string, blobWrtr io.Writer, token []string) error { + vars := map[string]string{ + "imagename": imageName, + "sumtype": sumType, + "sum": sum, + } + + routeURL, err := getV2URL(r.indexEndpoint, "downloadBlob", vars) + if err != nil { + return err + } + + method := "GET" + log.Debugf("[registry] Calling %q %s", method, routeURL.String()) + req, err := r.reqFactory.NewRequest(method, routeURL.String(), nil) + if err != nil { + return err + } + setTokenAuth(req, token) + res, _, err := r.doRequest(req) + if err != nil { + return err + } + defer res.Body.Close() + if res.StatusCode != 200 { + if res.StatusCode == 401 { + return errLoginRequired + } + return utils.NewHTTPRequestError(fmt.Sprintf("Server error: %d trying to pull %s blob", res.StatusCode, imageName), res) + } + + _, err = io.Copy(blobWrtr, res.Body) + return err +} + +func (r *Session) GetV2ImageBlobReader(imageName, sumType, sum string, token []string) (io.ReadCloser, int64, error) { + vars := map[string]string{ + "imagename": imageName, + "sumtype": sumType, + "sum": sum, + } + + routeURL, err := getV2URL(r.indexEndpoint, "downloadBlob", vars) + if err != nil { + return nil, 0, err + } + + method := "GET" + log.Debugf("[registry] Calling %q %s", method, routeURL.String()) + req, err := r.reqFactory.NewRequest(method, routeURL.String(), nil) + if err != nil { + return nil, 0, err + } + setTokenAuth(req, token) + res, _, err := r.doRequest(req) + if err != nil { + return nil, 0, err + } + if res.StatusCode != 200 { + if res.StatusCode == 401 { + return nil, 0, errLoginRequired + } + return nil, 0, utils.NewHTTPRequestError(fmt.Sprintf("Server error: %d trying to pull %s blob", res.StatusCode, imageName), res) + } + lenStr := res.Header.Get("Content-Length") + l, err := strconv.ParseInt(lenStr, 10, 64) + if err != nil { + return nil, 0, err + } + + return res.Body, l, err +} + +// Push the image to the server for storage. +// 'layer' is an uncompressed reader of the blob to be pushed. +// The server will generate it's own checksum calculation. +func (r *Session) PutV2ImageBlob(imageName, sumType string, blobRdr io.Reader, token []string) (serverChecksum string, err error) { + vars := map[string]string{ + "imagename": imageName, + "sumtype": sumType, + } + + routeURL, err := getV2URL(r.indexEndpoint, "uploadBlob", vars) + if err != nil { + return "", err + } + + method := "PUT" + log.Debugf("[registry] Calling %q %s", method, routeURL.String()) + req, err := r.reqFactory.NewRequest(method, routeURL.String(), blobRdr) + if err != nil { + return "", err + } + setTokenAuth(req, token) + res, _, err := r.doRequest(req) + if err != nil { + return "", err + } + defer res.Body.Close() + if res.StatusCode != 201 { + if res.StatusCode == 401 { + return "", errLoginRequired + } + return "", utils.NewHTTPRequestError(fmt.Sprintf("Server error: %d trying to push %s blob", res.StatusCode, imageName), res) + } + + type sumReturn struct { + Checksum string `json:"checksum"` + } + + decoder := json.NewDecoder(res.Body) + var sumInfo sumReturn + + err = decoder.Decode(&sumInfo) + if err != nil { + return "", fmt.Errorf("unable to decode PutV2ImageBlob JSON response: %s", err) + } + + // XXX this is a json struct from the registry, with its checksum + return sumInfo.Checksum, nil +} + +// Finally Push the (signed) manifest of the blobs we've just pushed +func (r *Session) PutV2ImageManifest(imageName, tagName string, manifestRdr io.Reader, token []string) error { + vars := map[string]string{ + "imagename": imageName, + "tagname": tagName, + } + + routeURL, err := getV2URL(r.indexEndpoint, "manifests", vars) + if err != nil { + return err + } + + method := "PUT" + log.Debugf("[registry] Calling %q %s", method, routeURL.String()) + req, err := r.reqFactory.NewRequest(method, routeURL.String(), manifestRdr) + if err != nil { + return err + } + setTokenAuth(req, token) + res, _, err := r.doRequest(req) + if err != nil { + return err + } + res.Body.Close() + if res.StatusCode != 201 { + if res.StatusCode == 401 { + return errLoginRequired + } + return utils.NewHTTPRequestError(fmt.Sprintf("Server error: %d trying to push %s:%s manifest", res.StatusCode, imageName, tagName), res) + } + + return nil +} + +// Given a repository name, returns a json array of string tags +func (r *Session) GetV2RemoteTags(imageName string, token []string) ([]string, error) { + vars := map[string]string{ + "imagename": imageName, + } + + routeURL, err := getV2URL(r.indexEndpoint, "tags", vars) + if err != nil { + return nil, err + } + + method := "GET" + log.Debugf("[registry] Calling %q %s", method, routeURL.String()) + + req, err := r.reqFactory.NewRequest(method, routeURL.String(), nil) + if err != nil { + return nil, err + } + setTokenAuth(req, token) + res, _, err := r.doRequest(req) + if err != nil { + return nil, err + } + defer res.Body.Close() + if res.StatusCode != 200 { + if res.StatusCode == 401 { + return nil, errLoginRequired + } else if res.StatusCode == 404 { + return nil, ErrDoesNotExist + } + return nil, utils.NewHTTPRequestError(fmt.Sprintf("Server error: %d trying to fetch for %s", res.StatusCode, imageName), res) + } + + decoder := json.NewDecoder(res.Body) + var tags []string + err = decoder.Decode(&tags) + if err != nil { + return nil, fmt.Errorf("Error while decoding the http response: %s", err) + } + return tags, nil +} diff --git a/docs/types.go b/docs/types.go index 70d55e42..3b429f19 100644 --- a/docs/types.go +++ b/docs/types.go @@ -31,3 +31,37 @@ type RegistryInfo struct { Version string `json:"version"` Standalone bool `json:"standalone"` } + +type FSLayer struct { + BlobSum string `json:"blobSum"` +} + +type ManifestHistory struct { + V1Compatibility string `json:"v1Compatibility"` +} + +type ManifestData struct { + Name string `json:"name"` + Tag string `json:"tag"` + Architecture string `json:"architecture"` + FSLayers []*FSLayer `json:"fsLayers"` + History []*ManifestHistory `json:"history"` + SchemaVersion int `json:"schemaVersion"` +} + +type APIVersion int + +func (av APIVersion) String() string { + return apiVersions[av] +} + +var DefaultAPIVersion APIVersion = APIVersion1 +var apiVersions = map[APIVersion]string{ + 1: "v1", + 2: "v2", +} + +const ( + APIVersion1 = iota + 1 + APIVersion2 +)