From 36b58e5abf2fc632a6481348ad52a9ecb055715c Mon Sep 17 00:00:00 2001 From: "Guillaume J. Charmes" Date: Wed, 15 May 2013 01:41:39 +0000 Subject: [PATCH 001/375] Split registry into subpackage --- docs/registry.go | 659 ++++++++++++++++++++++++++++++++++++++++++ docs/registry_test.go | 151 ++++++++++ 2 files changed, 810 insertions(+) create mode 100644 docs/registry.go create mode 100644 docs/registry_test.go diff --git a/docs/registry.go b/docs/registry.go new file mode 100644 index 00000000..8900f275 --- /dev/null +++ b/docs/registry.go @@ -0,0 +1,659 @@ +package registry + +import ( + "encoding/json" + "fmt" + "github.com/dotcloud/docker/auth" + "github.com/dotcloud/docker/utils" + "github.com/shin-/cookiejar" + "io" + "io/ioutil" + "net/http" + "net/url" + "strings" +) + +func doWithCookies(c *http.Client, req *http.Request) (*http.Response, error) { + for _, cookie := range c.Jar.Cookies(req.URL) { + req.AddCookie(cookie) + } + return c.Do(req) +} + +// Retrieve the history of a given image from the Registry. +// Return a list of the parent's json (requested image included) +func (r *Registry) GetRemoteHistory(imgId, registry string, token []string) ([]string, error) { + client := r.getHttpClient() + + req, err := http.NewRequest("GET", registry+"/images/"+imgId+"/ancestry", nil) + if err != nil { + return nil, err + } + req.Header.Set("Authorization", "Token "+strings.Join(token, ", ")) + res, err := client.Do(req) + if err != nil || res.StatusCode != 200 { + if res != nil { + return nil, fmt.Errorf("Internal server error: %d trying to fetch remote history for %s", res.StatusCode, imgId) + } + return nil, err + } + defer res.Body.Close() + + jsonString, err := ioutil.ReadAll(res.Body) + if err != nil { + return nil, fmt.Errorf("Error while reading the http response: %s\n", err) + } + + utils.Debugf("Ancestry: %s", jsonString) + history := new([]string) + if err := json.Unmarshal(jsonString, history); err != nil { + return nil, err + } + return *history, nil +} + +func (r *Registry) getHttpClient() *http.Client { + if r.httpClient == nil { + r.httpClient = &http.Client{} + r.httpClient.Jar = cookiejar.NewCookieJar() + } + return r.httpClient +} + +// Check if an image exists in the Registry +func (r *Registry) LookupRemoteImage(imgId, registry string, authConfig *auth.AuthConfig) bool { + rt := &http.Transport{Proxy: http.ProxyFromEnvironment} + + req, err := http.NewRequest("GET", registry+"/images/"+imgId+"/json", nil) + if err != nil { + return false + } + req.SetBasicAuth(authConfig.Username, authConfig.Password) + res, err := rt.RoundTrip(req) + return err == nil && res.StatusCode == 307 +} + +func (r *Registry) getImagesInRepository(repository string, authConfig *auth.AuthConfig) ([]map[string]string, error) { + u := auth.IndexServerAddress() + "/repositories/" + repository + "/images" + req, err := http.NewRequest("GET", u, nil) + if err != nil { + return nil, err + } + if authConfig != nil && len(authConfig.Username) > 0 { + req.SetBasicAuth(authConfig.Username, authConfig.Password) + } + res, err := r.getHttpClient().Do(req) + if err != nil { + return nil, err + } + defer res.Body.Close() + + // Repository doesn't exist yet + if res.StatusCode == 404 { + return nil, nil + } + + jsonData, err := ioutil.ReadAll(res.Body) + if err != nil { + return nil, err + } + + imageList := []map[string]string{} + + err = json.Unmarshal(jsonData, &imageList) + if err != nil { + utils.Debugf("Body: %s (%s)\n", res.Body, u) + return nil, err + } + + return imageList, nil +} + +// Retrieve an image from the Registry. +// Returns the Image object as well as the layer as an Archive (io.Reader) +func (r *Registry) GetRemoteImageJson(stdout io.Writer, imgId, registry string, token []string) ([]byte, error) { + client := r.getHttpClient() + + fmt.Fprintf(stdout, "Pulling %s metadata\r\n", imgId) + // Get the Json + req, err := http.NewRequest("GET", registry+"/images/"+imgId+"/json", nil) + if err != nil { + return nil, fmt.Errorf("Failed to download json: %s", err) + } + req.Header.Set("Authorization", "Token "+strings.Join(token, ", ")) + res, err := client.Do(req) + if err != nil { + return nil, fmt.Errorf("Failed to download json: %s", err) + } + defer res.Body.Close() + if res.StatusCode != 200 { + return nil, fmt.Errorf("HTTP code %d", res.StatusCode) + } + jsonString, err := ioutil.ReadAll(res.Body) + if err != nil { + return nil, fmt.Errorf("Failed to parse downloaded json: %s (%s)", err, jsonString) + } + return jsonString, nil +} + +func (r *Registry) GetRemoteImageLayer(stdout io.Writer, imgId, registry string, token []string) (io.Reader, error) { + client := r.getHttpClient() + + req, err := http.NewRequest("GET", registry+"/images/"+imgId+"/layer", nil) + if err != nil { + return nil, fmt.Errorf("Error while getting from the server: %s\n", err) + } + req.Header.Set("Authorization", "Token "+strings.Join(token, ", ")) + res, err := client.Do(req) + if err != nil { + return nil, err + } + return utils.ProgressReader(res.Body, int(res.ContentLength), stdout, "Downloading %v/%v (%v)"), nil +} + +func (r *Registry) GetRemoteTags(stdout io.Writer, registries []string, repository string, token []string) (map[string]string, error) { + client := r.getHttpClient() + if strings.Count(repository, "/") == 0 { + // This will be removed once the Registry supports auto-resolution on + // the "library" namespace + repository = "library/" + repository + } + for _, host := range registries { + endpoint := fmt.Sprintf("https://%s/v1/repositories/%s/tags", host, repository) + req, err := http.NewRequest("GET", endpoint, nil) + if err != nil { + return nil, err + } + req.Header.Set("Authorization", "Token "+strings.Join(token, ", ")) + res, err := client.Do(req) + defer res.Body.Close() + utils.Debugf("Got status code %d from %s", res.StatusCode, endpoint) + if err != nil || (res.StatusCode != 200 && res.StatusCode != 404) { + continue + } else if res.StatusCode == 404 { + return nil, fmt.Errorf("Repository not found") + } + + result := make(map[string]string) + + rawJson, err := ioutil.ReadAll(res.Body) + if err != nil { + return nil, err + } + if err := json.Unmarshal(rawJson, &result); err != nil { + return nil, err + } + return result, nil + } + return nil, fmt.Errorf("Could not reach any registry endpoint") +} + +func (r *Registry) getImageForTag(stdout io.Writer, tag, remote, registry string, token []string) (string, error) { + client := r.getHttpClient() + + if !strings.Contains(remote, "/") { + remote = "library/" + remote + } + + registryEndpoint := "https://" + registry + "/v1" + repositoryTarget := registryEndpoint + "/repositories/" + remote + "/tags/" + tag + + req, err := http.NewRequest("GET", repositoryTarget, nil) + if err != nil { + return "", err + } + req.Header.Set("Authorization", "Token "+strings.Join(token, ", ")) + res, err := client.Do(req) + if err != nil { + return "", fmt.Errorf("Error while retrieving repository info: %v", err) + } + defer res.Body.Close() + if res.StatusCode == 403 { + return "", fmt.Errorf("You aren't authorized to access this resource") + } else if res.StatusCode != 200 { + return "", fmt.Errorf("HTTP code: %d", res.StatusCode) + } + + var imgId string + rawJson, err := ioutil.ReadAll(res.Body) + if err != nil { + return "", err + } + if err = json.Unmarshal(rawJson, &imgId); err != nil { + return "", err + } + return imgId, nil +} + +func (r *Registry) GetRepositoryData(remote string) (*RepositoryData, error) { + client := r.getHttpClient() + + utils.Debugf("Pulling repository %s from %s\r\n", remote, auth.IndexServerAddress()) + repositoryTarget := auth.IndexServerAddress() + "/repositories/" + remote + "/images" + + req, err := http.NewRequest("GET", repositoryTarget, nil) + if err != nil { + return nil, err + } + if r.authConfig != nil && len(r.authConfig.Username) > 0 { + req.SetBasicAuth(r.authConfig.Username, r.authConfig.Password) + } + req.Header.Set("X-Docker-Token", "true") + + res, err := client.Do(req) + if err != nil { + return nil, err + } + defer res.Body.Close() + if res.StatusCode == 401 { + return nil, fmt.Errorf("Please login first (HTTP code %d)", res.StatusCode) + } + // TODO: Right now we're ignoring checksums in the response body. + // In the future, we need to use them to check image validity. + if res.StatusCode != 200 { + return nil, fmt.Errorf("HTTP code: %d", res.StatusCode) + } + + var tokens []string + if res.Header.Get("X-Docker-Token") != "" { + tokens = res.Header["X-Docker-Token"] + } + + var endpoints []string + if res.Header.Get("X-Docker-Endpoints") != "" { + endpoints = res.Header["X-Docker-Endpoints"] + } else { + return nil, fmt.Errorf("Index response didn't contain any endpoints") + } + + checksumsJson, err := ioutil.ReadAll(res.Body) + if err != nil { + return nil, err + } + remoteChecksums := []*ImgData{} + if err := json.Unmarshal(checksumsJson, &remoteChecksums); err != nil { + return nil, err + } + + // Forge a better object from the retrieved data + imgsData := make(map[string]*ImgData) + for _, elem := range remoteChecksums { + imgsData[elem.Id] = elem + } + + return &RepositoryData{ + ImgList: imgsData, + Endpoints: endpoints, + Tokens: tokens, + }, nil +} + +// // Push a local image to the registry +// func (r *Registry) PushImage(stdout io.Writer, img *Image, registry string, token []string) error { +// registry = "https://" + registry + "/v1" + +// client := graph.getHttpClient() +// jsonRaw, err := ioutil.ReadFile(path.Join(graph.Root, img.Id, "json")) +// if err != nil { +// return fmt.Errorf("Error while retreiving the path for {%s}: %s", img.Id, err) +// } + +// fmt.Fprintf(stdout, "Pushing %s metadata\r\n", img.Id) + +// // FIXME: try json with UTF8 +// jsonData := strings.NewReader(string(jsonRaw)) +// req, err := http.NewRequest("PUT", registry+"/images/"+img.Id+"/json", jsonData) +// if err != nil { +// return err +// } +// req.Header.Add("Content-type", "application/json") +// req.Header.Set("Authorization", "Token "+strings.Join(token, ",")) + +// checksum, err := img.Checksum() +// if err != nil { +// return fmt.Errorf("Error while retrieving checksum for %s: %v", img.Id, err) +// } +// req.Header.Set("X-Docker-Checksum", checksum) +// utils.Debugf("Setting checksum for %s: %s", img.ShortId(), checksum) +// res, err := doWithCookies(client, req) +// if err != nil { +// return fmt.Errorf("Failed to upload metadata: %s", err) +// } +// defer res.Body.Close() +// if len(res.Cookies()) > 0 { +// client.Jar.SetCookies(req.URL, res.Cookies()) +// } +// if res.StatusCode != 200 { +// errBody, err := ioutil.ReadAll(res.Body) +// if err != nil { +// return fmt.Errorf("HTTP code %d while uploading metadata and error when"+ +// " trying to parse response body: %v", res.StatusCode, err) +// } +// var jsonBody map[string]string +// if err := json.Unmarshal(errBody, &jsonBody); err != nil { +// errBody = []byte(err.Error()) +// } else if jsonBody["error"] == "Image already exists" { +// fmt.Fprintf(stdout, "Image %v already uploaded ; skipping\n", img.Id) +// return nil +// } +// return fmt.Errorf("HTTP code %d while uploading metadata: %s", res.StatusCode, errBody) +// } + +// fmt.Fprintf(stdout, "Pushing %s fs layer\r\n", img.Id) +// root, err := img.root() +// if err != nil { +// return err +// } + +// var layerData *TempArchive +// // If the archive exists, use it +// file, err := os.Open(layerArchivePath(root)) +// if err != nil { +// if os.IsNotExist(err) { +// // If the archive does not exist, create one from the layer +// layerData, err = graph.TempLayerArchive(img.Id, Xz, stdout) +// if err != nil { +// return fmt.Errorf("Failed to generate layer archive: %s", err) +// } +// } else { +// return err +// } +// } else { +// defer file.Close() +// st, err := file.Stat() +// if err != nil { +// return err +// } +// layerData = &TempArchive{file, st.Size()} +// } + +// req3, err := http.NewRequest("PUT", registry+"/images/"+img.Id+"/layer", utils.ProgressReader(layerData, int(layerData.Size), stdout, "")) +// if err != nil { +// return err +// } + +// req3.ContentLength = -1 +// req3.TransferEncoding = []string{"chunked"} +// req3.Header.Set("Authorization", "Token "+strings.Join(token, ",")) +// res3, err := doWithCookies(client, req3) +// if err != nil { +// return fmt.Errorf("Failed to upload layer: %s", err) +// } +// defer res3.Body.Close() + +// if res3.StatusCode != 200 { +// errBody, err := ioutil.ReadAll(res3.Body) +// if err != nil { +// return fmt.Errorf("HTTP code %d while uploading metadata and error when"+ +// " trying to parse response body: %v", res.StatusCode, err) +// } +// return fmt.Errorf("Received HTTP code %d while uploading layer: %s", res3.StatusCode, errBody) +// } +// return nil +// } + +// // push a tag on the registry. +// // Remote has the format '/ +// func (r *Registry) pushTag(remote, revision, tag, registry string, token []string) error { +// // "jsonify" the string +// revision = "\"" + revision + "\"" +// registry = "https://" + registry + "/v1" + +// utils.Debugf("Pushing tags for rev [%s] on {%s}\n", revision, registry+"/users/"+remote+"/"+tag) + +// client := graph.getHttpClient() +// req, err := http.NewRequest("PUT", registry+"/repositories/"+remote+"/tags/"+tag, strings.NewReader(revision)) +// if err != nil { +// return err +// } +// req.Header.Add("Content-type", "application/json") +// req.Header.Set("Authorization", "Token "+strings.Join(token, ",")) +// req.ContentLength = int64(len(revision)) +// res, err := doWithCookies(client, req) +// if err != nil { +// return err +// } +// res.Body.Close() +// if res.StatusCode != 200 && res.StatusCode != 201 { +// return fmt.Errorf("Internal server error: %d trying to push tag %s on %s", res.StatusCode, tag, remote) +// } +// return nil +// } + +// // FIXME: this should really be PushTag +// func (r *Registry) pushPrimitive(stdout io.Writer, remote, tag, imgId, registry string, token []string) error { +// // Check if the local impage exists +// img, err := graph.Get(imgId) +// if err != nil { +// fmt.Fprintf(stdout, "Skipping tag %s:%s: %s does not exist\r\n", remote, tag, imgId) +// return nil +// } +// fmt.Fprintf(stdout, "Pushing image %s:%s\r\n", remote, tag) +// // Push the image +// if err = graph.PushImage(stdout, img, registry, token); err != nil { +// return err +// } +// fmt.Fprintf(stdout, "Registering tag %s:%s\r\n", remote, tag) +// // And then the tag +// if err = graph.pushTag(remote, imgId, tag, registry, token); err != nil { +// return err +// } +// return nil +// } + +// // Retrieve the checksum of an image +// // Priority: +// // - Check on the stored checksums +// // - Check if the archive exists, if it does not, ask the registry +// // - If the archive does exists, process the checksum from it +// // - If the archive does not exists and not found on registry, process checksum from layer +// func (r *Registry) getChecksum(imageId string) (string, error) { +// // FIXME: Use in-memory map instead of reading the file each time +// if sums, err := graph.getStoredChecksums(); err != nil { +// return "", err +// } else if checksum, exists := sums[imageId]; exists { +// return checksum, nil +// } + +// img, err := graph.Get(imageId) +// if err != nil { +// return "", err +// } + +// if _, err := os.Stat(layerArchivePath(graph.imageRoot(imageId))); err != nil { +// if os.IsNotExist(err) { +// // TODO: Ask the registry for the checksum +// // As the archive is not there, it is supposed to come from a pull. +// } else { +// return "", err +// } +// } + +// checksum, err := img.Checksum() +// if err != nil { +// return "", err +// } +// return checksum, nil +// } + +// // Push a repository to the registry. +// // Remote has the format '/ +// func (r *Registry) PushRepository(stdout io.Writer, remote string, localRepo Repository, authConfig *auth.AuthConfig) error { +// client := graph.getHttpClient() +// // FIXME: Do not reset the cookie each time? (need to reset it in case updating latest of a repo and repushing) +// client.Jar = cookiejar.NewCookieJar() +// var imgList []*ImgListJson + +// fmt.Fprintf(stdout, "Processing checksums\n") +// imageSet := make(map[string]struct{}) + +// for tag, id := range localRepo { +// img, err := graph.Get(id) +// if err != nil { +// return err +// } +// img.WalkHistory(func(img *Image) error { +// if _, exists := imageSet[img.Id]; exists { +// return nil +// } +// imageSet[img.Id] = struct{}{} +// checksum, err := graph.getChecksum(img.Id) +// if err != nil { +// return err +// } +// imgList = append([]*ImgListJson{{ +// Id: img.Id, +// Checksum: checksum, +// tag: tag, +// }}, imgList...) +// return nil +// }) +// } + +// imgListJson, err := json.Marshal(imgList) +// if err != nil { +// return err +// } + +// utils.Debugf("json sent: %s\n", imgListJson) + +// fmt.Fprintf(stdout, "Sending image list\n") +// req, err := http.NewRequest("PUT", auth.IndexServerAddress()+"/repositories/"+remote+"/", bytes.NewReader(imgListJson)) +// if err != nil { +// return err +// } +// req.SetBasicAuth(authConfig.Username, authConfig.Password) +// req.ContentLength = int64(len(imgListJson)) +// req.Header.Set("X-Docker-Token", "true") + +// res, err := client.Do(req) +// if err != nil { +// return err +// } +// defer res.Body.Close() + +// for res.StatusCode >= 300 && res.StatusCode < 400 { +// utils.Debugf("Redirected to %s\n", res.Header.Get("Location")) +// req, err = http.NewRequest("PUT", res.Header.Get("Location"), bytes.NewReader(imgListJson)) +// if err != nil { +// return err +// } +// req.SetBasicAuth(authConfig.Username, authConfig.Password) +// req.ContentLength = int64(len(imgListJson)) +// req.Header.Set("X-Docker-Token", "true") + +// res, err = client.Do(req) +// if err != nil { +// return err +// } +// defer res.Body.Close() +// } + +// if res.StatusCode != 200 && res.StatusCode != 201 { +// errBody, err := ioutil.ReadAll(res.Body) +// if err != nil { +// return err +// } +// return fmt.Errorf("Error: Status %d trying to push repository %s: %s", res.StatusCode, remote, errBody) +// } + +// var token, endpoints []string +// if res.Header.Get("X-Docker-Token") != "" { +// token = res.Header["X-Docker-Token"] +// utils.Debugf("Auth token: %v", token) +// } else { +// return fmt.Errorf("Index response didn't contain an access token") +// } +// if res.Header.Get("X-Docker-Endpoints") != "" { +// endpoints = res.Header["X-Docker-Endpoints"] +// } else { +// return fmt.Errorf("Index response didn't contain any endpoints") +// } + +// // FIXME: Send only needed images +// for _, registry := range endpoints { +// fmt.Fprintf(stdout, "Pushing repository %s to %s (%d tags)\r\n", remote, registry, len(localRepo)) +// // For each image within the repo, push them +// for _, elem := range imgList { +// if err := graph.pushPrimitive(stdout, remote, elem.tag, elem.Id, registry, token); err != nil { +// // FIXME: Continue on error? +// return err +// } +// } +// } + +// req2, err := http.NewRequest("PUT", auth.IndexServerAddress()+"/repositories/"+remote+"/images", bytes.NewReader(imgListJson)) +// if err != nil { +// return err +// } +// req2.SetBasicAuth(authConfig.Username, authConfig.Password) +// req2.Header["X-Docker-Endpoints"] = endpoints +// req2.ContentLength = int64(len(imgListJson)) +// res2, err := client.Do(req2) +// if err != nil { +// return err +// } +// defer res2.Body.Close() +// if res2.StatusCode != 204 { +// if errBody, err := ioutil.ReadAll(res2.Body); err != nil { +// return err +// } else { +// return fmt.Errorf("Error: Status %d trying to push checksums %s: %s", res2.StatusCode, remote, errBody) +// } +// } + +// return nil +// } + +func (r *Registry) SearchRepositories(stdout io.Writer, term string) (*SearchResults, error) { + client := r.getHttpClient() + u := auth.IndexServerAddress() + "/search?q=" + url.QueryEscape(term) + req, err := http.NewRequest("GET", u, nil) + if err != nil { + return nil, err + } + res, err := client.Do(req) + if err != nil { + return nil, err + } + defer res.Body.Close() + if res.StatusCode != 200 { + return nil, fmt.Errorf("Unexepected status code %d", res.StatusCode) + } + rawData, err := ioutil.ReadAll(res.Body) + if err != nil { + return nil, err + } + result := new(SearchResults) + err = json.Unmarshal(rawData, result) + return result, err +} + +type SearchResults struct { + Query string `json:"query"` + NumResults int `json:"num_results"` + Results []map[string]string `json:"results"` +} + +type RepositoryData struct { + ImgList map[string]*ImgData + Endpoints []string + Tokens []string +} + +type ImgData struct { + Id string `json:"id"` + Checksum string `json:"checksum,omitempty"` + Tag string `json:",omitempty"` +} + +type Registry struct { + httpClient *http.Client + authConfig *auth.AuthConfig +} + +func NewRegistry(authConfig *auth.AuthConfig) *Registry { + return &Registry{ + authConfig: authConfig, + } +} diff --git a/docs/registry_test.go b/docs/registry_test.go new file mode 100644 index 00000000..cead591a --- /dev/null +++ b/docs/registry_test.go @@ -0,0 +1,151 @@ +package registry + +import ( + "crypto/rand" + "encoding/hex" + "github.com/dotcloud/docker/auth" + "io/ioutil" + "os" + "path" + "testing" +) + +func TestPull(t *testing.T) { + os.Setenv("DOCKER_INDEX_URL", "") + runtime, err := newTestRuntime() + if err != nil { + t.Fatal(err) + } + defer nuke(runtime) + + err = runtime.graph.PullRepository(ioutil.Discard, "busybox", "", runtime.repositories, nil) + if err != nil { + t.Fatal(err) + } + img, err := runtime.repositories.LookupImage("busybox") + if err != nil { + t.Fatal(err) + } + + // Try to run something on this image to make sure the layer's been downloaded properly. + config, _, err := ParseRun([]string{img.Id, "echo", "Hello World"}, runtime.capabilities) + if err != nil { + t.Fatal(err) + } + + b := NewBuilder(runtime) + container, err := b.Create(config) + if err != nil { + t.Fatal(err) + } + if err := container.Start(); err != nil { + t.Fatal(err) + } + + if status := container.Wait(); status != 0 { + t.Fatalf("Expected status code 0, found %d instead", status) + } +} + +func TestPullTag(t *testing.T) { + os.Setenv("DOCKER_INDEX_URL", "") + runtime, err := newTestRuntime() + if err != nil { + t.Fatal(err) + } + defer nuke(runtime) + + err = runtime.graph.PullRepository(ioutil.Discard, "ubuntu", "12.04", runtime.repositories, nil) + if err != nil { + t.Fatal(err) + } + _, err = runtime.repositories.LookupImage("ubuntu:12.04") + if err != nil { + t.Fatal(err) + } + + img2, err := runtime.repositories.LookupImage("ubuntu:12.10") + if img2 != nil { + t.Fatalf("Expected nil image but found %v instead", img2.Id) + } +} + +func login(runtime *Runtime) error { + authConfig := auth.NewAuthConfig("unittester", "surlautrerivejetattendrai", "noise+unittester@dotcloud.com", runtime.root) + runtime.authConfig = authConfig + _, err := auth.Login(authConfig) + return err +} + +func TestPush(t *testing.T) { + os.Setenv("DOCKER_INDEX_URL", "https://indexstaging-docker.dotcloud.com") + defer os.Setenv("DOCKER_INDEX_URL", "") + runtime, err := newTestRuntime() + if err != nil { + t.Fatal(err) + } + defer nuke(runtime) + + err = login(runtime) + if err != nil { + t.Fatal(err) + } + + err = runtime.graph.PullRepository(ioutil.Discard, "joffrey/busybox", "", runtime.repositories, nil) + if err != nil { + t.Fatal(err) + } + tokenBuffer := make([]byte, 16) + _, err = rand.Read(tokenBuffer) + if err != nil { + t.Fatal(err) + } + token := hex.EncodeToString(tokenBuffer)[:29] + config, _, err := ParseRun([]string{"joffrey/busybox", "touch", "/" + token}, runtime.capabilities) + if err != nil { + t.Fatal(err) + } + + b := NewBuilder(runtime) + container, err := b.Create(config) + if err != nil { + t.Fatal(err) + } + if err := container.Start(); err != nil { + t.Fatal(err) + } + + if status := container.Wait(); status != 0 { + t.Fatalf("Expected status code 0, found %d instead", status) + } + + img, err := b.Commit(container, "unittester/"+token, "", "", "", nil) + if err != nil { + t.Fatal(err) + } + + repo := runtime.repositories.Repositories["unittester/"+token] + err = runtime.graph.PushRepository(ioutil.Discard, "unittester/"+token, repo, runtime.authConfig) + if err != nil { + t.Fatal(err) + } + + // Remove image so we can pull it again + if err := runtime.graph.Delete(img.Id); err != nil { + t.Fatal(err) + } + + err = runtime.graph.PullRepository(ioutil.Discard, "unittester/"+token, "", runtime.repositories, runtime.authConfig) + if err != nil { + t.Fatal(err) + } + + layerPath, err := img.layer() + if err != nil { + t.Fatal(err) + } + + if _, err := os.Stat(path.Join(layerPath, token)); err != nil { + t.Fatalf("Error while trying to retrieve token file: %v", err) + } +} From 1b23cb09da4b88e1aa57cb8ba663bef27b17db8e Mon Sep 17 00:00:00 2001 From: "Guillaume J. Charmes" Date: Wed, 15 May 2013 03:27:15 +0000 Subject: [PATCH 002/375] Begin to implement push with new project structure --- docs/registry.go | 467 ++++++++++++++++++----------------------------- 1 file changed, 173 insertions(+), 294 deletions(-) diff --git a/docs/registry.go b/docs/registry.go index 8900f275..7254b49e 100644 --- a/docs/registry.go +++ b/docs/registry.go @@ -1,6 +1,7 @@ package registry import ( + "bytes" "encoding/json" "fmt" "github.com/dotcloud/docker/auth" @@ -288,322 +289,200 @@ func (r *Registry) GetRepositoryData(remote string) (*RepositoryData, error) { }, nil } -// // Push a local image to the registry -// func (r *Registry) PushImage(stdout io.Writer, img *Image, registry string, token []string) error { -// registry = "https://" + registry + "/v1" +// Push a local image to the registry +func (r *Registry) PushImage(imgData *ImgData, jsonRaw []byte, layer io.Reader, registry string, token []string) error { + registry = "https://" + registry + "/v1" -// client := graph.getHttpClient() -// jsonRaw, err := ioutil.ReadFile(path.Join(graph.Root, img.Id, "json")) -// if err != nil { -// return fmt.Errorf("Error while retreiving the path for {%s}: %s", img.Id, err) -// } + client := r.getHttpClient() -// fmt.Fprintf(stdout, "Pushing %s metadata\r\n", img.Id) + // FIXME: try json with UTF8 + req, err := http.NewRequest("PUT", registry+"/images/"+imgData.Id+"/json", strings.NewReader(string(jsonRaw))) + if err != nil { + return err + } + req.Header.Add("Content-type", "application/json") + req.Header.Set("Authorization", "Token "+strings.Join(token, ",")) -// // FIXME: try json with UTF8 -// jsonData := strings.NewReader(string(jsonRaw)) -// req, err := http.NewRequest("PUT", registry+"/images/"+img.Id+"/json", jsonData) -// if err != nil { -// return err -// } -// req.Header.Add("Content-type", "application/json") -// req.Header.Set("Authorization", "Token "+strings.Join(token, ",")) + req.Header.Set("X-Docker-Checksum", imgData.Checksum) + utils.Debugf("Setting checksum for %s: %s", imgData.Id, imgData.Checksum) + res, err := doWithCookies(client, req) + if err != nil { + return fmt.Errorf("Failed to upload metadata: %s", err) + } + defer res.Body.Close() + if len(res.Cookies()) > 0 { + client.Jar.SetCookies(req.URL, res.Cookies()) + } + if res.StatusCode != 200 { + errBody, err := ioutil.ReadAll(res.Body) + if err != nil { + return fmt.Errorf("HTTP code %d while uploading metadata and error when"+ + " trying to parse response body: %v", res.StatusCode, err) + } + var jsonBody map[string]string + if err := json.Unmarshal(errBody, &jsonBody); err != nil { + errBody = []byte(err.Error()) + } else if jsonBody["error"] == "Image already exists" { + utils.Debugf("Image %s already uploaded ; skipping\n", imgData.Id) + return nil + } + return fmt.Errorf("HTTP code %d while uploading metadata: %s", res.StatusCode, errBody) + } -// checksum, err := img.Checksum() -// if err != nil { -// return fmt.Errorf("Error while retrieving checksum for %s: %v", img.Id, err) -// } -// req.Header.Set("X-Docker-Checksum", checksum) -// utils.Debugf("Setting checksum for %s: %s", img.ShortId(), checksum) -// res, err := doWithCookies(client, req) -// if err != nil { -// return fmt.Errorf("Failed to upload metadata: %s", err) -// } -// defer res.Body.Close() -// if len(res.Cookies()) > 0 { -// client.Jar.SetCookies(req.URL, res.Cookies()) -// } -// if res.StatusCode != 200 { -// errBody, err := ioutil.ReadAll(res.Body) -// if err != nil { -// return fmt.Errorf("HTTP code %d while uploading metadata and error when"+ -// " trying to parse response body: %v", res.StatusCode, err) -// } -// var jsonBody map[string]string -// if err := json.Unmarshal(errBody, &jsonBody); err != nil { -// errBody = []byte(err.Error()) -// } else if jsonBody["error"] == "Image already exists" { -// fmt.Fprintf(stdout, "Image %v already uploaded ; skipping\n", img.Id) -// return nil -// } -// return fmt.Errorf("HTTP code %d while uploading metadata: %s", res.StatusCode, errBody) -// } + req3, err := http.NewRequest("PUT", registry+"/images/"+imgData.Id+"/layer", layer) + if err != nil { + return err + } -// fmt.Fprintf(stdout, "Pushing %s fs layer\r\n", img.Id) -// root, err := img.root() -// if err != nil { -// return err -// } + req3.ContentLength = -1 + req3.TransferEncoding = []string{"chunked"} + req3.Header.Set("Authorization", "Token "+strings.Join(token, ",")) + res3, err := doWithCookies(client, req3) + if err != nil { + return fmt.Errorf("Failed to upload layer: %s", err) + } + defer res3.Body.Close() -// var layerData *TempArchive -// // If the archive exists, use it -// file, err := os.Open(layerArchivePath(root)) -// if err != nil { -// if os.IsNotExist(err) { -// // If the archive does not exist, create one from the layer -// layerData, err = graph.TempLayerArchive(img.Id, Xz, stdout) -// if err != nil { -// return fmt.Errorf("Failed to generate layer archive: %s", err) -// } -// } else { -// return err -// } -// } else { -// defer file.Close() -// st, err := file.Stat() -// if err != nil { -// return err -// } -// layerData = &TempArchive{file, st.Size()} -// } + if res3.StatusCode != 200 { + errBody, err := ioutil.ReadAll(res3.Body) + if err != nil { + return fmt.Errorf("HTTP code %d while uploading metadata and error when"+ + " trying to parse response body: %v", res.StatusCode, err) + } + return fmt.Errorf("Received HTTP code %d while uploading layer: %s", res3.StatusCode, errBody) + } + return nil +} -// req3, err := http.NewRequest("PUT", registry+"/images/"+img.Id+"/layer", utils.ProgressReader(layerData, int(layerData.Size), stdout, "")) -// if err != nil { -// return err -// } +// push a tag on the registry. +// Remote has the format '/ +func (r *Registry) pushTag(remote, revision, tag, registry string, token []string) error { + // "jsonify" the string + revision = "\"" + revision + "\"" + registry = "https://" + registry + "/v1" -// req3.ContentLength = -1 -// req3.TransferEncoding = []string{"chunked"} -// req3.Header.Set("Authorization", "Token "+strings.Join(token, ",")) -// res3, err := doWithCookies(client, req3) -// if err != nil { -// return fmt.Errorf("Failed to upload layer: %s", err) -// } -// defer res3.Body.Close() + utils.Debugf("Pushing tags for rev [%s] on {%s}\n", revision, registry+"/users/"+remote+"/"+tag) -// if res3.StatusCode != 200 { -// errBody, err := ioutil.ReadAll(res3.Body) -// if err != nil { -// return fmt.Errorf("HTTP code %d while uploading metadata and error when"+ -// " trying to parse response body: %v", res.StatusCode, err) -// } -// return fmt.Errorf("Received HTTP code %d while uploading layer: %s", res3.StatusCode, errBody) -// } -// return nil -// } + client := r.getHttpClient() + req, err := http.NewRequest("PUT", registry+"/repositories/"+remote+"/tags/"+tag, strings.NewReader(revision)) + if err != nil { + return err + } + req.Header.Add("Content-type", "application/json") + req.Header.Set("Authorization", "Token "+strings.Join(token, ",")) + req.ContentLength = int64(len(revision)) + res, err := doWithCookies(client, req) + if err != nil { + return err + } + res.Body.Close() + if res.StatusCode != 200 && res.StatusCode != 201 { + return fmt.Errorf("Internal server error: %d trying to push tag %s on %s", res.StatusCode, tag, remote) + } + return nil +} -// // push a tag on the registry. -// // Remote has the format '/ -// func (r *Registry) pushTag(remote, revision, tag, registry string, token []string) error { -// // "jsonify" the string -// revision = "\"" + revision + "\"" -// registry = "https://" + registry + "/v1" +// FIXME: this should really be PushTag +func (r *Registry) PushLayer(remote, tag, imgId, registry string, token []string) error { + // Check if the local impage exists + img, err := graph.Get(imgId) + if err != nil { + fmt.Fprintf(stdout, "Skipping tag %s:%s: %s does not exist\r\n", remote, tag, imgId) + return nil + } + fmt.Fprintf(stdout, "Pushing image %s:%s\r\n", remote, tag) + // Push the image + if err = graph.PushImage(stdout, img, registry, token); err != nil { + return err + } + fmt.Fprintf(stdout, "Registering tag %s:%s\r\n", remote, tag) + // And then the tag + if err = graph.pushTag(remote, imgId, tag, registry, token); err != nil { + return err + } + return nil +} -// utils.Debugf("Pushing tags for rev [%s] on {%s}\n", revision, registry+"/users/"+remote+"/"+tag) +func (r *Registry) PushJsonIndex(remote string, imgList []*ImgData, validate bool) (*RepositoryData, error) { + client := r.getHttpClient() -// client := graph.getHttpClient() -// req, err := http.NewRequest("PUT", registry+"/repositories/"+remote+"/tags/"+tag, strings.NewReader(revision)) -// if err != nil { -// return err -// } -// req.Header.Add("Content-type", "application/json") -// req.Header.Set("Authorization", "Token "+strings.Join(token, ",")) -// req.ContentLength = int64(len(revision)) -// res, err := doWithCookies(client, req) -// if err != nil { -// return err -// } -// res.Body.Close() -// if res.StatusCode != 200 && res.StatusCode != 201 { -// return fmt.Errorf("Internal server error: %d trying to push tag %s on %s", res.StatusCode, tag, remote) -// } -// return nil -// } + imgListJson, err := json.Marshal(imgList) + if err != nil { + return nil, err + } -// // FIXME: this should really be PushTag -// func (r *Registry) pushPrimitive(stdout io.Writer, remote, tag, imgId, registry string, token []string) error { -// // Check if the local impage exists -// img, err := graph.Get(imgId) -// if err != nil { -// fmt.Fprintf(stdout, "Skipping tag %s:%s: %s does not exist\r\n", remote, tag, imgId) -// return nil -// } -// fmt.Fprintf(stdout, "Pushing image %s:%s\r\n", remote, tag) -// // Push the image -// if err = graph.PushImage(stdout, img, registry, token); err != nil { -// return err -// } -// fmt.Fprintf(stdout, "Registering tag %s:%s\r\n", remote, tag) -// // And then the tag -// if err = graph.pushTag(remote, imgId, tag, registry, token); err != nil { -// return err -// } -// return nil -// } + utils.Debugf("json sent: %s\n", imgListJson) -// // Retrieve the checksum of an image -// // Priority: -// // - Check on the stored checksums -// // - Check if the archive exists, if it does not, ask the registry -// // - If the archive does exists, process the checksum from it -// // - If the archive does not exists and not found on registry, process checksum from layer -// func (r *Registry) getChecksum(imageId string) (string, error) { -// // FIXME: Use in-memory map instead of reading the file each time -// if sums, err := graph.getStoredChecksums(); err != nil { -// return "", err -// } else if checksum, exists := sums[imageId]; exists { -// return checksum, nil -// } + req, err := http.NewRequest("PUT", auth.IndexServerAddress()+"/repositories/"+remote+"/", bytes.NewReader(imgListJson)) + if err != nil { + return nil, err + } + req.SetBasicAuth(r.authConfig.Username, r.authConfig.Password) + req.ContentLength = int64(len(imgListJson)) + req.Header.Set("X-Docker-Token", "true") -// img, err := graph.Get(imageId) -// if err != nil { -// return "", err -// } + res, err := client.Do(req) + if err != nil { + return nil, err + } + defer res.Body.Close() -// if _, err := os.Stat(layerArchivePath(graph.imageRoot(imageId))); err != nil { -// if os.IsNotExist(err) { -// // TODO: Ask the registry for the checksum -// // As the archive is not there, it is supposed to come from a pull. -// } else { -// return "", err -// } -// } + // Redirect if necessary + for res.StatusCode >= 300 && res.StatusCode < 400 { + utils.Debugf("Redirected to %s\n", res.Header.Get("Location")) + req, err = http.NewRequest("PUT", res.Header.Get("Location"), bytes.NewReader(imgListJson)) + if err != nil { + return nil, err + } + req.SetBasicAuth(r.authConfig.Username, r.authConfig.Password) + req.ContentLength = int64(len(imgListJson)) + req.Header.Set("X-Docker-Token", "true") -// checksum, err := img.Checksum() -// if err != nil { -// return "", err -// } -// return checksum, nil -// } + res, err = client.Do(req) + if err != nil { + return nil, err + } + defer res.Body.Close() + } -// // Push a repository to the registry. -// // Remote has the format '/ -// func (r *Registry) PushRepository(stdout io.Writer, remote string, localRepo Repository, authConfig *auth.AuthConfig) error { -// client := graph.getHttpClient() -// // FIXME: Do not reset the cookie each time? (need to reset it in case updating latest of a repo and repushing) -// client.Jar = cookiejar.NewCookieJar() -// var imgList []*ImgListJson + if res.StatusCode != 200 && res.StatusCode != 201 { + errBody, err := ioutil.ReadAll(res.Body) + if err != nil { + return nil, err + } + return nil, fmt.Errorf("Error: Status %d trying to push repository %s: %s", res.StatusCode, remote, errBody) + } -// fmt.Fprintf(stdout, "Processing checksums\n") -// imageSet := make(map[string]struct{}) + var tokens []string + if res.Header.Get("X-Docker-Token") != "" { + tokens = res.Header["X-Docker-Token"] + utils.Debugf("Auth token: %v", tokens) + } else { + return nil, fmt.Errorf("Index response didn't contain an access token") + } -// for tag, id := range localRepo { -// img, err := graph.Get(id) -// if err != nil { -// return err -// } -// img.WalkHistory(func(img *Image) error { -// if _, exists := imageSet[img.Id]; exists { -// return nil -// } -// imageSet[img.Id] = struct{}{} -// checksum, err := graph.getChecksum(img.Id) -// if err != nil { -// return err -// } -// imgList = append([]*ImgListJson{{ -// Id: img.Id, -// Checksum: checksum, -// tag: tag, -// }}, imgList...) -// return nil -// }) -// } + var endpoints []string + if res.Header.Get("X-Docker-Endpoints") != "" { + endpoints = res.Header["X-Docker-Endpoints"] + } else { + return nil, fmt.Errorf("Index response didn't contain any endpoints") + } -// imgListJson, err := json.Marshal(imgList) -// if err != nil { -// return err -// } + if validate { + if res.StatusCode != 204 { + if errBody, err := ioutil.ReadAll(res.Body); err != nil { + return nil, err + } else { + return nil, fmt.Errorf("Error: Status %d trying to push checksums %s: %s", res.StatusCode, remote, errBody) + } + } + } -// utils.Debugf("json sent: %s\n", imgListJson) - -// fmt.Fprintf(stdout, "Sending image list\n") -// req, err := http.NewRequest("PUT", auth.IndexServerAddress()+"/repositories/"+remote+"/", bytes.NewReader(imgListJson)) -// if err != nil { -// return err -// } -// req.SetBasicAuth(authConfig.Username, authConfig.Password) -// req.ContentLength = int64(len(imgListJson)) -// req.Header.Set("X-Docker-Token", "true") - -// res, err := client.Do(req) -// if err != nil { -// return err -// } -// defer res.Body.Close() - -// for res.StatusCode >= 300 && res.StatusCode < 400 { -// utils.Debugf("Redirected to %s\n", res.Header.Get("Location")) -// req, err = http.NewRequest("PUT", res.Header.Get("Location"), bytes.NewReader(imgListJson)) -// if err != nil { -// return err -// } -// req.SetBasicAuth(authConfig.Username, authConfig.Password) -// req.ContentLength = int64(len(imgListJson)) -// req.Header.Set("X-Docker-Token", "true") - -// res, err = client.Do(req) -// if err != nil { -// return err -// } -// defer res.Body.Close() -// } - -// if res.StatusCode != 200 && res.StatusCode != 201 { -// errBody, err := ioutil.ReadAll(res.Body) -// if err != nil { -// return err -// } -// return fmt.Errorf("Error: Status %d trying to push repository %s: %s", res.StatusCode, remote, errBody) -// } - -// var token, endpoints []string -// if res.Header.Get("X-Docker-Token") != "" { -// token = res.Header["X-Docker-Token"] -// utils.Debugf("Auth token: %v", token) -// } else { -// return fmt.Errorf("Index response didn't contain an access token") -// } -// if res.Header.Get("X-Docker-Endpoints") != "" { -// endpoints = res.Header["X-Docker-Endpoints"] -// } else { -// return fmt.Errorf("Index response didn't contain any endpoints") -// } - -// // FIXME: Send only needed images -// for _, registry := range endpoints { -// fmt.Fprintf(stdout, "Pushing repository %s to %s (%d tags)\r\n", remote, registry, len(localRepo)) -// // For each image within the repo, push them -// for _, elem := range imgList { -// if err := graph.pushPrimitive(stdout, remote, elem.tag, elem.Id, registry, token); err != nil { -// // FIXME: Continue on error? -// return err -// } -// } -// } - -// req2, err := http.NewRequest("PUT", auth.IndexServerAddress()+"/repositories/"+remote+"/images", bytes.NewReader(imgListJson)) -// if err != nil { -// return err -// } -// req2.SetBasicAuth(authConfig.Username, authConfig.Password) -// req2.Header["X-Docker-Endpoints"] = endpoints -// req2.ContentLength = int64(len(imgListJson)) -// res2, err := client.Do(req2) -// if err != nil { -// return err -// } -// defer res2.Body.Close() -// if res2.StatusCode != 204 { -// if errBody, err := ioutil.ReadAll(res2.Body); err != nil { -// return err -// } else { -// return fmt.Errorf("Error: Status %d trying to push checksums %s: %s", res2.StatusCode, remote, errBody) -// } -// } - -// return nil -// } + return &RepositoryData{ + Tokens: tokens, + Endpoints: endpoints, + }, nil +} func (r *Registry) SearchRepositories(stdout io.Writer, term string) (*SearchResults, error) { client := r.getHttpClient() From a2e94b289c620edd4d9998cb4ca347bde44c1eec Mon Sep 17 00:00:00 2001 From: "Guillaume J. Charmes" Date: Wed, 15 May 2013 18:30:40 +0000 Subject: [PATCH 003/375] Refactor registry Push --- docs/registry.go | 58 +++++++++++++++++------------------------------- 1 file changed, 20 insertions(+), 38 deletions(-) diff --git a/docs/registry.go b/docs/registry.go index 7254b49e..1c75e836 100644 --- a/docs/registry.go +++ b/docs/registry.go @@ -290,9 +290,8 @@ func (r *Registry) GetRepositoryData(remote string) (*RepositoryData, error) { } // Push a local image to the registry -func (r *Registry) PushImage(imgData *ImgData, jsonRaw []byte, layer io.Reader, registry string, token []string) error { +func (r *Registry) PushImageJsonRegistry(imgData *ImgData, jsonRaw []byte, registry string, token []string) error { registry = "https://" + registry + "/v1" - client := r.getHttpClient() // FIXME: try json with UTF8 @@ -302,8 +301,8 @@ func (r *Registry) PushImage(imgData *ImgData, jsonRaw []byte, layer io.Reader, } req.Header.Add("Content-type", "application/json") req.Header.Set("Authorization", "Token "+strings.Join(token, ",")) - req.Header.Set("X-Docker-Checksum", imgData.Checksum) + utils.Debugf("Setting checksum for %s: %s", imgData.Id, imgData.Checksum) res, err := doWithCookies(client, req) if err != nil { @@ -328,35 +327,39 @@ func (r *Registry) PushImage(imgData *ImgData, jsonRaw []byte, layer io.Reader, } return fmt.Errorf("HTTP code %d while uploading metadata: %s", res.StatusCode, errBody) } + return nil +} - req3, err := http.NewRequest("PUT", registry+"/images/"+imgData.Id+"/layer", layer) +func (r *Registry) PushImageLayerRegistry(imgId string, layer io.Reader, registry string, token []string) error { + registry = "https://" + registry + "/v1" + client := r.getHttpClient() + + req, err := http.NewRequest("PUT", registry+"/images/"+imgId+"/layer", layer) if err != nil { return err } - - req3.ContentLength = -1 - req3.TransferEncoding = []string{"chunked"} - req3.Header.Set("Authorization", "Token "+strings.Join(token, ",")) - res3, err := doWithCookies(client, req3) + req.ContentLength = -1 + req.TransferEncoding = []string{"chunked"} + req.Header.Set("Authorization", "Token "+strings.Join(token, ",")) + res, err := doWithCookies(client, req) if err != nil { return fmt.Errorf("Failed to upload layer: %s", err) } - defer res3.Body.Close() + defer res.Body.Close() - if res3.StatusCode != 200 { - errBody, err := ioutil.ReadAll(res3.Body) + if res.StatusCode != 200 { + errBody, err := ioutil.ReadAll(res.Body) if err != nil { - return fmt.Errorf("HTTP code %d while uploading metadata and error when"+ - " trying to parse response body: %v", res.StatusCode, err) + return fmt.Errorf("HTTP code %d while uploading metadata and error when trying to parse response body: %s", res.StatusCode, err) } - return fmt.Errorf("Received HTTP code %d while uploading layer: %s", res3.StatusCode, errBody) + return fmt.Errorf("Received HTTP code %d while uploading layer: %s", res.StatusCode, errBody) } return nil } // push a tag on the registry. // Remote has the format '/ -func (r *Registry) pushTag(remote, revision, tag, registry string, token []string) error { +func (r *Registry) PushRegistryTag(remote, revision, tag, registry string, token []string) error { // "jsonify" the string revision = "\"" + revision + "\"" registry = "https://" + registry + "/v1" @@ -382,28 +385,7 @@ func (r *Registry) pushTag(remote, revision, tag, registry string, token []strin return nil } -// FIXME: this should really be PushTag -func (r *Registry) PushLayer(remote, tag, imgId, registry string, token []string) error { - // Check if the local impage exists - img, err := graph.Get(imgId) - if err != nil { - fmt.Fprintf(stdout, "Skipping tag %s:%s: %s does not exist\r\n", remote, tag, imgId) - return nil - } - fmt.Fprintf(stdout, "Pushing image %s:%s\r\n", remote, tag) - // Push the image - if err = graph.PushImage(stdout, img, registry, token); err != nil { - return err - } - fmt.Fprintf(stdout, "Registering tag %s:%s\r\n", remote, tag) - // And then the tag - if err = graph.pushTag(remote, imgId, tag, registry, token); err != nil { - return err - } - return nil -} - -func (r *Registry) PushJsonIndex(remote string, imgList []*ImgData, validate bool) (*RepositoryData, error) { +func (r *Registry) PushImageJsonIndex(remote string, imgList []*ImgData, validate bool) (*RepositoryData, error) { client := r.getHttpClient() imgListJson, err := json.Marshal(imgList) From b5d8930631ad44ca7d6f6ae50b233b64b4cb7796 Mon Sep 17 00:00:00 2001 From: "Guillaume J. Charmes" Date: Wed, 15 May 2013 18:50:52 +0000 Subject: [PATCH 004/375] Remove stdout from registry --- docs/registry.go | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-) diff --git a/docs/registry.go b/docs/registry.go index 1c75e836..8a5fc1ac 100644 --- a/docs/registry.go +++ b/docs/registry.go @@ -112,10 +112,9 @@ func (r *Registry) getImagesInRepository(repository string, authConfig *auth.Aut // Retrieve an image from the Registry. // Returns the Image object as well as the layer as an Archive (io.Reader) -func (r *Registry) GetRemoteImageJson(stdout io.Writer, imgId, registry string, token []string) ([]byte, error) { +func (r *Registry) GetRemoteImageJson(imgId, registry string, token []string) ([]byte, error) { client := r.getHttpClient() - fmt.Fprintf(stdout, "Pulling %s metadata\r\n", imgId) // Get the Json req, err := http.NewRequest("GET", registry+"/images/"+imgId+"/json", nil) if err != nil { @@ -137,22 +136,22 @@ func (r *Registry) GetRemoteImageJson(stdout io.Writer, imgId, registry string, return jsonString, nil } -func (r *Registry) GetRemoteImageLayer(stdout io.Writer, imgId, registry string, token []string) (io.Reader, error) { +func (r *Registry) GetRemoteImageLayer(imgId, registry string, token []string) (io.ReadCloser, int, error) { client := r.getHttpClient() req, err := http.NewRequest("GET", registry+"/images/"+imgId+"/layer", nil) if err != nil { - return nil, fmt.Errorf("Error while getting from the server: %s\n", err) + return nil, -1, fmt.Errorf("Error while getting from the server: %s\n", err) } req.Header.Set("Authorization", "Token "+strings.Join(token, ", ")) res, err := client.Do(req) if err != nil { - return nil, err + return nil, -1, err } - return utils.ProgressReader(res.Body, int(res.ContentLength), stdout, "Downloading %v/%v (%v)"), nil + return res.Body, int(res.ContentLength), nil } -func (r *Registry) GetRemoteTags(stdout io.Writer, registries []string, repository string, token []string) (map[string]string, error) { +func (r *Registry) GetRemoteTags(registries []string, repository string, token []string) (map[string]string, error) { client := r.getHttpClient() if strings.Count(repository, "/") == 0 { // This will be removed once the Registry supports auto-resolution on @@ -189,7 +188,7 @@ func (r *Registry) GetRemoteTags(stdout io.Writer, registries []string, reposito return nil, fmt.Errorf("Could not reach any registry endpoint") } -func (r *Registry) getImageForTag(stdout io.Writer, tag, remote, registry string, token []string) (string, error) { +func (r *Registry) getImageForTag(tag, remote, registry string, token []string) (string, error) { client := r.getHttpClient() if !strings.Contains(remote, "/") { @@ -466,7 +465,7 @@ func (r *Registry) PushImageJsonIndex(remote string, imgList []*ImgData, validat }, nil } -func (r *Registry) SearchRepositories(stdout io.Writer, term string) (*SearchResults, error) { +func (r *Registry) SearchRepositories(term string) (*SearchResults, error) { client := r.getHttpClient() u := auth.IndexServerAddress() + "/search?q=" + url.QueryEscape(term) req, err := http.NewRequest("GET", u, nil) From ffa1e56748ef01bdc054825b5a4d5403bba1970b Mon Sep 17 00:00:00 2001 From: "Guillaume J. Charmes" Date: Wed, 15 May 2013 19:22:08 +0000 Subject: [PATCH 005/375] Move httpClient within registry object --- docs/registry.go | 72 ++++++++++++++++-------------------------------- 1 file changed, 23 insertions(+), 49 deletions(-) diff --git a/docs/registry.go b/docs/registry.go index 8a5fc1ac..03e977c3 100644 --- a/docs/registry.go +++ b/docs/registry.go @@ -24,14 +24,12 @@ func doWithCookies(c *http.Client, req *http.Request) (*http.Response, error) { // Retrieve the history of a given image from the Registry. // Return a list of the parent's json (requested image included) func (r *Registry) GetRemoteHistory(imgId, registry string, token []string) ([]string, error) { - client := r.getHttpClient() - req, err := http.NewRequest("GET", registry+"/images/"+imgId+"/ancestry", nil) if err != nil { return nil, err } req.Header.Set("Authorization", "Token "+strings.Join(token, ", ")) - res, err := client.Do(req) + res, err := r.client.Do(req) if err != nil || res.StatusCode != 200 { if res != nil { return nil, fmt.Errorf("Internal server error: %d trying to fetch remote history for %s", res.StatusCode, imgId) @@ -42,7 +40,7 @@ func (r *Registry) GetRemoteHistory(imgId, registry string, token []string) ([]s jsonString, err := ioutil.ReadAll(res.Body) if err != nil { - return nil, fmt.Errorf("Error while reading the http response: %s\n", err) + return nil, fmt.Errorf("Error while reading the http response: %s", err) } utils.Debugf("Ancestry: %s", jsonString) @@ -53,14 +51,6 @@ func (r *Registry) GetRemoteHistory(imgId, registry string, token []string) ([]s return *history, nil } -func (r *Registry) getHttpClient() *http.Client { - if r.httpClient == nil { - r.httpClient = &http.Client{} - r.httpClient.Jar = cookiejar.NewCookieJar() - } - return r.httpClient -} - // Check if an image exists in the Registry func (r *Registry) LookupRemoteImage(imgId, registry string, authConfig *auth.AuthConfig) bool { rt := &http.Transport{Proxy: http.ProxyFromEnvironment} @@ -83,7 +73,7 @@ func (r *Registry) getImagesInRepository(repository string, authConfig *auth.Aut if authConfig != nil && len(authConfig.Username) > 0 { req.SetBasicAuth(authConfig.Username, authConfig.Password) } - res, err := r.getHttpClient().Do(req) + res, err := r.client.Do(req) if err != nil { return nil, err } @@ -100,9 +90,7 @@ func (r *Registry) getImagesInRepository(repository string, authConfig *auth.Aut } imageList := []map[string]string{} - - err = json.Unmarshal(jsonData, &imageList) - if err != nil { + if err := json.Unmarshal(jsonData, &imageList); err != nil { utils.Debugf("Body: %s (%s)\n", res.Body, u) return nil, err } @@ -113,15 +101,13 @@ func (r *Registry) getImagesInRepository(repository string, authConfig *auth.Aut // Retrieve an image from the Registry. // Returns the Image object as well as the layer as an Archive (io.Reader) func (r *Registry) GetRemoteImageJson(imgId, registry string, token []string) ([]byte, error) { - client := r.getHttpClient() - // Get the Json req, err := http.NewRequest("GET", registry+"/images/"+imgId+"/json", nil) if err != nil { return nil, fmt.Errorf("Failed to download json: %s", err) } req.Header.Set("Authorization", "Token "+strings.Join(token, ", ")) - res, err := client.Do(req) + res, err := r.client.Do(req) if err != nil { return nil, fmt.Errorf("Failed to download json: %s", err) } @@ -137,14 +123,12 @@ func (r *Registry) GetRemoteImageJson(imgId, registry string, token []string) ([ } func (r *Registry) GetRemoteImageLayer(imgId, registry string, token []string) (io.ReadCloser, int, error) { - client := r.getHttpClient() - req, err := http.NewRequest("GET", registry+"/images/"+imgId+"/layer", nil) if err != nil { return nil, -1, fmt.Errorf("Error while getting from the server: %s\n", err) } req.Header.Set("Authorization", "Token "+strings.Join(token, ", ")) - res, err := client.Do(req) + res, err := r.client.Do(req) if err != nil { return nil, -1, err } @@ -152,7 +136,6 @@ func (r *Registry) GetRemoteImageLayer(imgId, registry string, token []string) ( } func (r *Registry) GetRemoteTags(registries []string, repository string, token []string) (map[string]string, error) { - client := r.getHttpClient() if strings.Count(repository, "/") == 0 { // This will be removed once the Registry supports auto-resolution on // the "library" namespace @@ -165,7 +148,7 @@ func (r *Registry) GetRemoteTags(registries []string, repository string, token [ return nil, err } req.Header.Set("Authorization", "Token "+strings.Join(token, ", ")) - res, err := client.Do(req) + res, err := r.client.Do(req) defer res.Body.Close() utils.Debugf("Got status code %d from %s", res.StatusCode, endpoint) if err != nil || (res.StatusCode != 200 && res.StatusCode != 404) { @@ -189,8 +172,6 @@ func (r *Registry) GetRemoteTags(registries []string, repository string, token [ } func (r *Registry) getImageForTag(tag, remote, registry string, token []string) (string, error) { - client := r.getHttpClient() - if !strings.Contains(remote, "/") { remote = "library/" + remote } @@ -203,9 +184,9 @@ func (r *Registry) getImageForTag(tag, remote, registry string, token []string) return "", err } req.Header.Set("Authorization", "Token "+strings.Join(token, ", ")) - res, err := client.Do(req) + res, err := r.client.Do(req) if err != nil { - return "", fmt.Errorf("Error while retrieving repository info: %v", err) + return "", fmt.Errorf("Error while retrieving repository info: %s", err) } defer res.Body.Close() if res.StatusCode == 403 { @@ -219,15 +200,13 @@ func (r *Registry) getImageForTag(tag, remote, registry string, token []string) if err != nil { return "", err } - if err = json.Unmarshal(rawJson, &imgId); err != nil { + if err := json.Unmarshal(rawJson, &imgId); err != nil { return "", err } return imgId, nil } func (r *Registry) GetRepositoryData(remote string) (*RepositoryData, error) { - client := r.getHttpClient() - utils.Debugf("Pulling repository %s from %s\r\n", remote, auth.IndexServerAddress()) repositoryTarget := auth.IndexServerAddress() + "/repositories/" + remote + "/images" @@ -240,7 +219,7 @@ func (r *Registry) GetRepositoryData(remote string) (*RepositoryData, error) { } req.Header.Set("X-Docker-Token", "true") - res, err := client.Do(req) + res, err := r.client.Do(req) if err != nil { return nil, err } @@ -291,8 +270,6 @@ func (r *Registry) GetRepositoryData(remote string) (*RepositoryData, error) { // Push a local image to the registry func (r *Registry) PushImageJsonRegistry(imgData *ImgData, jsonRaw []byte, registry string, token []string) error { registry = "https://" + registry + "/v1" - client := r.getHttpClient() - // FIXME: try json with UTF8 req, err := http.NewRequest("PUT", registry+"/images/"+imgData.Id+"/json", strings.NewReader(string(jsonRaw))) if err != nil { @@ -303,13 +280,13 @@ func (r *Registry) PushImageJsonRegistry(imgData *ImgData, jsonRaw []byte, regis req.Header.Set("X-Docker-Checksum", imgData.Checksum) utils.Debugf("Setting checksum for %s: %s", imgData.Id, imgData.Checksum) - res, err := doWithCookies(client, req) + res, err := doWithCookies(r.client, req) if err != nil { return fmt.Errorf("Failed to upload metadata: %s", err) } defer res.Body.Close() if len(res.Cookies()) > 0 { - client.Jar.SetCookies(req.URL, res.Cookies()) + r.client.Jar.SetCookies(req.URL, res.Cookies()) } if res.StatusCode != 200 { errBody, err := ioutil.ReadAll(res.Body) @@ -331,8 +308,6 @@ func (r *Registry) PushImageJsonRegistry(imgData *ImgData, jsonRaw []byte, regis func (r *Registry) PushImageLayerRegistry(imgId string, layer io.Reader, registry string, token []string) error { registry = "https://" + registry + "/v1" - client := r.getHttpClient() - req, err := http.NewRequest("PUT", registry+"/images/"+imgId+"/layer", layer) if err != nil { return err @@ -340,7 +315,7 @@ func (r *Registry) PushImageLayerRegistry(imgId string, layer io.Reader, registr req.ContentLength = -1 req.TransferEncoding = []string{"chunked"} req.Header.Set("Authorization", "Token "+strings.Join(token, ",")) - res, err := doWithCookies(client, req) + res, err := doWithCookies(r.client, req) if err != nil { return fmt.Errorf("Failed to upload layer: %s", err) } @@ -365,7 +340,6 @@ func (r *Registry) PushRegistryTag(remote, revision, tag, registry string, token utils.Debugf("Pushing tags for rev [%s] on {%s}\n", revision, registry+"/users/"+remote+"/"+tag) - client := r.getHttpClient() req, err := http.NewRequest("PUT", registry+"/repositories/"+remote+"/tags/"+tag, strings.NewReader(revision)) if err != nil { return err @@ -373,7 +347,7 @@ func (r *Registry) PushRegistryTag(remote, revision, tag, registry string, token req.Header.Add("Content-type", "application/json") req.Header.Set("Authorization", "Token "+strings.Join(token, ",")) req.ContentLength = int64(len(revision)) - res, err := doWithCookies(client, req) + res, err := doWithCookies(r.client, req) if err != nil { return err } @@ -385,8 +359,6 @@ func (r *Registry) PushRegistryTag(remote, revision, tag, registry string, token } func (r *Registry) PushImageJsonIndex(remote string, imgList []*ImgData, validate bool) (*RepositoryData, error) { - client := r.getHttpClient() - imgListJson, err := json.Marshal(imgList) if err != nil { return nil, err @@ -402,7 +374,7 @@ func (r *Registry) PushImageJsonIndex(remote string, imgList []*ImgData, validat req.ContentLength = int64(len(imgListJson)) req.Header.Set("X-Docker-Token", "true") - res, err := client.Do(req) + res, err := r.client.Do(req) if err != nil { return nil, err } @@ -419,7 +391,7 @@ func (r *Registry) PushImageJsonIndex(remote string, imgList []*ImgData, validat req.ContentLength = int64(len(imgListJson)) req.Header.Set("X-Docker-Token", "true") - res, err = client.Do(req) + res, err = r.client.Do(req) if err != nil { return nil, err } @@ -466,13 +438,12 @@ func (r *Registry) PushImageJsonIndex(remote string, imgList []*ImgData, validat } func (r *Registry) SearchRepositories(term string) (*SearchResults, error) { - client := r.getHttpClient() u := auth.IndexServerAddress() + "/search?q=" + url.QueryEscape(term) req, err := http.NewRequest("GET", u, nil) if err != nil { return nil, err } - res, err := client.Do(req) + res, err := r.client.Do(req) if err != nil { return nil, err } @@ -508,12 +479,15 @@ type ImgData struct { } type Registry struct { - httpClient *http.Client + client *http.Client authConfig *auth.AuthConfig } func NewRegistry(authConfig *auth.AuthConfig) *Registry { - return &Registry{ + r := &Registry{ authConfig: authConfig, + client: &http.Client{}, } + r.client.Jar = cookiejar.NewCookieJar() + return r } From a82a6bfdffed4453a99c2d42b6646cab3a234569 Mon Sep 17 00:00:00 2001 From: "Guillaume J. Charmes" Date: Wed, 15 May 2013 13:22:57 -0700 Subject: [PATCH 006/375] Upload images only when necessary --- docs/registry.go | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/docs/registry.go b/docs/registry.go index 03e977c3..d79b3e9f 100644 --- a/docs/registry.go +++ b/docs/registry.go @@ -3,6 +3,7 @@ package registry import ( "bytes" "encoding/json" + "errors" "fmt" "github.com/dotcloud/docker/auth" "github.com/dotcloud/docker/utils" @@ -14,6 +15,8 @@ import ( "strings" ) +var ErrAlreadyExists error = errors.New("Image already exists") + func doWithCookies(c *http.Client, req *http.Request) (*http.Response, error) { for _, cookie := range c.Jar.Cookies(req.URL) { req.AddCookie(cookie) @@ -291,15 +294,13 @@ func (r *Registry) PushImageJsonRegistry(imgData *ImgData, jsonRaw []byte, regis if res.StatusCode != 200 { errBody, err := ioutil.ReadAll(res.Body) if err != nil { - return fmt.Errorf("HTTP code %d while uploading metadata and error when"+ - " trying to parse response body: %v", res.StatusCode, err) + return fmt.Errorf("HTTP code %d while uploading metadata and error when trying to parse response body: %s", res.StatusCode, err) } var jsonBody map[string]string if err := json.Unmarshal(errBody, &jsonBody); err != nil { errBody = []byte(err.Error()) } else if jsonBody["error"] == "Image already exists" { - utils.Debugf("Image %s already uploaded ; skipping\n", imgData.Id) - return nil + return ErrAlreadyExists } return fmt.Errorf("HTTP code %d while uploading metadata: %s", res.StatusCode, errBody) } @@ -338,8 +339,6 @@ func (r *Registry) PushRegistryTag(remote, revision, tag, registry string, token revision = "\"" + revision + "\"" registry = "https://" + registry + "/v1" - utils.Debugf("Pushing tags for rev [%s] on {%s}\n", revision, registry+"/users/"+remote+"/"+tag) - req, err := http.NewRequest("PUT", registry+"/repositories/"+remote+"/tags/"+tag, strings.NewReader(revision)) if err != nil { return err From 4a0228fd8e329138986c449670310864b9e5a52b Mon Sep 17 00:00:00 2001 From: "Guillaume J. Charmes" Date: Wed, 15 May 2013 13:39:24 -0700 Subject: [PATCH 007/375] Allow to change login --- docs/registry.go | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/docs/registry.go b/docs/registry.go index d79b3e9f..7a075e0e 100644 --- a/docs/registry.go +++ b/docs/registry.go @@ -459,6 +459,11 @@ func (r *Registry) SearchRepositories(term string) (*SearchResults, error) { return result, err } +func (r *Registry) ResetClient(authConfig *auth.AuthConfig) { + r.authConfig = authConfig + r.client.Jar = cookiejar.NewCookieJar() +} + type SearchResults struct { Query string `json:"query"` NumResults int `json:"num_results"` From 0933aa442492508bd6724cf883e6e423cd687090 Mon Sep 17 00:00:00 2001 From: "Guillaume J. Charmes" Date: Wed, 15 May 2013 17:17:33 -0700 Subject: [PATCH 008/375] Move authConfig from runtime to registry --- docs/registry.go | 47 +++++++++++------------------------------------ 1 file changed, 11 insertions(+), 36 deletions(-) diff --git a/docs/registry.go b/docs/registry.go index 7a075e0e..b6a641a8 100644 --- a/docs/registry.go +++ b/docs/registry.go @@ -174,41 +174,6 @@ func (r *Registry) GetRemoteTags(registries []string, repository string, token [ return nil, fmt.Errorf("Could not reach any registry endpoint") } -func (r *Registry) getImageForTag(tag, remote, registry string, token []string) (string, error) { - if !strings.Contains(remote, "/") { - remote = "library/" + remote - } - - registryEndpoint := "https://" + registry + "/v1" - repositoryTarget := registryEndpoint + "/repositories/" + remote + "/tags/" + tag - - req, err := http.NewRequest("GET", repositoryTarget, nil) - if err != nil { - return "", err - } - req.Header.Set("Authorization", "Token "+strings.Join(token, ", ")) - res, err := r.client.Do(req) - if err != nil { - return "", fmt.Errorf("Error while retrieving repository info: %s", err) - } - defer res.Body.Close() - if res.StatusCode == 403 { - return "", fmt.Errorf("You aren't authorized to access this resource") - } else if res.StatusCode != 200 { - return "", fmt.Errorf("HTTP code: %d", res.StatusCode) - } - - var imgId string - rawJson, err := ioutil.ReadAll(res.Body) - if err != nil { - return "", err - } - if err := json.Unmarshal(rawJson, &imgId); err != nil { - return "", err - } - return imgId, nil -} - func (r *Registry) GetRepositoryData(remote string) (*RepositoryData, error) { utils.Debugf("Pulling repository %s from %s\r\n", remote, auth.IndexServerAddress()) repositoryTarget := auth.IndexServerAddress() + "/repositories/" + remote + "/images" @@ -464,6 +429,13 @@ func (r *Registry) ResetClient(authConfig *auth.AuthConfig) { r.client.Jar = cookiejar.NewCookieJar() } +func (r *Registry) GetAuthConfig() *auth.AuthConfig { + return &auth.AuthConfig{ + Username: r.authConfig.Username, + Email: r.authConfig.Password, + } +} + type SearchResults struct { Query string `json:"query"` NumResults int `json:"num_results"` @@ -487,7 +459,10 @@ type Registry struct { authConfig *auth.AuthConfig } -func NewRegistry(authConfig *auth.AuthConfig) *Registry { +func NewRegistry(root string) *Registry { + // If the auth file does not exist, keep going + authConfig, _ := auth.LoadConfig(root) + r := &Registry{ authConfig: authConfig, client: &http.Client{}, From 5e6d1a0d5679539d2868ee619f6592e97dc07bd9 Mon Sep 17 00:00:00 2001 From: "Guillaume J. Charmes" Date: Wed, 15 May 2013 17:31:11 -0700 Subject: [PATCH 009/375] Update tests to reflect new AuthConfig --- docs/registry.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/registry.go b/docs/registry.go index b6a641a8..e2ffb292 100644 --- a/docs/registry.go +++ b/docs/registry.go @@ -432,7 +432,7 @@ func (r *Registry) ResetClient(authConfig *auth.AuthConfig) { func (r *Registry) GetAuthConfig() *auth.AuthConfig { return &auth.AuthConfig{ Username: r.authConfig.Username, - Email: r.authConfig.Password, + Email: r.authConfig.Email, } } From c8c892fec4f2c85b2b1880a527488cedf3ba3e63 Mon Sep 17 00:00:00 2001 From: "Guillaume J. Charmes" Date: Wed, 15 May 2013 17:57:53 -0700 Subject: [PATCH 010/375] Disable registry unit tests --- docs/registry_test.go | 275 ++++++++++++++++++++++-------------------- 1 file changed, 146 insertions(+), 129 deletions(-) diff --git a/docs/registry_test.go b/docs/registry_test.go index cead591a..fd955b7b 100644 --- a/docs/registry_test.go +++ b/docs/registry_test.go @@ -1,151 +1,168 @@ package registry -import ( - "crypto/rand" - "encoding/hex" - "github.com/dotcloud/docker/auth" - "io/ioutil" - "os" - "path" - "testing" -) +// import ( +// "crypto/rand" +// "encoding/hex" +// "github.com/dotcloud/docker" +// "github.com/dotcloud/docker/auth" +// "io/ioutil" +// "os" +// "path" +// "testing" +// ) -func TestPull(t *testing.T) { - os.Setenv("DOCKER_INDEX_URL", "") - runtime, err := newTestRuntime() - if err != nil { - t.Fatal(err) - } - defer nuke(runtime) +// func newTestRuntime() (*Runtime, error) { +// root, err := ioutil.TempDir("", "docker-test") +// if err != nil { +// return nil, err +// } +// if err := os.Remove(root); err != nil { +// return nil, err +// } - err = runtime.graph.PullRepository(ioutil.Discard, "busybox", "", runtime.repositories, nil) - if err != nil { - t.Fatal(err) - } - img, err := runtime.repositories.LookupImage("busybox") - if err != nil { - t.Fatal(err) - } +// if err := os.MkdirAll(root, 0700); err != nil && !os.IsExist(err) { +// return nil, err +// } - // Try to run something on this image to make sure the layer's been downloaded properly. - config, _, err := ParseRun([]string{img.Id, "echo", "Hello World"}, runtime.capabilities) - if err != nil { - t.Fatal(err) - } +// return runtime, nil +// } - b := NewBuilder(runtime) - container, err := b.Create(config) - if err != nil { - t.Fatal(err) - } - if err := container.Start(); err != nil { - t.Fatal(err) - } +// func TestPull(t *testing.T) { +// os.Setenv("DOCKER_INDEX_URL", "") +// runtime, err := newTestRuntime() +// if err != nil { +// t.Fatal(err) +// } +// defer nuke(runtime) - if status := container.Wait(); status != 0 { - t.Fatalf("Expected status code 0, found %d instead", status) - } -} +// err = runtime.graph.PullRepository(ioutil.Discard, "busybox", "", runtime.repositories, nil) +// if err != nil { +// t.Fatal(err) +// } +// img, err := runtime.repositories.LookupImage("busybox") +// if err != nil { +// t.Fatal(err) +// } -func TestPullTag(t *testing.T) { - os.Setenv("DOCKER_INDEX_URL", "") - runtime, err := newTestRuntime() - if err != nil { - t.Fatal(err) - } - defer nuke(runtime) +// // Try to run something on this image to make sure the layer's been downloaded properly. +// config, _, err := docker.ParseRun([]string{img.Id, "echo", "Hello World"}, runtime.capabilities) +// if err != nil { +// t.Fatal(err) +// } - err = runtime.graph.PullRepository(ioutil.Discard, "ubuntu", "12.04", runtime.repositories, nil) - if err != nil { - t.Fatal(err) - } - _, err = runtime.repositories.LookupImage("ubuntu:12.04") - if err != nil { - t.Fatal(err) - } +// b := NewBuilder(runtime) +// container, err := b.Create(config) +// if err != nil { +// t.Fatal(err) +// } +// if err := container.Start(); err != nil { +// t.Fatal(err) +// } - img2, err := runtime.repositories.LookupImage("ubuntu:12.10") - if img2 != nil { - t.Fatalf("Expected nil image but found %v instead", img2.Id) - } -} +// if status := container.Wait(); status != 0 { +// t.Fatalf("Expected status code 0, found %d instead", status) +// } +// } -func login(runtime *Runtime) error { - authConfig := auth.NewAuthConfig("unittester", "surlautrerivejetattendrai", "noise+unittester@dotcloud.com", runtime.root) - runtime.authConfig = authConfig - _, err := auth.Login(authConfig) - return err -} +// func TestPullTag(t *testing.T) { +// os.Setenv("DOCKER_INDEX_URL", "") +// runtime, err := newTestRuntime() +// if err != nil { +// t.Fatal(err) +// } +// defer nuke(runtime) -func TestPush(t *testing.T) { - os.Setenv("DOCKER_INDEX_URL", "https://indexstaging-docker.dotcloud.com") - defer os.Setenv("DOCKER_INDEX_URL", "") - runtime, err := newTestRuntime() - if err != nil { - t.Fatal(err) - } - defer nuke(runtime) +// err = runtime.graph.PullRepository(ioutil.Discard, "ubuntu", "12.04", runtime.repositories, nil) +// if err != nil { +// t.Fatal(err) +// } +// _, err = runtime.repositories.LookupImage("ubuntu:12.04") +// if err != nil { +// t.Fatal(err) +// } - err = login(runtime) - if err != nil { - t.Fatal(err) - } +// img2, err := runtime.repositories.LookupImage("ubuntu:12.10") +// if img2 != nil { +// t.Fatalf("Expected nil image but found %v instead", img2.Id) +// } +// } - err = runtime.graph.PullRepository(ioutil.Discard, "joffrey/busybox", "", runtime.repositories, nil) - if err != nil { - t.Fatal(err) - } - tokenBuffer := make([]byte, 16) - _, err = rand.Read(tokenBuffer) - if err != nil { - t.Fatal(err) - } - token := hex.EncodeToString(tokenBuffer)[:29] - config, _, err := ParseRun([]string{"joffrey/busybox", "touch", "/" + token}, runtime.capabilities) - if err != nil { - t.Fatal(err) - } +// func login(runtime *Runtime) error { +// authConfig := auth.NewAuthConfig("unittester", "surlautrerivejetattendrai", "noise+unittester@dotcloud.com", runtime.root) +// runtime.authConfig = authConfig +// _, err := auth.Login(authConfig) +// return err +// } - b := NewBuilder(runtime) - container, err := b.Create(config) - if err != nil { - t.Fatal(err) - } - if err := container.Start(); err != nil { - t.Fatal(err) - } +// func TestPush(t *testing.T) { +// os.Setenv("DOCKER_INDEX_URL", "https://indexstaging-docker.dotcloud.com") +// defer os.Setenv("DOCKER_INDEX_URL", "") +// runtime, err := newTestRuntime() +// if err != nil { +// t.Fatal(err) +// } +// defer nuke(runtime) - if status := container.Wait(); status != 0 { - t.Fatalf("Expected status code 0, found %d instead", status) - } +// err = login(runtime) +// if err != nil { +// t.Fatal(err) +// } - img, err := b.Commit(container, "unittester/"+token, "", "", "", nil) - if err != nil { - t.Fatal(err) - } +// err = runtime.graph.PullRepository(ioutil.Discard, "joffrey/busybox", "", runtime.repositories, nil) +// if err != nil { +// t.Fatal(err) +// } +// tokenBuffer := make([]byte, 16) +// _, err = rand.Read(tokenBuffer) +// if err != nil { +// t.Fatal(err) +// } +// token := hex.EncodeToString(tokenBuffer)[:29] +// config, _, err := ParseRun([]string{"joffrey/busybox", "touch", "/" + token}, runtime.capabilities) +// if err != nil { +// t.Fatal(err) +// } - repo := runtime.repositories.Repositories["unittester/"+token] - err = runtime.graph.PushRepository(ioutil.Discard, "unittester/"+token, repo, runtime.authConfig) - if err != nil { - t.Fatal(err) - } +// b := NewBuilder(runtime) +// container, err := b.Create(config) +// if err != nil { +// t.Fatal(err) +// } +// if err := container.Start(); err != nil { +// t.Fatal(err) +// } - // Remove image so we can pull it again - if err := runtime.graph.Delete(img.Id); err != nil { - t.Fatal(err) - } +// if status := container.Wait(); status != 0 { +// t.Fatalf("Expected status code 0, found %d instead", status) +// } - err = runtime.graph.PullRepository(ioutil.Discard, "unittester/"+token, "", runtime.repositories, runtime.authConfig) - if err != nil { - t.Fatal(err) - } +// img, err := b.Commit(container, "unittester/"+token, "", "", "", nil) +// if err != nil { +// t.Fatal(err) +// } - layerPath, err := img.layer() - if err != nil { - t.Fatal(err) - } +// repo := runtime.repositories.Repositories["unittester/"+token] +// err = runtime.graph.PushRepository(ioutil.Discard, "unittester/"+token, repo, runtime.authConfig) +// if err != nil { +// t.Fatal(err) +// } - if _, err := os.Stat(path.Join(layerPath, token)); err != nil { - t.Fatalf("Error while trying to retrieve token file: %v", err) - } -} +// // Remove image so we can pull it again +// if err := runtime.graph.Delete(img.Id); err != nil { +// t.Fatal(err) +// } + +// err = runtime.graph.PullRepository(ioutil.Discard, "unittester/"+token, "", runtime.repositories, runtime.authConfig) +// if err != nil { +// t.Fatal(err) +// } + +// layerPath, err := img.layer() +// if err != nil { +// t.Fatal(err) +// } + +// if _, err := os.Stat(path.Join(layerPath, token)); err != nil { +// t.Fatalf("Error while trying to retrieve token file: %v", err) +// } +// } From 40ccd26d824c5fce4dcfa5d8fc03ad09e755d387 Mon Sep 17 00:00:00 2001 From: "Guillaume J. Charmes" Date: Thu, 16 May 2013 12:09:06 -0700 Subject: [PATCH 011/375] Remove hijack from api when not necessary --- docs/registry.go | 1 - 1 file changed, 1 deletion(-) diff --git a/docs/registry.go b/docs/registry.go index e2ffb292..71648d18 100644 --- a/docs/registry.go +++ b/docs/registry.go @@ -175,7 +175,6 @@ func (r *Registry) GetRemoteTags(registries []string, repository string, token [ } func (r *Registry) GetRepositoryData(remote string) (*RepositoryData, error) { - utils.Debugf("Pulling repository %s from %s\r\n", remote, auth.IndexServerAddress()) repositoryTarget := auth.IndexServerAddress() + "/repositories/" + remote + "/images" req, err := http.NewRequest("GET", repositoryTarget, nil) From 9373c8e4599de15889c6309375e220f4a6feb846 Mon Sep 17 00:00:00 2001 From: "Guillaume J. Charmes" Date: Thu, 16 May 2013 14:33:29 -0700 Subject: [PATCH 012/375] Update Push to reflect the correct API --- docs/registry.go | 48 ++++++++++++++++++++++++------------------------ 1 file changed, 24 insertions(+), 24 deletions(-) diff --git a/docs/registry.go b/docs/registry.go index 71648d18..ce9b4b4a 100644 --- a/docs/registry.go +++ b/docs/registry.go @@ -326,10 +326,11 @@ func (r *Registry) PushImageJsonIndex(remote string, imgList []*ImgData, validat if err != nil { return nil, err } - - utils.Debugf("json sent: %s\n", imgListJson) - - req, err := http.NewRequest("PUT", auth.IndexServerAddress()+"/repositories/"+remote+"/", bytes.NewReader(imgListJson)) + var suffix string + if validate { + suffix = "images" + } + req, err := http.NewRequest("PUT", auth.IndexServerAddress()+"/repositories/"+remote+"/"+suffix, bytes.NewReader(imgListJson)) if err != nil { return nil, err } @@ -361,29 +362,28 @@ func (r *Registry) PushImageJsonIndex(remote string, imgList []*ImgData, validat defer res.Body.Close() } - if res.StatusCode != 200 && res.StatusCode != 201 { - errBody, err := ioutil.ReadAll(res.Body) - if err != nil { - return nil, err + var tokens, endpoints []string + if !validate { + if res.StatusCode != 200 && res.StatusCode != 201 { + errBody, err := ioutil.ReadAll(res.Body) + if err != nil { + return nil, err + } + return nil, fmt.Errorf("Error: Status %d trying to push repository %s: %s", res.StatusCode, remote, errBody) + } + if res.Header.Get("X-Docker-Token") != "" { + tokens = res.Header["X-Docker-Token"] + utils.Debugf("Auth token: %v", tokens) + } else { + return nil, fmt.Errorf("Index response didn't contain an access token") } - return nil, fmt.Errorf("Error: Status %d trying to push repository %s: %s", res.StatusCode, remote, errBody) - } - var tokens []string - if res.Header.Get("X-Docker-Token") != "" { - tokens = res.Header["X-Docker-Token"] - utils.Debugf("Auth token: %v", tokens) - } else { - return nil, fmt.Errorf("Index response didn't contain an access token") + if res.Header.Get("X-Docker-Endpoints") != "" { + endpoints = res.Header["X-Docker-Endpoints"] + } else { + return nil, fmt.Errorf("Index response didn't contain any endpoints") + } } - - var endpoints []string - if res.Header.Get("X-Docker-Endpoints") != "" { - endpoints = res.Header["X-Docker-Endpoints"] - } else { - return nil, fmt.Errorf("Index response didn't contain any endpoints") - } - if validate { if res.StatusCode != 204 { if errBody, err := ioutil.ReadAll(res.Body); err != nil { From 6bd45ee686efd524d5820e708b461d68387fe413 Mon Sep 17 00:00:00 2001 From: Victor Vieux Date: Fri, 24 May 2013 14:23:43 +0000 Subject: [PATCH 013/375] fix docker login when same username --- docs/registry.go | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/docs/registry.go b/docs/registry.go index ce9b4b4a..bd361b5e 100644 --- a/docs/registry.go +++ b/docs/registry.go @@ -428,9 +428,14 @@ func (r *Registry) ResetClient(authConfig *auth.AuthConfig) { r.client.Jar = cookiejar.NewCookieJar() } -func (r *Registry) GetAuthConfig() *auth.AuthConfig { +func (r *Registry) GetAuthConfig(withPasswd bool) *auth.AuthConfig { + password := "" + if withPasswd { + password = r.authConfig.Password + } return &auth.AuthConfig{ Username: r.authConfig.Username, + Password: password, Email: r.authConfig.Email, } } From 2312a0e491868ac3d25cb6140d7ecffd6cdcff69 Mon Sep 17 00:00:00 2001 From: "Guillaume J. Charmes" Date: Tue, 28 May 2013 17:12:24 -0700 Subject: [PATCH 014/375] Cereate a new registry object for each request (~session) --- docs/registry.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/docs/registry.go b/docs/registry.go index bd361b5e..36b01d64 100644 --- a/docs/registry.go +++ b/docs/registry.go @@ -330,6 +330,9 @@ func (r *Registry) PushImageJsonIndex(remote string, imgList []*ImgData, validat if validate { suffix = "images" } + + utils.Debugf("Image list pushed to index:\n%s\n", imgListJson) + req, err := http.NewRequest("PUT", auth.IndexServerAddress()+"/repositories/"+remote+"/"+suffix, bytes.NewReader(imgListJson)) if err != nil { return nil, err From 3e3a7c03aeb290d3f52052ab79b9eb51ae81aa30 Mon Sep 17 00:00:00 2001 From: Solomon Hykes Date: Tue, 28 May 2013 19:39:09 -0700 Subject: [PATCH 015/375] Documented who decides what and how. --- docs/MAINTAINERS | 3 +++ 1 file changed, 3 insertions(+) create mode 100644 docs/MAINTAINERS diff --git a/docs/MAINTAINERS b/docs/MAINTAINERS new file mode 100644 index 00000000..b11dfc06 --- /dev/null +++ b/docs/MAINTAINERS @@ -0,0 +1,3 @@ +Sam Alba +Joffrey Fuhrer +Ken Cochrane From e6cc4ff646a8b7ab42356d3cad72e93e28a6c3d3 Mon Sep 17 00:00:00 2001 From: Victor Vieux Date: Thu, 30 May 2013 15:39:43 +0000 Subject: [PATCH 016/375] move auth to the client WIP --- docs/registry.go | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/docs/registry.go b/docs/registry.go index 36b01d64..cc5e7496 100644 --- a/docs/registry.go +++ b/docs/registry.go @@ -466,10 +466,7 @@ type Registry struct { authConfig *auth.AuthConfig } -func NewRegistry(root string) *Registry { - // If the auth file does not exist, keep going - authConfig, _ := auth.LoadConfig(root) - +func NewRegistry(root string, authConfig *auth.AuthConfig) *Registry { r := &Registry{ authConfig: authConfig, client: &http.Client{}, From fc340ec9667b9adb3df1ff6599b7bbc64b6838b7 Mon Sep 17 00:00:00 2001 From: Sam Alba Date: Mon, 3 Jun 2013 12:14:57 -0700 Subject: [PATCH 017/375] Fixed missing Body.Close when doing some HTTP requests. It should improve some request issues. --- docs/registry.go | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/docs/registry.go b/docs/registry.go index 36b01d64..d9f53ee9 100644 --- a/docs/registry.go +++ b/docs/registry.go @@ -64,6 +64,9 @@ func (r *Registry) LookupRemoteImage(imgId, registry string, authConfig *auth.Au } req.SetBasicAuth(authConfig.Username, authConfig.Password) res, err := rt.RoundTrip(req) + if err == nil { + defer res.Body.Close() + } return err == nil && res.StatusCode == 307 } @@ -152,7 +155,9 @@ func (r *Registry) GetRemoteTags(registries []string, repository string, token [ } req.Header.Set("Authorization", "Token "+strings.Join(token, ", ")) res, err := r.client.Do(req) - defer res.Body.Close() + if err == nil { + defer res.Body.Close() + } utils.Debugf("Got status code %d from %s", res.StatusCode, endpoint) if err != nil || (res.StatusCode != 200 && res.StatusCode != 404) { continue From 6189c3cb0b517586ddab9db5a16bbed039b68a83 Mon Sep 17 00:00:00 2001 From: "Guillaume J. Charmes" Date: Mon, 3 Jun 2013 12:20:52 -0700 Subject: [PATCH 018/375] Minor changes in registry.go --- docs/registry.go | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/docs/registry.go b/docs/registry.go index d9f53ee9..8283bf44 100644 --- a/docs/registry.go +++ b/docs/registry.go @@ -64,10 +64,11 @@ func (r *Registry) LookupRemoteImage(imgId, registry string, authConfig *auth.Au } req.SetBasicAuth(authConfig.Username, authConfig.Password) res, err := rt.RoundTrip(req) - if err == nil { - defer res.Body.Close() + if err != nil { + return false } - return err == nil && res.StatusCode == 307 + res.Body.Close() + return res.StatusCode == 307 } func (r *Registry) getImagesInRepository(repository string, authConfig *auth.AuthConfig) ([]map[string]string, error) { @@ -155,18 +156,19 @@ func (r *Registry) GetRemoteTags(registries []string, repository string, token [ } req.Header.Set("Authorization", "Token "+strings.Join(token, ", ")) res, err := r.client.Do(req) - if err == nil { - defer res.Body.Close() - } utils.Debugf("Got status code %d from %s", res.StatusCode, endpoint) - if err != nil || (res.StatusCode != 200 && res.StatusCode != 404) { + if err != nil { + return nil, err + } + defer res.Body.Close() + + if res.StatusCode != 200 && res.StatusCode != 404 { continue } else if res.StatusCode == 404 { return nil, fmt.Errorf("Repository not found") } result := make(map[string]string) - rawJson, err := ioutil.ReadAll(res.Body) if err != nil { return nil, err From b9e67a8884b1d03536e0a50373aba6340d8bd892 Mon Sep 17 00:00:00 2001 From: Sam Alba Date: Mon, 3 Jun 2013 14:42:21 -0700 Subject: [PATCH 019/375] Disabled HTTP keep-alive in the default HTTP client for Registry calls --- docs/registry.go | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/docs/registry.go b/docs/registry.go index 8283bf44..aeae3fe4 100644 --- a/docs/registry.go +++ b/docs/registry.go @@ -477,9 +477,15 @@ func NewRegistry(root string) *Registry { // If the auth file does not exist, keep going authConfig, _ := auth.LoadConfig(root) + httpTransport := &http.Transport{ + DisableKeepAlives: true, + } + r := &Registry{ authConfig: authConfig, - client: &http.Client{}, + client: &http.Client{ + Transport: httpTransport, + }, } r.client.Jar = cookiejar.NewCookieJar() return r From f085aa4adceff90e5690599e7996340ce2e68cd6 Mon Sep 17 00:00:00 2001 From: Victor Vieux Date: Tue, 4 Jun 2013 13:51:12 +0000 Subject: [PATCH 020/375] drop/omit --- docs/registry.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/docs/registry.go b/docs/registry.go index aeae3fe4..b6cda928 100644 --- a/docs/registry.go +++ b/docs/registry.go @@ -15,7 +15,7 @@ import ( "strings" ) -var ErrAlreadyExists error = errors.New("Image already exists") +var ErrAlreadyExists = errors.New("Image already exists") func doWithCookies(c *http.Client, req *http.Request) (*http.Response, error) { for _, cookie := range c.Jar.Cookies(req.URL) { @@ -396,11 +396,11 @@ func (r *Registry) PushImageJsonIndex(remote string, imgList []*ImgData, validat } if validate { if res.StatusCode != 204 { - if errBody, err := ioutil.ReadAll(res.Body); err != nil { + errBody, err := ioutil.ReadAll(res.Body) + if err != nil { return nil, err - } else { - return nil, fmt.Errorf("Error: Status %d trying to push checksums %s: %s", res.StatusCode, remote, errBody) } + return nil, fmt.Errorf("Error: Status %d trying to push checksums %s: %s", res.StatusCode, remote, errBody) } } From 93c7079f8903be906e59dca89c5b23dff534e4f2 Mon Sep 17 00:00:00 2001 From: Victor Vieux Date: Tue, 4 Jun 2013 15:44:27 +0000 Subject: [PATCH 021/375] fix proxy --- docs/registry.go | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/registry.go b/docs/registry.go index aeae3fe4..ede8eda7 100644 --- a/docs/registry.go +++ b/docs/registry.go @@ -479,6 +479,7 @@ func NewRegistry(root string) *Registry { httpTransport := &http.Transport{ DisableKeepAlives: true, + Proxy: http.ProxyFromEnvironment, } r := &Registry{ From ead91d946e4dc870983ffbc19947924161401430 Mon Sep 17 00:00:00 2001 From: Victor Vieux Date: Tue, 4 Jun 2013 18:00:22 +0000 Subject: [PATCH 022/375] linted names --- docs/registry.go | 36 ++++++++++++++++++------------------ 1 file changed, 18 insertions(+), 18 deletions(-) diff --git a/docs/registry.go b/docs/registry.go index b6cda928..befceb8e 100644 --- a/docs/registry.go +++ b/docs/registry.go @@ -107,8 +107,8 @@ func (r *Registry) getImagesInRepository(repository string, authConfig *auth.Aut // Retrieve an image from the Registry. // Returns the Image object as well as the layer as an Archive (io.Reader) -func (r *Registry) GetRemoteImageJson(imgId, registry string, token []string) ([]byte, error) { - // Get the Json +func (r *Registry) GetRemoteImageJSON(imgId, registry string, token []string) ([]byte, error) { + // Get the JSON req, err := http.NewRequest("GET", registry+"/images/"+imgId+"/json", nil) if err != nil { return nil, fmt.Errorf("Failed to download json: %s", err) @@ -169,11 +169,11 @@ func (r *Registry) GetRemoteTags(registries []string, repository string, token [ } result := make(map[string]string) - rawJson, err := ioutil.ReadAll(res.Body) + rawJSON, err := ioutil.ReadAll(res.Body) if err != nil { return nil, err } - if err := json.Unmarshal(rawJson, &result); err != nil { + if err := json.Unmarshal(rawJSON, &result); err != nil { return nil, err } return result, nil @@ -219,19 +219,19 @@ func (r *Registry) GetRepositoryData(remote string) (*RepositoryData, error) { return nil, fmt.Errorf("Index response didn't contain any endpoints") } - checksumsJson, err := ioutil.ReadAll(res.Body) + checksumsJSON, err := ioutil.ReadAll(res.Body) if err != nil { return nil, err } remoteChecksums := []*ImgData{} - if err := json.Unmarshal(checksumsJson, &remoteChecksums); err != nil { + if err := json.Unmarshal(checksumsJSON, &remoteChecksums); err != nil { return nil, err } // Forge a better object from the retrieved data imgsData := make(map[string]*ImgData) for _, elem := range remoteChecksums { - imgsData[elem.Id] = elem + imgsData[elem.ID] = elem } return &RepositoryData{ @@ -242,10 +242,10 @@ func (r *Registry) GetRepositoryData(remote string) (*RepositoryData, error) { } // Push a local image to the registry -func (r *Registry) PushImageJsonRegistry(imgData *ImgData, jsonRaw []byte, registry string, token []string) error { +func (r *Registry) PushImageJSONRegistry(imgData *ImgData, jsonRaw []byte, registry string, token []string) error { registry = "https://" + registry + "/v1" // FIXME: try json with UTF8 - req, err := http.NewRequest("PUT", registry+"/images/"+imgData.Id+"/json", strings.NewReader(string(jsonRaw))) + req, err := http.NewRequest("PUT", registry+"/images/"+imgData.ID+"/json", strings.NewReader(string(jsonRaw))) if err != nil { return err } @@ -253,7 +253,7 @@ func (r *Registry) PushImageJsonRegistry(imgData *ImgData, jsonRaw []byte, regis req.Header.Set("Authorization", "Token "+strings.Join(token, ",")) req.Header.Set("X-Docker-Checksum", imgData.Checksum) - utils.Debugf("Setting checksum for %s: %s", imgData.Id, imgData.Checksum) + utils.Debugf("Setting checksum for %s: %s", imgData.ID, imgData.Checksum) res, err := doWithCookies(r.client, req) if err != nil { return fmt.Errorf("Failed to upload metadata: %s", err) @@ -328,8 +328,8 @@ func (r *Registry) PushRegistryTag(remote, revision, tag, registry string, token return nil } -func (r *Registry) PushImageJsonIndex(remote string, imgList []*ImgData, validate bool) (*RepositoryData, error) { - imgListJson, err := json.Marshal(imgList) +func (r *Registry) PushImageJSONIndex(remote string, imgList []*ImgData, validate bool) (*RepositoryData, error) { + imgListJSON, err := json.Marshal(imgList) if err != nil { return nil, err } @@ -338,14 +338,14 @@ func (r *Registry) PushImageJsonIndex(remote string, imgList []*ImgData, validat suffix = "images" } - utils.Debugf("Image list pushed to index:\n%s\n", imgListJson) + utils.Debugf("Image list pushed to index:\n%s\n", imgListJSON) - req, err := http.NewRequest("PUT", auth.IndexServerAddress()+"/repositories/"+remote+"/"+suffix, bytes.NewReader(imgListJson)) + req, err := http.NewRequest("PUT", auth.IndexServerAddress()+"/repositories/"+remote+"/"+suffix, bytes.NewReader(imgListJSON)) if err != nil { return nil, err } req.SetBasicAuth(r.authConfig.Username, r.authConfig.Password) - req.ContentLength = int64(len(imgListJson)) + req.ContentLength = int64(len(imgListJSON)) req.Header.Set("X-Docker-Token", "true") res, err := r.client.Do(req) @@ -357,12 +357,12 @@ func (r *Registry) PushImageJsonIndex(remote string, imgList []*ImgData, validat // Redirect if necessary for res.StatusCode >= 300 && res.StatusCode < 400 { utils.Debugf("Redirected to %s\n", res.Header.Get("Location")) - req, err = http.NewRequest("PUT", res.Header.Get("Location"), bytes.NewReader(imgListJson)) + req, err = http.NewRequest("PUT", res.Header.Get("Location"), bytes.NewReader(imgListJSON)) if err != nil { return nil, err } req.SetBasicAuth(r.authConfig.Username, r.authConfig.Password) - req.ContentLength = int64(len(imgListJson)) + req.ContentLength = int64(len(imgListJSON)) req.Header.Set("X-Docker-Token", "true") res, err = r.client.Do(req) @@ -463,7 +463,7 @@ type RepositoryData struct { } type ImgData struct { - Id string `json:"id"` + ID string `json:"id"` Checksum string `json:"checksum,omitempty"` Tag string `json:",omitempty"` } From deddb3c757d3cd5d4cfbc66d67c45641310ae777 Mon Sep 17 00:00:00 2001 From: "Guillaume J. Charmes" Date: Thu, 6 Jun 2013 18:16:16 -0700 Subject: [PATCH 023/375] Make the progressbar take the image size into consideration --- docs/registry.go | 30 ++++++++++++++++++------------ 1 file changed, 18 insertions(+), 12 deletions(-) diff --git a/docs/registry.go b/docs/registry.go index bd5c6b79..a2b43eed 100644 --- a/docs/registry.go +++ b/docs/registry.go @@ -12,6 +12,7 @@ import ( "io/ioutil" "net/http" "net/url" + "strconv" "strings" ) @@ -106,40 +107,45 @@ func (r *Registry) getImagesInRepository(repository string, authConfig *auth.Aut } // Retrieve an image from the Registry. -// Returns the Image object as well as the layer as an Archive (io.Reader) -func (r *Registry) GetRemoteImageJSON(imgId, registry string, token []string) ([]byte, error) { +func (r *Registry) GetRemoteImageJSON(imgId, registry string, token []string) ([]byte, int, error) { // Get the JSON req, err := http.NewRequest("GET", registry+"/images/"+imgId+"/json", nil) if err != nil { - return nil, fmt.Errorf("Failed to download json: %s", err) + return nil, -1, fmt.Errorf("Failed to download json: %s", err) } req.Header.Set("Authorization", "Token "+strings.Join(token, ", ")) res, err := r.client.Do(req) if err != nil { - return nil, fmt.Errorf("Failed to download json: %s", err) + return nil, -1, fmt.Errorf("Failed to download json: %s", err) } defer res.Body.Close() if res.StatusCode != 200 { - return nil, fmt.Errorf("HTTP code %d", res.StatusCode) + return nil, -1, fmt.Errorf("HTTP code %d", res.StatusCode) } + + imageSize, err := strconv.Atoi(res.Header.Get("X-Docker-Size")) + if err != nil { + return nil, -1, err + } + jsonString, err := ioutil.ReadAll(res.Body) if err != nil { - return nil, fmt.Errorf("Failed to parse downloaded json: %s (%s)", err, jsonString) + return nil, -1, fmt.Errorf("Failed to parse downloaded json: %s (%s)", err, jsonString) } - return jsonString, nil + return jsonString, imageSize, nil } -func (r *Registry) GetRemoteImageLayer(imgId, registry string, token []string) (io.ReadCloser, int, error) { +func (r *Registry) GetRemoteImageLayer(imgId, registry string, token []string) (io.ReadCloser, error) { req, err := http.NewRequest("GET", registry+"/images/"+imgId+"/layer", nil) if err != nil { - return nil, -1, fmt.Errorf("Error while getting from the server: %s\n", err) + return nil, fmt.Errorf("Error while getting from the server: %s\n", err) } req.Header.Set("Authorization", "Token "+strings.Join(token, ", ")) res, err := r.client.Do(req) if err != nil { - return nil, -1, err + return nil, err } - return res.Body, int(res.ContentLength), nil + return res.Body, nil } func (r *Registry) GetRemoteTags(registries []string, repository string, token []string) (map[string]string, error) { @@ -479,7 +485,7 @@ func NewRegistry(root string) *Registry { httpTransport := &http.Transport{ DisableKeepAlives: true, - Proxy: http.ProxyFromEnvironment, + Proxy: http.ProxyFromEnvironment, } r := &Registry{ From ca71aa4f8da365b2668577a811a42575d482fef8 Mon Sep 17 00:00:00 2001 From: shin- Date: Mon, 10 Jun 2013 11:21:56 -0700 Subject: [PATCH 024/375] Send X-Docker-Endpoints header when validating the images upload with the index at the end of a push --- docs/registry.go | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/docs/registry.go b/docs/registry.go index bd5c6b79..0ae37f7a 100644 --- a/docs/registry.go +++ b/docs/registry.go @@ -328,7 +328,7 @@ func (r *Registry) PushRegistryTag(remote, revision, tag, registry string, token return nil } -func (r *Registry) PushImageJSONIndex(remote string, imgList []*ImgData, validate bool) (*RepositoryData, error) { +func (r *Registry) PushImageJSONIndex(remote string, imgList []*ImgData, validate bool, regs []string) (*RepositoryData, error) { imgListJSON, err := json.Marshal(imgList) if err != nil { return nil, err @@ -347,6 +347,9 @@ func (r *Registry) PushImageJSONIndex(remote string, imgList []*ImgData, validat req.SetBasicAuth(r.authConfig.Username, r.authConfig.Password) req.ContentLength = int64(len(imgListJSON)) req.Header.Set("X-Docker-Token", "true") + if validate { + req.Header["X-Docker-Endpoints"] = regs + } res, err := r.client.Do(req) if err != nil { @@ -364,7 +367,9 @@ func (r *Registry) PushImageJSONIndex(remote string, imgList []*ImgData, validat req.SetBasicAuth(r.authConfig.Username, r.authConfig.Password) req.ContentLength = int64(len(imgListJSON)) req.Header.Set("X-Docker-Token", "true") - + if validate { + req.Header["X-Docker-Endpoints"] = regs + } res, err = r.client.Do(req) if err != nil { return nil, err From c7e86e5eabb9ad59dbe45ee958174933fa6838e3 Mon Sep 17 00:00:00 2001 From: Victor Vieux Date: Mon, 17 Jun 2013 18:13:40 +0000 Subject: [PATCH 025/375] use go 1.1 cookiejar and revome ResetClient --- docs/registry.go | 15 +++++---------- 1 file changed, 5 insertions(+), 10 deletions(-) diff --git a/docs/registry.go b/docs/registry.go index 131b0270..21979fad 100644 --- a/docs/registry.go +++ b/docs/registry.go @@ -7,10 +7,10 @@ import ( "fmt" "github.com/dotcloud/docker/auth" "github.com/dotcloud/docker/utils" - "github.com/shin-/cookiejar" "io" "io/ioutil" "net/http" + "net/http/cookiejar" "net/url" "strings" ) @@ -438,11 +438,6 @@ func (r *Registry) SearchRepositories(term string) (*SearchResults, error) { return result, err } -func (r *Registry) ResetClient(authConfig *auth.AuthConfig) { - r.authConfig = authConfig - r.client.Jar = cookiejar.NewCookieJar() -} - func (r *Registry) GetAuthConfig(withPasswd bool) *auth.AuthConfig { password := "" if withPasswd { @@ -478,18 +473,18 @@ type Registry struct { authConfig *auth.AuthConfig } -func NewRegistry(root string, authConfig *auth.AuthConfig) *Registry { +func NewRegistry(root string, authConfig *auth.AuthConfig) (r *Registry, err error) { httpTransport := &http.Transport{ DisableKeepAlives: true, Proxy: http.ProxyFromEnvironment, } - r := &Registry{ + r = &Registry{ authConfig: authConfig, client: &http.Client{ Transport: httpTransport, }, } - r.client.Jar = cookiejar.NewCookieJar() - return r + r.client.Jar, err = cookiejar.New(nil) + return r, err } From ff418e9c369b60e2e0e59d6fa077aeb5b0163114 Mon Sep 17 00:00:00 2001 From: Victor Vieux Date: Wed, 19 Jun 2013 14:50:58 +0000 Subject: [PATCH 026/375] gofmt and test sub directories in makefile --- docs/registry.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/registry.go b/docs/registry.go index 131b0270..23aef432 100644 --- a/docs/registry.go +++ b/docs/registry.go @@ -481,7 +481,7 @@ type Registry struct { func NewRegistry(root string, authConfig *auth.AuthConfig) *Registry { httpTransport := &http.Transport{ DisableKeepAlives: true, - Proxy: http.ProxyFromEnvironment, + Proxy: http.ProxyFromEnvironment, } r := &Registry{ From 7e78627908b160eff557e11369f29171ee840fb3 Mon Sep 17 00:00:00 2001 From: "Guillaume J. Charmes" Date: Wed, 19 Jun 2013 11:07:36 -0700 Subject: [PATCH 027/375] hotfix: nil pointer uppon some registry error --- docs/registry.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/registry.go b/docs/registry.go index 18bdad26..276c9f86 100644 --- a/docs/registry.go +++ b/docs/registry.go @@ -162,10 +162,10 @@ func (r *Registry) GetRemoteTags(registries []string, repository string, token [ } req.Header.Set("Authorization", "Token "+strings.Join(token, ", ")) res, err := r.client.Do(req) - utils.Debugf("Got status code %d from %s", res.StatusCode, endpoint) if err != nil { return nil, err } + utils.Debugf("Got status code %d from %s", res.StatusCode, endpoint) defer res.Body.Close() if res.StatusCode != 200 && res.StatusCode != 404 { From 3238f3ea49a0560df4ff1875aef13c4a4de1efcb Mon Sep 17 00:00:00 2001 From: shin- Date: Wed, 19 Jun 2013 13:48:49 -0700 Subject: [PATCH 028/375] Use opaque requests when we need to preserve urlencoding in registry requests --- docs/registry.go | 19 ++++++++++++++----- 1 file changed, 14 insertions(+), 5 deletions(-) diff --git a/docs/registry.go b/docs/registry.go index 276c9f86..81b16d8d 100644 --- a/docs/registry.go +++ b/docs/registry.go @@ -156,7 +156,7 @@ func (r *Registry) GetRemoteTags(registries []string, repository string, token [ } for _, host := range registries { endpoint := fmt.Sprintf("https://%s/v1/repositories/%s/tags", host, repository) - req, err := http.NewRequest("GET", endpoint, nil) + req, err := r.opaqueRequest("GET", endpoint, nil) if err != nil { return nil, err } @@ -190,7 +190,7 @@ func (r *Registry) GetRemoteTags(registries []string, repository string, token [ func (r *Registry) GetRepositoryData(remote string) (*RepositoryData, error) { repositoryTarget := auth.IndexServerAddress() + "/repositories/" + remote + "/images" - req, err := http.NewRequest("GET", repositoryTarget, nil) + req, err := r.opaqueRequest("GET", repositoryTarget, nil) if err != nil { return nil, err } @@ -309,6 +309,15 @@ func (r *Registry) PushImageLayerRegistry(imgId string, layer io.Reader, registr return nil } +func (r *Registry) opaqueRequest(method, urlStr string, body io.Reader) (*http.Request, error) { + req, err := http.NewRequest(method, urlStr, body) + if err != nil { + return nil, err + } + req.URL.Opaque = strings.Replace(urlStr, req.URL.Scheme + ":", "", 1) + return req, err +} + // push a tag on the registry. // Remote has the format '/ func (r *Registry) PushRegistryTag(remote, revision, tag, registry string, token []string) error { @@ -316,7 +325,7 @@ func (r *Registry) PushRegistryTag(remote, revision, tag, registry string, token revision = "\"" + revision + "\"" registry = "https://" + registry + "/v1" - req, err := http.NewRequest("PUT", registry+"/repositories/"+remote+"/tags/"+tag, strings.NewReader(revision)) + req, err := r.opaqueRequest("PUT", registry+"/repositories/"+remote+"/tags/"+tag, strings.NewReader(revision)) if err != nil { return err } @@ -346,7 +355,7 @@ func (r *Registry) PushImageJSONIndex(remote string, imgList []*ImgData, validat utils.Debugf("Image list pushed to index:\n%s\n", imgListJSON) - req, err := http.NewRequest("PUT", auth.IndexServerAddress()+"/repositories/"+remote+"/"+suffix, bytes.NewReader(imgListJSON)) + req, err := r.opaqueRequest("PUT", auth.IndexServerAddress()+"/repositories/"+remote+"/"+suffix, bytes.NewReader(imgListJSON)) if err != nil { return nil, err } @@ -366,7 +375,7 @@ func (r *Registry) PushImageJSONIndex(remote string, imgList []*ImgData, validat // Redirect if necessary for res.StatusCode >= 300 && res.StatusCode < 400 { utils.Debugf("Redirected to %s\n", res.Header.Get("Location")) - req, err = http.NewRequest("PUT", res.Header.Get("Location"), bytes.NewReader(imgListJSON)) + req, err = r.opaqueRequest("PUT", res.Header.Get("Location"), bytes.NewReader(imgListJSON)) if err != nil { return nil, err } From 0d85570c9b6d1a0cbe2221b77170f29f8a1a4d0e Mon Sep 17 00:00:00 2001 From: Sam Alba Date: Thu, 27 Jun 2013 17:55:17 -0700 Subject: [PATCH 029/375] URL schemes of both Registry and Index are now consistent --- docs/registry.go | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/docs/registry.go b/docs/registry.go index c565c299..ed1cd405 100644 --- a/docs/registry.go +++ b/docs/registry.go @@ -18,6 +18,14 @@ import ( var ErrAlreadyExists = errors.New("Image already exists") +func UrlScheme() string { + u, err := url.Parse(auth.IndexServerAddress()) + if err != nil { + return "https" + } + return u.Scheme +} + func doWithCookies(c *http.Client, req *http.Request) (*http.Response, error) { for _, cookie := range c.Jar.Cookies(req.URL) { req.AddCookie(cookie) @@ -155,7 +163,7 @@ func (r *Registry) GetRemoteTags(registries []string, repository string, token [ repository = "library/" + repository } for _, host := range registries { - endpoint := fmt.Sprintf("https://%s/v1/repositories/%s/tags", host, repository) + endpoint := fmt.Sprintf("%s://%s/v1/repositories/%s/tags", UrlScheme(), host, repository) req, err := r.opaqueRequest("GET", endpoint, nil) if err != nil { return nil, err @@ -249,7 +257,7 @@ func (r *Registry) GetRepositoryData(remote string) (*RepositoryData, error) { // Push a local image to the registry func (r *Registry) PushImageJSONRegistry(imgData *ImgData, jsonRaw []byte, registry string, token []string) error { - registry = "https://" + registry + "/v1" + registry = fmt.Sprintf("%s://%s/v1", UrlScheme(), registry) // FIXME: try json with UTF8 req, err := http.NewRequest("PUT", registry+"/images/"+imgData.ID+"/json", strings.NewReader(string(jsonRaw))) if err != nil { @@ -285,7 +293,7 @@ func (r *Registry) PushImageJSONRegistry(imgData *ImgData, jsonRaw []byte, regis } func (r *Registry) PushImageLayerRegistry(imgId string, layer io.Reader, registry string, token []string) error { - registry = "https://" + registry + "/v1" + registry = fmt.Sprintf("%s://%s/v1", UrlScheme(), registry) req, err := http.NewRequest("PUT", registry+"/images/"+imgId+"/layer", layer) if err != nil { return err @@ -323,7 +331,7 @@ func (r *Registry) opaqueRequest(method, urlStr string, body io.Reader) (*http.R func (r *Registry) PushRegistryTag(remote, revision, tag, registry string, token []string) error { // "jsonify" the string revision = "\"" + revision + "\"" - registry = "https://" + registry + "/v1" + registry = fmt.Sprintf("%s://%s/v1", UrlScheme(), registry) req, err := r.opaqueRequest("PUT", registry+"/repositories/"+remote+"/tags/"+tag, strings.NewReader(revision)) if err != nil { From 7a664e6a5f0b8b8a11167789b817b2dd5185940d Mon Sep 17 00:00:00 2001 From: shin- Date: Fri, 24 May 2013 10:37:34 -0700 Subject: [PATCH 030/375] Tentative support for independent registries --- docs/registry.go | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/docs/registry.go b/docs/registry.go index c565c299..5d642b39 100644 --- a/docs/registry.go +++ b/docs/registry.go @@ -56,20 +56,19 @@ func (r *Registry) GetRemoteHistory(imgId, registry string, token []string) ([]s } // Check if an image exists in the Registry -func (r *Registry) LookupRemoteImage(imgId, registry string, authConfig *auth.AuthConfig) bool { +func (r *Registry) LookupRemoteImage(imgId, registry string, token []string) bool { rt := &http.Transport{Proxy: http.ProxyFromEnvironment} req, err := http.NewRequest("GET", registry+"/images/"+imgId+"/json", nil) if err != nil { return false } - req.SetBasicAuth(authConfig.Username, authConfig.Password) res, err := rt.RoundTrip(req) if err != nil { return false } res.Body.Close() - return res.StatusCode == 307 + return res.StatusCode == 200 } func (r *Registry) getImagesInRepository(repository string, authConfig *auth.AuthConfig) ([]map[string]string, error) { From dc97156c832f7262196d141844d9752299086b20 Mon Sep 17 00:00:00 2001 From: shin- Date: Wed, 29 May 2013 11:24:50 -0700 Subject: [PATCH 031/375] Skip certificate check (don't error out on self-signed certs) --- docs/registry.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/registry.go b/docs/registry.go index 5d642b39..864f3e8b 100644 --- a/docs/registry.go +++ b/docs/registry.go @@ -2,6 +2,7 @@ package registry import ( "bytes" + "crypto/tls" "encoding/json" "errors" "fmt" @@ -164,6 +165,7 @@ func (r *Registry) GetRemoteTags(registries []string, repository string, token [ if err != nil { return nil, err } + utils.Debugf("Got status code %d from %s", res.StatusCode, endpoint) defer res.Body.Close() From e1d8d0245fbb8b48546431cc938262c1a28bb8e2 Mon Sep 17 00:00:00 2001 From: shin- Date: Wed, 29 May 2013 11:39:31 -0700 Subject: [PATCH 032/375] Rolled back of previous commit (skip cert verification) --- docs/registry.go | 1 - 1 file changed, 1 deletion(-) diff --git a/docs/registry.go b/docs/registry.go index 864f3e8b..30b91cef 100644 --- a/docs/registry.go +++ b/docs/registry.go @@ -2,7 +2,6 @@ package registry import ( "bytes" - "crypto/tls" "encoding/json" "errors" "fmt" From 259eeb382c03fd672e83a67a95a7384d3b370019 Mon Sep 17 00:00:00 2001 From: "Guillaume J. Charmes" Date: Wed, 5 Jun 2013 15:12:50 -0700 Subject: [PATCH 033/375] Remove https prefix from registry --- docs/registry.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/docs/registry.go b/docs/registry.go index 30b91cef..29a1f29e 100644 --- a/docs/registry.go +++ b/docs/registry.go @@ -154,7 +154,7 @@ func (r *Registry) GetRemoteTags(registries []string, repository string, token [ repository = "library/" + repository } for _, host := range registries { - endpoint := fmt.Sprintf("https://%s/v1/repositories/%s/tags", host, repository) + endpoint := fmt.Sprintf("%s/v1/repositories/%s/tags", host, repository) req, err := r.opaqueRequest("GET", endpoint, nil) if err != nil { return nil, err @@ -249,7 +249,7 @@ func (r *Registry) GetRepositoryData(remote string) (*RepositoryData, error) { // Push a local image to the registry func (r *Registry) PushImageJSONRegistry(imgData *ImgData, jsonRaw []byte, registry string, token []string) error { - registry = "https://" + registry + "/v1" + registry = registry + "/v1" // FIXME: try json with UTF8 req, err := http.NewRequest("PUT", registry+"/images/"+imgData.ID+"/json", strings.NewReader(string(jsonRaw))) if err != nil { @@ -285,7 +285,7 @@ func (r *Registry) PushImageJSONRegistry(imgData *ImgData, jsonRaw []byte, regis } func (r *Registry) PushImageLayerRegistry(imgId string, layer io.Reader, registry string, token []string) error { - registry = "https://" + registry + "/v1" + registry = registry + "/v1" req, err := http.NewRequest("PUT", registry+"/images/"+imgId+"/layer", layer) if err != nil { return err @@ -323,7 +323,7 @@ func (r *Registry) opaqueRequest(method, urlStr string, body io.Reader) (*http.R func (r *Registry) PushRegistryTag(remote, revision, tag, registry string, token []string) error { // "jsonify" the string revision = "\"" + revision + "\"" - registry = "https://" + registry + "/v1" + registry = registry + "/v1" req, err := r.opaqueRequest("PUT", registry+"/repositories/"+remote+"/tags/"+tag, strings.NewReader(revision)) if err != nil { From 03a77bd8511bf59481524bd70c5d313ae863cfeb Mon Sep 17 00:00:00 2001 From: shin- Date: Fri, 28 Jun 2013 18:42:37 +0200 Subject: [PATCH 034/375] Fixed issue in registry.GetRemoteTags --- docs/registry.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/docs/registry.go b/docs/registry.go index 29a1f29e..456e4321 100644 --- a/docs/registry.go +++ b/docs/registry.go @@ -155,6 +155,9 @@ func (r *Registry) GetRemoteTags(registries []string, repository string, token [ } for _, host := range registries { endpoint := fmt.Sprintf("%s/v1/repositories/%s/tags", host, repository) + if !(strings.HasPrefix(endpoint, "http://") || strings.HasPrefix(endpoint, "https://")) { + endpoint = "https://" + endpoint + } req, err := r.opaqueRequest("GET", endpoint, nil) if err != nil { return nil, err From 7e215123fea8096228c18616adf328f7f92565f2 Mon Sep 17 00:00:00 2001 From: Tobias Schwab Date: Tue, 2 Jul 2013 22:07:02 +0000 Subject: [PATCH 035/375] fix two obvious bugs??? --- docs/registry.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/registry.go b/docs/registry.go index 622c09b3..0853a68e 100644 --- a/docs/registry.go +++ b/docs/registry.go @@ -67,7 +67,7 @@ func (r *Registry) GetRemoteHistory(imgId, registry string, token []string) ([]s func (r *Registry) LookupRemoteImage(imgId, registry string, token []string) bool { rt := &http.Transport{Proxy: http.ProxyFromEnvironment} - req, err := http.NewRequest("GET", registry+"/images/"+imgId+"/json", nil) + req, err := http.NewRequest("GET", registry+"/v1/images/"+imgId+"/json", nil) if err != nil { return false } From 7df93a5ab391184ffb0cb399e45a11a4f7767a09 Mon Sep 17 00:00:00 2001 From: Caleb Spare Date: Tue, 2 Jul 2013 15:27:22 -0700 Subject: [PATCH 036/375] Implement several golint suggestions, including: * Removing type declarations where they're inferred * Changing Url -> URL, Id -> ID in names * Fixing snake-case names --- docs/registry.go | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/docs/registry.go b/docs/registry.go index 0853a68e..584e3824 100644 --- a/docs/registry.go +++ b/docs/registry.go @@ -18,7 +18,7 @@ import ( var ErrAlreadyExists = errors.New("Image already exists") -func UrlScheme() string { +func URLScheme() string { u, err := url.Parse(auth.IndexServerAddress()) if err != nil { return "https" @@ -35,8 +35,8 @@ func doWithCookies(c *http.Client, req *http.Request) (*http.Response, error) { // Retrieve the history of a given image from the Registry. // Return a list of the parent's json (requested image included) -func (r *Registry) GetRemoteHistory(imgId, registry string, token []string) ([]string, error) { - req, err := http.NewRequest("GET", registry+"/images/"+imgId+"/ancestry", nil) +func (r *Registry) GetRemoteHistory(imgID, registry string, token []string) ([]string, error) { + req, err := http.NewRequest("GET", registry+"/images/"+imgID+"/ancestry", nil) if err != nil { return nil, err } @@ -44,7 +44,7 @@ func (r *Registry) GetRemoteHistory(imgId, registry string, token []string) ([]s res, err := r.client.Do(req) if err != nil || res.StatusCode != 200 { if res != nil { - return nil, fmt.Errorf("Internal server error: %d trying to fetch remote history for %s", res.StatusCode, imgId) + return nil, fmt.Errorf("Internal server error: %d trying to fetch remote history for %s", res.StatusCode, imgID) } return nil, err } @@ -64,10 +64,10 @@ func (r *Registry) GetRemoteHistory(imgId, registry string, token []string) ([]s } // Check if an image exists in the Registry -func (r *Registry) LookupRemoteImage(imgId, registry string, token []string) bool { +func (r *Registry) LookupRemoteImage(imgID, registry string, token []string) bool { rt := &http.Transport{Proxy: http.ProxyFromEnvironment} - req, err := http.NewRequest("GET", registry+"/v1/images/"+imgId+"/json", nil) + req, err := http.NewRequest("GET", registry+"/v1/images/"+imgID+"/json", nil) if err != nil { return false } @@ -114,9 +114,9 @@ func (r *Registry) getImagesInRepository(repository string, authConfig *auth.Aut } // Retrieve an image from the Registry. -func (r *Registry) GetRemoteImageJSON(imgId, registry string, token []string) ([]byte, int, error) { +func (r *Registry) GetRemoteImageJSON(imgID, registry string, token []string) ([]byte, int, error) { // Get the JSON - req, err := http.NewRequest("GET", registry+"/images/"+imgId+"/json", nil) + req, err := http.NewRequest("GET", registry+"/images/"+imgID+"/json", nil) if err != nil { return nil, -1, fmt.Errorf("Failed to download json: %s", err) } @@ -142,8 +142,8 @@ func (r *Registry) GetRemoteImageJSON(imgId, registry string, token []string) ([ return jsonString, imageSize, nil } -func (r *Registry) GetRemoteImageLayer(imgId, registry string, token []string) (io.ReadCloser, error) { - req, err := http.NewRequest("GET", registry+"/images/"+imgId+"/layer", nil) +func (r *Registry) GetRemoteImageLayer(imgID, registry string, token []string) (io.ReadCloser, error) { + req, err := http.NewRequest("GET", registry+"/images/"+imgID+"/layer", nil) if err != nil { return nil, fmt.Errorf("Error while getting from the server: %s\n", err) } @@ -164,7 +164,7 @@ func (r *Registry) GetRemoteTags(registries []string, repository string, token [ for _, host := range registries { endpoint := fmt.Sprintf("%s/v1/repositories/%s/tags", host, repository) if !(strings.HasPrefix(endpoint, "http://") || strings.HasPrefix(endpoint, "https://")) { - endpoint = fmt.Sprintf("%s://%s", UrlScheme(), endpoint) + endpoint = fmt.Sprintf("%s://%s", URLScheme(), endpoint) } req, err := r.opaqueRequest("GET", endpoint, nil) if err != nil { @@ -295,9 +295,9 @@ func (r *Registry) PushImageJSONRegistry(imgData *ImgData, jsonRaw []byte, regis return nil } -func (r *Registry) PushImageLayerRegistry(imgId string, layer io.Reader, registry string, token []string) error { +func (r *Registry) PushImageLayerRegistry(imgID string, layer io.Reader, registry string, token []string) error { registry = registry + "/v1" - req, err := http.NewRequest("PUT", registry+"/images/"+imgId+"/layer", layer) + req, err := http.NewRequest("PUT", registry+"/images/"+imgID+"/layer", layer) if err != nil { return err } From ec6d1d60201a3f6a3efd3684db85c6505f57602d Mon Sep 17 00:00:00 2001 From: Sam Alba Date: Fri, 5 Jul 2013 12:20:58 -0700 Subject: [PATCH 037/375] Adding support for nicer URLs to support standalone registry (+ some registry code cleaning) --- docs/registry.go | 141 +++++++++++++++++++++++++++-------------------- 1 file changed, 81 insertions(+), 60 deletions(-) diff --git a/docs/registry.go b/docs/registry.go index 622c09b3..e9d7b2b8 100644 --- a/docs/registry.go +++ b/docs/registry.go @@ -12,18 +12,70 @@ import ( "net/http" "net/http/cookiejar" "net/url" + "regexp" "strconv" "strings" ) var ErrAlreadyExists = errors.New("Image already exists") -func UrlScheme() string { - u, err := url.Parse(auth.IndexServerAddress()) +func pingRegistryEndpoint(endpoint string) error { + // FIXME: implement the check to discover if it should be http or https + resp, err := http.Get(endpoint) if err != nil { - return "https" + return err } - return u.Scheme + if resp.Header.Get("X-Docker-Registry-Version") == "" { + return errors.New("This does not look like a Registry server (\"X-Docker-Registry-Version\" header not found in the response)") + } + return nil +} + +func validateRepositoryName(namespace, name string) error { + validNamespace := regexp.MustCompile(`^([a-z0-9_]{4,30})$`) + if !validNamespace.MatchString(namespace) { + return fmt.Errorf("Invalid namespace name (%s), only [a-z0-9_] are allowed, size between 4 and 30", namespace) + } + validRepo := regexp.MustCompile(`^([a-zA-Z0-9-_.]+)$`) + if !validRepo.MatchString(name) { + return fmt.Errorf("Invalid repository name (%s), only [a-zA-Z0-9-_.] are allowed", name) + } + return nil +} + +// Resolves a repository name to a endpoint + name +func ResolveRepositoryName(reposName string) (string, string, error) { + nameParts := strings.SplitN(reposName, "/", 2) + if !strings.Contains(nameParts[0], ".") { + // This is a Docker Index repos (ex: samalba/hipache or ubuntu) + var err error + if len(nameParts) < 2 { + err = validateRepositoryName("library", nameParts[0]) + } else { + err = validateRepositoryName(nameParts[0], nameParts[1]) + } + return "https://index.docker.io/v1/", reposName, err + } + if len(nameParts) < 2 { + // There is a dot in repos name (and no registry address) + // Is it a Registry address without repos name? + return "", "", errors.New("Invalid repository name (ex: \"registry.domain.tld/myrepos\")") + } + n := strings.LastIndex(reposName, "/") + hostname := nameParts[0] + path := reposName[len(nameParts[0]):n] + reposName = reposName[n+1:] + endpoint := fmt.Sprintf("https://%s%s/v1/", hostname, path) + if err := pingRegistryEndpoint(endpoint); err != nil { + utils.Debugf("Registry %s does not work (%s), falling back to http", endpoint, err) + endpoint = fmt.Sprintf("http://%s%s/v1/", hostname, path) + if err = pingRegistryEndpoint(endpoint); err != nil { + //TODO: triggering highland build can be done there without "failing" + return "", "", errors.New("Invalid Registry endpoint: " + err.Error()) + } + } + err := validateRepositoryName("library", reposName) + return endpoint, reposName, err } func doWithCookies(c *http.Client, req *http.Request) (*http.Response, error) { @@ -36,7 +88,7 @@ func doWithCookies(c *http.Client, req *http.Request) (*http.Response, error) { // Retrieve the history of a given image from the Registry. // Return a list of the parent's json (requested image included) func (r *Registry) GetRemoteHistory(imgId, registry string, token []string) ([]string, error) { - req, err := http.NewRequest("GET", registry+"/images/"+imgId+"/ancestry", nil) + req, err := http.NewRequest("GET", registry+"images/"+imgId+"/ancestry", nil) if err != nil { return nil, err } @@ -67,7 +119,7 @@ func (r *Registry) GetRemoteHistory(imgId, registry string, token []string) ([]s func (r *Registry) LookupRemoteImage(imgId, registry string, token []string) bool { rt := &http.Transport{Proxy: http.ProxyFromEnvironment} - req, err := http.NewRequest("GET", registry+"/images/"+imgId+"/json", nil) + req, err := http.NewRequest("GET", registry+"images/"+imgId+"/json", nil) if err != nil { return false } @@ -79,44 +131,10 @@ func (r *Registry) LookupRemoteImage(imgId, registry string, token []string) boo return res.StatusCode == 200 } -func (r *Registry) getImagesInRepository(repository string, authConfig *auth.AuthConfig) ([]map[string]string, error) { - u := auth.IndexServerAddress() + "/repositories/" + repository + "/images" - req, err := http.NewRequest("GET", u, nil) - if err != nil { - return nil, err - } - if authConfig != nil && len(authConfig.Username) > 0 { - req.SetBasicAuth(authConfig.Username, authConfig.Password) - } - res, err := r.client.Do(req) - if err != nil { - return nil, err - } - defer res.Body.Close() - - // Repository doesn't exist yet - if res.StatusCode == 404 { - return nil, nil - } - - jsonData, err := ioutil.ReadAll(res.Body) - if err != nil { - return nil, err - } - - imageList := []map[string]string{} - if err := json.Unmarshal(jsonData, &imageList); err != nil { - utils.Debugf("Body: %s (%s)\n", res.Body, u) - return nil, err - } - - return imageList, nil -} - // Retrieve an image from the Registry. func (r *Registry) GetRemoteImageJSON(imgId, registry string, token []string) ([]byte, int, error) { // Get the JSON - req, err := http.NewRequest("GET", registry+"/images/"+imgId+"/json", nil) + req, err := http.NewRequest("GET", registry+"images/"+imgId+"/json", nil) if err != nil { return nil, -1, fmt.Errorf("Failed to download json: %s", err) } @@ -143,7 +161,7 @@ func (r *Registry) GetRemoteImageJSON(imgId, registry string, token []string) ([ } func (r *Registry) GetRemoteImageLayer(imgId, registry string, token []string) (io.ReadCloser, error) { - req, err := http.NewRequest("GET", registry+"/images/"+imgId+"/layer", nil) + req, err := http.NewRequest("GET", registry+"images/"+imgId+"/layer", nil) if err != nil { return nil, fmt.Errorf("Error while getting from the server: %s\n", err) } @@ -162,10 +180,7 @@ func (r *Registry) GetRemoteTags(registries []string, repository string, token [ repository = "library/" + repository } for _, host := range registries { - endpoint := fmt.Sprintf("%s/v1/repositories/%s/tags", host, repository) - if !(strings.HasPrefix(endpoint, "http://") || strings.HasPrefix(endpoint, "https://")) { - endpoint = fmt.Sprintf("%s://%s", UrlScheme(), endpoint) - } + endpoint := fmt.Sprintf("%srepositories/%s/tags", host, repository) req, err := r.opaqueRequest("GET", endpoint, nil) if err != nil { return nil, err @@ -198,8 +213,8 @@ func (r *Registry) GetRemoteTags(registries []string, repository string, token [ return nil, fmt.Errorf("Could not reach any registry endpoint") } -func (r *Registry) GetRepositoryData(remote string) (*RepositoryData, error) { - repositoryTarget := auth.IndexServerAddress() + "/repositories/" + remote + "/images" +func (r *Registry) GetRepositoryData(indexEp, remote string) (*RepositoryData, error) { + repositoryTarget := fmt.Sprintf("%srepositories/%s/images", indexEp, remote) req, err := r.opaqueRequest("GET", repositoryTarget, nil) if err != nil { @@ -230,8 +245,12 @@ func (r *Registry) GetRepositoryData(remote string) (*RepositoryData, error) { } var endpoints []string + var urlScheme = indexEp[:strings.Index(indexEp, ":")] if res.Header.Get("X-Docker-Endpoints") != "" { - endpoints = res.Header["X-Docker-Endpoints"] + // The Registry's URL scheme has to match the Index' + for _, ep := range res.Header["X-Docker-Endpoints"] { + endpoints = append(endpoints, fmt.Sprintf("%s://%s/v1/", urlScheme, ep)) + } } else { return nil, fmt.Errorf("Index response didn't contain any endpoints") } @@ -260,9 +279,8 @@ func (r *Registry) GetRepositoryData(remote string) (*RepositoryData, error) { // Push a local image to the registry func (r *Registry) PushImageJSONRegistry(imgData *ImgData, jsonRaw []byte, registry string, token []string) error { - registry = registry + "/v1" // FIXME: try json with UTF8 - req, err := http.NewRequest("PUT", registry+"/images/"+imgData.ID+"/json", strings.NewReader(string(jsonRaw))) + req, err := http.NewRequest("PUT", registry+"images/"+imgData.ID+"/json", strings.NewReader(string(jsonRaw))) if err != nil { return err } @@ -296,8 +314,7 @@ func (r *Registry) PushImageJSONRegistry(imgData *ImgData, jsonRaw []byte, regis } func (r *Registry) PushImageLayerRegistry(imgId string, layer io.Reader, registry string, token []string) error { - registry = registry + "/v1" - req, err := http.NewRequest("PUT", registry+"/images/"+imgId+"/layer", layer) + req, err := http.NewRequest("PUT", registry+"images/"+imgId+"/layer", layer) if err != nil { return err } @@ -334,9 +351,8 @@ func (r *Registry) opaqueRequest(method, urlStr string, body io.Reader) (*http.R func (r *Registry) PushRegistryTag(remote, revision, tag, registry string, token []string) error { // "jsonify" the string revision = "\"" + revision + "\"" - registry = registry + "/v1" - req, err := r.opaqueRequest("PUT", registry+"/repositories/"+remote+"/tags/"+tag, strings.NewReader(revision)) + req, err := r.opaqueRequest("PUT", registry+"repositories/"+remote+"/tags/"+tag, strings.NewReader(revision)) if err != nil { return err } @@ -354,7 +370,7 @@ func (r *Registry) PushRegistryTag(remote, revision, tag, registry string, token return nil } -func (r *Registry) PushImageJSONIndex(remote string, imgList []*ImgData, validate bool, regs []string) (*RepositoryData, error) { +func (r *Registry) PushImageJSONIndex(indexEp, remote string, imgList []*ImgData, validate bool, regs []string) (*RepositoryData, error) { imgListJSON, err := json.Marshal(imgList) if err != nil { return nil, err @@ -364,9 +380,10 @@ func (r *Registry) PushImageJSONIndex(remote string, imgList []*ImgData, validat suffix = "images" } + u := fmt.Sprintf("%srepositories/%s/%s", indexEp, remote, suffix) + utils.Debugf("PUT %s", u) utils.Debugf("Image list pushed to index:\n%s\n", imgListJSON) - - req, err := r.opaqueRequest("PUT", auth.IndexServerAddress()+"/repositories/"+remote+"/"+suffix, bytes.NewReader(imgListJSON)) + req, err := r.opaqueRequest("PUT", u, bytes.NewReader(imgListJSON)) if err != nil { return nil, err } @@ -404,6 +421,7 @@ func (r *Registry) PushImageJSONIndex(remote string, imgList []*ImgData, validat } var tokens, endpoints []string + var urlScheme = indexEp[:strings.Index(indexEp, ":")] if !validate { if res.StatusCode != 200 && res.StatusCode != 201 { errBody, err := ioutil.ReadAll(res.Body) @@ -420,7 +438,10 @@ func (r *Registry) PushImageJSONIndex(remote string, imgList []*ImgData, validat } if res.Header.Get("X-Docker-Endpoints") != "" { - endpoints = res.Header["X-Docker-Endpoints"] + // The Registry's URL scheme has to match the Index' + for _, ep := range res.Header["X-Docker-Endpoints"] { + endpoints = append(endpoints, fmt.Sprintf("%s://%s/v1/", urlScheme, ep)) + } } else { return nil, fmt.Errorf("Index response didn't contain any endpoints") } @@ -442,7 +463,7 @@ func (r *Registry) PushImageJSONIndex(remote string, imgList []*ImgData, validat } func (r *Registry) SearchRepositories(term string) (*SearchResults, error) { - u := auth.IndexServerAddress() + "/search?q=" + url.QueryEscape(term) + u := auth.IndexServerAddress() + "search?q=" + url.QueryEscape(term) req, err := http.NewRequest("GET", u, nil) if err != nil { return nil, err From c6068feffab5e2351a0bc7a395173103be531829 Mon Sep 17 00:00:00 2001 From: Sam Alba Date: Fri, 5 Jul 2013 12:37:07 -0700 Subject: [PATCH 038/375] Restoring old changeset lost by previous merge --- docs/registry.go | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/docs/registry.go b/docs/registry.go index ee473493..4bd2a5ad 100644 --- a/docs/registry.go +++ b/docs/registry.go @@ -87,8 +87,8 @@ func doWithCookies(c *http.Client, req *http.Request) (*http.Response, error) { // Retrieve the history of a given image from the Registry. // Return a list of the parent's json (requested image included) -func (r *Registry) GetRemoteHistory(imgId, registry string, token []string) ([]string, error) { - req, err := http.NewRequest("GET", registry+"images/"+imgId+"/ancestry", nil) +func (r *Registry) GetRemoteHistory(imgID, registry string, token []string) ([]string, error) { + req, err := http.NewRequest("GET", registry+"images/"+imgID+"/ancestry", nil) if err != nil { return nil, err } @@ -119,7 +119,7 @@ func (r *Registry) GetRemoteHistory(imgId, registry string, token []string) ([]s func (r *Registry) LookupRemoteImage(imgID, registry string, token []string) bool { rt := &http.Transport{Proxy: http.ProxyFromEnvironment} - req, err := http.NewRequest("GET", registry+"images/"+imgId+"/json", nil) + req, err := http.NewRequest("GET", registry+"images/"+imgID+"/json", nil) if err != nil { return false } @@ -134,7 +134,7 @@ func (r *Registry) LookupRemoteImage(imgID, registry string, token []string) boo // Retrieve an image from the Registry. func (r *Registry) GetRemoteImageJSON(imgID, registry string, token []string) ([]byte, int, error) { // Get the JSON - req, err := http.NewRequest("GET", registry+"images/"+imgId+"/json", nil) + req, err := http.NewRequest("GET", registry+"images/"+imgID+"/json", nil) if err != nil { return nil, -1, fmt.Errorf("Failed to download json: %s", err) } @@ -160,8 +160,8 @@ func (r *Registry) GetRemoteImageJSON(imgID, registry string, token []string) ([ return jsonString, imageSize, nil } -func (r *Registry) GetRemoteImageLayer(imgId, registry string, token []string) (io.ReadCloser, error) { - req, err := http.NewRequest("GET", registry+"images/"+imgId+"/layer", nil) +func (r *Registry) GetRemoteImageLayer(imgID, registry string, token []string) (io.ReadCloser, error) { + req, err := http.NewRequest("GET", registry+"images/"+imgID+"/layer", nil) if err != nil { return nil, fmt.Errorf("Error while getting from the server: %s\n", err) } @@ -313,8 +313,8 @@ func (r *Registry) PushImageJSONRegistry(imgData *ImgData, jsonRaw []byte, regis return nil } -func (r *Registry) PushImageLayerRegistry(imgId string, layer io.Reader, registry string, token []string) error { - req, err := http.NewRequest("PUT", registry+"images/"+imgId+"/layer", layer) +func (r *Registry) PushImageLayerRegistry(imgID string, layer io.Reader, registry string, token []string) error { + req, err := http.NewRequest("PUT", registry+"images/"+imgID+"/layer", layer) if err != nil { return err } From 16fa043e344eafa67121ffea9ed0032081653f59 Mon Sep 17 00:00:00 2001 From: Sam Alba Date: Fri, 5 Jul 2013 14:30:43 -0700 Subject: [PATCH 039/375] Allowing namespaces in standalone registry --- docs/registry.go | 34 +++++++++++++++++++--------------- 1 file changed, 19 insertions(+), 15 deletions(-) diff --git a/docs/registry.go b/docs/registry.go index 4bd2a5ad..72521a31 100644 --- a/docs/registry.go +++ b/docs/registry.go @@ -20,8 +20,7 @@ import ( var ErrAlreadyExists = errors.New("Image already exists") func pingRegistryEndpoint(endpoint string) error { - // FIXME: implement the check to discover if it should be http or https - resp, err := http.Get(endpoint) + resp, err := http.Get(endpoint + "/_ping") if err != nil { return err } @@ -31,7 +30,19 @@ func pingRegistryEndpoint(endpoint string) error { return nil } -func validateRepositoryName(namespace, name string) error { +func validateRepositoryName(repositoryName string) error { + var ( + namespace string + name string + ) + nameParts := strings.SplitN(repositoryName, "/", 2) + if len(nameParts) < 2 { + namespace = "library" + name = nameParts[0] + } else { + namespace = nameParts[0] + name = nameParts[1] + } validNamespace := regexp.MustCompile(`^([a-z0-9_]{4,30})$`) if !validNamespace.MatchString(namespace) { return fmt.Errorf("Invalid namespace name (%s), only [a-z0-9_] are allowed, size between 4 and 30", namespace) @@ -48,12 +59,7 @@ func ResolveRepositoryName(reposName string) (string, string, error) { nameParts := strings.SplitN(reposName, "/", 2) if !strings.Contains(nameParts[0], ".") { // This is a Docker Index repos (ex: samalba/hipache or ubuntu) - var err error - if len(nameParts) < 2 { - err = validateRepositoryName("library", nameParts[0]) - } else { - err = validateRepositoryName(nameParts[0], nameParts[1]) - } + err := validateRepositoryName(reposName) return "https://index.docker.io/v1/", reposName, err } if len(nameParts) < 2 { @@ -61,20 +67,18 @@ func ResolveRepositoryName(reposName string) (string, string, error) { // Is it a Registry address without repos name? return "", "", errors.New("Invalid repository name (ex: \"registry.domain.tld/myrepos\")") } - n := strings.LastIndex(reposName, "/") hostname := nameParts[0] - path := reposName[len(nameParts[0]):n] - reposName = reposName[n+1:] - endpoint := fmt.Sprintf("https://%s%s/v1/", hostname, path) + reposName = nameParts[1] + endpoint := fmt.Sprintf("https://%s/v1/", hostname) if err := pingRegistryEndpoint(endpoint); err != nil { utils.Debugf("Registry %s does not work (%s), falling back to http", endpoint, err) - endpoint = fmt.Sprintf("http://%s%s/v1/", hostname, path) + endpoint = fmt.Sprintf("http://%s/v1/", hostname) if err = pingRegistryEndpoint(endpoint); err != nil { //TODO: triggering highland build can be done there without "failing" return "", "", errors.New("Invalid Registry endpoint: " + err.Error()) } } - err := validateRepositoryName("library", reposName) + err := validateRepositoryName(reposName) return endpoint, reposName, err } From 98060903a9d86f96a9ed96c64a310d3c947910d2 Mon Sep 17 00:00:00 2001 From: Sam Alba Date: Fri, 5 Jul 2013 14:55:48 -0700 Subject: [PATCH 040/375] Fixed ping URL --- docs/registry.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/registry.go b/docs/registry.go index 72521a31..730fcf6e 100644 --- a/docs/registry.go +++ b/docs/registry.go @@ -20,7 +20,7 @@ import ( var ErrAlreadyExists = errors.New("Image already exists") func pingRegistryEndpoint(endpoint string) error { - resp, err := http.Get(endpoint + "/_ping") + resp, err := http.Get(endpoint + "_ping") if err != nil { return err } From 67115ec4794a5d45fcaff332a732350f9e233b55 Mon Sep 17 00:00:00 2001 From: Sam Alba Date: Fri, 5 Jul 2013 14:56:56 -0700 Subject: [PATCH 041/375] fmt.Errorf instead of errors.New --- docs/registry.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/registry.go b/docs/registry.go index 730fcf6e..c458f616 100644 --- a/docs/registry.go +++ b/docs/registry.go @@ -65,7 +65,7 @@ func ResolveRepositoryName(reposName string) (string, string, error) { if len(nameParts) < 2 { // There is a dot in repos name (and no registry address) // Is it a Registry address without repos name? - return "", "", errors.New("Invalid repository name (ex: \"registry.domain.tld/myrepos\")") + return "", "", fmt.Errorf("Invalid repository name (ex: \"registry.domain.tld/myrepos\")") } hostname := nameParts[0] reposName = nameParts[1] From bf8d59a1d434be76a0d15cfa85d8221b7780d4fb Mon Sep 17 00:00:00 2001 From: Sam Alba Date: Tue, 9 Jul 2013 11:30:12 -0700 Subject: [PATCH 042/375] Fixed potential security issue (never try http on official index when polling the endpoint). Also fixed local repos name when pulling index.docker.io/foo/bar --- docs/registry.go | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/docs/registry.go b/docs/registry.go index c458f616..2f225aed 100644 --- a/docs/registry.go +++ b/docs/registry.go @@ -18,8 +18,14 @@ import ( ) var ErrAlreadyExists = errors.New("Image already exists") +var ErrInvalidRepositoryName = errors.New("Invalid repository name (ex: \"registry.domain.tld/myrepos\")") func pingRegistryEndpoint(endpoint string) error { + if endpoint == auth.IndexServerAddress() { + // Skip the check, we now this one is valid + // (and we never want to fallback to http in case of error) + return nil + } resp, err := http.Get(endpoint + "_ping") if err != nil { return err @@ -56,16 +62,20 @@ func validateRepositoryName(repositoryName string) error { // Resolves a repository name to a endpoint + name func ResolveRepositoryName(reposName string) (string, string, error) { + if strings.Contains(reposName, "://") { + // It cannot contain a scheme! + return "", "", ErrInvalidRepositoryName + } nameParts := strings.SplitN(reposName, "/", 2) if !strings.Contains(nameParts[0], ".") { // This is a Docker Index repos (ex: samalba/hipache or ubuntu) err := validateRepositoryName(reposName) - return "https://index.docker.io/v1/", reposName, err + return auth.IndexServerAddress(), reposName, err } if len(nameParts) < 2 { // There is a dot in repos name (and no registry address) // Is it a Registry address without repos name? - return "", "", fmt.Errorf("Invalid repository name (ex: \"registry.domain.tld/myrepos\")") + return "", "", ErrInvalidRepositoryName } hostname := nameParts[0] reposName = nameParts[1] From 358574ab57fea861789057e092813402f82b8af6 Mon Sep 17 00:00:00 2001 From: Sam Alba Date: Tue, 9 Jul 2013 16:46:55 -0700 Subject: [PATCH 043/375] Hardened repos name validation --- docs/registry.go | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/docs/registry.go b/docs/registry.go index 2f225aed..fc84f19e 100644 --- a/docs/registry.go +++ b/docs/registry.go @@ -67,7 +67,7 @@ func ResolveRepositoryName(reposName string) (string, string, error) { return "", "", ErrInvalidRepositoryName } nameParts := strings.SplitN(reposName, "/", 2) - if !strings.Contains(nameParts[0], ".") { + if !strings.Contains(nameParts[0], ".") && !strings.Contains(nameParts[0], ":") { // This is a Docker Index repos (ex: samalba/hipache or ubuntu) err := validateRepositoryName(reposName) return auth.IndexServerAddress(), reposName, err @@ -79,6 +79,12 @@ func ResolveRepositoryName(reposName string) (string, string, error) { } hostname := nameParts[0] reposName = nameParts[1] + if strings.Contains(hostname, "index.docker.io") { + return "", "", fmt.Errorf("Invalid repository name, try \"%s\" instead", reposName) + } + if err := validateRepositoryName(reposName); err != nil { + return "", "", err + } endpoint := fmt.Sprintf("https://%s/v1/", hostname) if err := pingRegistryEndpoint(endpoint); err != nil { utils.Debugf("Registry %s does not work (%s), falling back to http", endpoint, err) From 2e95c379d16d7902a9337ecac46c0a46ddc2f2c4 Mon Sep 17 00:00:00 2001 From: Nan Monnand Deng Date: Fri, 28 Jun 2013 17:12:12 -0400 Subject: [PATCH 044/375] Added version checker interface --- docs/registry.go | 42 ++++++++++++++++++++++++++++++++++++++---- 1 file changed, 38 insertions(+), 4 deletions(-) diff --git a/docs/registry.go b/docs/registry.go index fc84f19e..12ca3c4b 100644 --- a/docs/registry.go +++ b/docs/registry.go @@ -98,6 +98,35 @@ func ResolveRepositoryName(reposName string) (string, string, error) { return endpoint, reposName, err } +// VersionChecker is used to model entities which has a version. +// It is basically a tupple with name and version. +type VersionChecker interface { + Name() string + Version() string +} + +func setUserAgentHeader(req *http.Request, baseVersions []VersionChecker, extra ...VersionChecker) error { + if len(baseVersions)+len(extra) == 0 { + return nil + } + userAgent := make(map[string]string, len(baseVersions)+len(extra)) + + for _, v := range baseVersions { + userAgent[v.Name()] = v.Version() + } + for _, v := range extra { + userAgent[v.Name()] = v.Version() + } + + header, err := json.Marshal(userAgent) + userAgent = nil + if err != nil { + return err + } + req.Header.Set("User-Agent", string(header)) + return nil +} + func doWithCookies(c *http.Client, req *http.Request) (*http.Response, error) { for _, cookie := range c.Jar.Cookies(req.URL) { req.AddCookie(cookie) @@ -536,11 +565,12 @@ type ImgData struct { } type Registry struct { - client *http.Client - authConfig *auth.AuthConfig + client *http.Client + authConfig *auth.AuthConfig + baseVersions []VersionChecker } -func NewRegistry(root string, authConfig *auth.AuthConfig) (r *Registry, err error) { +func NewRegistry(root string, authConfig *auth.AuthConfig, baseVersions ...VersionChecker) (r *Registry, err error) { httpTransport := &http.Transport{ DisableKeepAlives: true, Proxy: http.ProxyFromEnvironment, @@ -553,5 +583,9 @@ func NewRegistry(root string, authConfig *auth.AuthConfig) (r *Registry, err err }, } r.client.Jar, err = cookiejar.New(nil) - return r, err + if err != nil { + return nil, err + } + r.baseVersions = baseVersions + return r, nil } From 342460ed9aaf7f7cf8f92ba13ee0787308694988 Mon Sep 17 00:00:00 2001 From: Nan Monnand Deng Date: Fri, 28 Jun 2013 17:24:54 -0400 Subject: [PATCH 045/375] inserted setUserAgent in each HTTP request --- docs/registry.go | 43 ++++++++++++++++++++++++++----------------- 1 file changed, 26 insertions(+), 17 deletions(-) diff --git a/docs/registry.go b/docs/registry.go index 12ca3c4b..c51df1ac 100644 --- a/docs/registry.go +++ b/docs/registry.go @@ -105,33 +105,30 @@ type VersionChecker interface { Version() string } -func setUserAgentHeader(req *http.Request, baseVersions []VersionChecker, extra ...VersionChecker) error { - if len(baseVersions)+len(extra) == 0 { - return nil +func doWithCookies(c *http.Client, req *http.Request) (*http.Response, error) { + for _, cookie := range c.Jar.Cookies(req.URL) { + req.AddCookie(cookie) } - userAgent := make(map[string]string, len(baseVersions)+len(extra)) + return c.Do(req) +} - for _, v := range baseVersions { +func (r *Registry) setUserAgent(req *http.Request, extra ...VersionChecker) { + if len(r.baseVersions)+len(extra) == 0 { + return + } + userAgent := make(map[string]string, len(r.baseVersions)+len(extra)) + + for _, v := range r.baseVersions { userAgent[v.Name()] = v.Version() } for _, v := range extra { userAgent[v.Name()] = v.Version() } - header, err := json.Marshal(userAgent) + header, _ := json.Marshal(userAgent) userAgent = nil - if err != nil { - return err - } req.Header.Set("User-Agent", string(header)) - return nil -} - -func doWithCookies(c *http.Client, req *http.Request) (*http.Response, error) { - for _, cookie := range c.Jar.Cookies(req.URL) { - req.AddCookie(cookie) - } - return c.Do(req) + return } // Retrieve the history of a given image from the Registry. @@ -142,6 +139,9 @@ func (r *Registry) GetRemoteHistory(imgID, registry string, token []string) ([]s return nil, err } req.Header.Set("Authorization", "Token "+strings.Join(token, ", ")) + if err != nil { + return nil, err + } res, err := r.client.Do(req) if err != nil || res.StatusCode != 200 { if res != nil { @@ -188,6 +188,7 @@ func (r *Registry) GetRemoteImageJSON(imgID, registry string, token []string) ([ return nil, -1, fmt.Errorf("Failed to download json: %s", err) } req.Header.Set("Authorization", "Token "+strings.Join(token, ", ")) + r.setUserAgent(req, nil) res, err := r.client.Do(req) if err != nil { return nil, -1, fmt.Errorf("Failed to download json: %s", err) @@ -215,6 +216,7 @@ func (r *Registry) GetRemoteImageLayer(imgID, registry string, token []string) ( return nil, fmt.Errorf("Error while getting from the server: %s\n", err) } req.Header.Set("Authorization", "Token "+strings.Join(token, ", ")) + r.setUserAgent(req, nil) res, err := r.client.Do(req) if err != nil { return nil, err @@ -235,6 +237,7 @@ func (r *Registry) GetRemoteTags(registries []string, repository string, token [ return nil, err } req.Header.Set("Authorization", "Token "+strings.Join(token, ", ")) + r.setUserAgent(req, nil) res, err := r.client.Do(req) if err != nil { return nil, err @@ -273,6 +276,7 @@ func (r *Registry) GetRepositoryData(indexEp, remote string) (*RepositoryData, e req.SetBasicAuth(r.authConfig.Username, r.authConfig.Password) } req.Header.Set("X-Docker-Token", "true") + r.setUserAgent(req, nil) res, err := r.client.Do(req) if err != nil { @@ -336,6 +340,7 @@ func (r *Registry) PushImageJSONRegistry(imgData *ImgData, jsonRaw []byte, regis req.Header.Add("Content-type", "application/json") req.Header.Set("Authorization", "Token "+strings.Join(token, ",")) req.Header.Set("X-Docker-Checksum", imgData.Checksum) + r.setUserAgent(req, nil) utils.Debugf("Setting checksum for %s: %s", imgData.ID, imgData.Checksum) res, err := doWithCookies(r.client, req) @@ -370,6 +375,7 @@ func (r *Registry) PushImageLayerRegistry(imgID string, layer io.Reader, registr req.ContentLength = -1 req.TransferEncoding = []string{"chunked"} req.Header.Set("Authorization", "Token "+strings.Join(token, ",")) + r.setUserAgent(req, nil) res, err := doWithCookies(r.client, req) if err != nil { return fmt.Errorf("Failed to upload layer: %s", err) @@ -407,6 +413,7 @@ func (r *Registry) PushRegistryTag(remote, revision, tag, registry string, token } req.Header.Add("Content-type", "application/json") req.Header.Set("Authorization", "Token "+strings.Join(token, ",")) + r.setUserAgent(req, nil) req.ContentLength = int64(len(revision)) res, err := doWithCookies(r.client, req) if err != nil { @@ -439,6 +446,7 @@ func (r *Registry) PushImageJSONIndex(indexEp, remote string, imgList []*ImgData req.SetBasicAuth(r.authConfig.Username, r.authConfig.Password) req.ContentLength = int64(len(imgListJSON)) req.Header.Set("X-Docker-Token", "true") + r.setUserAgent(req, nil) if validate { req.Header["X-Docker-Endpoints"] = regs } @@ -459,6 +467,7 @@ func (r *Registry) PushImageJSONIndex(indexEp, remote string, imgList []*ImgData req.SetBasicAuth(r.authConfig.Username, r.authConfig.Password) req.ContentLength = int64(len(imgListJSON)) req.Header.Set("X-Docker-Token", "true") + r.setUserAgent(req, nil) if validate { req.Header["X-Docker-Endpoints"] = regs } From cf8afcf647aa1a49a118b2891ec545ed8ad04a1f Mon Sep 17 00:00:00 2001 From: Nan Monnand Deng Date: Fri, 28 Jun 2013 17:48:37 -0400 Subject: [PATCH 046/375] added client's kernel version --- docs/registry.go | 24 +++++++++++++++--------- 1 file changed, 15 insertions(+), 9 deletions(-) diff --git a/docs/registry.go b/docs/registry.go index c51df1ac..683a64ab 100644 --- a/docs/registry.go +++ b/docs/registry.go @@ -119,9 +119,15 @@ func (r *Registry) setUserAgent(req *http.Request, extra ...VersionChecker) { userAgent := make(map[string]string, len(r.baseVersions)+len(extra)) for _, v := range r.baseVersions { + if v == nil { + continue + } userAgent[v.Name()] = v.Version() } for _, v := range extra { + if v == nil { + continue + } userAgent[v.Name()] = v.Version() } @@ -188,7 +194,7 @@ func (r *Registry) GetRemoteImageJSON(imgID, registry string, token []string) ([ return nil, -1, fmt.Errorf("Failed to download json: %s", err) } req.Header.Set("Authorization", "Token "+strings.Join(token, ", ")) - r.setUserAgent(req, nil) + r.setUserAgent(req) res, err := r.client.Do(req) if err != nil { return nil, -1, fmt.Errorf("Failed to download json: %s", err) @@ -216,7 +222,7 @@ func (r *Registry) GetRemoteImageLayer(imgID, registry string, token []string) ( return nil, fmt.Errorf("Error while getting from the server: %s\n", err) } req.Header.Set("Authorization", "Token "+strings.Join(token, ", ")) - r.setUserAgent(req, nil) + r.setUserAgent(req) res, err := r.client.Do(req) if err != nil { return nil, err @@ -237,7 +243,7 @@ func (r *Registry) GetRemoteTags(registries []string, repository string, token [ return nil, err } req.Header.Set("Authorization", "Token "+strings.Join(token, ", ")) - r.setUserAgent(req, nil) + r.setUserAgent(req) res, err := r.client.Do(req) if err != nil { return nil, err @@ -276,7 +282,7 @@ func (r *Registry) GetRepositoryData(indexEp, remote string) (*RepositoryData, e req.SetBasicAuth(r.authConfig.Username, r.authConfig.Password) } req.Header.Set("X-Docker-Token", "true") - r.setUserAgent(req, nil) + r.setUserAgent(req) res, err := r.client.Do(req) if err != nil { @@ -340,7 +346,7 @@ func (r *Registry) PushImageJSONRegistry(imgData *ImgData, jsonRaw []byte, regis req.Header.Add("Content-type", "application/json") req.Header.Set("Authorization", "Token "+strings.Join(token, ",")) req.Header.Set("X-Docker-Checksum", imgData.Checksum) - r.setUserAgent(req, nil) + r.setUserAgent(req) utils.Debugf("Setting checksum for %s: %s", imgData.ID, imgData.Checksum) res, err := doWithCookies(r.client, req) @@ -375,7 +381,7 @@ func (r *Registry) PushImageLayerRegistry(imgID string, layer io.Reader, registr req.ContentLength = -1 req.TransferEncoding = []string{"chunked"} req.Header.Set("Authorization", "Token "+strings.Join(token, ",")) - r.setUserAgent(req, nil) + r.setUserAgent(req) res, err := doWithCookies(r.client, req) if err != nil { return fmt.Errorf("Failed to upload layer: %s", err) @@ -413,7 +419,7 @@ func (r *Registry) PushRegistryTag(remote, revision, tag, registry string, token } req.Header.Add("Content-type", "application/json") req.Header.Set("Authorization", "Token "+strings.Join(token, ",")) - r.setUserAgent(req, nil) + r.setUserAgent(req) req.ContentLength = int64(len(revision)) res, err := doWithCookies(r.client, req) if err != nil { @@ -446,7 +452,7 @@ func (r *Registry) PushImageJSONIndex(indexEp, remote string, imgList []*ImgData req.SetBasicAuth(r.authConfig.Username, r.authConfig.Password) req.ContentLength = int64(len(imgListJSON)) req.Header.Set("X-Docker-Token", "true") - r.setUserAgent(req, nil) + r.setUserAgent(req) if validate { req.Header["X-Docker-Endpoints"] = regs } @@ -467,7 +473,7 @@ func (r *Registry) PushImageJSONIndex(indexEp, remote string, imgList []*ImgData req.SetBasicAuth(r.authConfig.Username, r.authConfig.Password) req.ContentLength = int64(len(imgListJSON)) req.Header.Set("X-Docker-Token", "true") - r.setUserAgent(req, nil) + r.setUserAgent(req) if validate { req.Header["X-Docker-Endpoints"] = regs } From 6a2aee3043508bee5cfe515468d27b1e10cee939 Mon Sep 17 00:00:00 2001 From: Nan Monnand Deng Date: Fri, 28 Jun 2013 18:45:45 -0400 Subject: [PATCH 047/375] Removed an unnecessary error check. --- docs/registry.go | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/docs/registry.go b/docs/registry.go index 683a64ab..26cefbbd 100644 --- a/docs/registry.go +++ b/docs/registry.go @@ -145,9 +145,7 @@ func (r *Registry) GetRemoteHistory(imgID, registry string, token []string) ([]s return nil, err } req.Header.Set("Authorization", "Token "+strings.Join(token, ", ")) - if err != nil { - return nil, err - } + r.setUserAgent(req) res, err := r.client.Do(req) if err != nil || res.StatusCode != 200 { if res != nil { From e9e0d3c1c55140e04dbee1eb3d0069805663f7ca Mon Sep 17 00:00:00 2001 From: Nan Monnand Deng Date: Fri, 28 Jun 2013 18:46:25 -0400 Subject: [PATCH 048/375] Removed an unnecessary nil assignment --- docs/registry.go | 1 - 1 file changed, 1 deletion(-) diff --git a/docs/registry.go b/docs/registry.go index 26cefbbd..920f9459 100644 --- a/docs/registry.go +++ b/docs/registry.go @@ -132,7 +132,6 @@ func (r *Registry) setUserAgent(req *http.Request, extra ...VersionChecker) { } header, _ := json.Marshal(userAgent) - userAgent = nil req.Header.Set("User-Agent", string(header)) return } From 14155d603146104ede45471f754069598746315b Mon Sep 17 00:00:00 2001 From: Nan Monnand Deng Date: Fri, 28 Jun 2013 19:29:02 -0400 Subject: [PATCH 049/375] format in the user agent header should follow RFC 2616 --- docs/registry.go | 59 ++++++++++++++++++++++++++++++++---------------- 1 file changed, 40 insertions(+), 19 deletions(-) diff --git a/docs/registry.go b/docs/registry.go index 920f9459..0840ffbb 100644 --- a/docs/registry.go +++ b/docs/registry.go @@ -116,23 +116,9 @@ func (r *Registry) setUserAgent(req *http.Request, extra ...VersionChecker) { if len(r.baseVersions)+len(extra) == 0 { return } - userAgent := make(map[string]string, len(r.baseVersions)+len(extra)) - for _, v := range r.baseVersions { - if v == nil { - continue - } - userAgent[v.Name()] = v.Version() - } - for _, v := range extra { - if v == nil { - continue - } - userAgent[v.Name()] = v.Version() - } - - header, _ := json.Marshal(userAgent) - req.Header.Set("User-Agent", string(header)) + userAgent := appendVersions(r.baseVersionsStr, extra...) + req.Header.Set("User-Agent", userAgent) return } @@ -577,9 +563,43 @@ type ImgData struct { } type Registry struct { - client *http.Client - authConfig *auth.AuthConfig - baseVersions []VersionChecker + client *http.Client + authConfig *auth.AuthConfig + baseVersions []VersionChecker + baseVersionsStr string +} + +func validVersion(version VersionChecker) bool { + stopChars := " \t\r\n/" + if strings.ContainsAny(version.Name(), stopChars) { + return false + } + if strings.ContainsAny(version.Version(), stopChars) { + return false + } + return true +} + +func appendVersions(base string, versions ...VersionChecker) string { + if len(versions) == 0 { + return base + } + + var buf bytes.Buffer + if len(base) > 0 { + buf.Write([]byte(base)) + } + + for _, v := range versions { + if !validVersion(v) { + continue + } + buf.Write([]byte(v.Name())) + buf.Write([]byte("/")) + buf.Write([]byte(v.Version())) + buf.Write([]byte(" ")) + } + return buf.String() } func NewRegistry(root string, authConfig *auth.AuthConfig, baseVersions ...VersionChecker) (r *Registry, err error) { @@ -599,5 +619,6 @@ func NewRegistry(root string, authConfig *auth.AuthConfig, baseVersions ...Versi return nil, err } r.baseVersions = baseVersions + r.baseVersionsStr = appendVersions("", baseVersions...) return r, nil } From 4b7dbfbcc3dc481756106aee5bec2c6f84ade40e Mon Sep 17 00:00:00 2001 From: Nan Monnand Deng Date: Mon, 1 Jul 2013 17:57:56 -0400 Subject: [PATCH 050/375] reduce the number of string copy operations. --- docs/registry.go | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/docs/registry.go b/docs/registry.go index 0840ffbb..03a28901 100644 --- a/docs/registry.go +++ b/docs/registry.go @@ -116,9 +116,11 @@ func (r *Registry) setUserAgent(req *http.Request, extra ...VersionChecker) { if len(r.baseVersions)+len(extra) == 0 { return } - - userAgent := appendVersions(r.baseVersionsStr, extra...) - req.Header.Set("User-Agent", userAgent) + if len(extra) == 0 { + req.Header.Set("User-Agent", r.baseVersionsStr) + } else { + req.Header.Set("User-Agent", appendVersions(r.baseVersionsStr, extra...)) + } return } From 5f13f19407a99995726909789883ea154c9a92f7 Mon Sep 17 00:00:00 2001 From: Nan Monnand Deng Date: Thu, 18 Jul 2013 14:22:49 -0400 Subject: [PATCH 051/375] documentation. --- docs/registry.go | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/docs/registry.go b/docs/registry.go index 03a28901..6ba80cbe 100644 --- a/docs/registry.go +++ b/docs/registry.go @@ -112,6 +112,8 @@ func doWithCookies(c *http.Client, req *http.Request) (*http.Response, error) { return c.Do(req) } +// Set the user agent field in the header based on the versions provided +// in NewRegistry() and extra. func (r *Registry) setUserAgent(req *http.Request, extra ...VersionChecker) { if len(r.baseVersions)+len(extra) == 0 { return @@ -582,6 +584,12 @@ func validVersion(version VersionChecker) bool { return true } +// Convert versions to a string and append the string to the string base. +// +// Each VersionChecker will be converted to a string in the format of +// "product/version", where the "product" is get from the Name() method, while +// version is get from the Version() method. Several pieces of verson information +// will be concatinated and separated by space. func appendVersions(base string, versions ...VersionChecker) string { if len(versions) == 0 { return base From 262838e069651f3d9c119eb79aab1eab4ca354b0 Mon Sep 17 00:00:00 2001 From: Nan Monnand Deng Date: Tue, 23 Jul 2013 17:05:13 -0400 Subject: [PATCH 052/375] Rename: VersionChecker->VersionInfo. --- docs/registry.go | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/docs/registry.go b/docs/registry.go index 6ba80cbe..e6f4f592 100644 --- a/docs/registry.go +++ b/docs/registry.go @@ -98,9 +98,9 @@ func ResolveRepositoryName(reposName string) (string, string, error) { return endpoint, reposName, err } -// VersionChecker is used to model entities which has a version. +// VersionInfo is used to model entities which has a version. // It is basically a tupple with name and version. -type VersionChecker interface { +type VersionInfo interface { Name() string Version() string } @@ -114,7 +114,7 @@ func doWithCookies(c *http.Client, req *http.Request) (*http.Response, error) { // Set the user agent field in the header based on the versions provided // in NewRegistry() and extra. -func (r *Registry) setUserAgent(req *http.Request, extra ...VersionChecker) { +func (r *Registry) setUserAgent(req *http.Request, extra ...VersionInfo) { if len(r.baseVersions)+len(extra) == 0 { return } @@ -569,11 +569,11 @@ type ImgData struct { type Registry struct { client *http.Client authConfig *auth.AuthConfig - baseVersions []VersionChecker + baseVersions []VersionInfo baseVersionsStr string } -func validVersion(version VersionChecker) bool { +func validVersion(version VersionInfo) bool { stopChars := " \t\r\n/" if strings.ContainsAny(version.Name(), stopChars) { return false @@ -586,11 +586,11 @@ func validVersion(version VersionChecker) bool { // Convert versions to a string and append the string to the string base. // -// Each VersionChecker will be converted to a string in the format of +// Each VersionInfo will be converted to a string in the format of // "product/version", where the "product" is get from the Name() method, while // version is get from the Version() method. Several pieces of verson information // will be concatinated and separated by space. -func appendVersions(base string, versions ...VersionChecker) string { +func appendVersions(base string, versions ...VersionInfo) string { if len(versions) == 0 { return base } @@ -612,7 +612,7 @@ func appendVersions(base string, versions ...VersionChecker) string { return buf.String() } -func NewRegistry(root string, authConfig *auth.AuthConfig, baseVersions ...VersionChecker) (r *Registry, err error) { +func NewRegistry(root string, authConfig *auth.AuthConfig, baseVersions ...VersionInfo) (r *Registry, err error) { httpTransport := &http.Transport{ DisableKeepAlives: true, Proxy: http.ProxyFromEnvironment, From 64a8dea9d7a15e261ee16579991e51fbd48572bb Mon Sep 17 00:00:00 2001 From: "Guillaume J. Charmes" Date: Tue, 23 Jul 2013 11:37:13 -0700 Subject: [PATCH 053/375] Make sure the cookie is used in all registry queries --- docs/registry.go | 22 +++++++++++++--------- 1 file changed, 13 insertions(+), 9 deletions(-) diff --git a/docs/registry.go b/docs/registry.go index e6f4f592..adef1c7b 100644 --- a/docs/registry.go +++ b/docs/registry.go @@ -109,7 +109,14 @@ func doWithCookies(c *http.Client, req *http.Request) (*http.Response, error) { for _, cookie := range c.Jar.Cookies(req.URL) { req.AddCookie(cookie) } - return c.Do(req) + res, err := c.Do(req) + if err != nil { + return nil, err + } + if len(res.Cookies()) > 0 { + c.Jar.SetCookies(req.URL, res.Cookies()) + } + return res, err } // Set the user agent field in the header based on the versions provided @@ -135,7 +142,7 @@ func (r *Registry) GetRemoteHistory(imgID, registry string, token []string) ([]s } req.Header.Set("Authorization", "Token "+strings.Join(token, ", ")) r.setUserAgent(req) - res, err := r.client.Do(req) + res, err := doWithCookies(r.client, req) if err != nil || res.StatusCode != 200 { if res != nil { return nil, fmt.Errorf("Internal server error: %d trying to fetch remote history for %s", res.StatusCode, imgID) @@ -182,7 +189,7 @@ func (r *Registry) GetRemoteImageJSON(imgID, registry string, token []string) ([ } req.Header.Set("Authorization", "Token "+strings.Join(token, ", ")) r.setUserAgent(req) - res, err := r.client.Do(req) + res, err := doWithCookies(r.client, req) if err != nil { return nil, -1, fmt.Errorf("Failed to download json: %s", err) } @@ -210,7 +217,7 @@ func (r *Registry) GetRemoteImageLayer(imgID, registry string, token []string) ( } req.Header.Set("Authorization", "Token "+strings.Join(token, ", ")) r.setUserAgent(req) - res, err := r.client.Do(req) + res, err := doWithCookies(r.client, req) if err != nil { return nil, err } @@ -231,7 +238,7 @@ func (r *Registry) GetRemoteTags(registries []string, repository string, token [ } req.Header.Set("Authorization", "Token "+strings.Join(token, ", ")) r.setUserAgent(req) - res, err := r.client.Do(req) + res, err := doWithCookies(r.client, req) if err != nil { return nil, err } @@ -326,7 +333,7 @@ func (r *Registry) GetRepositoryData(indexEp, remote string) (*RepositoryData, e // Push a local image to the registry func (r *Registry) PushImageJSONRegistry(imgData *ImgData, jsonRaw []byte, registry string, token []string) error { // FIXME: try json with UTF8 - req, err := http.NewRequest("PUT", registry+"images/"+imgData.ID+"/json", strings.NewReader(string(jsonRaw))) + req, err := http.NewRequest("PUT", registry+"images/"+imgData.ID+"/json", bytes.NewReader(jsonRaw)) if err != nil { return err } @@ -341,9 +348,6 @@ func (r *Registry) PushImageJSONRegistry(imgData *ImgData, jsonRaw []byte, regis return fmt.Errorf("Failed to upload metadata: %s", err) } defer res.Body.Close() - if len(res.Cookies()) > 0 { - r.client.Jar.SetCookies(req.URL, res.Cookies()) - } if res.StatusCode != 200 { errBody, err := ioutil.ReadAll(res.Body) if err != nil { From 4a818a5e7343dc4a65c564269877a0c5d23c77dc Mon Sep 17 00:00:00 2001 From: "Guillaume J. Charmes" Date: Wed, 17 Jul 2013 12:13:22 -0700 Subject: [PATCH 054/375] Refactor checksum --- docs/registry.go | 58 ++++++++++++++++++++++++++++++++++++++++-------- 1 file changed, 49 insertions(+), 9 deletions(-) diff --git a/docs/registry.go b/docs/registry.go index adef1c7b..cac77ba0 100644 --- a/docs/registry.go +++ b/docs/registry.go @@ -330,16 +330,52 @@ func (r *Registry) GetRepositoryData(indexEp, remote string) (*RepositoryData, e }, nil } +func (r *Registry) PushImageChecksumRegistry(imgData *ImgData, registry string, token []string) error { + + utils.Debugf("[registry] Calling PUT %s", registry+"images/"+imgData.ID+"/checksum") + + req, err := http.NewRequest("PUT", registry+"images/"+imgData.ID+"/checksum", nil) + if err != nil { + return err + } + req.Header.Set("Authorization", "Token "+strings.Join(token, ",")) + req.Header.Set("X-Docker-Checksum", imgData.Checksum) + + res, err := doWithCookies(r.client, req) + if err != nil { + return fmt.Errorf("Failed to upload metadata: %s", err) + } + defer res.Body.Close() + if len(res.Cookies()) > 0 { + r.client.Jar.SetCookies(req.URL, res.Cookies()) + } + if res.StatusCode != 200 { + errBody, err := ioutil.ReadAll(res.Body) + if err != nil { + return fmt.Errorf("HTTP code %d while uploading metadata and error when trying to parse response body: %s", res.StatusCode, err) + } + var jsonBody map[string]string + if err := json.Unmarshal(errBody, &jsonBody); err != nil { + errBody = []byte(err.Error()) + } else if jsonBody["error"] == "Image already exists" { + return ErrAlreadyExists + } + return fmt.Errorf("HTTP code %d while uploading metadata: %s", res.StatusCode, errBody) + } + return nil +} + // Push a local image to the registry func (r *Registry) PushImageJSONRegistry(imgData *ImgData, jsonRaw []byte, registry string, token []string) error { - // FIXME: try json with UTF8 + + utils.Debugf("[registry] Calling PUT %s", registry+"images/"+imgData.ID+"/json") + req, err := http.NewRequest("PUT", registry+"images/"+imgData.ID+"/json", bytes.NewReader(jsonRaw)) if err != nil { return err } req.Header.Add("Content-type", "application/json") req.Header.Set("Authorization", "Token "+strings.Join(token, ",")) - req.Header.Set("X-Docker-Checksum", imgData.Checksum) r.setUserAgent(req) utils.Debugf("Setting checksum for %s: %s", imgData.ID, imgData.Checksum) @@ -364,10 +400,14 @@ func (r *Registry) PushImageJSONRegistry(imgData *ImgData, jsonRaw []byte, regis return nil } -func (r *Registry) PushImageLayerRegistry(imgID string, layer io.Reader, registry string, token []string) error { - req, err := http.NewRequest("PUT", registry+"images/"+imgID+"/layer", layer) +func (r *Registry) PushImageLayerRegistry(imgID string, layer io.Reader, registry string, token []string) (checksum string, err error) { + + utils.Debugf("[registry] Calling PUT %s", registry+"images/"+imgID+"/layer") + + tarsumLayer := &utils.TarSum{Reader: layer} + req, err := http.NewRequest("PUT", registry+"images/"+imgID+"/layer", tarsumLayer) if err != nil { - return err + return "", err } req.ContentLength = -1 req.TransferEncoding = []string{"chunked"} @@ -375,18 +415,18 @@ func (r *Registry) PushImageLayerRegistry(imgID string, layer io.Reader, registr r.setUserAgent(req) res, err := doWithCookies(r.client, req) if err != nil { - return fmt.Errorf("Failed to upload layer: %s", err) + return "", fmt.Errorf("Failed to upload layer: %s", err) } defer res.Body.Close() if res.StatusCode != 200 { errBody, err := ioutil.ReadAll(res.Body) if err != nil { - return fmt.Errorf("HTTP code %d while uploading metadata and error when trying to parse response body: %s", res.StatusCode, err) + return "", fmt.Errorf("HTTP code %d while uploading metadata and error when trying to parse response body: %s", res.StatusCode, err) } - return fmt.Errorf("Received HTTP code %d while uploading layer: %s", res.StatusCode, errBody) + return "", fmt.Errorf("Received HTTP code %d while uploading layer: %s", res.StatusCode, errBody) } - return nil + return tarsumLayer.Sum(), nil } func (r *Registry) opaqueRequest(method, urlStr string, body io.Reader) (*http.Request, error) { From 1c62adeda765eb39d35c1672c9eed41248e17932 Mon Sep 17 00:00:00 2001 From: "Guillaume J. Charmes" Date: Mon, 22 Jul 2013 14:50:32 -0700 Subject: [PATCH 055/375] Handle extra-paremeter within checksum calculations --- docs/registry.go | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/docs/registry.go b/docs/registry.go index cac77ba0..40b9872a 100644 --- a/docs/registry.go +++ b/docs/registry.go @@ -17,8 +17,10 @@ import ( "strings" ) -var ErrAlreadyExists = errors.New("Image already exists") -var ErrInvalidRepositoryName = errors.New("Invalid repository name (ex: \"registry.domain.tld/myrepos\")") +var ( + ErrAlreadyExists = errors.New("Image already exists") + ErrInvalidRepositoryName = errors.New("Invalid repository name (ex: \"registry.domain.tld/myrepos\")") +) func pingRegistryEndpoint(endpoint string) error { if endpoint == auth.IndexServerAddress() { @@ -266,8 +268,11 @@ func (r *Registry) GetRemoteTags(registries []string, repository string, token [ } func (r *Registry) GetRepositoryData(indexEp, remote string) (*RepositoryData, error) { + repositoryTarget := fmt.Sprintf("%srepositories/%s/images", indexEp, remote) + utils.Debugf("[registry] Calling GET %s", repositoryTarget) + req, err := r.opaqueRequest("GET", repositoryTarget, nil) if err != nil { return nil, err @@ -378,7 +383,6 @@ func (r *Registry) PushImageJSONRegistry(imgData *ImgData, jsonRaw []byte, regis req.Header.Set("Authorization", "Token "+strings.Join(token, ",")) r.setUserAgent(req) - utils.Debugf("Setting checksum for %s: %s", imgData.ID, imgData.Checksum) res, err := doWithCookies(r.client, req) if err != nil { return fmt.Errorf("Failed to upload metadata: %s", err) @@ -400,11 +404,12 @@ func (r *Registry) PushImageJSONRegistry(imgData *ImgData, jsonRaw []byte, regis return nil } -func (r *Registry) PushImageLayerRegistry(imgID string, layer io.Reader, registry string, token []string) (checksum string, err error) { +func (r *Registry) PushImageLayerRegistry(imgID string, layer io.Reader, registry string, token []string, jsonRaw []byte) (checksum string, err error) { utils.Debugf("[registry] Calling PUT %s", registry+"images/"+imgID+"/layer") tarsumLayer := &utils.TarSum{Reader: layer} + req, err := http.NewRequest("PUT", registry+"images/"+imgID+"/layer", tarsumLayer) if err != nil { return "", err @@ -426,7 +431,7 @@ func (r *Registry) PushImageLayerRegistry(imgID string, layer io.Reader, registr } return "", fmt.Errorf("Received HTTP code %d while uploading layer: %s", res.StatusCode, errBody) } - return tarsumLayer.Sum(), nil + return tarsumLayer.Sum(jsonRaw), nil } func (r *Registry) opaqueRequest(method, urlStr string, body io.Reader) (*http.Request, error) { @@ -474,7 +479,7 @@ func (r *Registry) PushImageJSONIndex(indexEp, remote string, imgList []*ImgData } u := fmt.Sprintf("%srepositories/%s/%s", indexEp, remote, suffix) - utils.Debugf("PUT %s", u) + utils.Debugf("[registry] PUT %s", u) utils.Debugf("Image list pushed to index:\n%s\n", imgListJSON) req, err := r.opaqueRequest("PUT", u, bytes.NewReader(imgListJSON)) if err != nil { From 0b59dcfa2d594f095b8b2e21a7e6fd7f409b7bf2 Mon Sep 17 00:00:00 2001 From: "Guillaume J. Charmes" Date: Mon, 22 Jul 2013 16:44:34 -0700 Subject: [PATCH 056/375] Make sure the index also receives the checksums --- docs/registry.go | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/docs/registry.go b/docs/registry.go index 40b9872a..4e9dd889 100644 --- a/docs/registry.go +++ b/docs/registry.go @@ -469,7 +469,19 @@ func (r *Registry) PushRegistryTag(remote, revision, tag, registry string, token } func (r *Registry) PushImageJSONIndex(indexEp, remote string, imgList []*ImgData, validate bool, regs []string) (*RepositoryData, error) { - imgListJSON, err := json.Marshal(imgList) + cleanImgList := []*ImgData{} + + if validate { + for _, elem := range imgList { + if elem.Checksum != "" { + cleanImgList = append(cleanImgList, elem) + } + } + } else { + cleanImgList = imgList + } + + imgListJSON, err := json.Marshal(cleanImgList) if err != nil { return nil, err } From 11cd5760f9d866748866b9406318c37967bd05da Mon Sep 17 00:00:00 2001 From: Michael Crosby Date: Wed, 24 Jul 2013 03:01:24 +0000 Subject: [PATCH 057/375] Return registy status code in error Added Details map to the JSONMessage --- docs/registry.go | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/docs/registry.go b/docs/registry.go index 4e9dd889..ed6f4c7d 100644 --- a/docs/registry.go +++ b/docs/registry.go @@ -147,7 +147,7 @@ func (r *Registry) GetRemoteHistory(imgID, registry string, token []string) ([]s res, err := doWithCookies(r.client, req) if err != nil || res.StatusCode != 200 { if res != nil { - return nil, fmt.Errorf("Internal server error: %d trying to fetch remote history for %s", res.StatusCode, imgID) + return nil, utils.NewHTTPRequestError(fmt.Sprintf("Internal server error: %d trying to fetch remote history for %s", res.StatusCode, imgID), res) } return nil, err } @@ -197,7 +197,7 @@ func (r *Registry) GetRemoteImageJSON(imgID, registry string, token []string) ([ } defer res.Body.Close() if res.StatusCode != 200 { - return nil, -1, fmt.Errorf("HTTP code %d", res.StatusCode) + return nil, -1, utils.NewHTTPRequestError(fmt.Sprintf("HTTP code %d", res.StatusCode), res) } imageSize, err := strconv.Atoi(res.Header.Get("X-Docker-Size")) @@ -289,12 +289,12 @@ func (r *Registry) GetRepositoryData(indexEp, remote string) (*RepositoryData, e } defer res.Body.Close() if res.StatusCode == 401 { - return nil, fmt.Errorf("Please login first (HTTP code %d)", res.StatusCode) + return nil, utils.NewHTTPRequestError(fmt.Sprintf("Please login first (HTTP code %d)", res.StatusCode), res) } // TODO: Right now we're ignoring checksums in the response body. // In the future, we need to use them to check image validity. if res.StatusCode != 200 { - return nil, fmt.Errorf("HTTP code: %d", res.StatusCode) + return nil, utils.NewHTTPRequestError(fmt.Sprintf("HTTP code: %d", res.StatusCode), res) } var tokens []string @@ -391,7 +391,7 @@ func (r *Registry) PushImageJSONRegistry(imgData *ImgData, jsonRaw []byte, regis if res.StatusCode != 200 { errBody, err := ioutil.ReadAll(res.Body) if err != nil { - return fmt.Errorf("HTTP code %d while uploading metadata and error when trying to parse response body: %s", res.StatusCode, err) + return utils.NewHTTPRequestError(fmt.Sprint("HTTP code %d while uploading metadata and error when trying to parse response body: %s", res.StatusCode, err), res) } var jsonBody map[string]string if err := json.Unmarshal(errBody, &jsonBody); err != nil { @@ -399,7 +399,7 @@ func (r *Registry) PushImageJSONRegistry(imgData *ImgData, jsonRaw []byte, regis } else if jsonBody["error"] == "Image already exists" { return ErrAlreadyExists } - return fmt.Errorf("HTTP code %d while uploading metadata: %s", res.StatusCode, errBody) + return utils.NewHTTPRequestError(fmt.Sprintf("HTTP code %d while uploading metadata: %s", res.StatusCode, errBody), res) } return nil } @@ -427,9 +427,9 @@ func (r *Registry) PushImageLayerRegistry(imgID string, layer io.Reader, registr if res.StatusCode != 200 { errBody, err := ioutil.ReadAll(res.Body) if err != nil { - return "", fmt.Errorf("HTTP code %d while uploading metadata and error when trying to parse response body: %s", res.StatusCode, err) + return utils.NewHTTPRequestError(fmt.Sprintf("HTTP code %d while uploading metadata and error when trying to parse response body: %s", res.StatusCode, err), res) } - return "", fmt.Errorf("Received HTTP code %d while uploading layer: %s", res.StatusCode, errBody) + return utils.NewHTTPRequestError(fmt.Sprintf("Received HTTP code %d while uploading layer: %s", res.StatusCode, errBody), res) } return tarsumLayer.Sum(jsonRaw), nil } @@ -463,7 +463,7 @@ func (r *Registry) PushRegistryTag(remote, revision, tag, registry string, token } res.Body.Close() if res.StatusCode != 200 && res.StatusCode != 201 { - return fmt.Errorf("Internal server error: %d trying to push tag %s on %s", res.StatusCode, tag, remote) + return utils.NewHTTPRequestError(fmt.Sprintf("Internal server error: %d trying to push tag %s on %s", res.StatusCode, tag, remote), res) } return nil } @@ -540,7 +540,7 @@ func (r *Registry) PushImageJSONIndex(indexEp, remote string, imgList []*ImgData if err != nil { return nil, err } - return nil, fmt.Errorf("Error: Status %d trying to push repository %s: %s", res.StatusCode, remote, errBody) + return nil, utils.NewHTTPRequestError(fmt.Sprintf("Error: Status %d trying to push repository %s: %s", res.StatusCode, remote, errBody), res) } if res.Header.Get("X-Docker-Token") != "" { tokens = res.Header["X-Docker-Token"] @@ -564,7 +564,7 @@ func (r *Registry) PushImageJSONIndex(indexEp, remote string, imgList []*ImgData if err != nil { return nil, err } - return nil, fmt.Errorf("Error: Status %d trying to push checksums %s: %s", res.StatusCode, remote, errBody) + return nil, utils.NewHTTPRequestError(fmt.Sprintf("Error: Status %d trying to push checksums %s: %s", res.StatusCode, remote, errBody), res) } } @@ -586,7 +586,7 @@ func (r *Registry) SearchRepositories(term string) (*SearchResults, error) { } defer res.Body.Close() if res.StatusCode != 200 { - return nil, fmt.Errorf("Unexepected status code %d", res.StatusCode) + return nil, utils.NewHTTPRequestError(fmt.Sprintf("Unexepected status code %d", res.StatusCode), res) } rawData, err := ioutil.ReadAll(res.Body) if err != nil { From 762dfbfced100ecddb79a255bdfc40c09a127e5c Mon Sep 17 00:00:00 2001 From: Nan Monnand Deng Date: Fri, 2 Aug 2013 03:08:08 -0400 Subject: [PATCH 058/375] reqFactory in Registry --- docs/registry.go | 101 +++++++---------------------------------------- 1 file changed, 15 insertions(+), 86 deletions(-) diff --git a/docs/registry.go b/docs/registry.go index 4e9dd889..da5c83bf 100644 --- a/docs/registry.go +++ b/docs/registry.go @@ -100,13 +100,6 @@ func ResolveRepositoryName(reposName string) (string, string, error) { return endpoint, reposName, err } -// VersionInfo is used to model entities which has a version. -// It is basically a tupple with name and version. -type VersionInfo interface { - Name() string - Version() string -} - func doWithCookies(c *http.Client, req *http.Request) (*http.Response, error) { for _, cookie := range c.Jar.Cookies(req.URL) { req.AddCookie(cookie) @@ -121,29 +114,14 @@ func doWithCookies(c *http.Client, req *http.Request) (*http.Response, error) { return res, err } -// Set the user agent field in the header based on the versions provided -// in NewRegistry() and extra. -func (r *Registry) setUserAgent(req *http.Request, extra ...VersionInfo) { - if len(r.baseVersions)+len(extra) == 0 { - return - } - if len(extra) == 0 { - req.Header.Set("User-Agent", r.baseVersionsStr) - } else { - req.Header.Set("User-Agent", appendVersions(r.baseVersionsStr, extra...)) - } - return -} - // Retrieve the history of a given image from the Registry. // Return a list of the parent's json (requested image included) func (r *Registry) GetRemoteHistory(imgID, registry string, token []string) ([]string, error) { - req, err := http.NewRequest("GET", registry+"images/"+imgID+"/ancestry", nil) + req, err := r.reqFactory.NewRequest("GET", registry+"images/"+imgID+"/ancestry", nil) if err != nil { return nil, err } req.Header.Set("Authorization", "Token "+strings.Join(token, ", ")) - r.setUserAgent(req) res, err := doWithCookies(r.client, req) if err != nil || res.StatusCode != 200 { if res != nil { @@ -170,7 +148,7 @@ func (r *Registry) GetRemoteHistory(imgID, registry string, token []string) ([]s func (r *Registry) LookupRemoteImage(imgID, registry string, token []string) bool { rt := &http.Transport{Proxy: http.ProxyFromEnvironment} - req, err := http.NewRequest("GET", registry+"images/"+imgID+"/json", nil) + req, err := r.reqFactory.NewRequest("GET", registry+"images/"+imgID+"/json", nil) if err != nil { return false } @@ -185,12 +163,11 @@ func (r *Registry) LookupRemoteImage(imgID, registry string, token []string) boo // Retrieve an image from the Registry. func (r *Registry) GetRemoteImageJSON(imgID, registry string, token []string) ([]byte, int, error) { // Get the JSON - req, err := http.NewRequest("GET", registry+"images/"+imgID+"/json", nil) + req, err := r.reqFactory.NewRequest("GET", registry+"images/"+imgID+"/json", nil) if err != nil { return nil, -1, fmt.Errorf("Failed to download json: %s", err) } req.Header.Set("Authorization", "Token "+strings.Join(token, ", ")) - r.setUserAgent(req) res, err := doWithCookies(r.client, req) if err != nil { return nil, -1, fmt.Errorf("Failed to download json: %s", err) @@ -213,12 +190,11 @@ func (r *Registry) GetRemoteImageJSON(imgID, registry string, token []string) ([ } func (r *Registry) GetRemoteImageLayer(imgID, registry string, token []string) (io.ReadCloser, error) { - req, err := http.NewRequest("GET", registry+"images/"+imgID+"/layer", nil) + req, err := r.reqFactory.NewRequest("GET", registry+"images/"+imgID+"/layer", nil) if err != nil { return nil, fmt.Errorf("Error while getting from the server: %s\n", err) } req.Header.Set("Authorization", "Token "+strings.Join(token, ", ")) - r.setUserAgent(req) res, err := doWithCookies(r.client, req) if err != nil { return nil, err @@ -239,7 +215,6 @@ func (r *Registry) GetRemoteTags(registries []string, repository string, token [ return nil, err } req.Header.Set("Authorization", "Token "+strings.Join(token, ", ")) - r.setUserAgent(req) res, err := doWithCookies(r.client, req) if err != nil { return nil, err @@ -281,7 +256,6 @@ func (r *Registry) GetRepositoryData(indexEp, remote string) (*RepositoryData, e req.SetBasicAuth(r.authConfig.Username, r.authConfig.Password) } req.Header.Set("X-Docker-Token", "true") - r.setUserAgent(req) res, err := r.client.Do(req) if err != nil { @@ -339,7 +313,7 @@ func (r *Registry) PushImageChecksumRegistry(imgData *ImgData, registry string, utils.Debugf("[registry] Calling PUT %s", registry+"images/"+imgData.ID+"/checksum") - req, err := http.NewRequest("PUT", registry+"images/"+imgData.ID+"/checksum", nil) + req, err := r.reqFactory.NewRequest("PUT", registry+"images/"+imgData.ID+"/checksum", nil) if err != nil { return err } @@ -375,13 +349,12 @@ func (r *Registry) PushImageJSONRegistry(imgData *ImgData, jsonRaw []byte, regis utils.Debugf("[registry] Calling PUT %s", registry+"images/"+imgData.ID+"/json") - req, err := http.NewRequest("PUT", registry+"images/"+imgData.ID+"/json", bytes.NewReader(jsonRaw)) + req, err := r.reqFactory.NewRequest("PUT", registry+"images/"+imgData.ID+"/json", bytes.NewReader(jsonRaw)) if err != nil { return err } req.Header.Add("Content-type", "application/json") req.Header.Set("Authorization", "Token "+strings.Join(token, ",")) - r.setUserAgent(req) res, err := doWithCookies(r.client, req) if err != nil { @@ -410,14 +383,13 @@ func (r *Registry) PushImageLayerRegistry(imgID string, layer io.Reader, registr tarsumLayer := &utils.TarSum{Reader: layer} - req, err := http.NewRequest("PUT", registry+"images/"+imgID+"/layer", tarsumLayer) + req, err := r.reqFactory.NewRequest("PUT", registry+"images/"+imgID+"/layer", tarsumLayer) if err != nil { return "", err } req.ContentLength = -1 req.TransferEncoding = []string{"chunked"} req.Header.Set("Authorization", "Token "+strings.Join(token, ",")) - r.setUserAgent(req) res, err := doWithCookies(r.client, req) if err != nil { return "", fmt.Errorf("Failed to upload layer: %s", err) @@ -435,7 +407,7 @@ func (r *Registry) PushImageLayerRegistry(imgID string, layer io.Reader, registr } func (r *Registry) opaqueRequest(method, urlStr string, body io.Reader) (*http.Request, error) { - req, err := http.NewRequest(method, urlStr, body) + req, err := r.reqFactory.NewRequest(method, urlStr, body) if err != nil { return nil, err } @@ -455,7 +427,6 @@ func (r *Registry) PushRegistryTag(remote, revision, tag, registry string, token } req.Header.Add("Content-type", "application/json") req.Header.Set("Authorization", "Token "+strings.Join(token, ",")) - r.setUserAgent(req) req.ContentLength = int64(len(revision)) res, err := doWithCookies(r.client, req) if err != nil { @@ -500,7 +471,6 @@ func (r *Registry) PushImageJSONIndex(indexEp, remote string, imgList []*ImgData req.SetBasicAuth(r.authConfig.Username, r.authConfig.Password) req.ContentLength = int64(len(imgListJSON)) req.Header.Set("X-Docker-Token", "true") - r.setUserAgent(req) if validate { req.Header["X-Docker-Endpoints"] = regs } @@ -521,7 +491,6 @@ func (r *Registry) PushImageJSONIndex(indexEp, remote string, imgList []*ImgData req.SetBasicAuth(r.authConfig.Username, r.authConfig.Password) req.ContentLength = int64(len(imgListJSON)) req.Header.Set("X-Docker-Token", "true") - r.setUserAgent(req) if validate { req.Header["X-Docker-Endpoints"] = regs } @@ -576,7 +545,7 @@ func (r *Registry) PushImageJSONIndex(indexEp, remote string, imgList []*ImgData func (r *Registry) SearchRepositories(term string) (*SearchResults, error) { u := auth.IndexServerAddress() + "search?q=" + url.QueryEscape(term) - req, err := http.NewRequest("GET", u, nil) + req, err := r.reqFactory.NewRequest("GET", u, nil) if err != nil { return nil, err } @@ -628,52 +597,12 @@ type ImgData struct { } type Registry struct { - client *http.Client - authConfig *auth.AuthConfig - baseVersions []VersionInfo - baseVersionsStr string + client *http.Client + authConfig *auth.AuthConfig + reqFactory *utils.HTTPRequestFactory } -func validVersion(version VersionInfo) bool { - stopChars := " \t\r\n/" - if strings.ContainsAny(version.Name(), stopChars) { - return false - } - if strings.ContainsAny(version.Version(), stopChars) { - return false - } - return true -} - -// Convert versions to a string and append the string to the string base. -// -// Each VersionInfo will be converted to a string in the format of -// "product/version", where the "product" is get from the Name() method, while -// version is get from the Version() method. Several pieces of verson information -// will be concatinated and separated by space. -func appendVersions(base string, versions ...VersionInfo) string { - if len(versions) == 0 { - return base - } - - var buf bytes.Buffer - if len(base) > 0 { - buf.Write([]byte(base)) - } - - for _, v := range versions { - if !validVersion(v) { - continue - } - buf.Write([]byte(v.Name())) - buf.Write([]byte("/")) - buf.Write([]byte(v.Version())) - buf.Write([]byte(" ")) - } - return buf.String() -} - -func NewRegistry(root string, authConfig *auth.AuthConfig, baseVersions ...VersionInfo) (r *Registry, err error) { +func NewRegistry(root string, authConfig *auth.AuthConfig, factory *utils.HTTPRequestFactory) (r *Registry, err error) { httpTransport := &http.Transport{ DisableKeepAlives: true, Proxy: http.ProxyFromEnvironment, @@ -689,7 +618,7 @@ func NewRegistry(root string, authConfig *auth.AuthConfig, baseVersions ...Versi if err != nil { return nil, err } - r.baseVersions = baseVersions - r.baseVersionsStr = appendVersions("", baseVersions...) + + r.reqFactory = factory return r, nil } From 95b4a0c32a93250d1ca033870b0cff1d58cc7336 Mon Sep 17 00:00:00 2001 From: Michael Crosby Date: Tue, 30 Jul 2013 22:48:20 +0000 Subject: [PATCH 059/375] Return JSONError for HTTPResponse error --- docs/registry.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/registry.go b/docs/registry.go index ed6f4c7d..5b8480d1 100644 --- a/docs/registry.go +++ b/docs/registry.go @@ -427,9 +427,9 @@ func (r *Registry) PushImageLayerRegistry(imgID string, layer io.Reader, registr if res.StatusCode != 200 { errBody, err := ioutil.ReadAll(res.Body) if err != nil { - return utils.NewHTTPRequestError(fmt.Sprintf("HTTP code %d while uploading metadata and error when trying to parse response body: %s", res.StatusCode, err), res) + return "", utils.NewHTTPRequestError(fmt.Sprintf("HTTP code %d while uploading metadata and error when trying to parse response body: %s", res.StatusCode, err), res) } - return utils.NewHTTPRequestError(fmt.Sprintf("Received HTTP code %d while uploading layer: %s", res.StatusCode, errBody), res) + return "", utils.NewHTTPRequestError(fmt.Sprintf("Received HTTP code %d while uploading layer: %s", res.StatusCode, errBody), res) } return tarsumLayer.Sum(jsonRaw), nil } From 1fe03a4bf7d9a95f67f05375fee6d058880e96a9 Mon Sep 17 00:00:00 2001 From: Sam Alba Date: Sun, 4 Aug 2013 17:42:24 -0700 Subject: [PATCH 060/375] Reduce connect and read timeout when pinging the registry (fixes issue #1363) --- docs/registry.go | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/docs/registry.go b/docs/registry.go index 5b8480d1..aa7e5242 100644 --- a/docs/registry.go +++ b/docs/registry.go @@ -9,12 +9,14 @@ import ( "github.com/dotcloud/docker/utils" "io" "io/ioutil" + "net" "net/http" "net/http/cookiejar" "net/url" "regexp" "strconv" "strings" + "time" ) var ( @@ -28,7 +30,19 @@ func pingRegistryEndpoint(endpoint string) error { // (and we never want to fallback to http in case of error) return nil } - resp, err := http.Get(endpoint + "_ping") + httpDial := func(proto string, addr string) (net.Conn, error) { + // Set the connect timeout to 5 seconds + conn, err := net.DialTimeout(proto, addr, time.Duration(5)*time.Second) + if err != nil { + return nil, err + } + // Set the recv timeout to 10 seconds + conn.SetDeadline(time.Now().Add(time.Duration(10) * time.Second)) + return conn, nil + } + httpTransport := &http.Transport{Dial: httpDial} + client := &http.Client{Transport: httpTransport} + resp, err := client.Get(endpoint + "_ping") if err != nil { return err } From fec63826b9c2c7b626b30cf1bdb35535dfe45f17 Mon Sep 17 00:00:00 2001 From: Sam Alba Date: Sun, 4 Aug 2013 17:59:12 -0700 Subject: [PATCH 061/375] Always consider localhost as a domain name when parsing the FQN repos name --- docs/registry.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/docs/registry.go b/docs/registry.go index 5b8480d1..f23ef6c0 100644 --- a/docs/registry.go +++ b/docs/registry.go @@ -69,7 +69,8 @@ func ResolveRepositoryName(reposName string) (string, string, error) { return "", "", ErrInvalidRepositoryName } nameParts := strings.SplitN(reposName, "/", 2) - if !strings.Contains(nameParts[0], ".") && !strings.Contains(nameParts[0], ":") { + if !strings.Contains(nameParts[0], ".") && !strings.Contains(nameParts[0], ":") && + nameParts[0] != "localhost" { // This is a Docker Index repos (ex: samalba/hipache or ubuntu) err := validateRepositoryName(reposName) return auth.IndexServerAddress(), reposName, err From 14cc9fcfda6a8a6f12b44ef426b57f01c426ae91 Mon Sep 17 00:00:00 2001 From: Sam Alba Date: Wed, 24 Jul 2013 17:44:55 -0700 Subject: [PATCH 062/375] Implemented a Mocked version of the Registry server --- docs/registry_mock_test.go | 321 +++++++++++++++++++++++++++++++++++++ 1 file changed, 321 insertions(+) create mode 100644 docs/registry_mock_test.go diff --git a/docs/registry_mock_test.go b/docs/registry_mock_test.go new file mode 100644 index 00000000..f1e65cad --- /dev/null +++ b/docs/registry_mock_test.go @@ -0,0 +1,321 @@ +package registry + +import ( + "encoding/json" + "fmt" + "github.com/gorilla/mux" + "io" + "io/ioutil" + "net/http" + "net/http/httptest" + "testing" + "time" +) + +var ( + testHttpServer *httptest.Server + testLayers = map[string]map[string]string{ + "77dbf71da1d00e3fbddc480176eac8994025630c6590d11cfc8fe1209c2a1d20": { + "json": `{"id":"77dbf71da1d00e3fbddc480176eac8994025630c6590d11cfc8fe1209c2a1d20", + "comment":"test base image","created":"2013-03-23T12:53:11.10432-07:00", + "container_config":{"Hostname":"","User":"","Memory":0,"MemorySwap":0, + "CpuShares":0,"AttachStdin":false,"AttachStdout":false,"AttachStderr":false, + "PortSpecs":null,"Tty":false,"OpenStdin":false,"StdinOnce":false, + "Env":null,"Cmd":null,"Dns":null,"Image":"","Volumes":null, + "VolumesFrom":"","Entrypoint":null},"Size":424242}`, + "checksum_simple": "sha256:1ac330d56e05eef6d438586545ceff7550d3bdcb6b19961f12c5ba714ee1bb37", + "checksum_tarsum": "tarsum+sha256:4409a0685741ca86d38df878ed6f8cbba4c99de5dc73cd71aef04be3bb70be7c", + "ancestry": `["77dbf71da1d00e3fbddc480176eac8994025630c6590d11cfc8fe1209c2a1d20"]`, + "layer": string([]byte{ + 0x1f, 0x8b, 0x08, 0x08, 0x0e, 0xb0, 0xee, 0x51, 0x02, 0x03, 0x6c, 0x61, 0x79, 0x65, + 0x72, 0x2e, 0x74, 0x61, 0x72, 0x00, 0xed, 0xd2, 0x31, 0x0e, 0xc2, 0x30, 0x0c, 0x05, + 0x50, 0xcf, 0x9c, 0xc2, 0x27, 0x48, 0xed, 0x38, 0x4e, 0xce, 0x13, 0x44, 0x2b, 0x66, + 0x62, 0x24, 0x8e, 0x4f, 0xa0, 0x15, 0x63, 0xb6, 0x20, 0x21, 0xfc, 0x96, 0xbf, 0x78, + 0xb0, 0xf5, 0x1d, 0x16, 0x98, 0x8e, 0x88, 0x8a, 0x2a, 0xbe, 0x33, 0xef, 0x49, 0x31, + 0xed, 0x79, 0x40, 0x8e, 0x5c, 0x44, 0x85, 0x88, 0x33, 0x12, 0x73, 0x2c, 0x02, 0xa8, + 0xf0, 0x05, 0xf7, 0x66, 0xf5, 0xd6, 0x57, 0x69, 0xd7, 0x7a, 0x19, 0xcd, 0xf5, 0xb1, + 0x6d, 0x1b, 0x1f, 0xf9, 0xba, 0xe3, 0x93, 0x3f, 0x22, 0x2c, 0xb6, 0x36, 0x0b, 0xf6, + 0xb0, 0xa9, 0xfd, 0xe7, 0x94, 0x46, 0xfd, 0xeb, 0xd1, 0x7f, 0x2c, 0xc4, 0xd2, 0xfb, + 0x97, 0xfe, 0x02, 0x80, 0xe4, 0xfd, 0x4f, 0x77, 0xae, 0x6d, 0x3d, 0x81, 0x73, 0xce, + 0xb9, 0x7f, 0xf3, 0x04, 0x41, 0xc1, 0xab, 0xc6, 0x00, 0x0a, 0x00, 0x00, + }), + }, + "42d718c941f5c532ac049bf0b0ab53f0062f09a03afd4aa4a02c098e46032b9d": { + "json": `{"id":"42d718c941f5c532ac049bf0b0ab53f0062f09a03afd4aa4a02c098e46032b9d", + "parent":"77dbf71da1d00e3fbddc480176eac8994025630c6590d11cfc8fe1209c2a1d20", + "comment":"test base image","created":"2013-03-23T12:55:11.10432-07:00", + "container_config":{"Hostname":"","User":"","Memory":0,"MemorySwap":0, + "CpuShares":0,"AttachStdin":false,"AttachStdout":false,"AttachStderr":false, + "PortSpecs":null,"Tty":false,"OpenStdin":false,"StdinOnce":false, + "Env":null,"Cmd":null,"Dns":null,"Image":"","Volumes":null, + "VolumesFrom":"","Entrypoint":null},"Size":424242}`, + "checksum_simple": "sha256:bea7bf2e4bacd479344b737328db47b18880d09096e6674165533aa994f5e9f2", + "checksum_tarsum": "tarsum+sha256:68fdb56fb364f074eec2c9b3f85ca175329c4dcabc4a6a452b7272aa613a07a2", + "ancestry": `["42d718c941f5c532ac049bf0b0ab53f0062f09a03afd4aa4a02c098e46032b9d", + "77dbf71da1d00e3fbddc480176eac8994025630c6590d11cfc8fe1209c2a1d20"]`, + "layer": string([]byte{ + 0x1f, 0x8b, 0x08, 0x08, 0xbd, 0xb3, 0xee, 0x51, 0x02, 0x03, 0x6c, 0x61, 0x79, 0x65, + 0x72, 0x2e, 0x74, 0x61, 0x72, 0x00, 0xed, 0xd1, 0x31, 0x0e, 0xc2, 0x30, 0x0c, 0x05, + 0x50, 0xcf, 0x9c, 0xc2, 0x27, 0x48, 0x9d, 0x38, 0x8e, 0xcf, 0x53, 0x51, 0xaa, 0x56, + 0xea, 0x44, 0x82, 0xc4, 0xf1, 0x09, 0xb4, 0xea, 0x98, 0x2d, 0x48, 0x08, 0xbf, 0xe5, + 0x2f, 0x1e, 0xfc, 0xf5, 0xdd, 0x00, 0xdd, 0x11, 0x91, 0x8a, 0xe0, 0x27, 0xd3, 0x9e, + 0x14, 0xe2, 0x9e, 0x07, 0xf4, 0xc1, 0x2b, 0x0b, 0xfb, 0xa4, 0x82, 0xe4, 0x3d, 0x93, + 0x02, 0x0a, 0x7c, 0xc1, 0x23, 0x97, 0xf1, 0x5e, 0x5f, 0xc9, 0xcb, 0x38, 0xb5, 0xee, + 0xea, 0xd9, 0x3c, 0xb7, 0x4b, 0xbe, 0x7b, 0x9c, 0xf9, 0x23, 0xdc, 0x50, 0x6e, 0xb9, + 0xb8, 0xf2, 0x2c, 0x5d, 0xf7, 0x4f, 0x31, 0xb6, 0xf6, 0x4f, 0xc7, 0xfe, 0x41, 0x55, + 0x63, 0xdd, 0x9f, 0x89, 0x09, 0x90, 0x6c, 0xff, 0xee, 0xae, 0xcb, 0xba, 0x4d, 0x17, + 0x30, 0xc6, 0x18, 0xf3, 0x67, 0x5e, 0xc1, 0xed, 0x21, 0x5d, 0x00, 0x0a, 0x00, 0x00, + }), + }, + } + testRepositories = map[string]map[string]string{ + "foo/bar": { + "latest": "42d718c941f5c532ac049bf0b0ab53f0062f09a03afd4aa4a02c098e46032b9d", + }, + } +) + +func init() { + r := mux.NewRouter() + r.HandleFunc("/v1/_ping", handlerGetPing).Methods("GET") + r.HandleFunc("/v1/images/{image_id:[^/]+}/{action:json|layer|ancestry}", handlerGetImage).Methods("GET") + r.HandleFunc("/v1/images/{image_id:[^/]+}/{action:json|layer|checksum}", handlerPutImage).Methods("PUT") + r.HandleFunc("/v1/repositories/{repository:.+}/tags", handlerGetDeleteTags).Methods("GET", "DELETE") + r.HandleFunc("/v1/repositories/{repository:.+}/tags/{tag:.+}", handlerGetTag).Methods("GET") + r.HandleFunc("/v1/repositories/{repository:.+}/tags/{tag:.+}", handlerPutTag).Methods("PUT") + r.HandleFunc("/v1/users{null:.*}", handlerUsers).Methods("GET", "POST", "PUT") + r.HandleFunc("/v1/repositories/{repository:.+}{action:/images|/}", handlerImages).Method("GET", "PUT", "DELETE") + r.HandleFunc("/v1/repositories/{repository:.+}/auth", handlerAuth).Methods("PUT") + r.HandleFunc("/v1/search", handlerSearch).Methods("GET") + testHttpServer = httptest.NewServer(r) +} + +func makeURL(req string) string { + return testHttpServer.URL + req +} + +func writeHeaders(w http.ResponseWriter) { + h := w.Header() + h.Add("Server", "docker-tests/mock") + h.Add("Expires", "-1") + h.Add("Content-Type", "application/json") + h.Add("Pragma", "no-cache") + h.Add("Cache-Control", "no-cache") + h.Add("X-Docker-Registry-Version", "0.0.0") + h.Add("X-Docker-Registry-Config", "mock") +} + +func writeResponse(w http.ResponseWriter, message interface{}, code int) { + writeHeaders(w) + w.WriteHeader(code) + body, err := json.Marshal(message) + if err != nil { + io.WriteString(w, err.Error()) + return + } + w.Write(body) +} + +func readJSON(r *http.Request, dest interface{}) error { + body, err := ioutil.ReadAll(r.Body) + if err != nil { + return err + } + return json.Unmarshal(body, dest) +} + +func apiError(w http.ResponseWriter, message string, code int) { + body := map[string]string{ + "error": message, + } + writeResponse(w, body, code) +} + +func assertEqual(t *testing.T, a interface{}, b interface{}, message string) { + if a == b { + return + } + if len(message) == 0 { + message = fmt.Sprintf("%v != %v", a, b) + } + t.Fatal(message) +} + +func requiresAuth(w http.ResponseWriter, r *http.Request) bool { + writeCookie := func() { + value := fmt.Sprintf("FAKE-SESSION-%d", time.Now().UnixNano()) + cookie := &http.Cookie{Name: "session", Value: value, MaxAge: 3600} + http.SetCookie(w, cookie) + } + if len(r.Cookies()) > 0 { + writeCookie() + return true + } + if len(r.Header.Get("Authorization")) > 0 { + writeCookie() + return true + } + w.Header().Add("WWW-Authenticate", "token") + apiError(w, "Wrong auth", 401) + return false +} + +func handlerGetPing(w http.ResponseWriter, r *http.Request) { + writeResponse(w, true, 200) +} + +func handlerGetImage(w http.ResponseWriter, r *http.Request) { + if !requiresAuth(w, r) { + return + } + vars := mux.Vars(r) + layer, exists := testLayers[vars["image_id"]] + if !exists { + http.NotFound(w, r) + return + } + writeHeaders(w) + io.WriteString(w, layer[vars["action"]]) +} + +func handlerPutImage(w http.ResponseWriter, r *http.Request) { + if !requiresAuth(w, r) { + return + } + vars := mux.Vars(r) + image_id := vars["image_id"] + action := vars["action"] + layer, exists := testLayers[image_id] + if !exists { + if action != "json" { + http.NotFound(w, r) + return + } + layer = make(map[string]string) + testLayers[image_id] = layer + } + if checksum := r.Header.Get("X-Docker-Checksum"); checksum != "" { + if checksum != layer["checksum_simple"] && checksum != layer["checksum_tarsum"] { + apiError(w, "Wrong checksum", 400) + return + } + } + body, err := ioutil.ReadAll(r.Body) + if err != nil { + apiError(w, fmt.Sprintf("Error: %s", err), 500) + return + } + layer[action] = string(body) + writeResponse(w, true, 200) +} + +func handlerGetDeleteTags(w http.ResponseWriter, r *http.Request) { + if !requiresAuth(w, r) { + return + } + repositoryName := mux.Vars(r)["repository"] + tags, exists := testRepositories[repositoryName] + if !exists { + apiError(w, "Repository not found", 404) + } + if r.Method == "DELETE" { + delete(testRepositories, repositoryName) + writeResponse(w, true, 200) + return + } + writeResponse(w, tags, 200) +} + +func handlerGetTag(w http.ResponseWriter, r *http.Request) { + if !requiresAuth(w, r) { + return + } + vars := mux.Vars(r) + repositoryName := vars["repository"] + tagName := vars["tag"] + tags, exists := testRepositories[repositoryName] + if !exists { + apiError(w, "Repository not found", 404) + } + tag, exists := tags[tagName] + if !exists { + apiError(w, "Tag not found", 404) + } + writeResponse(w, tag, 200) +} + +func handlerPutTag(w http.ResponseWriter, r *http.Request) { + if !requiresAuth(w, r) { + return + } + vars := mux.Vars(r) + repositoryName := vars["repository"] + tagName := vars["tag"] + tags, exists := testRepositories[repositoryName] + if !exists { + tags := make(map[string]string) + testRepositories[repositoryName] = tags + } + tagValue := "" + readJSON(r, tagValue) + tags[tagName] = tagValue + writeResponse(w, true, 200) +} + +func handlerUsers(w http.ResponseWriter, r *http.Request) { + code := 200 + if r.Method == "POST" { + code = 201 + } else if r.Method == "PUT" { + code = 204 + } + writeResponse(w, "", code) +} + +func handlerImages(w http.ResponseWriter, r *http.Request) { + if r.Method == "PUT" { + writeResponse(w, "", 200) + return + } + if r.Method == "DELETE" { + writeResponse(w, "", 204) + return + } + images := make([]map[string]string) + for image_id, layer := range testLayers { + image := make(map[string]string) + image["id"] = image_id + image["checksum"] = layer["checksum_tarsum"] + append(images, image) + } + writeResponse(w, images, 200) +} + +func handlerAuth(w http.ResponseWriter, r *http.Request) { + writeResponse(w, "OK", 200) +} + +func handlerSearch(w http.ResponseWriter, r *http.Request) { + writeResponse(w, "{}", 200) +} + +func TestPing(t *testing.T) { + res, err := http.Get(makeURL("/v1/_ping")) + if err != nil { + t.Fatal(err) + } + assertEqual(t, res.StatusCode, 200, "") + assertEqual(t, res.Header.Get("X-Docker-Registry-Config"), "mock", + "This is not a Mocked Registry") +} + +/* Uncomment this to test Mocked Registry locally with curl + * WARNING: Don't push on the repos uncommented, it'll block the tests + * +func TestWait(t *testing.T) { + fmt.Println("Test HTTP server ready and waiting...") + fmt.Println(testHttpServer.URL) + c := make(chan int) + <-c +} +//*/ From 3ca4529fbe7bb5eed0fd0108b1ae8924b3d1c7e9 Mon Sep 17 00:00:00 2001 From: Sam Alba Date: Wed, 24 Jul 2013 19:22:36 -0700 Subject: [PATCH 063/375] Fixed mocked registry --- docs/registry_mock_test.go | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/docs/registry_mock_test.go b/docs/registry_mock_test.go index f1e65cad..3bbef25d 100644 --- a/docs/registry_mock_test.go +++ b/docs/registry_mock_test.go @@ -8,6 +8,7 @@ import ( "io/ioutil" "net/http" "net/http/httptest" + "net/url" "testing" "time" ) @@ -69,7 +70,7 @@ var ( }, } testRepositories = map[string]map[string]string{ - "foo/bar": { + "foo42/bar": { "latest": "42d718c941f5c532ac049bf0b0ab53f0062f09a03afd4aa4a02c098e46032b9d", }, } @@ -84,7 +85,7 @@ func init() { r.HandleFunc("/v1/repositories/{repository:.+}/tags/{tag:.+}", handlerGetTag).Methods("GET") r.HandleFunc("/v1/repositories/{repository:.+}/tags/{tag:.+}", handlerPutTag).Methods("PUT") r.HandleFunc("/v1/users{null:.*}", handlerUsers).Methods("GET", "POST", "PUT") - r.HandleFunc("/v1/repositories/{repository:.+}{action:/images|/}", handlerImages).Method("GET", "PUT", "DELETE") + r.HandleFunc("/v1/repositories/{repository:.+}{action:/images|/}", handlerImages).Methods("GET", "PUT", "DELETE") r.HandleFunc("/v1/repositories/{repository:.+}/auth", handlerAuth).Methods("PUT") r.HandleFunc("/v1/search", handlerSearch).Methods("GET") testHttpServer = httptest.NewServer(r) @@ -103,6 +104,8 @@ func writeHeaders(w http.ResponseWriter) { h.Add("Cache-Control", "no-cache") h.Add("X-Docker-Registry-Version", "0.0.0") h.Add("X-Docker-Registry-Config", "mock") + u, _ := url.Parse(testHttpServer.URL) + h.Add("X-Docker-Endpoints", u.Host) } func writeResponse(w http.ResponseWriter, message interface{}, code int) { @@ -146,6 +149,9 @@ func requiresAuth(w http.ResponseWriter, r *http.Request) bool { value := fmt.Sprintf("FAKE-SESSION-%d", time.Now().UnixNano()) cookie := &http.Cookie{Name: "session", Value: value, MaxAge: 3600} http.SetCookie(w, cookie) + //FIXME(sam): this should be sent only on Index routes + value = fmt.Sprintf("FAKE-TOKEN-%d", time.Now().UnixNano()) + w.Header().Add("X-Docker-Token", value) } if len(r.Cookies()) > 0 { writeCookie() @@ -281,12 +287,12 @@ func handlerImages(w http.ResponseWriter, r *http.Request) { writeResponse(w, "", 204) return } - images := make([]map[string]string) + images := []map[string]string{} for image_id, layer := range testLayers { image := make(map[string]string) image["id"] = image_id image["checksum"] = layer["checksum_tarsum"] - append(images, image) + images = append(images, image) } writeResponse(w, images, 200) } From 34fc4b84074f60e1b09765e233b39b7f85151f8b Mon Sep 17 00:00:00 2001 From: Sam Alba Date: Thu, 25 Jul 2013 12:57:09 -0700 Subject: [PATCH 064/375] Mocked registry: Added X-Docker-Size when fetching the layer --- docs/registry_mock_test.go | 25 +++++++++++++++++++------ 1 file changed, 19 insertions(+), 6 deletions(-) diff --git a/docs/registry_mock_test.go b/docs/registry_mock_test.go index 3bbef25d..57830672 100644 --- a/docs/registry_mock_test.go +++ b/docs/registry_mock_test.go @@ -6,9 +6,11 @@ import ( "github.com/gorilla/mux" "io" "io/ioutil" + "log" "net/http" "net/http/httptest" "net/url" + "strconv" "testing" "time" ) @@ -88,7 +90,15 @@ func init() { r.HandleFunc("/v1/repositories/{repository:.+}{action:/images|/}", handlerImages).Methods("GET", "PUT", "DELETE") r.HandleFunc("/v1/repositories/{repository:.+}/auth", handlerAuth).Methods("PUT") r.HandleFunc("/v1/search", handlerSearch).Methods("GET") - testHttpServer = httptest.NewServer(r) + testHttpServer = httptest.NewServer(handlerAccessLog(r)) +} + +func handlerAccessLog(handler http.Handler) http.Handler { + logHandler := func(w http.ResponseWriter, r *http.Request) { + log.Printf("%s \"%s %s\"", r.RemoteAddr, r.Method, r.URL) + handler.ServeHTTP(w, r) + } + return http.HandlerFunc(logHandler) } func makeURL(req string) string { @@ -104,8 +114,6 @@ func writeHeaders(w http.ResponseWriter) { h.Add("Cache-Control", "no-cache") h.Add("X-Docker-Registry-Version", "0.0.0") h.Add("X-Docker-Registry-Config", "mock") - u, _ := url.Parse(testHttpServer.URL) - h.Add("X-Docker-Endpoints", u.Host) } func writeResponse(w http.ResponseWriter, message interface{}, code int) { @@ -181,6 +189,8 @@ func handlerGetImage(w http.ResponseWriter, r *http.Request) { return } writeHeaders(w) + layer_size := len(layer["layer"]) + w.Header().Add("X-Docker-Size", strconv.Itoa(layer_size)) io.WriteString(w, layer[vars["action"]]) } @@ -279,6 +289,8 @@ func handlerUsers(w http.ResponseWriter, r *http.Request) { } func handlerImages(w http.ResponseWriter, r *http.Request) { + u, _ := url.Parse(testHttpServer.URL) + w.Header().Add("X-Docker-Endpoints", u.Host) if r.Method == "PUT" { writeResponse(w, "", 200) return @@ -292,6 +304,7 @@ func handlerImages(w http.ResponseWriter, r *http.Request) { image := make(map[string]string) image["id"] = image_id image["checksum"] = layer["checksum_tarsum"] + image["Tag"] = "latest" images = append(images, image) } writeResponse(w, images, 200) @@ -317,11 +330,11 @@ func TestPing(t *testing.T) { /* Uncomment this to test Mocked Registry locally with curl * WARNING: Don't push on the repos uncommented, it'll block the tests - * + */ func TestWait(t *testing.T) { - fmt.Println("Test HTTP server ready and waiting...") - fmt.Println(testHttpServer.URL) + log.Println("Test HTTP server ready and waiting:", testHttpServer.URL) c := make(chan int) <-c } + //*/ From 28f0f0ffb8ff48b25aeaa9f1c510cabf4294a9a7 Mon Sep 17 00:00:00 2001 From: Sam Alba Date: Fri, 26 Jul 2013 17:28:17 -0700 Subject: [PATCH 065/375] Disabled test server in the tests --- docs/registry_mock_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/registry_mock_test.go b/docs/registry_mock_test.go index 57830672..f634877b 100644 --- a/docs/registry_mock_test.go +++ b/docs/registry_mock_test.go @@ -330,7 +330,7 @@ func TestPing(t *testing.T) { /* Uncomment this to test Mocked Registry locally with curl * WARNING: Don't push on the repos uncommented, it'll block the tests - */ + * func TestWait(t *testing.T) { log.Println("Test HTTP server ready and waiting:", testHttpServer.URL) c := make(chan int) From 04cbff8d35ff871278bb60f7f85ca2df3eb59f4d Mon Sep 17 00:00:00 2001 From: shin- Date: Wed, 31 Jul 2013 19:03:14 +0200 Subject: [PATCH 066/375] registry: Fixed a bug where token and cookie info wouldn't be sent when using LookupRemoteImage(). Fixed a bug where no error would be reported when getting a non-200 status code in GetRemoteImageLayer() --- docs/registry.go | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/docs/registry.go b/docs/registry.go index 0d2a64e0..579b34e8 100644 --- a/docs/registry.go +++ b/docs/registry.go @@ -147,13 +147,14 @@ func (r *Registry) GetRemoteHistory(imgID, registry string, token []string) ([]s // Check if an image exists in the Registry func (r *Registry) LookupRemoteImage(imgID, registry string, token []string) bool { - rt := &http.Transport{Proxy: http.ProxyFromEnvironment} + req, err := r.reqFactory.NewRequest("GET", registry+"images/"+imgID+"/json", nil) if err != nil { return false } - res, err := rt.RoundTrip(req) + req.Header.Set("Authorization", "Token "+strings.Join(token, ", ")) + res, err := doWithCookies(r.client, req) if err != nil { return false } @@ -200,6 +201,10 @@ func (r *Registry) GetRemoteImageLayer(imgID, registry string, token []string) ( if err != nil { return nil, err } + if res.StatusCode != 200 { + return nil, fmt.Errorf("Server error: Status %d while fetching image layer (%s)", + res.StatusCode, imgID) + } return res.Body, nil } From 93877a859aed0c8ebeb0b73a0f27cc41a4f7a9c6 Mon Sep 17 00:00:00 2001 From: shin- Date: Wed, 31 Jul 2013 19:05:06 +0200 Subject: [PATCH 067/375] Mock registry: Fixed a bug where the index validation path would return a 200 status code instead of the expected 204 --- docs/registry_mock_test.go | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/docs/registry_mock_test.go b/docs/registry_mock_test.go index f634877b..236dc00d 100644 --- a/docs/registry_mock_test.go +++ b/docs/registry_mock_test.go @@ -11,6 +11,7 @@ import ( "net/http/httptest" "net/url" "strconv" + "strings" "testing" "time" ) @@ -291,7 +292,12 @@ func handlerUsers(w http.ResponseWriter, r *http.Request) { func handlerImages(w http.ResponseWriter, r *http.Request) { u, _ := url.Parse(testHttpServer.URL) w.Header().Add("X-Docker-Endpoints", u.Host) + w.Header().Add("X-Docker-Token", fmt.Sprintf("FAKE-SESSION-%d", time.Now().UnixNano())) if r.Method == "PUT" { + if strings.HasSuffix(r.URL.Path, "images") { + writeResponse(w, "", 204) + return + } writeResponse(w, "", 200) return } @@ -330,6 +336,7 @@ func TestPing(t *testing.T) { /* Uncomment this to test Mocked Registry locally with curl * WARNING: Don't push on the repos uncommented, it'll block the tests +<<<<<<< HEAD * func TestWait(t *testing.T) { log.Println("Test HTTP server ready and waiting:", testHttpServer.URL) @@ -338,3 +345,11 @@ func TestWait(t *testing.T) { } //*/ +======= + */ +// func TestWait(t *testing.T) { +// log.Println("Test HTTP server ready and waiting:", testHttpServer.URL) +// c := make(chan int) +// <-c +// } +>>>>>>> Mock registry: Fixed a bug where the index validation path would return a 200 status code instead of the expected 204 From 4d9dcc3cba3cc5ba3e7ec2eea2597c1fbd5f9d8c Mon Sep 17 00:00:00 2001 From: shin- Date: Wed, 31 Jul 2013 19:07:31 +0200 Subject: [PATCH 068/375] New registry unit tests remade from scratch, using the mock registry --- docs/registry_test.go | 331 ++++++++++++++++++++++++------------------ 1 file changed, 187 insertions(+), 144 deletions(-) diff --git a/docs/registry_test.go b/docs/registry_test.go index fd955b7b..68a5f75f 100644 --- a/docs/registry_test.go +++ b/docs/registry_test.go @@ -1,168 +1,211 @@ package registry -// import ( -// "crypto/rand" -// "encoding/hex" -// "github.com/dotcloud/docker" -// "github.com/dotcloud/docker/auth" -// "io/ioutil" -// "os" -// "path" -// "testing" -// ) +import ( + "github.com/dotcloud/docker/auth" + "strings" + "testing" +) +var ( + IMAGE_ID = "42d718c941f5c532ac049bf0b0ab53f0062f09a03afd4aa4a02c098e46032b9d" + TOKEN = []string{"fake-token"} + REPO = "foo42/bar" +) -// func newTestRuntime() (*Runtime, error) { -// root, err := ioutil.TempDir("", "docker-test") -// if err != nil { -// return nil, err -// } -// if err := os.Remove(root); err != nil { -// return nil, err -// } +type simpleVersionInfo struct { + name string + version string +} -// if err := os.MkdirAll(root, 0700); err != nil && !os.IsExist(err) { -// return nil, err -// } +func (v *simpleVersionInfo) Name() string { + return v.name +} -// return runtime, nil -// } +func (v *simpleVersionInfo) Version() string { + return v.version +} -// func TestPull(t *testing.T) { -// os.Setenv("DOCKER_INDEX_URL", "") -// runtime, err := newTestRuntime() -// if err != nil { -// t.Fatal(err) -// } -// defer nuke(runtime) +func spawnTestRegistry(t *testing.T) *Registry { + versionInfo := make([]VersionInfo, 0, 4) + versionInfo = append(versionInfo, &simpleVersionInfo{"docker", "0.0.0test"}) + versionInfo = append(versionInfo, &simpleVersionInfo{"go", "test"}) + versionInfo = append(versionInfo, &simpleVersionInfo{"git-commit", "test"}) + versionInfo = append(versionInfo, &simpleVersionInfo{"kernel", "test"}) + authConfig := &auth.AuthConfig{} + r, err := NewRegistry("", authConfig, versionInfo...) + if err != nil { + t.Fatal(err) + } + return r +} -// err = runtime.graph.PullRepository(ioutil.Discard, "busybox", "", runtime.repositories, nil) -// if err != nil { -// t.Fatal(err) -// } -// img, err := runtime.repositories.LookupImage("busybox") -// if err != nil { -// t.Fatal(err) -// } +func TestPingRegistryEndpoint(t *testing.T) { + err := pingRegistryEndpoint(makeURL("/v1/")) + if err != nil { + t.Fatal(err) + } +} -// // Try to run something on this image to make sure the layer's been downloaded properly. -// config, _, err := docker.ParseRun([]string{img.Id, "echo", "Hello World"}, runtime.capabilities) -// if err != nil { -// t.Fatal(err) -// } +func TestGetRemoteHistory(t *testing.T) { + r := spawnTestRegistry(t) + hist, err := r.GetRemoteHistory(IMAGE_ID, makeURL("/v1/"), TOKEN) + if err != nil { + t.Fatal(err) + } + assertEqual(t, len(hist), 2, "Expected 2 images in history") + assertEqual(t, hist[0], IMAGE_ID, "Expected " + IMAGE_ID + "as first ancestry") + assertEqual(t, hist[1], "77dbf71da1d00e3fbddc480176eac8994025630c6590d11cfc8fe1209c2a1d20", + "Unexpected second ancestry") +} -// b := NewBuilder(runtime) -// container, err := b.Create(config) -// if err != nil { -// t.Fatal(err) -// } -// if err := container.Start(); err != nil { -// t.Fatal(err) -// } +func TestLookupRemoteImage(t *testing.T) { + r := spawnTestRegistry(t) + found := r.LookupRemoteImage(IMAGE_ID, makeURL("/v1/"), TOKEN) + assertEqual(t, found, true, "Expected remote lookup to succeed") + found = r.LookupRemoteImage("abcdef", makeURL("/v1/"), TOKEN) + assertEqual(t, found, false, "Expected remote lookup to fail") +} -// if status := container.Wait(); status != 0 { -// t.Fatalf("Expected status code 0, found %d instead", status) -// } -// } +func TestGetRemoteImageJSON(t *testing.T) { + r := spawnTestRegistry(t) + json, size, err := r.GetRemoteImageJSON(IMAGE_ID, makeURL("/v1/"), TOKEN) + if err != nil { + t.Fatal(err) + } + assertEqual(t, size, 154, "Expected size 154") + if len(json) <= 0 { + t.Fatal("Expected non-empty json") + } -// func TestPullTag(t *testing.T) { -// os.Setenv("DOCKER_INDEX_URL", "") -// runtime, err := newTestRuntime() -// if err != nil { -// t.Fatal(err) -// } -// defer nuke(runtime) + _, _, err = r.GetRemoteImageJSON("abcdef", makeURL("/v1/"), TOKEN) + if err == nil { + t.Fatal("Expected image not found error") + } +} -// err = runtime.graph.PullRepository(ioutil.Discard, "ubuntu", "12.04", runtime.repositories, nil) -// if err != nil { -// t.Fatal(err) -// } -// _, err = runtime.repositories.LookupImage("ubuntu:12.04") -// if err != nil { -// t.Fatal(err) -// } +func TestGetRemoteImageLayer(t *testing.T) { + r := spawnTestRegistry(t) + data, err := r.GetRemoteImageLayer(IMAGE_ID, makeURL("/v1/"), TOKEN) + if err != nil { + t.Fatal(err) + } + if data == nil { + t.Fatal("Expected non-nil data result") + } -// img2, err := runtime.repositories.LookupImage("ubuntu:12.10") -// if img2 != nil { -// t.Fatalf("Expected nil image but found %v instead", img2.Id) -// } -// } + _, err = r.GetRemoteImageLayer("abcdef", makeURL("/v1/"), TOKEN) + if err == nil { + t.Fatal("Expected image not found error") + } +} -// func login(runtime *Runtime) error { -// authConfig := auth.NewAuthConfig("unittester", "surlautrerivejetattendrai", "noise+unittester@dotcloud.com", runtime.root) -// runtime.authConfig = authConfig -// _, err := auth.Login(authConfig) -// return err -// } +func TestGetRemoteTags(t *testing.T) { + r := spawnTestRegistry(t) + tags, err := r.GetRemoteTags([]string{makeURL("/v1/")}, REPO, TOKEN) + if err != nil { + t.Fatal(err) + } + assertEqual(t, len(tags), 1, "Expected one tag") + assertEqual(t, tags["latest"], IMAGE_ID, "Expected tag latest to map to " + IMAGE_ID) -// func TestPush(t *testing.T) { -// os.Setenv("DOCKER_INDEX_URL", "https://indexstaging-docker.dotcloud.com") -// defer os.Setenv("DOCKER_INDEX_URL", "") -// runtime, err := newTestRuntime() -// if err != nil { -// t.Fatal(err) -// } -// defer nuke(runtime) + _, err = r.GetRemoteTags([]string{makeURL("/v1/")}, "foo42/baz", TOKEN) + if err == nil { + t.Fatal("Expected error when fetching tags for bogus repo") + } +} -// err = login(runtime) -// if err != nil { -// t.Fatal(err) -// } +func TestGetRepositoryData(t *testing.T) { + r := spawnTestRegistry(t) + data, err := r.GetRepositoryData(makeURL("/v1/"), "foo42/bar") + if err != nil { + t.Fatal(err) + } + assertEqual(t, len(data.ImgList), 2, "Expected 2 images in ImgList") + assertEqual(t, len(data.Endpoints), 1, "Expected one endpoint in Endpoints") +} -// err = runtime.graph.PullRepository(ioutil.Discard, "joffrey/busybox", "", runtime.repositories, nil) -// if err != nil { -// t.Fatal(err) -// } -// tokenBuffer := make([]byte, 16) -// _, err = rand.Read(tokenBuffer) -// if err != nil { -// t.Fatal(err) -// } -// token := hex.EncodeToString(tokenBuffer)[:29] -// config, _, err := ParseRun([]string{"joffrey/busybox", "touch", "/" + token}, runtime.capabilities) -// if err != nil { -// t.Fatal(err) -// } +func TestPushImageJSONRegistry(t *testing.T) { + r := spawnTestRegistry(t) + imgData := &ImgData{ + ID: "77dbf71da1d00e3fbddc480176eac8994025630c6590d11cfc8fe1209c2a1d20", + Checksum: "sha256:1ac330d56e05eef6d438586545ceff7550d3bdcb6b19961f12c5ba714ee1bb37", + } -// b := NewBuilder(runtime) -// container, err := b.Create(config) -// if err != nil { -// t.Fatal(err) -// } -// if err := container.Start(); err != nil { -// t.Fatal(err) -// } + err := r.PushImageJSONRegistry(imgData, []byte{ 0x42, 0xdf, 0x0 }, makeURL("/v1/"), TOKEN) + if err != nil { + t.Fatal(err) + } +} -// if status := container.Wait(); status != 0 { -// t.Fatalf("Expected status code 0, found %d instead", status) -// } +func TestPushImageLayerRegistry(t *testing.T) { + r := spawnTestRegistry(t) + layer := strings.NewReader("FAKELAYER") + r.PushImageLayerRegistry(IMAGE_ID, layer, makeURL("/v1/"), TOKEN) +} -// img, err := b.Commit(container, "unittester/"+token, "", "", "", nil) -// if err != nil { -// t.Fatal(err) -// } +func TestResolveRepositoryName(t *testing.T) { + _, _, err := ResolveRepositoryName("https://github.com/dotcloud/docker") + assertEqual(t, err, ErrInvalidRepositoryName, "Expected error invalid repo name") + ep, repo, err := ResolveRepositoryName("fooo/bar") + if err != nil { + t.Fatal(err) + } + assertEqual(t, ep, auth.IndexServerAddress(), "Expected endpoint to be index server address") + assertEqual(t, repo, "fooo/bar", "Expected resolved repo to be foo/bar") -// repo := runtime.repositories.Repositories["unittester/"+token] -// err = runtime.graph.PushRepository(ioutil.Discard, "unittester/"+token, repo, runtime.authConfig) -// if err != nil { -// t.Fatal(err) -// } + u := makeURL("")[7:] + ep, repo, err = ResolveRepositoryName(u + "/private/moonbase") + if err != nil { + t.Fatal(err) + } + assertEqual(t, ep, "http://" + u + "/v1/", "Expected endpoint to be " + u) + assertEqual(t, repo, "private/moonbase", "Expected endpoint to be private/moonbase") +} -// // Remove image so we can pull it again -// if err := runtime.graph.Delete(img.Id); err != nil { -// t.Fatal(err) -// } +func TestPushRegistryTag(t *testing.T) { + r := spawnTestRegistry(t) + err := r.PushRegistryTag("foo42/bar", IMAGE_ID, "stable", makeURL("/v1/"), TOKEN) + if err != nil { + t.Fatal(err) + } +} -// err = runtime.graph.PullRepository(ioutil.Discard, "unittester/"+token, "", runtime.repositories, runtime.authConfig) -// if err != nil { -// t.Fatal(err) -// } +func TestPushImageJSONIndex(t *testing.T) { + r := spawnTestRegistry(t) + imgData := []*ImgData{ + &ImgData{ + ID: "77dbf71da1d00e3fbddc480176eac8994025630c6590d11cfc8fe1209c2a1d20", + Checksum: "sha256:1ac330d56e05eef6d438586545ceff7550d3bdcb6b19961f12c5ba714ee1bb37", + }, + &ImgData{ + ID: "42d718c941f5c532ac049bf0b0ab53f0062f09a03afd4aa4a02c098e46032b9d", + Checksum: "sha256:bea7bf2e4bacd479344b737328db47b18880d09096e6674165533aa994f5e9f2", + }, + } + ep := makeURL("/v1/") + repoData, err := r.PushImageJSONIndex(ep, "foo42/bar", imgData, false, nil) + if err != nil { + t.Fatal(err) + } + if repoData == nil { + t.Fatal("Expected RepositoryData object") + } + repoData, err = r.PushImageJSONIndex(ep, "foo42/bar", imgData, true, []string{ep}) + if err != nil { + t.Fatal(err) + } + if repoData == nil { + t.Fatal("Expected RepositoryData object") + } +} -// layerPath, err := img.layer() -// if err != nil { -// t.Fatal(err) -// } - -// if _, err := os.Stat(path.Join(layerPath, token)); err != nil { -// t.Fatalf("Error while trying to retrieve token file: %v", err) -// } -// } +func TestSearchRepositories(t *testing.T) { + r := spawnTestRegistry(t) + results, err := r.SearchRepositories("supercalifragilisticepsialidocious") + if err != nil { + t.Fatal(err) + } + if results == nil { + t.Fatal("Expected non-nil SearchResults object") + } + assertEqual(t, results.NumResults, 0, "Expected 0 search results") +} \ No newline at end of file From 7c3b31e5d482efdb7cc04c97d02cfa6abcb51fdd Mon Sep 17 00:00:00 2001 From: shin- Date: Wed, 31 Jul 2013 19:12:40 +0200 Subject: [PATCH 069/375] gofmt --- docs/registry_test.go | 21 +++++++++++---------- 1 file changed, 11 insertions(+), 10 deletions(-) diff --git a/docs/registry_test.go b/docs/registry_test.go index 68a5f75f..642fe1c0 100644 --- a/docs/registry_test.go +++ b/docs/registry_test.go @@ -5,10 +5,11 @@ import ( "strings" "testing" ) + var ( IMAGE_ID = "42d718c941f5c532ac049bf0b0ab53f0062f09a03afd4aa4a02c098e46032b9d" - TOKEN = []string{"fake-token"} - REPO = "foo42/bar" + TOKEN = []string{"fake-token"} + REPO = "foo42/bar" ) type simpleVersionInfo struct { @@ -52,7 +53,7 @@ func TestGetRemoteHistory(t *testing.T) { t.Fatal(err) } assertEqual(t, len(hist), 2, "Expected 2 images in history") - assertEqual(t, hist[0], IMAGE_ID, "Expected " + IMAGE_ID + "as first ancestry") + assertEqual(t, hist[0], IMAGE_ID, "Expected "+IMAGE_ID+"as first ancestry") assertEqual(t, hist[1], "77dbf71da1d00e3fbddc480176eac8994025630c6590d11cfc8fe1209c2a1d20", "Unexpected second ancestry") } @@ -105,7 +106,7 @@ func TestGetRemoteTags(t *testing.T) { t.Fatal(err) } assertEqual(t, len(tags), 1, "Expected one tag") - assertEqual(t, tags["latest"], IMAGE_ID, "Expected tag latest to map to " + IMAGE_ID) + assertEqual(t, tags["latest"], IMAGE_ID, "Expected tag latest to map to "+IMAGE_ID) _, err = r.GetRemoteTags([]string{makeURL("/v1/")}, "foo42/baz", TOKEN) if err == nil { @@ -126,11 +127,11 @@ func TestGetRepositoryData(t *testing.T) { func TestPushImageJSONRegistry(t *testing.T) { r := spawnTestRegistry(t) imgData := &ImgData{ - ID: "77dbf71da1d00e3fbddc480176eac8994025630c6590d11cfc8fe1209c2a1d20", + ID: "77dbf71da1d00e3fbddc480176eac8994025630c6590d11cfc8fe1209c2a1d20", Checksum: "sha256:1ac330d56e05eef6d438586545ceff7550d3bdcb6b19961f12c5ba714ee1bb37", } - err := r.PushImageJSONRegistry(imgData, []byte{ 0x42, 0xdf, 0x0 }, makeURL("/v1/"), TOKEN) + err := r.PushImageJSONRegistry(imgData, []byte{0x42, 0xdf, 0x0}, makeURL("/v1/"), TOKEN) if err != nil { t.Fatal(err) } @@ -157,7 +158,7 @@ func TestResolveRepositoryName(t *testing.T) { if err != nil { t.Fatal(err) } - assertEqual(t, ep, "http://" + u + "/v1/", "Expected endpoint to be " + u) + assertEqual(t, ep, "http://"+u+"/v1/", "Expected endpoint to be "+u) assertEqual(t, repo, "private/moonbase", "Expected endpoint to be private/moonbase") } @@ -173,11 +174,11 @@ func TestPushImageJSONIndex(t *testing.T) { r := spawnTestRegistry(t) imgData := []*ImgData{ &ImgData{ - ID: "77dbf71da1d00e3fbddc480176eac8994025630c6590d11cfc8fe1209c2a1d20", + ID: "77dbf71da1d00e3fbddc480176eac8994025630c6590d11cfc8fe1209c2a1d20", Checksum: "sha256:1ac330d56e05eef6d438586545ceff7550d3bdcb6b19961f12c5ba714ee1bb37", }, &ImgData{ - ID: "42d718c941f5c532ac049bf0b0ab53f0062f09a03afd4aa4a02c098e46032b9d", + ID: "42d718c941f5c532ac049bf0b0ab53f0062f09a03afd4aa4a02c098e46032b9d", Checksum: "sha256:bea7bf2e4bacd479344b737328db47b18880d09096e6674165533aa994f5e9f2", }, } @@ -208,4 +209,4 @@ func TestSearchRepositories(t *testing.T) { t.Fatal("Expected non-nil SearchResults object") } assertEqual(t, results.NumResults, 0, "Expected 0 search results") -} \ No newline at end of file +} From da046e945f322614728ba2be56d253ab819d8d65 Mon Sep 17 00:00:00 2001 From: shin- Date: Mon, 5 Aug 2013 16:22:46 +0200 Subject: [PATCH 070/375] Mock access logs don't show up in non-debug mode --- docs/registry_mock_test.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/registry_mock_test.go b/docs/registry_mock_test.go index 236dc00d..e39637c1 100644 --- a/docs/registry_mock_test.go +++ b/docs/registry_mock_test.go @@ -3,10 +3,10 @@ package registry import ( "encoding/json" "fmt" + "github.com/dotcloud/docker/utils" "github.com/gorilla/mux" "io" "io/ioutil" - "log" "net/http" "net/http/httptest" "net/url" @@ -96,7 +96,7 @@ func init() { func handlerAccessLog(handler http.Handler) http.Handler { logHandler := func(w http.ResponseWriter, r *http.Request) { - log.Printf("%s \"%s %s\"", r.RemoteAddr, r.Method, r.URL) + utils.Debugf("%s \"%s %s\"", r.RemoteAddr, r.Method, r.URL) handler.ServeHTTP(w, r) } return http.HandlerFunc(logHandler) From 5ea461f3005a41c5985511a0bf758e50b793d940 Mon Sep 17 00:00:00 2001 From: shin- Date: Mon, 5 Aug 2013 19:07:23 +0200 Subject: [PATCH 071/375] Cleanup --- docs/registry_mock_test.go | 11 +---------- 1 file changed, 1 insertion(+), 10 deletions(-) diff --git a/docs/registry_mock_test.go b/docs/registry_mock_test.go index e39637c1..e7523155 100644 --- a/docs/registry_mock_test.go +++ b/docs/registry_mock_test.go @@ -336,7 +336,6 @@ func TestPing(t *testing.T) { /* Uncomment this to test Mocked Registry locally with curl * WARNING: Don't push on the repos uncommented, it'll block the tests -<<<<<<< HEAD * func TestWait(t *testing.T) { log.Println("Test HTTP server ready and waiting:", testHttpServer.URL) @@ -344,12 +343,4 @@ func TestWait(t *testing.T) { <-c } -//*/ -======= - */ -// func TestWait(t *testing.T) { -// log.Println("Test HTTP server ready and waiting:", testHttpServer.URL) -// c := make(chan int) -// <-c -// } ->>>>>>> Mock registry: Fixed a bug where the index validation path would return a 200 status code instead of the expected 204 +//*/ \ No newline at end of file From 03c1bbbf6522a47a6f9589f9b7de13379f5f83fd Mon Sep 17 00:00:00 2001 From: shin- Date: Mon, 5 Aug 2013 20:28:05 +0200 Subject: [PATCH 072/375] Adapted tests to latest registry changes --- docs/registry_test.go | 28 +++++++--------------------- 1 file changed, 7 insertions(+), 21 deletions(-) diff --git a/docs/registry_test.go b/docs/registry_test.go index 642fe1c0..a8543f18 100644 --- a/docs/registry_test.go +++ b/docs/registry_test.go @@ -2,6 +2,7 @@ package registry import ( "github.com/dotcloud/docker/auth" + "github.com/dotcloud/docker/utils" "strings" "testing" ) @@ -12,27 +13,9 @@ var ( REPO = "foo42/bar" ) -type simpleVersionInfo struct { - name string - version string -} - -func (v *simpleVersionInfo) Name() string { - return v.name -} - -func (v *simpleVersionInfo) Version() string { - return v.version -} - func spawnTestRegistry(t *testing.T) *Registry { - versionInfo := make([]VersionInfo, 0, 4) - versionInfo = append(versionInfo, &simpleVersionInfo{"docker", "0.0.0test"}) - versionInfo = append(versionInfo, &simpleVersionInfo{"go", "test"}) - versionInfo = append(versionInfo, &simpleVersionInfo{"git-commit", "test"}) - versionInfo = append(versionInfo, &simpleVersionInfo{"kernel", "test"}) authConfig := &auth.AuthConfig{} - r, err := NewRegistry("", authConfig, versionInfo...) + r, err := NewRegistry("", authConfig, utils.NewHTTPRequestFactory()) if err != nil { t.Fatal(err) } @@ -139,8 +122,11 @@ func TestPushImageJSONRegistry(t *testing.T) { func TestPushImageLayerRegistry(t *testing.T) { r := spawnTestRegistry(t) - layer := strings.NewReader("FAKELAYER") - r.PushImageLayerRegistry(IMAGE_ID, layer, makeURL("/v1/"), TOKEN) + layer := strings.NewReader("") + _, err := r.PushImageLayerRegistry(IMAGE_ID, layer, makeURL("/v1/"), TOKEN, []byte{}) + if err != nil { + t.Fatal(err) + } } func TestResolveRepositoryName(t *testing.T) { From 42b6e56d193098b5db444f19ab10a79910d48a78 Mon Sep 17 00:00:00 2001 From: Jonathan Rudenberg Date: Fri, 9 Aug 2013 19:52:05 -0400 Subject: [PATCH 073/375] Fix typo: fmt.Sprint -> fmt.Sprintf --- docs/registry.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/registry.go b/docs/registry.go index 5b8480d1..1e0e1815 100644 --- a/docs/registry.go +++ b/docs/registry.go @@ -391,7 +391,7 @@ func (r *Registry) PushImageJSONRegistry(imgData *ImgData, jsonRaw []byte, regis if res.StatusCode != 200 { errBody, err := ioutil.ReadAll(res.Body) if err != nil { - return utils.NewHTTPRequestError(fmt.Sprint("HTTP code %d while uploading metadata and error when trying to parse response body: %s", res.StatusCode, err), res) + return utils.NewHTTPRequestError(fmt.Sprintf("HTTP code %d while uploading metadata and error when trying to parse response body: %s", res.StatusCode, err), res) } var jsonBody map[string]string if err := json.Unmarshal(errBody, &jsonBody); err != nil { From e55267bc9904d00fce99f283c3c7b4590f0be816 Mon Sep 17 00:00:00 2001 From: Jonathan Rudenberg Date: Fri, 9 Aug 2013 21:16:44 -0400 Subject: [PATCH 074/375] Add GitHub usernames to MAINTAINERS --- docs/MAINTAINERS | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/MAINTAINERS b/docs/MAINTAINERS index b11dfc06..bf3984f5 100644 --- a/docs/MAINTAINERS +++ b/docs/MAINTAINERS @@ -1,3 +1,3 @@ -Sam Alba -Joffrey Fuhrer -Ken Cochrane +Sam Alba (@samalba) +Joffrey Fuhrer (@shin-) +Ken Cochrane (@kencochrane) From ecd70a194853f494496b888e427984666361360e Mon Sep 17 00:00:00 2001 From: Victor Vieux Date: Thu, 15 Aug 2013 11:42:40 +0000 Subject: [PATCH 075/375] hot fix display in parallel pull and go fmt --- docs/registry.go | 1 - docs/registry_mock_test.go | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/docs/registry.go b/docs/registry.go index ba62b346..ab13000b 100644 --- a/docs/registry.go +++ b/docs/registry.go @@ -162,7 +162,6 @@ func (r *Registry) GetRemoteHistory(imgID, registry string, token []string) ([]s // Check if an image exists in the Registry func (r *Registry) LookupRemoteImage(imgID, registry string, token []string) bool { - req, err := r.reqFactory.NewRequest("GET", registry+"images/"+imgID+"/json", nil) if err != nil { return false diff --git a/docs/registry_mock_test.go b/docs/registry_mock_test.go index e7523155..6eb94b63 100644 --- a/docs/registry_mock_test.go +++ b/docs/registry_mock_test.go @@ -343,4 +343,4 @@ func TestWait(t *testing.T) { <-c } -//*/ \ No newline at end of file +//*/ From a7b3e7eb785edb9310b8eea91dc3d43fbeea4e89 Mon Sep 17 00:00:00 2001 From: shin- Date: Wed, 3 Jul 2013 17:24:43 +0200 Subject: [PATCH 076/375] registry: removing opaqueRequest --- docs/registry.go | 23 +++++++---------------- 1 file changed, 7 insertions(+), 16 deletions(-) diff --git a/docs/registry.go b/docs/registry.go index ab13000b..759652f0 100644 --- a/docs/registry.go +++ b/docs/registry.go @@ -229,7 +229,8 @@ func (r *Registry) GetRemoteTags(registries []string, repository string, token [ } for _, host := range registries { endpoint := fmt.Sprintf("%srepositories/%s/tags", host, repository) - req, err := r.opaqueRequest("GET", endpoint, nil) + req, err := r.reqFactory.NewRequest("GET", endpoint, nil) + if err != nil { return nil, err } @@ -262,12 +263,11 @@ func (r *Registry) GetRemoteTags(registries []string, repository string, token [ } func (r *Registry) GetRepositoryData(indexEp, remote string) (*RepositoryData, error) { - repositoryTarget := fmt.Sprintf("%srepositories/%s/images", indexEp, remote) utils.Debugf("[registry] Calling GET %s", repositoryTarget) - req, err := r.opaqueRequest("GET", repositoryTarget, nil) + req, err := r.reqFactory.NewRequest("GET", repositoryTarget, nil) if err != nil { return nil, err } @@ -425,22 +425,14 @@ func (r *Registry) PushImageLayerRegistry(imgID string, layer io.Reader, registr return tarsumLayer.Sum(jsonRaw), nil } -func (r *Registry) opaqueRequest(method, urlStr string, body io.Reader) (*http.Request, error) { - req, err := r.reqFactory.NewRequest(method, urlStr, body) - if err != nil { - return nil, err - } - req.URL.Opaque = strings.Replace(urlStr, req.URL.Scheme+":", "", 1) - return req, err -} - // push a tag on the registry. // Remote has the format '/ func (r *Registry) PushRegistryTag(remote, revision, tag, registry string, token []string) error { // "jsonify" the string revision = "\"" + revision + "\"" + path := fmt.Sprintf("repositories/%s/tags/%s", remote, tag) - req, err := r.opaqueRequest("PUT", registry+"repositories/"+remote+"/tags/"+tag, strings.NewReader(revision)) + req, err := r.reqFactory.NewRequest("PUT", registry+path, strings.NewReader(revision)) if err != nil { return err } @@ -479,11 +471,10 @@ func (r *Registry) PushImageJSONIndex(indexEp, remote string, imgList []*ImgData if validate { suffix = "images" } - u := fmt.Sprintf("%srepositories/%s/%s", indexEp, remote, suffix) utils.Debugf("[registry] PUT %s", u) utils.Debugf("Image list pushed to index:\n%s\n", imgListJSON) - req, err := r.opaqueRequest("PUT", u, bytes.NewReader(imgListJSON)) + req, err := r.reqFactory.NewRequest("PUT", u, bytes.NewReader(imgListJSON)) if err != nil { return nil, err } @@ -503,7 +494,7 @@ func (r *Registry) PushImageJSONIndex(indexEp, remote string, imgList []*ImgData // Redirect if necessary for res.StatusCode >= 300 && res.StatusCode < 400 { utils.Debugf("Redirected to %s\n", res.Header.Get("Location")) - req, err = r.opaqueRequest("PUT", res.Header.Get("Location"), bytes.NewReader(imgListJSON)) + req, err = r.reqFactory.NewRequest("PUT", res.Header.Get("Location"), bytes.NewReader(imgListJSON)) if err != nil { return nil, err } From ee38e49093fac2e99c256070948725e85994857c Mon Sep 17 00:00:00 2001 From: Marco Hennings Date: Tue, 3 Sep 2013 20:45:49 +0200 Subject: [PATCH 077/375] Login against private registry To improve the use of docker with a private registry the login command is extended with a parameter for the server address. While implementing i noticed that two problems hindered authentication to a private registry: 1. the resolve of the authentication did not match during push because the looked up key was for example localhost:8080 but the stored one would have been https://localhost:8080 Besides The lookup needs to still work if the https->http fallback is used 2. During pull of an image no authentication is sent, which means all repositories are expected to be private. These points are fixed now. The changes are implemented in a way to be compatible to existing behavior both in the API as also with the private registry. Update: - login does not require the full url any more, you can login to the repository prefix: example: docker logon localhost:8080 Fixed corner corner cases: - When login is done during pull and push the registry endpoint is used and not the central index - When Remote sends a 401 during pull, it is now correctly delegating to CmdLogin - After a Login is done pull and push are using the newly entered login data, and not the previous ones. This one seems to be also broken in master, too. - Auth config is now transfered in a parameter instead of the body when /images/create is called. --- docs/registry.go | 33 +++++++++++++++++++++++++++++---- 1 file changed, 29 insertions(+), 4 deletions(-) diff --git a/docs/registry.go b/docs/registry.go index 759652f0..f24e0b35 100644 --- a/docs/registry.go +++ b/docs/registry.go @@ -22,6 +22,7 @@ import ( var ( ErrAlreadyExists = errors.New("Image already exists") ErrInvalidRepositoryName = errors.New("Invalid repository name (ex: \"registry.domain.tld/myrepos\")") + ErrLoginRequired = errors.New("Authentication is required.") ) func pingRegistryEndpoint(endpoint string) error { @@ -102,17 +103,38 @@ func ResolveRepositoryName(reposName string) (string, string, error) { if err := validateRepositoryName(reposName); err != nil { return "", "", err } + endpoint, err := ExpandAndVerifyRegistryUrl(hostname) + if err != nil { + return "", "", err + } + return endpoint, reposName, err +} + +// this method expands the registry name as used in the prefix of a repo +// to a full url. if it already is a url, there will be no change. +// The registry is pinged to test if it http or https +func ExpandAndVerifyRegistryUrl(hostname string) (string, error) { + if strings.HasPrefix(hostname, "http:") || strings.HasPrefix(hostname, "https:") { + // if there is no slash after https:// (8 characters) then we have no path in the url + if strings.LastIndex(hostname, "/") < 9 { + // there is no path given. Expand with default path + hostname = hostname + "/v1/" + } + if err := pingRegistryEndpoint(hostname); err != nil { + return "", errors.New("Invalid Registry endpoint: " + err.Error()) + } + return hostname, nil + } endpoint := fmt.Sprintf("https://%s/v1/", hostname) if err := pingRegistryEndpoint(endpoint); err != nil { utils.Debugf("Registry %s does not work (%s), falling back to http", endpoint, err) endpoint = fmt.Sprintf("http://%s/v1/", hostname) if err = pingRegistryEndpoint(endpoint); err != nil { //TODO: triggering highland build can be done there without "failing" - return "", "", errors.New("Invalid Registry endpoint: " + err.Error()) + return "", errors.New("Invalid Registry endpoint: " + err.Error()) } } - err := validateRepositoryName(reposName) - return endpoint, reposName, err + return endpoint, nil } func doWithCookies(c *http.Client, req *http.Request) (*http.Response, error) { @@ -139,6 +161,9 @@ func (r *Registry) GetRemoteHistory(imgID, registry string, token []string) ([]s req.Header.Set("Authorization", "Token "+strings.Join(token, ", ")) res, err := doWithCookies(r.client, req) if err != nil || res.StatusCode != 200 { + if res.StatusCode == 401 { + return nil, ErrLoginRequired + } if res != nil { return nil, utils.NewHTTPRequestError(fmt.Sprintf("Internal server error: %d trying to fetch remote history for %s", res.StatusCode, imgID), res) } @@ -282,7 +307,7 @@ func (r *Registry) GetRepositoryData(indexEp, remote string) (*RepositoryData, e } defer res.Body.Close() if res.StatusCode == 401 { - return nil, utils.NewHTTPRequestError(fmt.Sprintf("Please login first (HTTP code %d)", res.StatusCode), res) + return nil, ErrLoginRequired } // TODO: Right now we're ignoring checksums in the response body. // In the future, we need to use them to check image validity. From 49736d5fc7f170788230c1f24eca3e903842fd69 Mon Sep 17 00:00:00 2001 From: "Guillaume J. Charmes" Date: Mon, 16 Sep 2013 16:18:25 -0700 Subject: [PATCH 078/375] Prevent panic upon error pulling registry --- docs/registry.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/registry.go b/docs/registry.go index f24e0b35..b3cb8660 100644 --- a/docs/registry.go +++ b/docs/registry.go @@ -161,10 +161,10 @@ func (r *Registry) GetRemoteHistory(imgID, registry string, token []string) ([]s req.Header.Set("Authorization", "Token "+strings.Join(token, ", ")) res, err := doWithCookies(r.client, req) if err != nil || res.StatusCode != 200 { - if res.StatusCode == 401 { - return nil, ErrLoginRequired - } if res != nil { + if res.StatusCode == 401 { + return nil, ErrLoginRequired + } return nil, utils.NewHTTPRequestError(fmt.Sprintf("Internal server error: %d trying to fetch remote history for %s", res.StatusCode, imgID), res) } return nil, err From 9c366e092dcd3a1abd8c7c8b4a43264615a64e27 Mon Sep 17 00:00:00 2001 From: Michael Crosby Date: Thu, 19 Sep 2013 20:25:00 -0700 Subject: [PATCH 079/375] Modify repository name regex to match index --- docs/registry.go | 2 +- docs/registry_test.go | 14 ++++++++++++-- 2 files changed, 13 insertions(+), 3 deletions(-) diff --git a/docs/registry.go b/docs/registry.go index b3cb8660..770399ea 100644 --- a/docs/registry.go +++ b/docs/registry.go @@ -70,7 +70,7 @@ func validateRepositoryName(repositoryName string) error { if !validNamespace.MatchString(namespace) { return fmt.Errorf("Invalid namespace name (%s), only [a-z0-9_] are allowed, size between 4 and 30", namespace) } - validRepo := regexp.MustCompile(`^([a-zA-Z0-9-_.]+)$`) + validRepo := regexp.MustCompile(`^([a-z0-9-_.]+)$`) if !validRepo.MatchString(name) { return fmt.Errorf("Invalid repository name (%s), only [a-zA-Z0-9-_.] are allowed", name) } diff --git a/docs/registry_test.go b/docs/registry_test.go index a8543f18..fb43da66 100644 --- a/docs/registry_test.go +++ b/docs/registry_test.go @@ -159,11 +159,11 @@ func TestPushRegistryTag(t *testing.T) { func TestPushImageJSONIndex(t *testing.T) { r := spawnTestRegistry(t) imgData := []*ImgData{ - &ImgData{ + { ID: "77dbf71da1d00e3fbddc480176eac8994025630c6590d11cfc8fe1209c2a1d20", Checksum: "sha256:1ac330d56e05eef6d438586545ceff7550d3bdcb6b19961f12c5ba714ee1bb37", }, - &ImgData{ + { ID: "42d718c941f5c532ac049bf0b0ab53f0062f09a03afd4aa4a02c098e46032b9d", Checksum: "sha256:bea7bf2e4bacd479344b737328db47b18880d09096e6674165533aa994f5e9f2", }, @@ -196,3 +196,13 @@ func TestSearchRepositories(t *testing.T) { } assertEqual(t, results.NumResults, 0, "Expected 0 search results") } + +func TestValidRepositoryName(t *testing.T) { + if err := validateRepositoryName("docker/docker"); err != nil { + t.Fatal(err) + } + if err := validateRepositoryName("docker/Docker"); err == nil { + t.Log("Repository name should be invalid") + t.Fail() + } +} From cbb906e41ff761af938a387f9be4dd94a9fd1b7d Mon Sep 17 00:00:00 2001 From: Ken Cochrane Date: Wed, 25 Sep 2013 11:33:09 -0400 Subject: [PATCH 080/375] fix the error message so it is the same as the regex issue #1999 --- docs/registry.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/registry.go b/docs/registry.go index 770399ea..e7bc5fea 100644 --- a/docs/registry.go +++ b/docs/registry.go @@ -72,7 +72,7 @@ func validateRepositoryName(repositoryName string) error { } validRepo := regexp.MustCompile(`^([a-z0-9-_.]+)$`) if !validRepo.MatchString(name) { - return fmt.Errorf("Invalid repository name (%s), only [a-zA-Z0-9-_.] are allowed", name) + return fmt.Errorf("Invalid repository name (%s), only [a-z0-9-_.] are allowed", name) } return nil } From 8d77082c92fdedd7d80702afc828873082884435 Mon Sep 17 00:00:00 2001 From: Jonathan Rudenberg Date: Tue, 8 Oct 2013 15:21:32 -0400 Subject: [PATCH 081/375] Fix some error cases where a HTTP body might not be closed Refs #2126 --- docs/registry.go | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/docs/registry.go b/docs/registry.go index e7bc5fea..0079215c 100644 --- a/docs/registry.go +++ b/docs/registry.go @@ -160,16 +160,16 @@ func (r *Registry) GetRemoteHistory(imgID, registry string, token []string) ([]s } req.Header.Set("Authorization", "Token "+strings.Join(token, ", ")) res, err := doWithCookies(r.client, req) - if err != nil || res.StatusCode != 200 { - if res != nil { - if res.StatusCode == 401 { - return nil, ErrLoginRequired - } - return nil, utils.NewHTTPRequestError(fmt.Sprintf("Internal server error: %d trying to fetch remote history for %s", res.StatusCode, imgID), res) - } + if err != nil { return nil, err } defer res.Body.Close() + if res.StatusCode != 200 { + if res.StatusCode == 401 { + return nil, ErrLoginRequired + } + return nil, utils.NewHTTPRequestError(fmt.Sprintf("Server error: %d trying to fetch remote history for %s", res.StatusCode, imgID), res) + } jsonString, err := ioutil.ReadAll(res.Body) if err != nil { @@ -240,6 +240,7 @@ func (r *Registry) GetRemoteImageLayer(imgID, registry string, token []string) ( return nil, err } if res.StatusCode != 200 { + res.Body.Close() return nil, fmt.Errorf("Server error: Status %d while fetching image layer (%s)", res.StatusCode, imgID) } From 2f94790d6718b9f6e7fb3c1032454266add5b440 Mon Sep 17 00:00:00 2001 From: Johan Euphrosine Date: Tue, 22 Oct 2013 11:56:48 -0700 Subject: [PATCH 082/375] registry: fix content-type for PushImageJSONIndex --- docs/registry.go | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/registry.go b/docs/registry.go index 0079215c..74e31739 100644 --- a/docs/registry.go +++ b/docs/registry.go @@ -504,6 +504,7 @@ func (r *Registry) PushImageJSONIndex(indexEp, remote string, imgList []*ImgData if err != nil { return nil, err } + req.Header.Add("Content-type", "application/json") req.SetBasicAuth(r.authConfig.Username, r.authConfig.Password) req.ContentLength = int64(len(imgListJSON)) req.Header.Set("X-Docker-Token", "true") From 77f6f327044b664514598bc241ef58b51ebe5653 Mon Sep 17 00:00:00 2001 From: Victor Vieux Date: Fri, 25 Oct 2013 17:50:40 -0700 Subject: [PATCH 083/375] Removes \\n from debugf calls --- docs/registry.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/registry.go b/docs/registry.go index 74e31739..9e49f356 100644 --- a/docs/registry.go +++ b/docs/registry.go @@ -499,7 +499,7 @@ func (r *Registry) PushImageJSONIndex(indexEp, remote string, imgList []*ImgData } u := fmt.Sprintf("%srepositories/%s/%s", indexEp, remote, suffix) utils.Debugf("[registry] PUT %s", u) - utils.Debugf("Image list pushed to index:\n%s\n", imgListJSON) + utils.Debugf("Image list pushed to index:\n%s", imgListJSON) req, err := r.reqFactory.NewRequest("PUT", u, bytes.NewReader(imgListJSON)) if err != nil { return nil, err @@ -520,7 +520,7 @@ func (r *Registry) PushImageJSONIndex(indexEp, remote string, imgList []*ImgData // Redirect if necessary for res.StatusCode >= 300 && res.StatusCode < 400 { - utils.Debugf("Redirected to %s\n", res.Header.Get("Location")) + utils.Debugf("Redirected to %s", res.Header.Get("Location")) req, err = r.reqFactory.NewRequest("PUT", res.Header.Get("Location"), bytes.NewReader(imgListJSON)) if err != nil { return nil, err From 2c26420bc45631a6c0e96a5a82203f72adbb5aee Mon Sep 17 00:00:00 2001 From: Victor Vieux Date: Thu, 24 Oct 2013 12:20:34 -0700 Subject: [PATCH 084/375] update docker search to reflect future changes of the api --- docs/registry.go | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/docs/registry.go b/docs/registry.go index 9e49f356..f02e3cf4 100644 --- a/docs/registry.go +++ b/docs/registry.go @@ -615,10 +615,18 @@ func (r *Registry) GetAuthConfig(withPasswd bool) *auth.AuthConfig { } } +type SearchResult struct { + StarCount int `json:"star_count"` + IsOfficial bool `json:"is_official"` + Name string `json:"name"` + IsTrusted bool `json:"is_trusted"` + Description string `json:"description"` +} + type SearchResults struct { - Query string `json:"query"` - NumResults int `json:"num_results"` - Results []map[string]string `json:"results"` + Query string `json:"query"` + NumResults int `json:"num_results"` + Results []SearchResult `json:"results"` } type RepositoryData struct { From c86cee210fe0ae19e9dd02cddf983c3bd8eba8bb Mon Sep 17 00:00:00 2001 From: cressie176 Date: Fri, 29 Nov 2013 10:02:53 +0000 Subject: [PATCH 085/375] Closing connection after ping --- docs/registry.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/registry.go b/docs/registry.go index f02e3cf4..d3d9f2be 100644 --- a/docs/registry.go +++ b/docs/registry.go @@ -47,6 +47,8 @@ func pingRegistryEndpoint(endpoint string) error { if err != nil { return err } + defer resp.Body.Close() + if resp.Header.Get("X-Docker-Registry-Version") == "" { return errors.New("This does not look like a Registry server (\"X-Docker-Registry-Version\" header not found in the response)") } From 52a0a052e8e3e1bfdede68820467767218eb6b60 Mon Sep 17 00:00:00 2001 From: Andrews Medina Date: Fri, 29 Nov 2013 22:20:59 -0200 Subject: [PATCH 086/375] go fmt. result of `gofmt -w -s .` without vendors. --- docs/registry.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/registry.go b/docs/registry.go index d3d9f2be..6aea458e 100644 --- a/docs/registry.go +++ b/docs/registry.go @@ -47,7 +47,7 @@ func pingRegistryEndpoint(endpoint string) error { if err != nil { return err } - defer resp.Body.Close() + defer resp.Body.Close() if resp.Header.Get("X-Docker-Registry-Version") == "" { return errors.New("This does not look like a Registry server (\"X-Docker-Registry-Version\" header not found in the response)") From 097f41245a2abdb9f017bdf55f45d1e53ba1f3ee Mon Sep 17 00:00:00 2001 From: shin- Date: Tue, 22 Oct 2013 20:49:13 +0200 Subject: [PATCH 087/375] Use basic auth for private registries when over HTTPS. RequestFactory is no longer a singleton (can be different for different instances of Registry) Registry now has an indexEndpoint member Registry methods that needed the indexEndpoint parameter no longer do so Registry methods will only use token auth where applicable if basic auth is not enabled. --- docs/registry.go | 62 ++++++++++++++++++++++++++++++++----------- docs/registry_test.go | 9 +++---- 2 files changed, 51 insertions(+), 20 deletions(-) diff --git a/docs/registry.go b/docs/registry.go index 6aea458e..c39ecfe5 100644 --- a/docs/registry.go +++ b/docs/registry.go @@ -160,7 +160,9 @@ func (r *Registry) GetRemoteHistory(imgID, registry string, token []string) ([]s if err != nil { return nil, err } - req.Header.Set("Authorization", "Token "+strings.Join(token, ", ")) + if req.Header.Get("Authorization") == "" { // Don't override + req.Header.Set("Authorization", "Token "+strings.Join(token, ",")) + } res, err := doWithCookies(r.client, req) if err != nil { return nil, err @@ -193,7 +195,9 @@ func (r *Registry) LookupRemoteImage(imgID, registry string, token []string) boo if err != nil { return false } - req.Header.Set("Authorization", "Token "+strings.Join(token, ", ")) + if req.Header.Get("Authorization") == "" { // Don't override + req.Header.Set("Authorization", "Token "+strings.Join(token, ",")) + } res, err := doWithCookies(r.client, req) if err != nil { return false @@ -209,7 +213,9 @@ func (r *Registry) GetRemoteImageJSON(imgID, registry string, token []string) ([ if err != nil { return nil, -1, fmt.Errorf("Failed to download json: %s", err) } - req.Header.Set("Authorization", "Token "+strings.Join(token, ", ")) + if req.Header.Get("Authorization") == "" { // Don't override + req.Header.Set("Authorization", "Token "+strings.Join(token, ",")) + } res, err := doWithCookies(r.client, req) if err != nil { return nil, -1, fmt.Errorf("Failed to download json: %s", err) @@ -236,7 +242,9 @@ func (r *Registry) GetRemoteImageLayer(imgID, registry string, token []string) ( if err != nil { return nil, fmt.Errorf("Error while getting from the server: %s\n", err) } - req.Header.Set("Authorization", "Token "+strings.Join(token, ", ")) + if req.Header.Get("Authorization") == "" { // Don't override + req.Header.Set("Authorization", "Token "+strings.Join(token, ",")) + } res, err := doWithCookies(r.client, req) if err != nil { return nil, err @@ -262,7 +270,9 @@ func (r *Registry) GetRemoteTags(registries []string, repository string, token [ if err != nil { return nil, err } - req.Header.Set("Authorization", "Token "+strings.Join(token, ", ")) + if req.Header.Get("Authorization") == "" { // Don't override + req.Header.Set("Authorization", "Token "+strings.Join(token, ",")) + } res, err := doWithCookies(r.client, req) if err != nil { return nil, err @@ -290,7 +300,8 @@ func (r *Registry) GetRemoteTags(registries []string, repository string, token [ return nil, fmt.Errorf("Could not reach any registry endpoint") } -func (r *Registry) GetRepositoryData(indexEp, remote string) (*RepositoryData, error) { +func (r *Registry) GetRepositoryData(remote string) (*RepositoryData, error) { + indexEp := r.indexEndpoint repositoryTarget := fmt.Sprintf("%srepositories/%s/images", indexEp, remote) utils.Debugf("[registry] Calling GET %s", repositoryTarget) @@ -364,7 +375,9 @@ func (r *Registry) PushImageChecksumRegistry(imgData *ImgData, registry string, if err != nil { return err } - req.Header.Set("Authorization", "Token "+strings.Join(token, ",")) + if req.Header.Get("Authorization") == "" { // Don't override + req.Header.Set("Authorization", "Token "+strings.Join(token, ",")) + } req.Header.Set("X-Docker-Checksum", imgData.Checksum) res, err := doWithCookies(r.client, req) @@ -401,7 +414,9 @@ func (r *Registry) PushImageJSONRegistry(imgData *ImgData, jsonRaw []byte, regis return err } req.Header.Add("Content-type", "application/json") - req.Header.Set("Authorization", "Token "+strings.Join(token, ",")) + if req.Header.Get("Authorization") == "" { // Don't override + req.Header.Set("Authorization", "Token "+strings.Join(token, ",")) + } res, err := doWithCookies(r.client, req) if err != nil { @@ -436,7 +451,9 @@ func (r *Registry) PushImageLayerRegistry(imgID string, layer io.Reader, registr } req.ContentLength = -1 req.TransferEncoding = []string{"chunked"} - req.Header.Set("Authorization", "Token "+strings.Join(token, ",")) + if req.Header.Get("Authorization") == "" { // Don't override + req.Header.Set("Authorization", "Token "+strings.Join(token, ",")) + } res, err := doWithCookies(r.client, req) if err != nil { return "", fmt.Errorf("Failed to upload layer: %s", err) @@ -465,7 +482,9 @@ func (r *Registry) PushRegistryTag(remote, revision, tag, registry string, token return err } req.Header.Add("Content-type", "application/json") - req.Header.Set("Authorization", "Token "+strings.Join(token, ",")) + if req.Header.Get("Authorization") == "" { // Don't override + req.Header.Set("Authorization", "Token "+strings.Join(token, ",")) + } req.ContentLength = int64(len(revision)) res, err := doWithCookies(r.client, req) if err != nil { @@ -478,8 +497,9 @@ func (r *Registry) PushRegistryTag(remote, revision, tag, registry string, token return nil } -func (r *Registry) PushImageJSONIndex(indexEp, remote string, imgList []*ImgData, validate bool, regs []string) (*RepositoryData, error) { +func (r *Registry) PushImageJSONIndex(remote string, imgList []*ImgData, validate bool, regs []string) (*RepositoryData, error) { cleanImgList := []*ImgData{} + indexEp := r.indexEndpoint if validate { for _, elem := range imgList { @@ -583,6 +603,7 @@ func (r *Registry) PushImageJSONIndex(indexEp, remote string, imgList []*ImgData } func (r *Registry) SearchRepositories(term string) (*SearchResults, error) { + utils.Debugf("Index server: %s", r.indexEndpoint) u := auth.IndexServerAddress() + "search?q=" + url.QueryEscape(term) req, err := r.reqFactory.NewRequest("GET", u, nil) if err != nil { @@ -644,12 +665,13 @@ type ImgData struct { } type Registry struct { - client *http.Client - authConfig *auth.AuthConfig - reqFactory *utils.HTTPRequestFactory + client *http.Client + authConfig *auth.AuthConfig + reqFactory *utils.HTTPRequestFactory + indexEndpoint string } -func NewRegistry(root string, authConfig *auth.AuthConfig, factory *utils.HTTPRequestFactory) (r *Registry, err error) { +func NewRegistry(authConfig *auth.AuthConfig, factory *utils.HTTPRequestFactory, indexEndpoint string) (r *Registry, err error) { httpTransport := &http.Transport{ DisableKeepAlives: true, Proxy: http.ProxyFromEnvironment, @@ -660,12 +682,22 @@ func NewRegistry(root string, authConfig *auth.AuthConfig, factory *utils.HTTPRe client: &http.Client{ Transport: httpTransport, }, + indexEndpoint: indexEndpoint, } r.client.Jar, err = cookiejar.New(nil) if err != nil { return nil, err } + // If we're working with a private registry over HTTPS, send Basic Auth headers + // alongside our requests. + if indexEndpoint != auth.IndexServerAddress() && strings.HasPrefix(indexEndpoint, "https://") { + utils.Debugf("Endpoint %s is eligible for private registry auth. Enabling decorator.", indexEndpoint) + dec := utils.NewHTTPAuthDecorator(authConfig.Username, authConfig.Password) + factory.AddDecorator(dec) + } + r.reqFactory = factory return r, nil } + diff --git a/docs/registry_test.go b/docs/registry_test.go index fb43da66..69eb25b2 100644 --- a/docs/registry_test.go +++ b/docs/registry_test.go @@ -15,7 +15,7 @@ var ( func spawnTestRegistry(t *testing.T) *Registry { authConfig := &auth.AuthConfig{} - r, err := NewRegistry("", authConfig, utils.NewHTTPRequestFactory()) + r, err := NewRegistry(authConfig, utils.NewHTTPRequestFactory(), makeURL("/v1/")) if err != nil { t.Fatal(err) } @@ -99,7 +99,7 @@ func TestGetRemoteTags(t *testing.T) { func TestGetRepositoryData(t *testing.T) { r := spawnTestRegistry(t) - data, err := r.GetRepositoryData(makeURL("/v1/"), "foo42/bar") + data, err := r.GetRepositoryData("foo42/bar") if err != nil { t.Fatal(err) } @@ -168,15 +168,14 @@ func TestPushImageJSONIndex(t *testing.T) { Checksum: "sha256:bea7bf2e4bacd479344b737328db47b18880d09096e6674165533aa994f5e9f2", }, } - ep := makeURL("/v1/") - repoData, err := r.PushImageJSONIndex(ep, "foo42/bar", imgData, false, nil) + repoData, err := r.PushImageJSONIndex("foo42/bar", imgData, false, nil) if err != nil { t.Fatal(err) } if repoData == nil { t.Fatal("Expected RepositoryData object") } - repoData, err = r.PushImageJSONIndex(ep, "foo42/bar", imgData, true, []string{ep}) + repoData, err = r.PushImageJSONIndex("foo42/bar", imgData, true, []string{r.indexEndpoint}) if err != nil { t.Fatal(err) } From d4a00ebecbc4967049e89f9b88b672e9e56e17ae Mon Sep 17 00:00:00 2001 From: shin- Date: Tue, 22 Oct 2013 20:57:48 +0200 Subject: [PATCH 088/375] gofmt --- docs/registry.go | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/docs/registry.go b/docs/registry.go index c39ecfe5..99f3403a 100644 --- a/docs/registry.go +++ b/docs/registry.go @@ -665,9 +665,9 @@ type ImgData struct { } type Registry struct { - client *http.Client - authConfig *auth.AuthConfig - reqFactory *utils.HTTPRequestFactory + client *http.Client + authConfig *auth.AuthConfig + reqFactory *utils.HTTPRequestFactory indexEndpoint string } @@ -700,4 +700,3 @@ func NewRegistry(authConfig *auth.AuthConfig, factory *utils.HTTPRequestFactory, r.reqFactory = factory return r, nil } - From 0fca0f12f6e356549a75d884e1bc13418d8629e7 Mon Sep 17 00:00:00 2001 From: shin- Date: Wed, 23 Oct 2013 17:56:40 +0200 Subject: [PATCH 089/375] Factorized auth token setting --- docs/registry.go | 39 +++++++++++++++------------------------ 1 file changed, 15 insertions(+), 24 deletions(-) diff --git a/docs/registry.go b/docs/registry.go index 99f3403a..ef561fea 100644 --- a/docs/registry.go +++ b/docs/registry.go @@ -153,6 +153,13 @@ func doWithCookies(c *http.Client, req *http.Request) (*http.Response, error) { return res, err } +func setTokenAuth(req *http.Request, token []string) (*http.Request) { + if req.Header.Get("Authorization") == "" { // Don't override + req.Header.Set("Authorization", "Token "+strings.Join(token, ",")) + } + return req +} + // Retrieve the history of a given image from the Registry. // Return a list of the parent's json (requested image included) func (r *Registry) GetRemoteHistory(imgID, registry string, token []string) ([]string, error) { @@ -160,9 +167,7 @@ func (r *Registry) GetRemoteHistory(imgID, registry string, token []string) ([]s if err != nil { return nil, err } - if req.Header.Get("Authorization") == "" { // Don't override - req.Header.Set("Authorization", "Token "+strings.Join(token, ",")) - } + req = setTokenAuth(req, token) res, err := doWithCookies(r.client, req) if err != nil { return nil, err @@ -195,9 +200,7 @@ func (r *Registry) LookupRemoteImage(imgID, registry string, token []string) boo if err != nil { return false } - if req.Header.Get("Authorization") == "" { // Don't override - req.Header.Set("Authorization", "Token "+strings.Join(token, ",")) - } + req = setTokenAuth(req, token) res, err := doWithCookies(r.client, req) if err != nil { return false @@ -213,9 +216,7 @@ func (r *Registry) GetRemoteImageJSON(imgID, registry string, token []string) ([ if err != nil { return nil, -1, fmt.Errorf("Failed to download json: %s", err) } - if req.Header.Get("Authorization") == "" { // Don't override - req.Header.Set("Authorization", "Token "+strings.Join(token, ",")) - } + req = setTokenAuth(req, token) res, err := doWithCookies(r.client, req) if err != nil { return nil, -1, fmt.Errorf("Failed to download json: %s", err) @@ -242,9 +243,7 @@ func (r *Registry) GetRemoteImageLayer(imgID, registry string, token []string) ( if err != nil { return nil, fmt.Errorf("Error while getting from the server: %s\n", err) } - if req.Header.Get("Authorization") == "" { // Don't override - req.Header.Set("Authorization", "Token "+strings.Join(token, ",")) - } + req = setTokenAuth(req, token) res, err := doWithCookies(r.client, req) if err != nil { return nil, err @@ -375,9 +374,7 @@ func (r *Registry) PushImageChecksumRegistry(imgData *ImgData, registry string, if err != nil { return err } - if req.Header.Get("Authorization") == "" { // Don't override - req.Header.Set("Authorization", "Token "+strings.Join(token, ",")) - } + req = setTokenAuth(req, token) req.Header.Set("X-Docker-Checksum", imgData.Checksum) res, err := doWithCookies(r.client, req) @@ -414,9 +411,7 @@ func (r *Registry) PushImageJSONRegistry(imgData *ImgData, jsonRaw []byte, regis return err } req.Header.Add("Content-type", "application/json") - if req.Header.Get("Authorization") == "" { // Don't override - req.Header.Set("Authorization", "Token "+strings.Join(token, ",")) - } + req = setTokenAuth(req, token) res, err := doWithCookies(r.client, req) if err != nil { @@ -451,9 +446,7 @@ func (r *Registry) PushImageLayerRegistry(imgID string, layer io.Reader, registr } req.ContentLength = -1 req.TransferEncoding = []string{"chunked"} - if req.Header.Get("Authorization") == "" { // Don't override - req.Header.Set("Authorization", "Token "+strings.Join(token, ",")) - } + req = setTokenAuth(req, token) res, err := doWithCookies(r.client, req) if err != nil { return "", fmt.Errorf("Failed to upload layer: %s", err) @@ -482,9 +475,7 @@ func (r *Registry) PushRegistryTag(remote, revision, tag, registry string, token return err } req.Header.Add("Content-type", "application/json") - if req.Header.Get("Authorization") == "" { // Don't override - req.Header.Set("Authorization", "Token "+strings.Join(token, ",")) - } + req = setTokenAuth(req, token) req.ContentLength = int64(len(revision)) res, err := doWithCookies(r.client, req) if err != nil { From 1ff180d1b4e1c9f52b15f4cdc562f1975b1510e5 Mon Sep 17 00:00:00 2001 From: shin- Date: Wed, 23 Oct 2013 18:00:40 +0200 Subject: [PATCH 090/375] missed one call to setTokenAuth --- docs/registry.go | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/docs/registry.go b/docs/registry.go index ef561fea..0a8d1dda 100644 --- a/docs/registry.go +++ b/docs/registry.go @@ -153,7 +153,7 @@ func doWithCookies(c *http.Client, req *http.Request) (*http.Response, error) { return res, err } -func setTokenAuth(req *http.Request, token []string) (*http.Request) { +func setTokenAuth(req *http.Request, token []string) *http.Request { if req.Header.Get("Authorization") == "" { // Don't override req.Header.Set("Authorization", "Token "+strings.Join(token, ",")) } @@ -269,9 +269,7 @@ func (r *Registry) GetRemoteTags(registries []string, repository string, token [ if err != nil { return nil, err } - if req.Header.Get("Authorization") == "" { // Don't override - req.Header.Set("Authorization", "Token "+strings.Join(token, ",")) - } + req = setTokenAuth(req, token) res, err := doWithCookies(r.client, req) if err != nil { return nil, err From d2f7d65d71fb9a8cca25371b0132089c626d140b Mon Sep 17 00:00:00 2001 From: shin- Date: Mon, 4 Nov 2013 21:49:34 +0100 Subject: [PATCH 091/375] Don't return req as result of setTokenAuth --- docs/registry.go | 21 ++++++++++----------- 1 file changed, 10 insertions(+), 11 deletions(-) diff --git a/docs/registry.go b/docs/registry.go index 0a8d1dda..6c9255aa 100644 --- a/docs/registry.go +++ b/docs/registry.go @@ -153,11 +153,10 @@ func doWithCookies(c *http.Client, req *http.Request) (*http.Response, error) { return res, err } -func setTokenAuth(req *http.Request, token []string) *http.Request { +func setTokenAuth(req *http.Request, token []string) { if req.Header.Get("Authorization") == "" { // Don't override req.Header.Set("Authorization", "Token "+strings.Join(token, ",")) } - return req } // Retrieve the history of a given image from the Registry. @@ -167,7 +166,7 @@ func (r *Registry) GetRemoteHistory(imgID, registry string, token []string) ([]s if err != nil { return nil, err } - req = setTokenAuth(req, token) + setTokenAuth(req, token) res, err := doWithCookies(r.client, req) if err != nil { return nil, err @@ -200,7 +199,7 @@ func (r *Registry) LookupRemoteImage(imgID, registry string, token []string) boo if err != nil { return false } - req = setTokenAuth(req, token) + setTokenAuth(req, token) res, err := doWithCookies(r.client, req) if err != nil { return false @@ -216,7 +215,7 @@ func (r *Registry) GetRemoteImageJSON(imgID, registry string, token []string) ([ if err != nil { return nil, -1, fmt.Errorf("Failed to download json: %s", err) } - req = setTokenAuth(req, token) + setTokenAuth(req, token) res, err := doWithCookies(r.client, req) if err != nil { return nil, -1, fmt.Errorf("Failed to download json: %s", err) @@ -243,7 +242,7 @@ func (r *Registry) GetRemoteImageLayer(imgID, registry string, token []string) ( if err != nil { return nil, fmt.Errorf("Error while getting from the server: %s\n", err) } - req = setTokenAuth(req, token) + setTokenAuth(req, token) res, err := doWithCookies(r.client, req) if err != nil { return nil, err @@ -269,7 +268,7 @@ func (r *Registry) GetRemoteTags(registries []string, repository string, token [ if err != nil { return nil, err } - req = setTokenAuth(req, token) + setTokenAuth(req, token) res, err := doWithCookies(r.client, req) if err != nil { return nil, err @@ -372,7 +371,7 @@ func (r *Registry) PushImageChecksumRegistry(imgData *ImgData, registry string, if err != nil { return err } - req = setTokenAuth(req, token) + setTokenAuth(req, token) req.Header.Set("X-Docker-Checksum", imgData.Checksum) res, err := doWithCookies(r.client, req) @@ -409,7 +408,7 @@ func (r *Registry) PushImageJSONRegistry(imgData *ImgData, jsonRaw []byte, regis return err } req.Header.Add("Content-type", "application/json") - req = setTokenAuth(req, token) + setTokenAuth(req, token) res, err := doWithCookies(r.client, req) if err != nil { @@ -444,7 +443,7 @@ func (r *Registry) PushImageLayerRegistry(imgID string, layer io.Reader, registr } req.ContentLength = -1 req.TransferEncoding = []string{"chunked"} - req = setTokenAuth(req, token) + setTokenAuth(req, token) res, err := doWithCookies(r.client, req) if err != nil { return "", fmt.Errorf("Failed to upload layer: %s", err) @@ -473,7 +472,7 @@ func (r *Registry) PushRegistryTag(remote, revision, tag, registry string, token return err } req.Header.Add("Content-type", "application/json") - req = setTokenAuth(req, token) + setTokenAuth(req, token) req.ContentLength = int64(len(revision)) res, err := doWithCookies(r.client, req) if err != nil { From 10eeaec70cd3a2c5d52a132009cd3a8920641e3a Mon Sep 17 00:00:00 2001 From: Victor Vieux Date: Thu, 19 Dec 2013 12:32:58 -0800 Subject: [PATCH 092/375] fix progressbar in docker push --- docs/registry.go | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/docs/registry.go b/docs/registry.go index 6c9255aa..3c793cf0 100644 --- a/docs/registry.go +++ b/docs/registry.go @@ -448,6 +448,11 @@ func (r *Registry) PushImageLayerRegistry(imgID string, layer io.Reader, registr if err != nil { return "", fmt.Errorf("Failed to upload layer: %s", err) } + if rc, ok := layer.(io.Closer); ok { + if err := rc.Close(); err != nil { + return "", err + } + } defer res.Body.Close() if res.StatusCode != 200 { From 79e0ed25dbc4b48987743eaefa86afa99fe09e5b Mon Sep 17 00:00:00 2001 From: shin- Date: Thu, 2 Jan 2014 17:51:42 +0100 Subject: [PATCH 093/375] Check standalone header when pinging a registry server. Standalone has to be true to use basic auth (in addition to previous requirements) --- docs/registry.go | 42 ++++++++++++++++++++++++++++++------------ 1 file changed, 30 insertions(+), 12 deletions(-) diff --git a/docs/registry.go b/docs/registry.go index 6c9255aa..17c2ccd8 100644 --- a/docs/registry.go +++ b/docs/registry.go @@ -25,11 +25,11 @@ var ( ErrLoginRequired = errors.New("Authentication is required.") ) -func pingRegistryEndpoint(endpoint string) error { +func pingRegistryEndpoint(endpoint string) (bool, error) { if endpoint == auth.IndexServerAddress() { // Skip the check, we now this one is valid // (and we never want to fallback to http in case of error) - return nil + return false, nil } httpDial := func(proto string, addr string) (net.Conn, error) { // Set the connect timeout to 5 seconds @@ -45,14 +45,26 @@ func pingRegistryEndpoint(endpoint string) error { client := &http.Client{Transport: httpTransport} resp, err := client.Get(endpoint + "_ping") if err != nil { - return err + return false, err } defer resp.Body.Close() if resp.Header.Get("X-Docker-Registry-Version") == "" { - return errors.New("This does not look like a Registry server (\"X-Docker-Registry-Version\" header not found in the response)") + return false, errors.New("This does not look like a Registry server (\"X-Docker-Registry-Version\" header not found in the response)") } - return nil + + standalone := resp.Header.Get("X-Docker-Registry-Standalone") + utils.Debugf("Registry standalone header: '%s'", standalone) + // If the header is absent, we assume true for compatibility with earlier + // versions of the registry + if standalone == "" { + return true, nil + // Accepted values are "true" (case-insensitive) and "1". + } else if strings.EqualFold(standalone, "true") || standalone == "1" { + return true, nil + } + // Otherwise, not standalone + return false, nil } func validateRepositoryName(repositoryName string) error { @@ -122,16 +134,16 @@ func ExpandAndVerifyRegistryUrl(hostname string) (string, error) { // there is no path given. Expand with default path hostname = hostname + "/v1/" } - if err := pingRegistryEndpoint(hostname); err != nil { + if _, err := pingRegistryEndpoint(hostname); err != nil { return "", errors.New("Invalid Registry endpoint: " + err.Error()) } return hostname, nil } endpoint := fmt.Sprintf("https://%s/v1/", hostname) - if err := pingRegistryEndpoint(endpoint); err != nil { + if _, err := pingRegistryEndpoint(endpoint); err != nil { utils.Debugf("Registry %s does not work (%s), falling back to http", endpoint, err) endpoint = fmt.Sprintf("http://%s/v1/", hostname) - if err = pingRegistryEndpoint(endpoint); err != nil { + if _, err = pingRegistryEndpoint(endpoint); err != nil { //TODO: triggering highland build can be done there without "failing" return "", errors.New("Invalid Registry endpoint: " + err.Error()) } @@ -677,12 +689,18 @@ func NewRegistry(authConfig *auth.AuthConfig, factory *utils.HTTPRequestFactory, return nil, err } - // If we're working with a private registry over HTTPS, send Basic Auth headers + // If we're working with a standalone private registry over HTTPS, send Basic Auth headers // alongside our requests. if indexEndpoint != auth.IndexServerAddress() && strings.HasPrefix(indexEndpoint, "https://") { - utils.Debugf("Endpoint %s is eligible for private registry auth. Enabling decorator.", indexEndpoint) - dec := utils.NewHTTPAuthDecorator(authConfig.Username, authConfig.Password) - factory.AddDecorator(dec) + standalone, err := pingRegistryEndpoint(indexEndpoint) + if err != nil { + return nil, err + } + if standalone { + utils.Debugf("Endpoint %s is eligible for private registry auth. Enabling decorator.", indexEndpoint) + dec := utils.NewHTTPAuthDecorator(authConfig.Username, authConfig.Password) + factory.AddDecorator(dec) + } } r.reqFactory = factory From 9bafa726be072c57d1570c5ea0275d3a1e66ea2c Mon Sep 17 00:00:00 2001 From: shin- Date: Mon, 6 Jan 2014 21:04:44 +0100 Subject: [PATCH 094/375] Fixed registry unit tests --- docs/registry_test.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/docs/registry_test.go b/docs/registry_test.go index 69eb25b2..16bc431e 100644 --- a/docs/registry_test.go +++ b/docs/registry_test.go @@ -23,10 +23,11 @@ func spawnTestRegistry(t *testing.T) *Registry { } func TestPingRegistryEndpoint(t *testing.T) { - err := pingRegistryEndpoint(makeURL("/v1/")) + standalone, err := pingRegistryEndpoint(makeURL("/v1/")) if err != nil { t.Fatal(err) } + assertEqual(t, standalone, true, "Expected standalone to be true (default)") } func TestGetRemoteHistory(t *testing.T) { From 78bc8d7377aa61d7667a96f8e8df9e176e6747f0 Mon Sep 17 00:00:00 2001 From: Victor Vieux Date: Mon, 13 Jan 2014 14:55:31 -0800 Subject: [PATCH 095/375] move legacy stuff outside the job Docker-DCO-1.1-Signed-off-by: Victor Vieux (github: vieux) --- docs/registry.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/registry.go b/docs/registry.go index a038fdfb..a0da733e 100644 --- a/docs/registry.go +++ b/docs/registry.go @@ -59,7 +59,7 @@ func pingRegistryEndpoint(endpoint string) (bool, error) { // versions of the registry if standalone == "" { return true, nil - // Accepted values are "true" (case-insensitive) and "1". + // Accepted values are "true" (case-insensitive) and "1". } else if strings.EqualFold(standalone, "true") || standalone == "1" { return true, nil } From 275109a6ad91374c72172d9e4e9a94fdbbf6014b Mon Sep 17 00:00:00 2001 From: Michael Crosby Date: Mon, 20 Jan 2014 13:39:35 -0800 Subject: [PATCH 096/375] Make sure new repositories can be pushed with multiple tags Docker-DCO-1.1-Signed-off-by: Michael Crosby (github: crosbymichael) --- docs/registry.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/docs/registry.go b/docs/registry.go index a0da733e..b2d26a2d 100644 --- a/docs/registry.go +++ b/docs/registry.go @@ -205,15 +205,18 @@ func (r *Registry) GetRemoteHistory(imgID, registry string, token []string) ([]s } // Check if an image exists in the Registry +// TODO: This method should return the errors instead of masking them and returning false func (r *Registry) LookupRemoteImage(imgID, registry string, token []string) bool { req, err := r.reqFactory.NewRequest("GET", registry+"images/"+imgID+"/json", nil) if err != nil { + utils.Errorf("Error in LookupRemoteImage %s", err) return false } setTokenAuth(req, token) res, err := doWithCookies(r.client, req) if err != nil { + utils.Errorf("Error in LookupRemoteImage %s", err) return false } res.Body.Close() From 4fe7a141bf34a911a58aa7ce2158f6702772a696 Mon Sep 17 00:00:00 2001 From: "Roberto G. Hashioka" Date: Tue, 21 Jan 2014 04:06:19 +0000 Subject: [PATCH 097/375] Added missing attributes to api search calls: - Added an argument to the call() method in order to control the auth sharing - Enabled it only for search. Pulls and pushes were enabled already. - Grouped a few variable declarations Docker-DCO-1.1-Signed-off-by: Roberto Hashioka (github: rogaha) --- docs/registry.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/docs/registry.go b/docs/registry.go index b2d26a2d..c0d8414d 100644 --- a/docs/registry.go +++ b/docs/registry.go @@ -617,6 +617,10 @@ func (r *Registry) SearchRepositories(term string) (*SearchResults, error) { if err != nil { return nil, err } + if r.authConfig != nil && len(r.authConfig.Username) > 0 { + req.SetBasicAuth(r.authConfig.Username, r.authConfig.Password) + } + req.Header.Set("X-Docker-Token", "true") res, err := r.client.Do(req) if err != nil { return nil, err From 9274def67d1bb1750fcccb8e1f0658afc16bee7b Mon Sep 17 00:00:00 2001 From: Michael Crosby Date: Mon, 3 Feb 2014 11:38:34 -0800 Subject: [PATCH 098/375] Fix login prompt on push and pull because of error message Docker-DCO-1.1-Signed-off-by: Michael Crosby (github: crosbymichael) --- docs/registry.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/registry.go b/docs/registry.go index c0d8414d..df943023 100644 --- a/docs/registry.go +++ b/docs/registry.go @@ -22,7 +22,7 @@ import ( var ( ErrAlreadyExists = errors.New("Image already exists") ErrInvalidRepositoryName = errors.New("Invalid repository name (ex: \"registry.domain.tld/myrepos\")") - ErrLoginRequired = errors.New("Authentication is required.") + errLoginRequired = errors.New("Authentication is required.") ) func pingRegistryEndpoint(endpoint string) (bool, error) { @@ -186,7 +186,7 @@ func (r *Registry) GetRemoteHistory(imgID, registry string, token []string) ([]s defer res.Body.Close() if res.StatusCode != 200 { if res.StatusCode == 401 { - return nil, ErrLoginRequired + return nil, errLoginRequired } return nil, utils.NewHTTPRequestError(fmt.Sprintf("Server error: %d trying to fetch remote history for %s", res.StatusCode, imgID), res) } @@ -332,7 +332,7 @@ func (r *Registry) GetRepositoryData(remote string) (*RepositoryData, error) { } defer res.Body.Close() if res.StatusCode == 401 { - return nil, ErrLoginRequired + return nil, errLoginRequired } // TODO: Right now we're ignoring checksums in the response body. // In the future, we need to use them to check image validity. From bac83c76084d2b8667b023b4506f385db993deb9 Mon Sep 17 00:00:00 2001 From: Jake Moshenko Date: Thu, 20 Feb 2014 17:57:58 -0500 Subject: [PATCH 099/375] Fix registry auth by storing the string passed on the command line, and allowing for credential selection by normalizing on hostname. Also, remove remote ping calls from CmdPush and CmdPull. Docker-DCO-1.1-Signed-off-by: Jake Moshenko (github: jakedt) --- docs/registry.go | 9 +++------ docs/registry_test.go | 2 +- 2 files changed, 4 insertions(+), 7 deletions(-) diff --git a/docs/registry.go b/docs/registry.go index df943023..37e107fa 100644 --- a/docs/registry.go +++ b/docs/registry.go @@ -91,7 +91,7 @@ func validateRepositoryName(repositoryName string) error { return nil } -// Resolves a repository name to a endpoint + name +// Resolves a repository name to a hostname + name func ResolveRepositoryName(reposName string) (string, string, error) { if strings.Contains(reposName, "://") { // It cannot contain a scheme! @@ -117,11 +117,8 @@ func ResolveRepositoryName(reposName string) (string, string, error) { if err := validateRepositoryName(reposName); err != nil { return "", "", err } - endpoint, err := ExpandAndVerifyRegistryUrl(hostname) - if err != nil { - return "", "", err - } - return endpoint, reposName, err + + return hostname, reposName, nil } // this method expands the registry name as used in the prefix of a repo diff --git a/docs/registry_test.go b/docs/registry_test.go index 16bc431e..5e398f99 100644 --- a/docs/registry_test.go +++ b/docs/registry_test.go @@ -145,7 +145,7 @@ func TestResolveRepositoryName(t *testing.T) { if err != nil { t.Fatal(err) } - assertEqual(t, ep, "http://"+u+"/v1/", "Expected endpoint to be "+u) + assertEqual(t, ep, u, "Expected endpoint to be "+u) assertEqual(t, repo, "private/moonbase", "Expected endpoint to be private/moonbase") } From 3bf0ee5e52b608b8b6d9bafa8a7fac6fb0fdf6f5 Mon Sep 17 00:00:00 2001 From: Sam Alba Date: Sun, 23 Feb 2014 18:33:46 -0800 Subject: [PATCH 100/375] registry: Added simple checksums (sha256) for layers Docker-DCO-1.1-Signed-off-by: Sam Alba (github: samalba) --- docs/registry.go | 29 ++++++++++++++++++----------- 1 file changed, 18 insertions(+), 11 deletions(-) diff --git a/docs/registry.go b/docs/registry.go index df943023..7bdf12c8 100644 --- a/docs/registry.go +++ b/docs/registry.go @@ -2,6 +2,7 @@ package registry import ( "bytes" + "crypto/sha256" "encoding/json" "errors" "fmt" @@ -388,6 +389,7 @@ func (r *Registry) PushImageChecksumRegistry(imgData *ImgData, registry string, } setTokenAuth(req, token) req.Header.Set("X-Docker-Checksum", imgData.Checksum) + req.Header.Set("X-Docker-Checksum-Payload", imgData.ChecksumPayload) res, err := doWithCookies(r.client, req) if err != nil { @@ -446,26 +448,28 @@ func (r *Registry) PushImageJSONRegistry(imgData *ImgData, jsonRaw []byte, regis return nil } -func (r *Registry) PushImageLayerRegistry(imgID string, layer io.Reader, registry string, token []string, jsonRaw []byte) (checksum string, err error) { +func (r *Registry) PushImageLayerRegistry(imgID string, layer io.Reader, registry string, token []string, jsonRaw []byte) (checksum string, checksumPayload string, err error) { utils.Debugf("[registry] Calling PUT %s", registry+"images/"+imgID+"/layer") - tarsumLayer := &utils.TarSum{Reader: layer} + h := sha256.New() + checksumLayer := &utils.CheckSum{Reader: layer, Hash: h} + tarsumLayer := &utils.TarSum{Reader: checksumLayer} req, err := r.reqFactory.NewRequest("PUT", registry+"images/"+imgID+"/layer", tarsumLayer) if err != nil { - return "", err + return "", "", err } req.ContentLength = -1 req.TransferEncoding = []string{"chunked"} setTokenAuth(req, token) res, err := doWithCookies(r.client, req) if err != nil { - return "", fmt.Errorf("Failed to upload layer: %s", err) + return "", "", fmt.Errorf("Failed to upload layer: %s", err) } if rc, ok := layer.(io.Closer); ok { if err := rc.Close(); err != nil { - return "", err + return "", "", err } } defer res.Body.Close() @@ -473,11 +477,13 @@ func (r *Registry) PushImageLayerRegistry(imgID string, layer io.Reader, registr if res.StatusCode != 200 { errBody, err := ioutil.ReadAll(res.Body) if err != nil { - return "", utils.NewHTTPRequestError(fmt.Sprintf("HTTP code %d while uploading metadata and error when trying to parse response body: %s", res.StatusCode, err), res) + return "", "", utils.NewHTTPRequestError(fmt.Sprintf("HTTP code %d while uploading metadata and error when trying to parse response body: %s", res.StatusCode, err), res) } - return "", utils.NewHTTPRequestError(fmt.Sprintf("Received HTTP code %d while uploading layer: %s", res.StatusCode, errBody), res) + return "", "", utils.NewHTTPRequestError(fmt.Sprintf("Received HTTP code %d while uploading layer: %s", res.StatusCode, errBody), res) } - return tarsumLayer.Sum(jsonRaw), nil + + checksumPayload = "sha256:" + checksumLayer.Sum() + return tarsumLayer.Sum(jsonRaw), checksumPayload, nil } // push a tag on the registry. @@ -671,9 +677,10 @@ type RepositoryData struct { } type ImgData struct { - ID string `json:"id"` - Checksum string `json:"checksum,omitempty"` - Tag string `json:",omitempty"` + ID string `json:"id"` + Checksum string `json:"checksum,omitempty"` + ChecksumPayload string `json:"checksum,omitempty"` + Tag string `json:",omitempty"` } type Registry struct { From bae6dc35bc18d86b0a62a48e5f8e1c0e6bea7e31 Mon Sep 17 00:00:00 2001 From: Sam Alba Date: Mon, 24 Feb 2014 09:04:27 -0800 Subject: [PATCH 101/375] registry: Fixed tests Docker-DCO-1.1-Signed-off-by: Sam Alba (github: samalba) --- docs/registry_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/registry_test.go b/docs/registry_test.go index 16bc431e..5b33485d 100644 --- a/docs/registry_test.go +++ b/docs/registry_test.go @@ -124,7 +124,7 @@ func TestPushImageJSONRegistry(t *testing.T) { func TestPushImageLayerRegistry(t *testing.T) { r := spawnTestRegistry(t) layer := strings.NewReader("") - _, err := r.PushImageLayerRegistry(IMAGE_ID, layer, makeURL("/v1/"), TOKEN, []byte{}) + _, _, err := r.PushImageLayerRegistry(IMAGE_ID, layer, makeURL("/v1/"), TOKEN, []byte{}) if err != nil { t.Fatal(err) } From ba8dbe4b9b6d280a2aeaeefbcd238c598bdb1a73 Mon Sep 17 00:00:00 2001 From: Sam Alba Date: Mon, 24 Feb 2014 12:40:33 -0800 Subject: [PATCH 102/375] registry: Removed checksumPayload from exported fields Docker-DCO-1.1-Signed-off-by: Sam Alba (github: samalba) --- docs/registry.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/registry.go b/docs/registry.go index 7bdf12c8..d0ddedb7 100644 --- a/docs/registry.go +++ b/docs/registry.go @@ -389,7 +389,7 @@ func (r *Registry) PushImageChecksumRegistry(imgData *ImgData, registry string, } setTokenAuth(req, token) req.Header.Set("X-Docker-Checksum", imgData.Checksum) - req.Header.Set("X-Docker-Checksum-Payload", imgData.ChecksumPayload) + req.Header.Set("X-Docker-Checksum-Payload", imgData.checksumPayload) res, err := doWithCookies(r.client, req) if err != nil { @@ -679,8 +679,8 @@ type RepositoryData struct { type ImgData struct { ID string `json:"id"` Checksum string `json:"checksum,omitempty"` - ChecksumPayload string `json:"checksum,omitempty"` Tag string `json:",omitempty"` + checksumPayload string } type Registry struct { From f29683f794763a1aacd96a4b5ac9a49db830a4b1 Mon Sep 17 00:00:00 2001 From: Sam Alba Date: Tue, 25 Feb 2014 16:06:04 -0800 Subject: [PATCH 103/375] registry: Fixed unexported field Docker-DCO-1.1-Signed-off-by: Sam Alba (github: samalba) --- docs/registry.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/registry.go b/docs/registry.go index d0ddedb7..c570e7b7 100644 --- a/docs/registry.go +++ b/docs/registry.go @@ -389,7 +389,7 @@ func (r *Registry) PushImageChecksumRegistry(imgData *ImgData, registry string, } setTokenAuth(req, token) req.Header.Set("X-Docker-Checksum", imgData.Checksum) - req.Header.Set("X-Docker-Checksum-Payload", imgData.checksumPayload) + req.Header.Set("X-Docker-Checksum-Payload", imgData.ChecksumPayload) res, err := doWithCookies(r.client, req) if err != nil { @@ -679,8 +679,8 @@ type RepositoryData struct { type ImgData struct { ID string `json:"id"` Checksum string `json:"checksum,omitempty"` + ChecksumPayload string `json:"-"` Tag string `json:",omitempty"` - checksumPayload string } type Registry struct { From 1c101d006bceabd606de6938219a4399a6195d71 Mon Sep 17 00:00:00 2001 From: Fabio Falci Date: Sun, 9 Mar 2014 01:49:36 +0000 Subject: [PATCH 104/375] Remove manual http cookie management Since docker uses cookiejar it doesn't need to manage cookies manually anymore. Managing cookie was duplicating it. Docker-DCO-1.1-Signed-off-by: Fabio Falci (github: fabiofalci) --- docs/registry.go | 32 +++++++++----------------------- 1 file changed, 9 insertions(+), 23 deletions(-) diff --git a/docs/registry.go b/docs/registry.go index 543dcea3..cc2e985c 100644 --- a/docs/registry.go +++ b/docs/registry.go @@ -149,20 +149,6 @@ func ExpandAndVerifyRegistryUrl(hostname string) (string, error) { return endpoint, nil } -func doWithCookies(c *http.Client, req *http.Request) (*http.Response, error) { - for _, cookie := range c.Jar.Cookies(req.URL) { - req.AddCookie(cookie) - } - res, err := c.Do(req) - if err != nil { - return nil, err - } - if len(res.Cookies()) > 0 { - c.Jar.SetCookies(req.URL, res.Cookies()) - } - return res, err -} - func setTokenAuth(req *http.Request, token []string) { if req.Header.Get("Authorization") == "" { // Don't override req.Header.Set("Authorization", "Token "+strings.Join(token, ",")) @@ -177,7 +163,7 @@ func (r *Registry) GetRemoteHistory(imgID, registry string, token []string) ([]s return nil, err } setTokenAuth(req, token) - res, err := doWithCookies(r.client, req) + res, err := r.client.Do(req) if err != nil { return nil, err } @@ -212,7 +198,7 @@ func (r *Registry) LookupRemoteImage(imgID, registry string, token []string) boo return false } setTokenAuth(req, token) - res, err := doWithCookies(r.client, req) + res, err := r.client.Do(req) if err != nil { utils.Errorf("Error in LookupRemoteImage %s", err) return false @@ -229,7 +215,7 @@ func (r *Registry) GetRemoteImageJSON(imgID, registry string, token []string) ([ return nil, -1, fmt.Errorf("Failed to download json: %s", err) } setTokenAuth(req, token) - res, err := doWithCookies(r.client, req) + res, err := r.client.Do(req) if err != nil { return nil, -1, fmt.Errorf("Failed to download json: %s", err) } @@ -256,7 +242,7 @@ func (r *Registry) GetRemoteImageLayer(imgID, registry string, token []string) ( return nil, fmt.Errorf("Error while getting from the server: %s\n", err) } setTokenAuth(req, token) - res, err := doWithCookies(r.client, req) + res, err := r.client.Do(req) if err != nil { return nil, err } @@ -282,7 +268,7 @@ func (r *Registry) GetRemoteTags(registries []string, repository string, token [ return nil, err } setTokenAuth(req, token) - res, err := doWithCookies(r.client, req) + res, err := r.client.Do(req) if err != nil { return nil, err } @@ -388,7 +374,7 @@ func (r *Registry) PushImageChecksumRegistry(imgData *ImgData, registry string, req.Header.Set("X-Docker-Checksum", imgData.Checksum) req.Header.Set("X-Docker-Checksum-Payload", imgData.ChecksumPayload) - res, err := doWithCookies(r.client, req) + res, err := r.client.Do(req) if err != nil { return fmt.Errorf("Failed to upload metadata: %s", err) } @@ -424,7 +410,7 @@ func (r *Registry) PushImageJSONRegistry(imgData *ImgData, jsonRaw []byte, regis req.Header.Add("Content-type", "application/json") setTokenAuth(req, token) - res, err := doWithCookies(r.client, req) + res, err := r.client.Do(req) if err != nil { return fmt.Errorf("Failed to upload metadata: %s", err) } @@ -460,7 +446,7 @@ func (r *Registry) PushImageLayerRegistry(imgID string, layer io.Reader, registr req.ContentLength = -1 req.TransferEncoding = []string{"chunked"} setTokenAuth(req, token) - res, err := doWithCookies(r.client, req) + res, err := r.client.Do(req) if err != nil { return "", "", fmt.Errorf("Failed to upload layer: %s", err) } @@ -497,7 +483,7 @@ func (r *Registry) PushRegistryTag(remote, revision, tag, registry string, token req.Header.Add("Content-type", "application/json") setTokenAuth(req, token) req.ContentLength = int64(len(revision)) - res, err := doWithCookies(r.client, req) + res, err := r.client.Do(req) if err != nil { return err } From f6fefb0bc1c18bc4889718dccf275c4ea3a41309 Mon Sep 17 00:00:00 2001 From: "Guillaume J. Charmes" Date: Mon, 10 Mar 2014 17:16:58 -0700 Subject: [PATCH 105/375] Merge auth package within registry Docker-DCO-1.1-Signed-off-by: Guillaume J. Charmes (github: creack) --- docs/auth.go | 290 ++++++++++++++++++++++++++++++++++++++++++ docs/auth_test.go | 149 ++++++++++++++++++++++ docs/registry.go | 19 ++- docs/registry_test.go | 5 +- 4 files changed, 450 insertions(+), 13 deletions(-) create mode 100644 docs/auth.go create mode 100644 docs/auth_test.go diff --git a/docs/auth.go b/docs/auth.go new file mode 100644 index 00000000..4fdd51fd --- /dev/null +++ b/docs/auth.go @@ -0,0 +1,290 @@ +package registry + +import ( + "encoding/base64" + "encoding/json" + "errors" + "fmt" + "github.com/dotcloud/docker/utils" + "io/ioutil" + "net/http" + "os" + "path" + "strings" +) + +// Where we store the config file +const CONFIGFILE = ".dockercfg" + +// Only used for user auth + account creation +const INDEXSERVER = "https://index.docker.io/v1/" + +//const INDEXSERVER = "https://indexstaging-docker.dotcloud.com/v1/" + +var ( + ErrConfigFileMissing = errors.New("The Auth config file is missing") +) + +type AuthConfig struct { + Username string `json:"username,omitempty"` + Password string `json:"password,omitempty"` + Auth string `json:"auth"` + Email string `json:"email"` + ServerAddress string `json:"serveraddress,omitempty"` +} + +type ConfigFile struct { + Configs map[string]AuthConfig `json:"configs,omitempty"` + rootPath string +} + +func IndexServerAddress() string { + return INDEXSERVER +} + +// create a base64 encoded auth string to store in config +func encodeAuth(authConfig *AuthConfig) string { + authStr := authConfig.Username + ":" + authConfig.Password + msg := []byte(authStr) + encoded := make([]byte, base64.StdEncoding.EncodedLen(len(msg))) + base64.StdEncoding.Encode(encoded, msg) + return string(encoded) +} + +// decode the auth string +func decodeAuth(authStr string) (string, string, error) { + decLen := base64.StdEncoding.DecodedLen(len(authStr)) + decoded := make([]byte, decLen) + authByte := []byte(authStr) + n, err := base64.StdEncoding.Decode(decoded, authByte) + if err != nil { + return "", "", err + } + if n > decLen { + return "", "", fmt.Errorf("Something went wrong decoding auth config") + } + arr := strings.SplitN(string(decoded), ":", 2) + if len(arr) != 2 { + return "", "", fmt.Errorf("Invalid auth configuration file") + } + password := strings.Trim(arr[1], "\x00") + return arr[0], password, nil +} + +// load up the auth config information and return values +// FIXME: use the internal golang config parser +func LoadConfig(rootPath string) (*ConfigFile, error) { + configFile := ConfigFile{Configs: make(map[string]AuthConfig), rootPath: rootPath} + confFile := path.Join(rootPath, CONFIGFILE) + if _, err := os.Stat(confFile); err != nil { + return &configFile, nil //missing file is not an error + } + b, err := ioutil.ReadFile(confFile) + if err != nil { + return &configFile, err + } + + if err := json.Unmarshal(b, &configFile.Configs); err != nil { + arr := strings.Split(string(b), "\n") + if len(arr) < 2 { + return &configFile, fmt.Errorf("The Auth config file is empty") + } + authConfig := AuthConfig{} + origAuth := strings.Split(arr[0], " = ") + if len(origAuth) != 2 { + return &configFile, fmt.Errorf("Invalid Auth config file") + } + authConfig.Username, authConfig.Password, err = decodeAuth(origAuth[1]) + if err != nil { + return &configFile, err + } + origEmail := strings.Split(arr[1], " = ") + if len(origEmail) != 2 { + return &configFile, fmt.Errorf("Invalid Auth config file") + } + authConfig.Email = origEmail[1] + authConfig.ServerAddress = IndexServerAddress() + configFile.Configs[IndexServerAddress()] = authConfig + } else { + for k, authConfig := range configFile.Configs { + authConfig.Username, authConfig.Password, err = decodeAuth(authConfig.Auth) + if err != nil { + return &configFile, err + } + authConfig.Auth = "" + configFile.Configs[k] = authConfig + authConfig.ServerAddress = k + } + } + return &configFile, nil +} + +// save the auth config +func SaveConfig(configFile *ConfigFile) error { + confFile := path.Join(configFile.rootPath, CONFIGFILE) + if len(configFile.Configs) == 0 { + os.Remove(confFile) + return nil + } + + configs := make(map[string]AuthConfig, len(configFile.Configs)) + for k, authConfig := range configFile.Configs { + authCopy := authConfig + + authCopy.Auth = encodeAuth(&authCopy) + authCopy.Username = "" + authCopy.Password = "" + authCopy.ServerAddress = "" + configs[k] = authCopy + } + + b, err := json.Marshal(configs) + if err != nil { + return err + } + err = ioutil.WriteFile(confFile, b, 0600) + if err != nil { + return err + } + return nil +} + +// try to register/login to the registry server +func Login(authConfig *AuthConfig, factory *utils.HTTPRequestFactory) (string, error) { + var ( + status string + reqBody []byte + err error + client = &http.Client{} + reqStatusCode = 0 + serverAddress = authConfig.ServerAddress + ) + + if serverAddress == "" { + serverAddress = IndexServerAddress() + } + + loginAgainstOfficialIndex := serverAddress == IndexServerAddress() + + // to avoid sending the server address to the server it should be removed before being marshalled + authCopy := *authConfig + authCopy.ServerAddress = "" + + jsonBody, err := json.Marshal(authCopy) + if err != nil { + return "", fmt.Errorf("Config Error: %s", err) + } + + // using `bytes.NewReader(jsonBody)` here causes the server to respond with a 411 status. + b := strings.NewReader(string(jsonBody)) + req1, err := http.Post(serverAddress+"users/", "application/json; charset=utf-8", b) + if err != nil { + return "", fmt.Errorf("Server Error: %s", err) + } + reqStatusCode = req1.StatusCode + defer req1.Body.Close() + reqBody, err = ioutil.ReadAll(req1.Body) + if err != nil { + return "", fmt.Errorf("Server Error: [%#v] %s", reqStatusCode, err) + } + + if reqStatusCode == 201 { + if loginAgainstOfficialIndex { + status = "Account created. Please use the confirmation link we sent" + + " to your e-mail to activate it." + } else { + status = "Account created. Please see the documentation of the registry " + serverAddress + " for instructions how to activate it." + } + } else if reqStatusCode == 400 { + if string(reqBody) == "\"Username or email already exists\"" { + req, err := factory.NewRequest("GET", serverAddress+"users/", nil) + req.SetBasicAuth(authConfig.Username, authConfig.Password) + resp, err := client.Do(req) + if err != nil { + return "", err + } + defer resp.Body.Close() + body, err := ioutil.ReadAll(resp.Body) + if err != nil { + return "", err + } + if resp.StatusCode == 200 { + status = "Login Succeeded" + } else if resp.StatusCode == 401 { + return "", fmt.Errorf("Wrong login/password, please try again") + } else if resp.StatusCode == 403 { + if loginAgainstOfficialIndex { + return "", fmt.Errorf("Login: Account is not Active. Please check your e-mail for a confirmation link.") + } + return "", fmt.Errorf("Login: Account is not Active. Please see the documentation of the registry %s for instructions how to activate it.", serverAddress) + } else { + return "", fmt.Errorf("Login: %s (Code: %d; Headers: %s)", body, resp.StatusCode, resp.Header) + } + } else { + return "", fmt.Errorf("Registration: %s", reqBody) + } + } else if reqStatusCode == 401 { + // This case would happen with private registries where /v1/users is + // protected, so people can use `docker login` as an auth check. + req, err := factory.NewRequest("GET", serverAddress+"users/", nil) + req.SetBasicAuth(authConfig.Username, authConfig.Password) + resp, err := client.Do(req) + if err != nil { + return "", err + } + defer resp.Body.Close() + body, err := ioutil.ReadAll(resp.Body) + if err != nil { + return "", err + } + if resp.StatusCode == 200 { + status = "Login Succeeded" + } else if resp.StatusCode == 401 { + return "", fmt.Errorf("Wrong login/password, please try again") + } else { + return "", fmt.Errorf("Login: %s (Code: %d; Headers: %s)", body, + resp.StatusCode, resp.Header) + } + } else { + return "", fmt.Errorf("Unexpected status code [%d] : %s", reqStatusCode, reqBody) + } + return status, nil +} + +// this method matches a auth configuration to a server address or a url +func (config *ConfigFile) ResolveAuthConfig(hostname string) AuthConfig { + if hostname == IndexServerAddress() || len(hostname) == 0 { + // default to the index server + return config.Configs[IndexServerAddress()] + } + + // First try the happy case + if c, found := config.Configs[hostname]; found { + return c + } + + convertToHostname := func(url string) string { + stripped := url + if strings.HasPrefix(url, "http://") { + stripped = strings.Replace(url, "http://", "", 1) + } else if strings.HasPrefix(url, "https://") { + stripped = strings.Replace(url, "https://", "", 1) + } + + nameParts := strings.SplitN(stripped, "/", 2) + + return nameParts[0] + } + + // Maybe they have a legacy config file, we will iterate the keys converting + // them to the new format and testing + normalizedHostename := convertToHostname(hostname) + for registry, config := range config.Configs { + if registryHostname := convertToHostname(registry); registryHostname == normalizedHostename { + return config + } + } + + // When all else fails, return an empty auth config + return AuthConfig{} +} diff --git a/docs/auth_test.go b/docs/auth_test.go new file mode 100644 index 00000000..3cb1a9ac --- /dev/null +++ b/docs/auth_test.go @@ -0,0 +1,149 @@ +package registry + +import ( + "io/ioutil" + "os" + "testing" +) + +func TestEncodeAuth(t *testing.T) { + newAuthConfig := &AuthConfig{Username: "ken", Password: "test", Email: "test@example.com"} + authStr := encodeAuth(newAuthConfig) + decAuthConfig := &AuthConfig{} + var err error + decAuthConfig.Username, decAuthConfig.Password, err = decodeAuth(authStr) + if err != nil { + t.Fatal(err) + } + if newAuthConfig.Username != decAuthConfig.Username { + t.Fatal("Encode Username doesn't match decoded Username") + } + if newAuthConfig.Password != decAuthConfig.Password { + t.Fatal("Encode Password doesn't match decoded Password") + } + if authStr != "a2VuOnRlc3Q=" { + t.Fatal("AuthString encoding isn't correct.") + } +} + +func setupTempConfigFile() (*ConfigFile, error) { + root, err := ioutil.TempDir("", "docker-test-auth") + if err != nil { + return nil, err + } + configFile := &ConfigFile{ + rootPath: root, + Configs: make(map[string]AuthConfig), + } + + for _, registry := range []string{"testIndex", IndexServerAddress()} { + configFile.Configs[registry] = AuthConfig{ + Username: "docker-user", + Password: "docker-pass", + Email: "docker@docker.io", + } + } + + return configFile, nil +} + +func TestSameAuthDataPostSave(t *testing.T) { + configFile, err := setupTempConfigFile() + if err != nil { + t.Fatal(err) + } + defer os.RemoveAll(configFile.rootPath) + + err = SaveConfig(configFile) + if err != nil { + t.Fatal(err) + } + + authConfig := configFile.Configs["testIndex"] + if authConfig.Username != "docker-user" { + t.Fail() + } + if authConfig.Password != "docker-pass" { + t.Fail() + } + if authConfig.Email != "docker@docker.io" { + t.Fail() + } + if authConfig.Auth != "" { + t.Fail() + } +} + +func TestResolveAuthConfigIndexServer(t *testing.T) { + configFile, err := setupTempConfigFile() + if err != nil { + t.Fatal(err) + } + defer os.RemoveAll(configFile.rootPath) + + for _, registry := range []string{"", IndexServerAddress()} { + resolved := configFile.ResolveAuthConfig(registry) + if resolved != configFile.Configs[IndexServerAddress()] { + t.Fail() + } + } +} + +func TestResolveAuthConfigFullURL(t *testing.T) { + configFile, err := setupTempConfigFile() + if err != nil { + t.Fatal(err) + } + defer os.RemoveAll(configFile.rootPath) + + registryAuth := AuthConfig{ + Username: "foo-user", + Password: "foo-pass", + Email: "foo@example.com", + } + localAuth := AuthConfig{ + Username: "bar-user", + Password: "bar-pass", + Email: "bar@example.com", + } + configFile.Configs["https://registry.example.com/v1/"] = registryAuth + configFile.Configs["http://localhost:8000/v1/"] = localAuth + configFile.Configs["registry.com"] = registryAuth + + validRegistries := map[string][]string{ + "https://registry.example.com/v1/": { + "https://registry.example.com/v1/", + "http://registry.example.com/v1/", + "registry.example.com", + "registry.example.com/v1/", + }, + "http://localhost:8000/v1/": { + "https://localhost:8000/v1/", + "http://localhost:8000/v1/", + "localhost:8000", + "localhost:8000/v1/", + }, + "registry.com": { + "https://registry.com/v1/", + "http://registry.com/v1/", + "registry.com", + "registry.com/v1/", + }, + } + + for configKey, registries := range validRegistries { + for _, registry := range registries { + var ( + configured AuthConfig + ok bool + ) + resolved := configFile.ResolveAuthConfig(registry) + if configured, ok = configFile.Configs[configKey]; !ok { + t.Fail() + } + if resolved.Email != configured.Email { + t.Errorf("%s -> %q != %q\n", registry, resolved.Email, configured.Email) + } + } + } +} diff --git a/docs/registry.go b/docs/registry.go index cc2e985c..dbf5d539 100644 --- a/docs/registry.go +++ b/docs/registry.go @@ -6,7 +6,6 @@ import ( "encoding/json" "errors" "fmt" - "github.com/dotcloud/docker/auth" "github.com/dotcloud/docker/utils" "io" "io/ioutil" @@ -27,7 +26,7 @@ var ( ) func pingRegistryEndpoint(endpoint string) (bool, error) { - if endpoint == auth.IndexServerAddress() { + if endpoint == IndexServerAddress() { // Skip the check, we now this one is valid // (and we never want to fallback to http in case of error) return false, nil @@ -103,7 +102,7 @@ func ResolveRepositoryName(reposName string) (string, string, error) { nameParts[0] != "localhost" { // This is a Docker Index repos (ex: samalba/hipache or ubuntu) err := validateRepositoryName(reposName) - return auth.IndexServerAddress(), reposName, err + return IndexServerAddress(), reposName, err } if len(nameParts) < 2 { // There is a dot in repos name (and no registry address) @@ -601,7 +600,7 @@ func (r *Registry) PushImageJSONIndex(remote string, imgList []*ImgData, validat func (r *Registry) SearchRepositories(term string) (*SearchResults, error) { utils.Debugf("Index server: %s", r.indexEndpoint) - u := auth.IndexServerAddress() + "search?q=" + url.QueryEscape(term) + u := IndexServerAddress() + "search?q=" + url.QueryEscape(term) req, err := r.reqFactory.NewRequest("GET", u, nil) if err != nil { return nil, err @@ -627,12 +626,12 @@ func (r *Registry) SearchRepositories(term string) (*SearchResults, error) { return result, err } -func (r *Registry) GetAuthConfig(withPasswd bool) *auth.AuthConfig { +func (r *Registry) GetAuthConfig(withPasswd bool) *AuthConfig { password := "" if withPasswd { password = r.authConfig.Password } - return &auth.AuthConfig{ + return &AuthConfig{ Username: r.authConfig.Username, Password: password, Email: r.authConfig.Email, @@ -668,12 +667,12 @@ type ImgData struct { type Registry struct { client *http.Client - authConfig *auth.AuthConfig + authConfig *AuthConfig reqFactory *utils.HTTPRequestFactory indexEndpoint string } -func NewRegistry(authConfig *auth.AuthConfig, factory *utils.HTTPRequestFactory, indexEndpoint string) (r *Registry, err error) { +func NewRegistry(authConfig *AuthConfig, factory *utils.HTTPRequestFactory, indexEndpoint string) (r *Registry, err error) { httpTransport := &http.Transport{ DisableKeepAlives: true, Proxy: http.ProxyFromEnvironment, @@ -693,13 +692,13 @@ func NewRegistry(authConfig *auth.AuthConfig, factory *utils.HTTPRequestFactory, // If we're working with a standalone private registry over HTTPS, send Basic Auth headers // alongside our requests. - if indexEndpoint != auth.IndexServerAddress() && strings.HasPrefix(indexEndpoint, "https://") { + if indexEndpoint != IndexServerAddress() && strings.HasPrefix(indexEndpoint, "https://") { standalone, err := pingRegistryEndpoint(indexEndpoint) if err != nil { return nil, err } if standalone { - utils.Debugf("Endpoint %s is eligible for private registry auth. Enabling decorator.", indexEndpoint) + utils.Debugf("Endpoint %s is eligible for private registry registry. Enabling decorator.", indexEndpoint) dec := utils.NewHTTPAuthDecorator(authConfig.Username, authConfig.Password) factory.AddDecorator(dec) } diff --git a/docs/registry_test.go b/docs/registry_test.go index 82a27a16..f21814c7 100644 --- a/docs/registry_test.go +++ b/docs/registry_test.go @@ -1,7 +1,6 @@ package registry import ( - "github.com/dotcloud/docker/auth" "github.com/dotcloud/docker/utils" "strings" "testing" @@ -14,7 +13,7 @@ var ( ) func spawnTestRegistry(t *testing.T) *Registry { - authConfig := &auth.AuthConfig{} + authConfig := &AuthConfig{} r, err := NewRegistry(authConfig, utils.NewHTTPRequestFactory(), makeURL("/v1/")) if err != nil { t.Fatal(err) @@ -137,7 +136,7 @@ func TestResolveRepositoryName(t *testing.T) { if err != nil { t.Fatal(err) } - assertEqual(t, ep, auth.IndexServerAddress(), "Expected endpoint to be index server address") + assertEqual(t, ep, IndexServerAddress(), "Expected endpoint to be index server address") assertEqual(t, repo, "fooo/bar", "Expected resolved repo to be foo/bar") u := makeURL("")[7:] From 471d923b1bca7ee48e30ff3b07ca8c63b2cde061 Mon Sep 17 00:00:00 2001 From: Vincent Batts Date: Mon, 10 Mar 2014 16:11:03 -0400 Subject: [PATCH 106/375] registry: make certain headers optional For a pull-only, static registry, there only a couple of headers that need to be optional (that are presently required. * X-Docker-Registry-Version * X-Docker-Size * X-Docker-Endpoints Docker-DCO-1.1-Signed-off-by: Vincent Batts (github: vbatts) --- docs/registry.go | 53 ++++++++++++++++++++++++++++++++++-------------- 1 file changed, 38 insertions(+), 15 deletions(-) diff --git a/docs/registry.go b/docs/registry.go index dbf5d539..30079e9a 100644 --- a/docs/registry.go +++ b/docs/registry.go @@ -25,12 +25,8 @@ var ( errLoginRequired = errors.New("Authentication is required.") ) -func pingRegistryEndpoint(endpoint string) (bool, error) { - if endpoint == IndexServerAddress() { - // Skip the check, we now this one is valid - // (and we never want to fallback to http in case of error) - return false, nil - } +// reuse this chunk of code +func newClient() *http.Client { httpDial := func(proto string, addr string) (net.Conn, error) { // Set the connect timeout to 5 seconds conn, err := net.DialTimeout(proto, addr, time.Duration(5)*time.Second) @@ -42,17 +38,39 @@ func pingRegistryEndpoint(endpoint string) (bool, error) { return conn, nil } httpTransport := &http.Transport{Dial: httpDial} - client := &http.Client{Transport: httpTransport} + return &http.Client{Transport: httpTransport} +} + +// Have an API to access the version of the registry +func getRegistryVersion(endpoint string) (string, error) { + + client := newClient() + resp, err := client.Get(endpoint + "_version") + if err != nil { + return "", err + } + defer resp.Body.Close() + + if hdr := resp.Header.Get("X-Docker-Registry-Version"); hdr != "" { + return hdr, nil + } + versionBody, err := ioutil.ReadAll(resp.Body) + return string(versionBody), err +} + +func pingRegistryEndpoint(endpoint string) (bool, error) { + if endpoint == IndexServerAddress() { + // Skip the check, we now this one is valid + // (and we never want to fallback to http in case of error) + return false, nil + } + client := newClient() resp, err := client.Get(endpoint + "_ping") if err != nil { return false, err } defer resp.Body.Close() - if resp.Header.Get("X-Docker-Registry-Version") == "" { - return false, errors.New("This does not look like a Registry server (\"X-Docker-Registry-Version\" header not found in the response)") - } - standalone := resp.Header.Get("X-Docker-Registry-Standalone") utils.Debugf("Registry standalone header: '%s'", standalone) // If the header is absent, we assume true for compatibility with earlier @@ -223,9 +241,13 @@ func (r *Registry) GetRemoteImageJSON(imgID, registry string, token []string) ([ return nil, -1, utils.NewHTTPRequestError(fmt.Sprintf("HTTP code %d", res.StatusCode), res) } - imageSize, err := strconv.Atoi(res.Header.Get("X-Docker-Size")) - if err != nil { - return nil, -1, err + // if the size header is not present, then set it to '-1' + imageSize := -1 + if hdr := res.Header.Get("X-Docker-Size"); hdr != "" { + imageSize, err = strconv.Atoi(hdr) + if err != nil { + return nil, -1, err + } } jsonString, err := ioutil.ReadAll(res.Body) @@ -336,7 +358,8 @@ func (r *Registry) GetRepositoryData(remote string) (*RepositoryData, error) { endpoints = append(endpoints, fmt.Sprintf("%s://%s/v1/", urlScheme, ep)) } } else { - return nil, fmt.Errorf("Index response didn't contain any endpoints") + // Assume the endpoint is on the same host + endpoints = append(endpoints, fmt.Sprintf("%s://%s/v1/", urlScheme, req.URL.Host)) } checksumsJSON, err := ioutil.ReadAll(res.Body) From c18c4b8d3c28a4fa2aa91c67633f55131068d4bc Mon Sep 17 00:00:00 2001 From: Vincent Batts Date: Tue, 11 Mar 2014 23:36:51 -0400 Subject: [PATCH 107/375] registry: Info collection roll version and standalone information into the _ping. And to support Headers they are checked after the JSON is loaded (if there is anything to load). To stay backwards compatible, if the _ping contents are not able to unmarshal to RegistryInfo, do not stop, but continue with the same behavior. Docker-DCO-1.1-Signed-off-by: Vincent Batts (github: vbatts) --- docs/registry.go | 84 +++++++++++++++++++++++++----------------------- 1 file changed, 43 insertions(+), 41 deletions(-) diff --git a/docs/registry.go b/docs/registry.go index 30079e9a..6040d750 100644 --- a/docs/registry.go +++ b/docs/registry.go @@ -25,8 +25,12 @@ var ( errLoginRequired = errors.New("Authentication is required.") ) -// reuse this chunk of code -func newClient() *http.Client { +func pingRegistryEndpoint(endpoint string) (RegistryInfo, error) { + if endpoint == IndexServerAddress() { + // Skip the check, we now this one is valid + // (and we never want to fallback to http in case of error) + return RegistryInfo{Standalone: false}, nil + } httpDial := func(proto string, addr string) (net.Conn, error) { // Set the connect timeout to 5 seconds conn, err := net.DialTimeout(proto, addr, time.Duration(5)*time.Second) @@ -38,51 +42,44 @@ func newClient() *http.Client { return conn, nil } httpTransport := &http.Transport{Dial: httpDial} - return &http.Client{Transport: httpTransport} -} - -// Have an API to access the version of the registry -func getRegistryVersion(endpoint string) (string, error) { - - client := newClient() - resp, err := client.Get(endpoint + "_version") - if err != nil { - return "", err - } - defer resp.Body.Close() - - if hdr := resp.Header.Get("X-Docker-Registry-Version"); hdr != "" { - return hdr, nil - } - versionBody, err := ioutil.ReadAll(resp.Body) - return string(versionBody), err -} - -func pingRegistryEndpoint(endpoint string) (bool, error) { - if endpoint == IndexServerAddress() { - // Skip the check, we now this one is valid - // (and we never want to fallback to http in case of error) - return false, nil - } - client := newClient() + client := &http.Client{Transport: httpTransport} resp, err := client.Get(endpoint + "_ping") if err != nil { - return false, err + return RegistryInfo{Standalone: false}, err } defer resp.Body.Close() + jsonString, err := ioutil.ReadAll(resp.Body) + if err != nil { + return RegistryInfo{Standalone: false}, fmt.Errorf("Error while reading the http response: %s", err) + } + + // If the header is absent, we assume true for compatibility with earlier + // versions of the registry. default to true + info := RegistryInfo{ + Standalone: true, + } + if err := json.Unmarshal(jsonString, &info); err != nil { + utils.Debugf("Error unmarshalling the _ping RegistryInfo: %s", err) + // don't stop here. Just assume sane defaults + } + if hdr := resp.Header.Get("X-Docker-Registry-Version"); hdr != "" { + utils.Debugf("Registry version header: '%s'", hdr) + info.Version = hdr + } + utils.Debugf("RegistryInfo.Version: %q", info.Version) + standalone := resp.Header.Get("X-Docker-Registry-Standalone") utils.Debugf("Registry standalone header: '%s'", standalone) - // If the header is absent, we assume true for compatibility with earlier - // versions of the registry - if standalone == "" { - return true, nil - // Accepted values are "true" (case-insensitive) and "1". - } else if strings.EqualFold(standalone, "true") || standalone == "1" { - return true, nil + // Accepted values are "true" (case-insensitive) and "1". + if strings.EqualFold(standalone, "true") || standalone == "1" { + info.Standalone = true + } else if len(standalone) > 0 { + // there is a header set, and it is not "true" or "1", so assume fails + info.Standalone = false } - // Otherwise, not standalone - return false, nil + utils.Debugf("RegistryInfo.Standalone: %q", info.Standalone) + return info, nil } func validateRepositoryName(repositoryName string) error { @@ -688,6 +685,11 @@ type ImgData struct { Tag string `json:",omitempty"` } +type RegistryInfo struct { + Version string `json:"version"` + Standalone bool `json:"standalone"` +} + type Registry struct { client *http.Client authConfig *AuthConfig @@ -716,11 +718,11 @@ func NewRegistry(authConfig *AuthConfig, factory *utils.HTTPRequestFactory, inde // If we're working with a standalone private registry over HTTPS, send Basic Auth headers // alongside our requests. if indexEndpoint != IndexServerAddress() && strings.HasPrefix(indexEndpoint, "https://") { - standalone, err := pingRegistryEndpoint(indexEndpoint) + info, err := pingRegistryEndpoint(indexEndpoint) if err != nil { return nil, err } - if standalone { + if info.Standalone { utils.Debugf("Endpoint %s is eligible for private registry registry. Enabling decorator.", indexEndpoint) dec := utils.NewHTTPAuthDecorator(authConfig.Username, authConfig.Password) factory.AddDecorator(dec) From 47c4e542ba329f6e1324fb5f3f468b6a8d434f5b Mon Sep 17 00:00:00 2001 From: Victor Vieux Date: Thu, 13 Mar 2014 17:40:34 +0000 Subject: [PATCH 108/375] use mock for search Docker-DCO-1.1-Signed-off-by: Victor Vieux (github: vieux) --- docs/registry.go | 2 +- docs/registry_mock_test.go | 7 ++++++- docs/registry_test.go | 6 ++++-- 3 files changed, 11 insertions(+), 4 deletions(-) diff --git a/docs/registry.go b/docs/registry.go index dbf5d539..346132bc 100644 --- a/docs/registry.go +++ b/docs/registry.go @@ -600,7 +600,7 @@ func (r *Registry) PushImageJSONIndex(remote string, imgList []*ImgData, validat func (r *Registry) SearchRepositories(term string) (*SearchResults, error) { utils.Debugf("Index server: %s", r.indexEndpoint) - u := IndexServerAddress() + "search?q=" + url.QueryEscape(term) + u := r.indexEndpoint + "search?q=" + url.QueryEscape(term) req, err := r.reqFactory.NewRequest("GET", u, nil) if err != nil { return nil, err diff --git a/docs/registry_mock_test.go b/docs/registry_mock_test.go index 6eb94b63..dd5da6bd 100644 --- a/docs/registry_mock_test.go +++ b/docs/registry_mock_test.go @@ -321,7 +321,12 @@ func handlerAuth(w http.ResponseWriter, r *http.Request) { } func handlerSearch(w http.ResponseWriter, r *http.Request) { - writeResponse(w, "{}", 200) + result := &SearchResults{ + Query: "fakequery", + NumResults: 1, + Results: []SearchResult{{Name: "fakeimage", StarCount: 42}}, + } + writeResponse(w, result, 200) } func TestPing(t *testing.T) { diff --git a/docs/registry_test.go b/docs/registry_test.go index f21814c7..ebfb99b4 100644 --- a/docs/registry_test.go +++ b/docs/registry_test.go @@ -186,14 +186,16 @@ func TestPushImageJSONIndex(t *testing.T) { func TestSearchRepositories(t *testing.T) { r := spawnTestRegistry(t) - results, err := r.SearchRepositories("supercalifragilisticepsialidocious") + results, err := r.SearchRepositories("fakequery") if err != nil { t.Fatal(err) } if results == nil { t.Fatal("Expected non-nil SearchResults object") } - assertEqual(t, results.NumResults, 0, "Expected 0 search results") + assertEqual(t, results.NumResults, 1, "Expected 1 search results") + assertEqual(t, results.Query, "fakequery", "Expected 'fakequery' as query") + assertEqual(t, results.Results[0].StarCount, 42, "Expected 'fakeimage' a ot hae 42 stars") } func TestValidRepositoryName(t *testing.T) { From 9bad706a1ffd5e5b21088e1dc8b1e29fe140f030 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Djibril=20Kon=C3=A9?= Date: Fri, 21 Mar 2014 00:40:58 +0100 Subject: [PATCH 109/375] Harmonize / across all name-related commands/Validate images names MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Docker-DCO-1.1-Signed-off-by: Djibril Koné (github: enokd) --- docs/registry_test.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/docs/registry_test.go b/docs/registry_test.go index ebfb99b4..c072da41 100644 --- a/docs/registry_test.go +++ b/docs/registry_test.go @@ -206,4 +206,8 @@ func TestValidRepositoryName(t *testing.T) { t.Log("Repository name should be invalid") t.Fail() } + if err := validateRepositoryName("docker///docker"); err == nil { + t.Log("Repository name should be invalid") + t.Fail() + } } From fffa920a895aa81b9d56b36474d98af1d3fbf39a Mon Sep 17 00:00:00 2001 From: Ryan Thomas Date: Tue, 25 Mar 2014 14:45:11 +1100 Subject: [PATCH 110/375] Docker-DCO-1.1-Signed-off-by: Ryan Thomas (github: rthomas) --- docs/registry.go | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/docs/registry.go b/docs/registry.go index 346132bc..01583f97 100644 --- a/docs/registry.go +++ b/docs/registry.go @@ -41,7 +41,10 @@ func pingRegistryEndpoint(endpoint string) (bool, error) { conn.SetDeadline(time.Now().Add(time.Duration(10) * time.Second)) return conn, nil } - httpTransport := &http.Transport{Dial: httpDial} + httpTransport := &http.Transport{ + Dial: httpDial, + Proxy: http.ProxyFromEnvironment, + } client := &http.Client{Transport: httpTransport} resp, err := client.Get(endpoint + "_ping") if err != nil { From 50ec0bbd4e5973cd42f8a61b0d5e8ca5e3a1fc71 Mon Sep 17 00:00:00 2001 From: Ryan Thomas Date: Fri, 28 Mar 2014 06:31:04 +1100 Subject: [PATCH 111/375] Docker-DCO-1.1-Signed-off-by: Ryan Thomas (github: rthomas) --- docs/registry.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/registry.go b/docs/registry.go index 01583f97..182ec78a 100644 --- a/docs/registry.go +++ b/docs/registry.go @@ -42,9 +42,9 @@ func pingRegistryEndpoint(endpoint string) (bool, error) { return conn, nil } httpTransport := &http.Transport{ - Dial: httpDial, - Proxy: http.ProxyFromEnvironment, - } + Dial: httpDial, + Proxy: http.ProxyFromEnvironment, + } client := &http.Client{Transport: httpTransport} resp, err := client.Get(endpoint + "_ping") if err != nil { From d2b2bf039386b25ed82dcb08649fb9532a44a02f Mon Sep 17 00:00:00 2001 From: Sam Alba Date: Mon, 31 Mar 2014 17:56:25 -0700 Subject: [PATCH 112/375] Inverted layer checksum and tarsum. The checksum of the payload has to be computed on the Gzip'ed content. Docker-DCO-1.1-Signed-off-by: Sam Alba (github: samalba) --- docs/registry.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/registry.go b/docs/registry.go index 182ec78a..414283b8 100644 --- a/docs/registry.go +++ b/docs/registry.go @@ -438,10 +438,10 @@ func (r *Registry) PushImageLayerRegistry(imgID string, layer io.Reader, registr utils.Debugf("[registry] Calling PUT %s", registry+"images/"+imgID+"/layer") h := sha256.New() - checksumLayer := &utils.CheckSum{Reader: layer, Hash: h} - tarsumLayer := &utils.TarSum{Reader: checksumLayer} + tarsumLayer := &utils.TarSum{Reader: layer} + checksumLayer := &utils.CheckSum{Reader: tarsumLayer, Hash: h} - req, err := r.reqFactory.NewRequest("PUT", registry+"images/"+imgID+"/layer", tarsumLayer) + req, err := r.reqFactory.NewRequest("PUT", registry+"images/"+imgID+"/layer", checksumLayer) if err != nil { return "", "", err } From 4f29181d9b516e006896bc0df8bc92a0e99b701a Mon Sep 17 00:00:00 2001 From: Sam Alba Date: Mon, 31 Mar 2014 18:31:15 -0700 Subject: [PATCH 113/375] Payload checksum now match the checksum simple Backported for backward compatibility. Docker-DCO-1.1-Signed-off-by: Sam Alba (github: samalba) --- docs/registry.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/docs/registry.go b/docs/registry.go index 414283b8..5ac04f9e 100644 --- a/docs/registry.go +++ b/docs/registry.go @@ -437,8 +437,10 @@ func (r *Registry) PushImageLayerRegistry(imgID string, layer io.Reader, registr utils.Debugf("[registry] Calling PUT %s", registry+"images/"+imgID+"/layer") - h := sha256.New() tarsumLayer := &utils.TarSum{Reader: layer} + h := sha256.New() + h.Write(jsonRaw) + h.Write([]byte{'\n'}) checksumLayer := &utils.CheckSum{Reader: tarsumLayer, Hash: h} req, err := r.reqFactory.NewRequest("PUT", registry+"images/"+imgID+"/layer", checksumLayer) From dbb929653108f390a3023bea6e2028a20478ba1c Mon Sep 17 00:00:00 2001 From: shin- Date: Tue, 8 Apr 2014 16:53:16 +0200 Subject: [PATCH 114/375] Added specific error message when hitting 401 over HTTP on push Docker-DCO-1.1-Signed-off-by: Joffrey F (github: shin-) --- docs/registry.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/docs/registry.go b/docs/registry.go index 5ac04f9e..817c08af 100644 --- a/docs/registry.go +++ b/docs/registry.go @@ -417,6 +417,9 @@ func (r *Registry) PushImageJSONRegistry(imgData *ImgData, jsonRaw []byte, regis return fmt.Errorf("Failed to upload metadata: %s", err) } defer res.Body.Close() + if res.StatusCode == 401 && strings.HasPrefix(registry, "http://") { + return utils.NewHTTPRequestError("HTTP code 401, Docker will not send auth headers over HTTP.", res) + } if res.StatusCode != 200 { errBody, err := ioutil.ReadAll(res.Body) if err != nil { From 4bc3522500d5c3f22e00e7c97f6e844c2bd5bb21 Mon Sep 17 00:00:00 2001 From: Victor Vieux Date: Mon, 14 Apr 2014 23:15:38 +0000 Subject: [PATCH 115/375] allow dot in repo name Docker-DCO-1.1-Signed-off-by: Victor Vieux (github: vieux) --- docs/registry.go | 9 ++------- docs/registry_test.go | 7 +++++++ 2 files changed, 9 insertions(+), 7 deletions(-) diff --git a/docs/registry.go b/docs/registry.go index 817c08af..451f30f6 100644 --- a/docs/registry.go +++ b/docs/registry.go @@ -101,17 +101,12 @@ func ResolveRepositoryName(reposName string) (string, string, error) { return "", "", ErrInvalidRepositoryName } nameParts := strings.SplitN(reposName, "/", 2) - if !strings.Contains(nameParts[0], ".") && !strings.Contains(nameParts[0], ":") && - nameParts[0] != "localhost" { + if len(nameParts) == 1 || (!strings.Contains(nameParts[0], ".") && !strings.Contains(nameParts[0], ":") && + nameParts[0] != "localhost") { // This is a Docker Index repos (ex: samalba/hipache or ubuntu) err := validateRepositoryName(reposName) return IndexServerAddress(), reposName, err } - if len(nameParts) < 2 { - // There is a dot in repos name (and no registry address) - // Is it a Registry address without repos name? - return "", "", ErrInvalidRepositoryName - } hostname := nameParts[0] reposName = nameParts[1] if strings.Contains(hostname, "index.docker.io") { diff --git a/docs/registry_test.go b/docs/registry_test.go index c072da41..cb56502f 100644 --- a/docs/registry_test.go +++ b/docs/registry_test.go @@ -146,6 +146,13 @@ func TestResolveRepositoryName(t *testing.T) { } assertEqual(t, ep, u, "Expected endpoint to be "+u) assertEqual(t, repo, "private/moonbase", "Expected endpoint to be private/moonbase") + + ep, repo, err = ResolveRepositoryName("ubuntu-12.04-base") + if err != nil { + t.Fatal(err) + } + assertEqual(t, ep, IndexServerAddress(), "Expected endpoint to be "+IndexServerAddress()) + assertEqual(t, repo, "ubuntu-12.04-base", "Expected endpoint to be ubuntu-12.04-base") } func TestPushRegistryTag(t *testing.T) { From 52893cae738b64dee20a435a68ab37cc4b84b9a8 Mon Sep 17 00:00:00 2001 From: shin- Date: Mon, 14 Apr 2014 20:32:47 +0200 Subject: [PATCH 116/375] Added support for multiple endpoints in X-Docker-Endpoints header Docker-DCO-1.1-Signed-off-by: Joffrey F (github: shin-) --- docs/registry.go | 33 +++++++++++++++++++++++++-------- docs/registry_mock_test.go | 2 +- docs/registry_test.go | 15 ++++++++++++++- 3 files changed, 40 insertions(+), 10 deletions(-) diff --git a/docs/registry.go b/docs/registry.go index 817c08af..3656032e 100644 --- a/docs/registry.go +++ b/docs/registry.go @@ -297,6 +297,25 @@ func (r *Registry) GetRemoteTags(registries []string, repository string, token [ return nil, fmt.Errorf("Could not reach any registry endpoint") } +func buildEndpointsList(headers []string, indexEp string) ([]string, error) { + var endpoints []string + parsedUrl, err := url.Parse(indexEp) + if err != nil { + return nil, err + } + var urlScheme = parsedUrl.Scheme + // The Registry's URL scheme has to match the Index' + for _, ep := range headers { + epList := strings.Split(ep, ",") + for _, epListElement := range epList { + endpoints = append( + endpoints, + fmt.Sprintf("%s://%s/v1/", urlScheme, strings.TrimSpace(epListElement))) + } + } + return endpoints, nil +} + func (r *Registry) GetRepositoryData(remote string) (*RepositoryData, error) { indexEp := r.indexEndpoint repositoryTarget := fmt.Sprintf("%srepositories/%s/images", indexEp, remote) @@ -332,11 +351,10 @@ func (r *Registry) GetRepositoryData(remote string) (*RepositoryData, error) { } var endpoints []string - var urlScheme = indexEp[:strings.Index(indexEp, ":")] if res.Header.Get("X-Docker-Endpoints") != "" { - // The Registry's URL scheme has to match the Index' - for _, ep := range res.Header["X-Docker-Endpoints"] { - endpoints = append(endpoints, fmt.Sprintf("%s://%s/v1/", urlScheme, ep)) + endpoints, err = buildEndpointsList(res.Header["X-Docker-Endpoints"], indexEp) + if err != nil { + return nil, err } } else { return nil, fmt.Errorf("Index response didn't contain any endpoints") @@ -565,7 +583,6 @@ func (r *Registry) PushImageJSONIndex(remote string, imgList []*ImgData, validat } var tokens, endpoints []string - var urlScheme = indexEp[:strings.Index(indexEp, ":")] if !validate { if res.StatusCode != 200 && res.StatusCode != 201 { errBody, err := ioutil.ReadAll(res.Body) @@ -582,9 +599,9 @@ func (r *Registry) PushImageJSONIndex(remote string, imgList []*ImgData, validat } if res.Header.Get("X-Docker-Endpoints") != "" { - // The Registry's URL scheme has to match the Index' - for _, ep := range res.Header["X-Docker-Endpoints"] { - endpoints = append(endpoints, fmt.Sprintf("%s://%s/v1/", urlScheme, ep)) + endpoints, err = buildEndpointsList(res.Header["X-Docker-Endpoints"], indexEp) + if err != nil { + return nil, err } } else { return nil, fmt.Errorf("Index response didn't contain any endpoints") diff --git a/docs/registry_mock_test.go b/docs/registry_mock_test.go index dd5da6bd..6b007513 100644 --- a/docs/registry_mock_test.go +++ b/docs/registry_mock_test.go @@ -291,7 +291,7 @@ func handlerUsers(w http.ResponseWriter, r *http.Request) { func handlerImages(w http.ResponseWriter, r *http.Request) { u, _ := url.Parse(testHttpServer.URL) - w.Header().Add("X-Docker-Endpoints", u.Host) + w.Header().Add("X-Docker-Endpoints", fmt.Sprintf("%s , %s ", u.Host, "test.example.com")) w.Header().Add("X-Docker-Token", fmt.Sprintf("FAKE-SESSION-%d", time.Now().UnixNano())) if r.Method == "PUT" { if strings.HasSuffix(r.URL.Path, "images") { diff --git a/docs/registry_test.go b/docs/registry_test.go index c072da41..ad64fb1f 100644 --- a/docs/registry_test.go +++ b/docs/registry_test.go @@ -1,7 +1,9 @@ package registry import ( + "fmt" "github.com/dotcloud/docker/utils" + "net/url" "strings" "testing" ) @@ -99,12 +101,23 @@ func TestGetRemoteTags(t *testing.T) { func TestGetRepositoryData(t *testing.T) { r := spawnTestRegistry(t) + parsedUrl, err := url.Parse(makeURL("/v1/")) + if err != nil { + t.Fatal(err) + } + host := "http://" + parsedUrl.Host + "/v1/" data, err := r.GetRepositoryData("foo42/bar") if err != nil { t.Fatal(err) } assertEqual(t, len(data.ImgList), 2, "Expected 2 images in ImgList") - assertEqual(t, len(data.Endpoints), 1, "Expected one endpoint in Endpoints") + assertEqual(t, len(data.Endpoints), 2, + fmt.Sprintf("Expected 2 endpoints in Endpoints, found %d instead", len(data.Endpoints))) + assertEqual(t, data.Endpoints[0], host, + fmt.Sprintf("Expected first endpoint to be %s but found %s instead", host, data.Endpoints[0])) + assertEqual(t, data.Endpoints[1], "http://test.example.com/v1/", + fmt.Sprintf("Expected first endpoint to be http://test.example.com/v1/ but found %s instead", data.Endpoints[1])) + } func TestPushImageJSONRegistry(t *testing.T) { From 2b89f57964786c91b5cfe5c7983e6b2ecb8570f7 Mon Sep 17 00:00:00 2001 From: Vincent Batts Date: Fri, 25 Apr 2014 20:01:25 -0400 Subject: [PATCH 117/375] static_registry: update the test for the new struct Docker-DCO-1.1-Signed-off-by: Vincent Batts (github: vbatts) --- docs/registry_test.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/registry_test.go b/docs/registry_test.go index f21814c7..f53345c1 100644 --- a/docs/registry_test.go +++ b/docs/registry_test.go @@ -22,11 +22,11 @@ func spawnTestRegistry(t *testing.T) *Registry { } func TestPingRegistryEndpoint(t *testing.T) { - standalone, err := pingRegistryEndpoint(makeURL("/v1/")) + regInfo, err := pingRegistryEndpoint(makeURL("/v1/")) if err != nil { t.Fatal(err) } - assertEqual(t, standalone, true, "Expected standalone to be true (default)") + assertEqual(t, regInfo.Standalone, true, "Expected standalone to be true (default)") } func TestGetRemoteHistory(t *testing.T) { From 3e064ac71cb318a3a60c2e058ecc7852fe50c208 Mon Sep 17 00:00:00 2001 From: Michael Crosby Date: Tue, 29 Apr 2014 02:01:07 -0700 Subject: [PATCH 118/375] Use proper scheme with static registry Docker-DCO-1.1-Signed-off-by: Michael Crosby (github: crosbymichael) --- docs/registry.go | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/docs/registry.go b/docs/registry.go index 88defdc7..1bd73cde 100644 --- a/docs/registry.go +++ b/docs/registry.go @@ -6,7 +6,6 @@ import ( "encoding/json" "errors" "fmt" - "github.com/dotcloud/docker/utils" "io" "io/ioutil" "net" @@ -17,6 +16,8 @@ import ( "strconv" "strings" "time" + + "github.com/dotcloud/docker/utils" ) var ( @@ -372,7 +373,11 @@ func (r *Registry) GetRepositoryData(remote string) (*RepositoryData, error) { } } else { // Assume the endpoint is on the same host - endpoints = append(endpoints, fmt.Sprintf("%s://%s/v1/", urlScheme, req.URL.Host)) + u, err := url.Parse(indexEp) + if err != nil { + return nil, err + } + endpoints = append(endpoints, fmt.Sprintf("%s://%s/v1/", u.Scheme, req.URL.Host)) } checksumsJSON, err := ioutil.ReadAll(res.Body) From 8934560bbc1212ea1c76fd8642985f0ad96fc935 Mon Sep 17 00:00:00 2001 From: Solomon Hykes Date: Sun, 27 Apr 2014 15:06:09 -0700 Subject: [PATCH 119/375] Move 'auth' to the registry subsystem This is the first step towards separating the registry subsystem from the deprecated `Server` object. * New service `github.com/dotcloud/docker/registry/Service` * The service is installed by default in `builtins` * The service only exposes `auth` for now... * ...Soon to be followed by `pull`, `push` and `search`. Docker-DCO-1.1-Signed-off-by: Solomon Hykes (github: shykes) --- docs/registry.go | 39 ++++++++++++++++++++++++++++++++++ docs/service.go | 54 ++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 93 insertions(+) create mode 100644 docs/service.go diff --git a/docs/registry.go b/docs/registry.go index 1bd73cde..55154e36 100644 --- a/docs/registry.go +++ b/docs/registry.go @@ -13,10 +13,12 @@ import ( "net/http/cookiejar" "net/url" "regexp" + "runtime" "strconv" "strings" "time" + "github.com/dotcloud/docker/dockerversion" "github.com/dotcloud/docker/utils" ) @@ -757,3 +759,40 @@ func NewRegistry(authConfig *AuthConfig, factory *utils.HTTPRequestFactory, inde r.reqFactory = factory return r, nil } + +func HTTPRequestFactory(metaHeaders map[string][]string) *utils.HTTPRequestFactory { + // FIXME: this replicates the 'info' job. + httpVersion := make([]utils.VersionInfo, 0, 4) + httpVersion = append(httpVersion, &simpleVersionInfo{"docker", dockerversion.VERSION}) + httpVersion = append(httpVersion, &simpleVersionInfo{"go", runtime.Version()}) + httpVersion = append(httpVersion, &simpleVersionInfo{"git-commit", dockerversion.GITCOMMIT}) + if kernelVersion, err := utils.GetKernelVersion(); err == nil { + httpVersion = append(httpVersion, &simpleVersionInfo{"kernel", kernelVersion.String()}) + } + httpVersion = append(httpVersion, &simpleVersionInfo{"os", runtime.GOOS}) + httpVersion = append(httpVersion, &simpleVersionInfo{"arch", runtime.GOARCH}) + ud := utils.NewHTTPUserAgentDecorator(httpVersion...) + md := &utils.HTTPMetaHeadersDecorator{ + Headers: metaHeaders, + } + factory := utils.NewHTTPRequestFactory(ud, md) + return factory +} + +// simpleVersionInfo is a simple implementation of +// the interface VersionInfo, which is used +// to provide version information for some product, +// component, etc. It stores the product name and the version +// in string and returns them on calls to Name() and Version(). +type simpleVersionInfo struct { + name string + version string +} + +func (v *simpleVersionInfo) Name() string { + return v.name +} + +func (v *simpleVersionInfo) Version() string { + return v.version +} diff --git a/docs/service.go b/docs/service.go new file mode 100644 index 00000000..530a7f7a --- /dev/null +++ b/docs/service.go @@ -0,0 +1,54 @@ +package registry + +import ( + "github.com/dotcloud/docker/engine" +) + +// Service exposes registry capabilities in the standard Engine +// interface. Once installed, it extends the engine with the +// following calls: +// +// 'auth': Authenticate against the public registry +// 'search': Search for images on the public registry (TODO) +// 'pull': Download images from any registry (TODO) +// 'push': Upload images to any registry (TODO) +type Service struct { +} + +// NewService returns a new instance of Service ready to be +// installed no an engine. +func NewService() *Service { + return &Service{} +} + +// Install installs registry capabilities to eng. +func (s *Service) Install(eng *engine.Engine) error { + eng.Register("auth", s.Auth) + return nil +} + +// Auth contacts the public registry with the provided credentials, +// and returns OK if authentication was sucessful. +// It can be used to verify the validity of a client's credentials. +func (s *Service) Auth(job *engine.Job) engine.Status { + var ( + err error + authConfig = &AuthConfig{} + ) + + job.GetenvJson("authConfig", authConfig) + // TODO: this is only done here because auth and registry need to be merged into one pkg + if addr := authConfig.ServerAddress; addr != "" && addr != IndexServerAddress() { + addr, err = ExpandAndVerifyRegistryUrl(addr) + if err != nil { + return job.Error(err) + } + authConfig.ServerAddress = addr + } + status, err := Login(authConfig, HTTPRequestFactory(nil)) + if err != nil { + return job.Error(err) + } + job.Printf("%s\n", status) + return engine.StatusOK +} From bbebff75b665c8bc632194d383503e1435d68011 Mon Sep 17 00:00:00 2001 From: Solomon Hykes Date: Sun, 27 Apr 2014 15:21:42 -0700 Subject: [PATCH 120/375] Move 'search' to the registry subsystem This continues the effort to separate all registry logic from the deprecated `Server` object. * 'search' is exposed by `github.com/dotcloud/docker/registry/Service` * Added proper documentation of Search while I was at it Docker-DCO-1.1-Signed-off-by: Solomon Hykes (github: shykes) --- docs/service.go | 52 ++++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 51 insertions(+), 1 deletion(-) diff --git a/docs/service.go b/docs/service.go index 530a7f7a..1c7a93de 100644 --- a/docs/service.go +++ b/docs/service.go @@ -9,7 +9,7 @@ import ( // following calls: // // 'auth': Authenticate against the public registry -// 'search': Search for images on the public registry (TODO) +// 'search': Search for images on the public registry // 'pull': Download images from any registry (TODO) // 'push': Upload images to any registry (TODO) type Service struct { @@ -24,6 +24,7 @@ func NewService() *Service { // Install installs registry capabilities to eng. func (s *Service) Install(eng *engine.Engine) error { eng.Register("auth", s.Auth) + eng.Register("search", s.Search) return nil } @@ -52,3 +53,52 @@ func (s *Service) Auth(job *engine.Job) engine.Status { job.Printf("%s\n", status) return engine.StatusOK } + +// Search queries the public registry for images matching the specified +// search terms, and returns the results. +// +// Argument syntax: search TERM +// +// Option environment: +// 'authConfig': json-encoded credentials to authenticate against the registry. +// The search extends to images only accessible via the credentials. +// +// 'metaHeaders': extra HTTP headers to include in the request to the registry. +// The headers should be passed as a json-encoded dictionary. +// +// Output: +// Results are sent as a collection of structured messages (using engine.Table). +// Each result is sent as a separate message. +// Results are ordered by number of stars on the public registry. +func (s *Service) Search(job *engine.Job) engine.Status { + if n := len(job.Args); n != 1 { + return job.Errorf("Usage: %s TERM", job.Name) + } + var ( + term = job.Args[0] + metaHeaders = map[string][]string{} + authConfig = &AuthConfig{} + ) + job.GetenvJson("authConfig", authConfig) + job.GetenvJson("metaHeaders", metaHeaders) + + r, err := NewRegistry(authConfig, HTTPRequestFactory(metaHeaders), IndexServerAddress()) + if err != nil { + return job.Error(err) + } + results, err := r.SearchRepositories(term) + if err != nil { + return job.Error(err) + } + outs := engine.NewTable("star_count", 0) + for _, result := range results.Results { + out := &engine.Env{} + out.Import(result) + outs.Add(out) + } + outs.ReverseSort() + if _, err := outs.WriteListTo(job.Stdout); err != nil { + return job.Error(err) + } + return engine.StatusOK +} From f293adf7f9b6077de409faedb135f5643fb7073b Mon Sep 17 00:00:00 2001 From: unclejack Date: Mon, 5 May 2014 20:29:20 +0300 Subject: [PATCH 121/375] import sha512 to make sha512 ssl certs work Docker-DCO-1.1-Signed-off-by: Cristian Staretu (github: unclejack) --- docs/registry.go | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/registry.go b/docs/registry.go index 1bd73cde..28b28c2b 100644 --- a/docs/registry.go +++ b/docs/registry.go @@ -3,6 +3,7 @@ package registry import ( "bytes" "crypto/sha256" + _ "crypto/sha512" "encoding/json" "errors" "fmt" From a9a754dad19368fa02f49f822b31358cc898f2f1 Mon Sep 17 00:00:00 2001 From: Vincent Batts Date: Tue, 6 May 2014 14:31:47 -0400 Subject: [PATCH 122/375] registry: adding vbatts to the MAINTAINERS Docker-DCO-1.1-Signed-off-by: Vincent Batts (github: vbatts) --- docs/MAINTAINERS | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/MAINTAINERS b/docs/MAINTAINERS index bf3984f5..af791fb4 100644 --- a/docs/MAINTAINERS +++ b/docs/MAINTAINERS @@ -1,3 +1,4 @@ Sam Alba (@samalba) Joffrey Fuhrer (@shin-) Ken Cochrane (@kencochrane) +Vincent Batts (@vbatts) From 3a21f339f1637aded4121715555c0e5fc7269f0e Mon Sep 17 00:00:00 2001 From: Derek Date: Thu, 22 May 2014 23:58:56 -0700 Subject: [PATCH 123/375] Use Timeout Conn wrapper to set read deadline for downloading layer Docker-DCO-1.1-Signed-off-by: Derek (github: crquan) --- docs/registry.go | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/docs/registry.go b/docs/registry.go index 2e3e7e03..3d0a3ed2 100644 --- a/docs/registry.go +++ b/docs/registry.go @@ -726,7 +726,17 @@ type Registry struct { } func NewRegistry(authConfig *AuthConfig, factory *utils.HTTPRequestFactory, indexEndpoint string) (r *Registry, err error) { + httpDial := func(proto string, addr string) (net.Conn, error) { + conn, err := net.Dial(proto, addr) + if err != nil { + return nil, err + } + conn = utils.NewTimeoutConn(conn, time.Duration(1)*time.Minute) + return conn, nil + } + httpTransport := &http.Transport{ + Dial: httpDial, DisableKeepAlives: true, Proxy: http.ProxyFromEnvironment, } @@ -738,6 +748,7 @@ func NewRegistry(authConfig *AuthConfig, factory *utils.HTTPRequestFactory, inde }, indexEndpoint: indexEndpoint, } + r.client.Jar, err = cookiejar.New(nil) if err != nil { return nil, err From 96412d40fd7bfbd47041f6b5a7805cd66bb4982c Mon Sep 17 00:00:00 2001 From: unclejack Date: Wed, 26 Mar 2014 02:33:17 +0200 Subject: [PATCH 124/375] resume pulling the layer on disconnect Docker-DCO-1.1-Signed-off-by: Cristian Staretu (github: unclejack) --- docs/registry.go | 45 ++++++++++++++++++++++++++++++++++++++++--- docs/registry_test.go | 4 ++-- 2 files changed, 44 insertions(+), 5 deletions(-) diff --git a/docs/registry.go b/docs/registry.go index 3d0a3ed2..7bcf0660 100644 --- a/docs/registry.go +++ b/docs/registry.go @@ -256,12 +256,43 @@ func (r *Registry) GetRemoteImageJSON(imgID, registry string, token []string) ([ return jsonString, imageSize, nil } -func (r *Registry) GetRemoteImageLayer(imgID, registry string, token []string) (io.ReadCloser, error) { - req, err := r.reqFactory.NewRequest("GET", registry+"images/"+imgID+"/layer", nil) +func (r *Registry) GetRemoteImageLayer(imgID, registry string, token []string, imgSize int64) (io.ReadCloser, error) { + var ( + retries = 5 + headRes *http.Response + hasResume bool = false + imageURL = fmt.Sprintf("%simages/%s/layer", registry, imgID) + ) + headReq, err := r.reqFactory.NewRequest("HEAD", imageURL, nil) + if err != nil { + return nil, fmt.Errorf("Error while getting from the server: %s\n", err) + } + setTokenAuth(headReq, token) + for i := 1; i <= retries; i++ { + headRes, err = r.client.Do(headReq) + if err != nil && i == retries { + return nil, fmt.Errorf("Eror while making head request: %s\n", err) + } else if err != nil { + time.Sleep(time.Duration(i) * 5 * time.Second) + continue + } + break + } + + if headRes.Header.Get("Accept-Ranges") == "bytes" && imgSize > 0 { + hasResume = true + } + + req, err := r.reqFactory.NewRequest("GET", imageURL, nil) if err != nil { return nil, fmt.Errorf("Error while getting from the server: %s\n", err) } setTokenAuth(req, token) + if hasResume { + utils.Debugf("server supports resume") + return utils.ResumableRequestReader(r.client, req, 5, imgSize), nil + } + utils.Debugf("server doesn't support resume") res, err := r.client.Do(req) if err != nil { return nil, err @@ -725,6 +756,13 @@ type Registry struct { indexEndpoint string } +func AddRequiredHeadersToRedirectedRequests(req *http.Request, via []*http.Request) error { + if via != nil && via[0] != nil { + req.Header = via[0].Header + } + return nil +} + func NewRegistry(authConfig *AuthConfig, factory *utils.HTTPRequestFactory, indexEndpoint string) (r *Registry, err error) { httpDial := func(proto string, addr string) (net.Conn, error) { conn, err := net.Dial(proto, addr) @@ -744,7 +782,8 @@ func NewRegistry(authConfig *AuthConfig, factory *utils.HTTPRequestFactory, inde r = &Registry{ authConfig: authConfig, client: &http.Client{ - Transport: httpTransport, + Transport: httpTransport, + CheckRedirect: AddRequiredHeadersToRedirectedRequests, }, indexEndpoint: indexEndpoint, } diff --git a/docs/registry_test.go b/docs/registry_test.go index 0a5be5e5..e207359e 100644 --- a/docs/registry_test.go +++ b/docs/registry_test.go @@ -70,7 +70,7 @@ func TestGetRemoteImageJSON(t *testing.T) { func TestGetRemoteImageLayer(t *testing.T) { r := spawnTestRegistry(t) - data, err := r.GetRemoteImageLayer(IMAGE_ID, makeURL("/v1/"), TOKEN) + data, err := r.GetRemoteImageLayer(IMAGE_ID, makeURL("/v1/"), TOKEN, 0) if err != nil { t.Fatal(err) } @@ -78,7 +78,7 @@ func TestGetRemoteImageLayer(t *testing.T) { t.Fatal("Expected non-nil data result") } - _, err = r.GetRemoteImageLayer("abcdef", makeURL("/v1/"), TOKEN) + _, err = r.GetRemoteImageLayer("abcdef", makeURL("/v1/"), TOKEN, 0) if err == nil { t.Fatal("Expected image not found error") } From 0ac3b3981fc85fcad9ce6c44531bc61ec746990f Mon Sep 17 00:00:00 2001 From: Victor Vieux Date: Tue, 3 Jun 2014 00:46:06 +0000 Subject: [PATCH 125/375] Add redirect and env proxy support to docker login Docker-DCO-1.1-Signed-off-by: Victor Vieux (github: vieux) --- docs/auth.go | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) diff --git a/docs/auth.go b/docs/auth.go index 4fdd51fd..7384efba 100644 --- a/docs/auth.go +++ b/docs/auth.go @@ -5,12 +5,13 @@ import ( "encoding/json" "errors" "fmt" - "github.com/dotcloud/docker/utils" "io/ioutil" "net/http" "os" "path" "strings" + + "github.com/dotcloud/docker/utils" ) // Where we store the config file @@ -152,10 +153,16 @@ func SaveConfig(configFile *ConfigFile) error { // try to register/login to the registry server func Login(authConfig *AuthConfig, factory *utils.HTTPRequestFactory) (string, error) { var ( - status string - reqBody []byte - err error - client = &http.Client{} + status string + reqBody []byte + err error + client = &http.Client{ + Transport: &http.Transport{ + DisableKeepAlives: true, + Proxy: http.ProxyFromEnvironment, + }, + CheckRedirect: AddRequiredHeadersToRedirectedRequests, + } reqStatusCode = 0 serverAddress = authConfig.ServerAddress ) From 8e8ffacf49a1c9128b56eeb2bfd3e5d20e8d67d8 Mon Sep 17 00:00:00 2001 From: Victor Vieux Date: Thu, 5 Jun 2014 18:37:37 +0000 Subject: [PATCH 126/375] only forward auth to trusted locations Docker-DCO-1.1-Signed-off-by: Victor Vieux (github: vieux) --- docs/registry.go | 29 +++++++++++++++++- docs/registry_test.go | 71 ++++++++++++++++++++++++++++++++++++++++++- 2 files changed, 98 insertions(+), 2 deletions(-) diff --git a/docs/registry.go b/docs/registry.go index 7bcf0660..8d1a9f22 100644 --- a/docs/registry.go +++ b/docs/registry.go @@ -756,9 +756,36 @@ type Registry struct { indexEndpoint string } +func trustedLocation(req *http.Request) bool { + var ( + trusteds = []string{"docker.com", "docker.io"} + hostname = strings.SplitN(req.Host, ":", 2)[0] + ) + if req.URL.Scheme != "https" { + return false + } + + for _, trusted := range trusteds { + if strings.HasSuffix(hostname, trusted) { + return true + } + } + return false +} + func AddRequiredHeadersToRedirectedRequests(req *http.Request, via []*http.Request) error { if via != nil && via[0] != nil { - req.Header = via[0].Header + if trustedLocation(req) && trustedLocation(via[0]) { + req.Header = via[0].Header + } else { + for k, v := range via[0].Header { + if k != "Authorization" { + for _, vv := range v { + req.Header.Add(k, vv) + } + } + } + } } return nil } diff --git a/docs/registry_test.go b/docs/registry_test.go index e207359e..2857ab4a 100644 --- a/docs/registry_test.go +++ b/docs/registry_test.go @@ -2,10 +2,12 @@ package registry import ( "fmt" - "github.com/dotcloud/docker/utils" + "net/http" "net/url" "strings" "testing" + + "github.com/dotcloud/docker/utils" ) var ( @@ -231,3 +233,70 @@ func TestValidRepositoryName(t *testing.T) { t.Fail() } } + +func TestTrustedLocation(t *testing.T) { + for _, url := range []string{"http://example.com", "https://example.com:7777", "http://docker.io", "http://test.docker.io"} { + req, _ := http.NewRequest("GET", url, nil) + if trustedLocation(req) == true { + t.Fatalf("'%s' shouldn't be detected as a trusted location", url) + } + } + + for _, url := range []string{"https://docker.io", "https://test.docker.io:80"} { + req, _ := http.NewRequest("GET", url, nil) + if trustedLocation(req) == false { + t.Fatalf("'%s' should be detected as a trusted location", url) + } + } +} + +func TestAddRequiredHeadersToRedirectedRequests(t *testing.T) { + for _, urls := range [][]string{ + {"http://docker.io", "https://docker.com"}, + {"https://foo.docker.io:7777", "http://bar.docker.com"}, + {"https://foo.docker.io", "https://example.com"}, + } { + reqFrom, _ := http.NewRequest("GET", urls[0], nil) + reqFrom.Header.Add("Content-Type", "application/json") + reqFrom.Header.Add("Authorization", "super_secret") + reqTo, _ := http.NewRequest("GET", urls[1], nil) + + AddRequiredHeadersToRedirectedRequests(reqTo, []*http.Request{reqFrom}) + + if len(reqTo.Header) != 1 { + t.Fatal("Expected 1 headers, got %d", len(reqTo.Header)) + } + + if reqTo.Header.Get("Content-Type") != "application/json" { + t.Fatal("'Content-Type' should be 'application/json'") + } + + if reqTo.Header.Get("Authorization") != "" { + t.Fatal("'Authorization' should be empty") + } + } + + for _, urls := range [][]string{ + {"https://docker.io", "https://docker.com"}, + {"https://foo.docker.io:7777", "https://bar.docker.com"}, + } { + reqFrom, _ := http.NewRequest("GET", urls[0], nil) + reqFrom.Header.Add("Content-Type", "application/json") + reqFrom.Header.Add("Authorization", "super_secret") + reqTo, _ := http.NewRequest("GET", urls[1], nil) + + AddRequiredHeadersToRedirectedRequests(reqTo, []*http.Request{reqFrom}) + + if len(reqTo.Header) != 2 { + t.Fatal("Expected 2 headers, got %d", len(reqTo.Header)) + } + + if reqTo.Header.Get("Content-Type") != "application/json" { + t.Fatal("'Content-Type' should be 'application/json'") + } + + if reqTo.Header.Get("Authorization") != "super_secret" { + t.Fatal("'Authorization' should be 'super_secret'") + } + } +} From 5cef006c5a8ac0c0b771d78999119ff2db029e10 Mon Sep 17 00:00:00 2001 From: Victor Vieux Date: Sat, 7 Jun 2014 21:17:56 +0000 Subject: [PATCH 127/375] improve trusted location detection Docker-DCO-1.1-Signed-off-by: Victor Vieux (github: vieux) --- docs/registry.go | 2 +- docs/registry_test.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/registry.go b/docs/registry.go index 8d1a9f22..95cc7406 100644 --- a/docs/registry.go +++ b/docs/registry.go @@ -766,7 +766,7 @@ func trustedLocation(req *http.Request) bool { } for _, trusted := range trusteds { - if strings.HasSuffix(hostname, trusted) { + if hostname == trusted || strings.HasSuffix(hostname, "."+trusted) { return true } } diff --git a/docs/registry_test.go b/docs/registry_test.go index 2857ab4a..91a5ffa1 100644 --- a/docs/registry_test.go +++ b/docs/registry_test.go @@ -235,7 +235,7 @@ func TestValidRepositoryName(t *testing.T) { } func TestTrustedLocation(t *testing.T) { - for _, url := range []string{"http://example.com", "https://example.com:7777", "http://docker.io", "http://test.docker.io"} { + for _, url := range []string{"http://example.com", "https://example.com:7777", "http://docker.io", "http://test.docker.io", "https://fakedocker.com"} { req, _ := http.NewRequest("GET", url, nil) if trustedLocation(req) == true { t.Fatalf("'%s' shouldn't be detected as a trusted location", url) From 4ec6e68e04a58a1abce9cd14967047ec6feeb334 Mon Sep 17 00:00:00 2001 From: Victor Vieux Date: Sat, 7 Jun 2014 23:48:25 +0000 Subject: [PATCH 128/375] Disable timeout for push Docker-DCO-1.1-Signed-off-by: Victor Vieux (github: vieux) --- docs/registry.go | 23 +++++++++++------------ docs/registry_test.go | 2 +- docs/service.go | 2 +- 3 files changed, 13 insertions(+), 14 deletions(-) diff --git a/docs/registry.go b/docs/registry.go index 95cc7406..e91e7d12 100644 --- a/docs/registry.go +++ b/docs/registry.go @@ -790,22 +790,21 @@ func AddRequiredHeadersToRedirectedRequests(req *http.Request, via []*http.Reque return nil } -func NewRegistry(authConfig *AuthConfig, factory *utils.HTTPRequestFactory, indexEndpoint string) (r *Registry, err error) { - httpDial := func(proto string, addr string) (net.Conn, error) { - conn, err := net.Dial(proto, addr) - if err != nil { - return nil, err - } - conn = utils.NewTimeoutConn(conn, time.Duration(1)*time.Minute) - return conn, nil - } - +func NewRegistry(authConfig *AuthConfig, factory *utils.HTTPRequestFactory, indexEndpoint string, timeout bool) (r *Registry, err error) { httpTransport := &http.Transport{ - Dial: httpDial, DisableKeepAlives: true, Proxy: http.ProxyFromEnvironment, } - + if timeout { + httpTransport.Dial = func(proto string, addr string) (net.Conn, error) { + conn, err := net.Dial(proto, addr) + if err != nil { + return nil, err + } + conn = utils.NewTimeoutConn(conn, time.Duration(1)*time.Minute) + return conn, nil + } + } r = &Registry{ authConfig: authConfig, client: &http.Client{ diff --git a/docs/registry_test.go b/docs/registry_test.go index 91a5ffa1..2aae80ed 100644 --- a/docs/registry_test.go +++ b/docs/registry_test.go @@ -18,7 +18,7 @@ var ( func spawnTestRegistry(t *testing.T) *Registry { authConfig := &AuthConfig{} - r, err := NewRegistry(authConfig, utils.NewHTTPRequestFactory(), makeURL("/v1/")) + r, err := NewRegistry(authConfig, utils.NewHTTPRequestFactory(), makeURL("/v1/"), true) if err != nil { t.Fatal(err) } diff --git a/docs/service.go b/docs/service.go index 1c7a93de..89a4baa7 100644 --- a/docs/service.go +++ b/docs/service.go @@ -82,7 +82,7 @@ func (s *Service) Search(job *engine.Job) engine.Status { job.GetenvJson("authConfig", authConfig) job.GetenvJson("metaHeaders", metaHeaders) - r, err := NewRegistry(authConfig, HTTPRequestFactory(metaHeaders), IndexServerAddress()) + r, err := NewRegistry(authConfig, HTTPRequestFactory(metaHeaders), IndexServerAddress(), true) if err != nil { return job.Error(err) } From 46cc7603d4d7cc0e018b0abf2aacf1f9366c510f Mon Sep 17 00:00:00 2001 From: Brandon Philips Date: Sun, 8 Jun 2014 11:01:07 -0700 Subject: [PATCH 129/375] registry: remove unneeded time.Duration() These constants don't need to use time.Duration(). Fixup this file since it seems to be the only one using this style. Docker-DCO-1.1-Signed-off-by: Brandon Philips (github: philips) --- docs/registry.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/registry.go b/docs/registry.go index e91e7d12..24c55125 100644 --- a/docs/registry.go +++ b/docs/registry.go @@ -37,12 +37,12 @@ func pingRegistryEndpoint(endpoint string) (RegistryInfo, error) { } httpDial := func(proto string, addr string) (net.Conn, error) { // Set the connect timeout to 5 seconds - conn, err := net.DialTimeout(proto, addr, time.Duration(5)*time.Second) + conn, err := net.DialTimeout(proto, addr, 5*time.Second) if err != nil { return nil, err } // Set the recv timeout to 10 seconds - conn.SetDeadline(time.Now().Add(time.Duration(10) * time.Second)) + conn.SetDeadline(time.Now().Add(10 * time.Second)) return conn, nil } httpTransport := &http.Transport{ @@ -801,7 +801,7 @@ func NewRegistry(authConfig *AuthConfig, factory *utils.HTTPRequestFactory, inde if err != nil { return nil, err } - conn = utils.NewTimeoutConn(conn, time.Duration(1)*time.Minute) + conn = utils.NewTimeoutConn(conn, 1*time.Minute) return conn, nil } } From d95235cc502ea2067d3aaed24b79e4fd578c45ab Mon Sep 17 00:00:00 2001 From: Alexander Larsson Date: Wed, 4 Dec 2013 15:03:51 +0100 Subject: [PATCH 130/375] Add support for client certificates for registries MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This lets you specify custom client TLS certificates and CA root for a specific registry hostname. Docker will then verify the registry against the CA and present the client cert when talking to that registry. This allows the registry to verify that the client has a proper key, indicating that the client is allowed to access the images. A custom cert is configured by creating a directory in /etc/docker/certs.d with the same name as the registry hostname. Inside this directory all *.crt files are added as CA Roots (if none exists, the system default is used) and pair of files .key and .cert indicate a custom certificate to present to the registry. If there are multiple certificates each one will be tried in alphabetical order, proceeding to the next if we get a 403 of 5xx response. So, an example setup would be: /etc/docker/certs.d/ └── localhost ├── client.cert ├── client.key └── localhost.crt A simple way to test this setup is to use an apache server to host a registry. Just copy a registry tree into the apache root, here is an example one containing the busybox image: http://people.gnome.org/~alexl/v1.tar.gz Then add this conf file as /etc/httpd/conf.d/registry.conf: # This must be in the root context, otherwise it causes a re-negotiation # which is not supported by the tls implementation in go SSLVerifyClient optional_no_ca Action cert-protected /cgi-bin/cert.cgi SetHandler cert-protected Header set x-docker-registry-version "0.6.2" SetEnvIf Host (.*) custom_host=$1 Header set X-Docker-Endpoints "%{custom_host}e" And this as /var/www/cgi-bin/cert.cgi #!/bin/bash if [ "$HTTPS" != "on" ]; then echo "Status: 403 Not using SSL" echo "x-docker-registry-version: 0.6.2" echo exit 0 fi if [ "$SSL_CLIENT_VERIFY" == "NONE" ]; then echo "Status: 403 Client certificate invalid" echo "x-docker-registry-version: 0.6.2" echo exit 0 fi echo "Content-length: $(stat --printf='%s' $PATH_TRANSLATED)" echo "x-docker-registry-version: 0.6.2" echo "X-Docker-Endpoints: $SERVER_NAME" echo "X-Docker-Size: 0" echo cat $PATH_TRANSLATED This will return 403 for all accessed to /v1 unless *any* client cert is presented. Obviously a real implementation would verify more details about the certificate. Example client certs can be generated with: openssl genrsa -out client.key 1024 openssl req -new -x509 -text -key client.key -out client.cert Docker-DCO-1.1-Signed-off-by: Alexander Larsson (github: alexlarsson) --- docs/registry.go | 227 ++++++++++++++++++++++++++++++++++++----------- 1 file changed, 174 insertions(+), 53 deletions(-) diff --git a/docs/registry.go b/docs/registry.go index 24c55125..748636dc 100644 --- a/docs/registry.go +++ b/docs/registry.go @@ -4,6 +4,8 @@ import ( "bytes" "crypto/sha256" _ "crypto/sha512" + "crypto/tls" + "crypto/x509" "encoding/json" "errors" "fmt" @@ -13,6 +15,8 @@ import ( "net/http" "net/http/cookiejar" "net/url" + "os" + "path" "regexp" "runtime" "strconv" @@ -29,31 +33,155 @@ var ( errLoginRequired = errors.New("Authentication is required.") ) +type TimeoutType uint32 + +const ( + NoTimeout TimeoutType = iota + ReceiveTimeout + ConnectTimeout +) + +func newClient(jar http.CookieJar, roots *x509.CertPool, cert *tls.Certificate, timeout TimeoutType) *http.Client { + tlsConfig := tls.Config{RootCAs: roots} + + if cert != nil { + tlsConfig.Certificates = append(tlsConfig.Certificates, *cert) + } + + httpTransport := &http.Transport{ + DisableKeepAlives: true, + Proxy: http.ProxyFromEnvironment, + TLSClientConfig: &tlsConfig, + } + + switch timeout { + case ConnectTimeout: + httpTransport.Dial = func(proto string, addr string) (net.Conn, error) { + // Set the connect timeout to 5 seconds + conn, err := net.DialTimeout(proto, addr, 5*time.Second) + if err != nil { + return nil, err + } + // Set the recv timeout to 10 seconds + conn.SetDeadline(time.Now().Add(10 * time.Second)) + return conn, nil + } + case ReceiveTimeout: + httpTransport.Dial = func(proto string, addr string) (net.Conn, error) { + conn, err := net.Dial(proto, addr) + if err != nil { + return nil, err + } + conn = utils.NewTimeoutConn(conn, 1*time.Minute) + return conn, nil + } + } + + return &http.Client{ + Transport: httpTransport, + CheckRedirect: AddRequiredHeadersToRedirectedRequests, + Jar: jar, + } +} + +func doRequest(req *http.Request, jar http.CookieJar, timeout TimeoutType) (*http.Response, *http.Client, error) { + hasFile := func(files []os.FileInfo, name string) bool { + for _, f := range files { + if f.Name() == name { + return true + } + } + return false + } + + hostDir := path.Join("/etc/docker/certs.d", req.URL.Host) + fs, err := ioutil.ReadDir(hostDir) + if err != nil && !os.IsNotExist(err) { + return nil, nil, err + } + + var ( + pool *x509.CertPool + certs []*tls.Certificate + ) + + for _, f := range fs { + if strings.HasSuffix(f.Name(), ".crt") { + if pool == nil { + pool = x509.NewCertPool() + } + data, err := ioutil.ReadFile(path.Join(hostDir, f.Name())) + if err != nil { + return nil, nil, err + } else { + pool.AppendCertsFromPEM(data) + } + } + if strings.HasSuffix(f.Name(), ".cert") { + certName := f.Name() + keyName := certName[:len(certName)-5] + ".key" + if !hasFile(fs, keyName) { + return nil, nil, fmt.Errorf("Missing key %s for certificate %s", keyName, certName) + } else { + cert, err := tls.LoadX509KeyPair(path.Join(hostDir, certName), path.Join(hostDir, keyName)) + if err != nil { + return nil, nil, err + } + certs = append(certs, &cert) + } + } + if strings.HasSuffix(f.Name(), ".key") { + keyName := f.Name() + certName := keyName[:len(keyName)-4] + ".cert" + if !hasFile(fs, certName) { + return nil, nil, fmt.Errorf("Missing certificate %s for key %s", certName, keyName) + } + } + } + + if len(certs) == 0 { + client := newClient(jar, pool, nil, timeout) + res, err := client.Do(req) + if err != nil { + return nil, nil, err + } + return res, client, nil + } else { + for i, cert := range certs { + client := newClient(jar, pool, cert, timeout) + res, err := client.Do(req) + if i == len(certs)-1 { + // If this is the last cert, always return the result + return res, client, err + } else { + // Otherwise, continue to next cert if 403 or 5xx + if err == nil && res.StatusCode != 403 && !(res.StatusCode >= 500 && res.StatusCode < 600) { + return res, client, err + } + } + } + } + + return nil, nil, nil +} + func pingRegistryEndpoint(endpoint string) (RegistryInfo, error) { if endpoint == IndexServerAddress() { // Skip the check, we now this one is valid // (and we never want to fallback to http in case of error) return RegistryInfo{Standalone: false}, nil } - httpDial := func(proto string, addr string) (net.Conn, error) { - // Set the connect timeout to 5 seconds - conn, err := net.DialTimeout(proto, addr, 5*time.Second) - if err != nil { - return nil, err - } - // Set the recv timeout to 10 seconds - conn.SetDeadline(time.Now().Add(10 * time.Second)) - return conn, nil - } - httpTransport := &http.Transport{ - Dial: httpDial, - Proxy: http.ProxyFromEnvironment, - } - client := &http.Client{Transport: httpTransport} - resp, err := client.Get(endpoint + "_ping") + + req, err := http.NewRequest("GET", endpoint+"_ping", nil) if err != nil { return RegistryInfo{Standalone: false}, err } + + resp, _, err := doRequest(req, nil, ConnectTimeout) + if err != nil { + return RegistryInfo{Standalone: false}, err + } + defer resp.Body.Close() jsonString, err := ioutil.ReadAll(resp.Body) @@ -171,6 +299,10 @@ func setTokenAuth(req *http.Request, token []string) { } } +func (r *Registry) doRequest(req *http.Request) (*http.Response, *http.Client, error) { + return doRequest(req, r.jar, r.timeout) +} + // Retrieve the history of a given image from the Registry. // Return a list of the parent's json (requested image included) func (r *Registry) GetRemoteHistory(imgID, registry string, token []string) ([]string, error) { @@ -179,7 +311,7 @@ func (r *Registry) GetRemoteHistory(imgID, registry string, token []string) ([]s return nil, err } setTokenAuth(req, token) - res, err := r.client.Do(req) + res, _, err := r.doRequest(req) if err != nil { return nil, err } @@ -214,7 +346,7 @@ func (r *Registry) LookupRemoteImage(imgID, registry string, token []string) boo return false } setTokenAuth(req, token) - res, err := r.client.Do(req) + res, _, err := r.doRequest(req) if err != nil { utils.Errorf("Error in LookupRemoteImage %s", err) return false @@ -231,7 +363,7 @@ func (r *Registry) GetRemoteImageJSON(imgID, registry string, token []string) ([ return nil, -1, fmt.Errorf("Failed to download json: %s", err) } setTokenAuth(req, token) - res, err := r.client.Do(req) + res, _, err := r.doRequest(req) if err != nil { return nil, -1, fmt.Errorf("Failed to download json: %s", err) } @@ -260,6 +392,7 @@ func (r *Registry) GetRemoteImageLayer(imgID, registry string, token []string, i var ( retries = 5 headRes *http.Response + client *http.Client hasResume bool = false imageURL = fmt.Sprintf("%simages/%s/layer", registry, imgID) ) @@ -267,9 +400,10 @@ func (r *Registry) GetRemoteImageLayer(imgID, registry string, token []string, i if err != nil { return nil, fmt.Errorf("Error while getting from the server: %s\n", err) } + setTokenAuth(headReq, token) for i := 1; i <= retries; i++ { - headRes, err = r.client.Do(headReq) + headRes, client, err = r.doRequest(headReq) if err != nil && i == retries { return nil, fmt.Errorf("Eror while making head request: %s\n", err) } else if err != nil { @@ -290,10 +424,10 @@ func (r *Registry) GetRemoteImageLayer(imgID, registry string, token []string, i setTokenAuth(req, token) if hasResume { utils.Debugf("server supports resume") - return utils.ResumableRequestReader(r.client, req, 5, imgSize), nil + return utils.ResumableRequestReader(client, req, 5, imgSize), nil } utils.Debugf("server doesn't support resume") - res, err := r.client.Do(req) + res, _, err := r.doRequest(req) if err != nil { return nil, err } @@ -319,7 +453,7 @@ func (r *Registry) GetRemoteTags(registries []string, repository string, token [ return nil, err } setTokenAuth(req, token) - res, err := r.client.Do(req) + res, _, err := r.doRequest(req) if err != nil { return nil, err } @@ -380,7 +514,7 @@ func (r *Registry) GetRepositoryData(remote string) (*RepositoryData, error) { } req.Header.Set("X-Docker-Token", "true") - res, err := r.client.Do(req) + res, _, err := r.doRequest(req) if err != nil { return nil, err } @@ -448,13 +582,13 @@ func (r *Registry) PushImageChecksumRegistry(imgData *ImgData, registry string, req.Header.Set("X-Docker-Checksum", imgData.Checksum) req.Header.Set("X-Docker-Checksum-Payload", imgData.ChecksumPayload) - res, err := r.client.Do(req) + res, _, err := r.doRequest(req) if err != nil { return fmt.Errorf("Failed to upload metadata: %s", err) } defer res.Body.Close() if len(res.Cookies()) > 0 { - r.client.Jar.SetCookies(req.URL, res.Cookies()) + r.jar.SetCookies(req.URL, res.Cookies()) } if res.StatusCode != 200 { errBody, err := ioutil.ReadAll(res.Body) @@ -484,7 +618,7 @@ func (r *Registry) PushImageJSONRegistry(imgData *ImgData, jsonRaw []byte, regis req.Header.Add("Content-type", "application/json") setTokenAuth(req, token) - res, err := r.client.Do(req) + res, _, err := r.doRequest(req) if err != nil { return fmt.Errorf("Failed to upload metadata: %s", err) } @@ -525,7 +659,7 @@ func (r *Registry) PushImageLayerRegistry(imgID string, layer io.Reader, registr req.ContentLength = -1 req.TransferEncoding = []string{"chunked"} setTokenAuth(req, token) - res, err := r.client.Do(req) + res, _, err := r.doRequest(req) if err != nil { return "", "", fmt.Errorf("Failed to upload layer: %s", err) } @@ -562,7 +696,7 @@ func (r *Registry) PushRegistryTag(remote, revision, tag, registry string, token req.Header.Add("Content-type", "application/json") setTokenAuth(req, token) req.ContentLength = int64(len(revision)) - res, err := r.client.Do(req) + res, _, err := r.doRequest(req) if err != nil { return err } @@ -610,7 +744,7 @@ func (r *Registry) PushImageJSONIndex(remote string, imgList []*ImgData, validat req.Header["X-Docker-Endpoints"] = regs } - res, err := r.client.Do(req) + res, _, err := r.doRequest(req) if err != nil { return nil, err } @@ -629,7 +763,7 @@ func (r *Registry) PushImageJSONIndex(remote string, imgList []*ImgData, validat if validate { req.Header["X-Docker-Endpoints"] = regs } - res, err = r.client.Do(req) + res, _, err := r.doRequest(req) if err != nil { return nil, err } @@ -688,7 +822,7 @@ func (r *Registry) SearchRepositories(term string) (*SearchResults, error) { req.SetBasicAuth(r.authConfig.Username, r.authConfig.Password) } req.Header.Set("X-Docker-Token", "true") - res, err := r.client.Do(req) + res, _, err := r.doRequest(req) if err != nil { return nil, err } @@ -750,10 +884,11 @@ type RegistryInfo struct { } type Registry struct { - client *http.Client authConfig *AuthConfig reqFactory *utils.HTTPRequestFactory indexEndpoint string + jar *cookiejar.Jar + timeout TimeoutType } func trustedLocation(req *http.Request) bool { @@ -791,30 +926,16 @@ func AddRequiredHeadersToRedirectedRequests(req *http.Request, via []*http.Reque } func NewRegistry(authConfig *AuthConfig, factory *utils.HTTPRequestFactory, indexEndpoint string, timeout bool) (r *Registry, err error) { - httpTransport := &http.Transport{ - DisableKeepAlives: true, - Proxy: http.ProxyFromEnvironment, - } - if timeout { - httpTransport.Dial = func(proto string, addr string) (net.Conn, error) { - conn, err := net.Dial(proto, addr) - if err != nil { - return nil, err - } - conn = utils.NewTimeoutConn(conn, 1*time.Minute) - return conn, nil - } - } r = &Registry{ - authConfig: authConfig, - client: &http.Client{ - Transport: httpTransport, - CheckRedirect: AddRequiredHeadersToRedirectedRequests, - }, + authConfig: authConfig, indexEndpoint: indexEndpoint, } - r.client.Jar, err = cookiejar.New(nil) + if timeout { + r.timeout = ReceiveTimeout + } + + r.jar, err = cookiejar.New(nil) if err != nil { return nil, err } From 7cd8de1329d3f893c140c04443467026df54b3e3 Mon Sep 17 00:00:00 2001 From: LK4D4 Date: Thu, 12 Jun 2014 09:15:53 +0400 Subject: [PATCH 131/375] Fix go vet errors Docker-DCO-1.1-Signed-off-by: Alexandr Morozov (github: LK4D4) Docker-DCO-1.1-Signed-off-by: Victor Vieux (github: vieux) --- docs/registry_test.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/registry_test.go b/docs/registry_test.go index 2aae80ed..5cec0595 100644 --- a/docs/registry_test.go +++ b/docs/registry_test.go @@ -264,7 +264,7 @@ func TestAddRequiredHeadersToRedirectedRequests(t *testing.T) { AddRequiredHeadersToRedirectedRequests(reqTo, []*http.Request{reqFrom}) if len(reqTo.Header) != 1 { - t.Fatal("Expected 1 headers, got %d", len(reqTo.Header)) + t.Fatalf("Expected 1 headers, got %d", len(reqTo.Header)) } if reqTo.Header.Get("Content-Type") != "application/json" { @@ -288,7 +288,7 @@ func TestAddRequiredHeadersToRedirectedRequests(t *testing.T) { AddRequiredHeadersToRedirectedRequests(reqTo, []*http.Request{reqFrom}) if len(reqTo.Header) != 2 { - t.Fatal("Expected 2 headers, got %d", len(reqTo.Header)) + t.Fatalf("Expected 2 headers, got %d", len(reqTo.Header)) } if reqTo.Header.Get("Content-Type") != "application/json" { From 19b4616baa3e502e3d630e8f7dd7709560457fc5 Mon Sep 17 00:00:00 2001 From: Gabor Nagy Date: Wed, 16 Jul 2014 12:22:13 +0200 Subject: [PATCH 132/375] Add Content-Type header in PushImageLayerRegistry Docker-DCO-1.1-Signed-off-by: Gabor Nagy (github: Aigeruth) --- docs/registry.go | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/registry.go b/docs/registry.go index 24c55125..c8d00082 100644 --- a/docs/registry.go +++ b/docs/registry.go @@ -522,6 +522,7 @@ func (r *Registry) PushImageLayerRegistry(imgID string, layer io.Reader, registr if err != nil { return "", "", err } + req.Header.Add("Content-Type", "application/octet-stream") req.ContentLength = -1 req.TransferEncoding = []string{"chunked"} setTokenAuth(req, token) From 78a499ac67b8f96c6aa1cb6ec4fd781d36f14c18 Mon Sep 17 00:00:00 2001 From: unclejack Date: Fri, 27 Jun 2014 15:10:30 +0300 Subject: [PATCH 133/375] get layer: remove HEAD req & pass down response Docker-DCO-1.1-Signed-off-by: Cristian Staretu (github: unclejack) --- docs/registry.go | 56 ++++++++++++++++++++---------------------------- 1 file changed, 23 insertions(+), 33 deletions(-) diff --git a/docs/registry.go b/docs/registry.go index 748636dc..57795f1c 100644 --- a/docs/registry.go +++ b/docs/registry.go @@ -390,52 +390,42 @@ func (r *Registry) GetRemoteImageJSON(imgID, registry string, token []string) ([ func (r *Registry) GetRemoteImageLayer(imgID, registry string, token []string, imgSize int64) (io.ReadCloser, error) { var ( - retries = 5 - headRes *http.Response - client *http.Client - hasResume bool = false - imageURL = fmt.Sprintf("%simages/%s/layer", registry, imgID) + retries = 5 + client *http.Client + res *http.Response + imageURL = fmt.Sprintf("%simages/%s/layer", registry, imgID) ) - headReq, err := r.reqFactory.NewRequest("HEAD", imageURL, nil) - if err != nil { - return nil, fmt.Errorf("Error while getting from the server: %s\n", err) - } - - setTokenAuth(headReq, token) - for i := 1; i <= retries; i++ { - headRes, client, err = r.doRequest(headReq) - if err != nil && i == retries { - return nil, fmt.Errorf("Eror while making head request: %s\n", err) - } else if err != nil { - time.Sleep(time.Duration(i) * 5 * time.Second) - continue - } - break - } - - if headRes.Header.Get("Accept-Ranges") == "bytes" && imgSize > 0 { - hasResume = true - } req, err := r.reqFactory.NewRequest("GET", imageURL, nil) if err != nil { return nil, fmt.Errorf("Error while getting from the server: %s\n", err) } setTokenAuth(req, token) - if hasResume { - utils.Debugf("server supports resume") - return utils.ResumableRequestReader(client, req, 5, imgSize), nil - } - utils.Debugf("server doesn't support resume") - res, _, err := r.doRequest(req) - if err != nil { - return nil, err + for i := 1; i <= retries; i++ { + res, client, err = r.doRequest(req) + if err != nil { + res.Body.Close() + if i == retries { + return nil, fmt.Errorf("Server error: Status %d while fetching image layer (%s)", + res.StatusCode, imgID) + } + time.Sleep(time.Duration(i) * 5 * time.Second) + continue + } + break } + if res.StatusCode != 200 { res.Body.Close() return nil, fmt.Errorf("Server error: Status %d while fetching image layer (%s)", res.StatusCode, imgID) } + + if res.Header.Get("Accept-Ranges") == "bytes" && imgSize > 0 { + utils.Debugf("server supports resume") + return utils.ResumableRequestReaderWithInitialResponse(client, req, 5, imgSize, res), nil + } + utils.Debugf("server doesn't support resume") return res.Body, nil } From 6365d94ef4a6f1a38611b9f2f04fc6bc8ad4fa99 Mon Sep 17 00:00:00 2001 From: Olivier Gambier Date: Tue, 22 Jul 2014 01:26:14 +0200 Subject: [PATCH 134/375] Joining registry maintainers Docker-DCO-1.1-Signed-off-by: Olivier Gambier (github: dmp42) --- docs/MAINTAINERS | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/MAINTAINERS b/docs/MAINTAINERS index af791fb4..6ed4e9d6 100644 --- a/docs/MAINTAINERS +++ b/docs/MAINTAINERS @@ -2,3 +2,4 @@ Sam Alba (@samalba) Joffrey Fuhrer (@shin-) Ken Cochrane (@kencochrane) Vincent Batts (@vbatts) +Olivier Gambier (@dmp42) From 822f8c1b5277c61d5e16777724b03094853f862d Mon Sep 17 00:00:00 2001 From: Victor Vieux Date: Thu, 24 Jul 2014 22:19:50 +0000 Subject: [PATCH 135/375] update go import path and libcontainer Docker-DCO-1.1-Signed-off-by: Victor Vieux (github: vieux) --- docs/MAINTAINERS | 6 +++--- docs/auth.go | 4 ++-- docs/registry.go | 4 ++-- docs/registry_mock_test.go | 2 +- docs/registry_test.go | 4 ++-- docs/service.go | 2 +- 6 files changed, 11 insertions(+), 11 deletions(-) diff --git a/docs/MAINTAINERS b/docs/MAINTAINERS index 6ed4e9d6..fdb03ed5 100644 --- a/docs/MAINTAINERS +++ b/docs/MAINTAINERS @@ -1,5 +1,5 @@ -Sam Alba (@samalba) -Joffrey Fuhrer (@shin-) -Ken Cochrane (@kencochrane) +Sam Alba (@samalba) +Joffrey Fuhrer (@shin-) +Ken Cochrane (@kencochrane) Vincent Batts (@vbatts) Olivier Gambier (@dmp42) diff --git a/docs/auth.go b/docs/auth.go index 7384efba..906a37dd 100644 --- a/docs/auth.go +++ b/docs/auth.go @@ -11,7 +11,7 @@ import ( "path" "strings" - "github.com/dotcloud/docker/utils" + "github.com/docker/docker/utils" ) // Where we store the config file @@ -20,7 +20,7 @@ const CONFIGFILE = ".dockercfg" // Only used for user auth + account creation const INDEXSERVER = "https://index.docker.io/v1/" -//const INDEXSERVER = "https://indexstaging-docker.dotcloud.com/v1/" +//const INDEXSERVER = "https://registry-stage.hub.docker.com/v1/" var ( ErrConfigFileMissing = errors.New("The Auth config file is missing") diff --git a/docs/registry.go b/docs/registry.go index 974e7fb9..567cd9f7 100644 --- a/docs/registry.go +++ b/docs/registry.go @@ -23,8 +23,8 @@ import ( "strings" "time" - "github.com/dotcloud/docker/dockerversion" - "github.com/dotcloud/docker/utils" + "github.com/docker/docker/dockerversion" + "github.com/docker/docker/utils" ) var ( diff --git a/docs/registry_mock_test.go b/docs/registry_mock_test.go index 6b007513..1a622228 100644 --- a/docs/registry_mock_test.go +++ b/docs/registry_mock_test.go @@ -3,7 +3,7 @@ package registry import ( "encoding/json" "fmt" - "github.com/dotcloud/docker/utils" + "github.com/docker/docker/utils" "github.com/gorilla/mux" "io" "io/ioutil" diff --git a/docs/registry_test.go b/docs/registry_test.go index 5cec0595..12dc7a28 100644 --- a/docs/registry_test.go +++ b/docs/registry_test.go @@ -7,7 +7,7 @@ import ( "strings" "testing" - "github.com/dotcloud/docker/utils" + "github.com/docker/docker/utils" ) var ( @@ -145,7 +145,7 @@ func TestPushImageLayerRegistry(t *testing.T) { } func TestResolveRepositoryName(t *testing.T) { - _, _, err := ResolveRepositoryName("https://github.com/dotcloud/docker") + _, _, err := ResolveRepositoryName("https://github.com/docker/docker") assertEqual(t, err, ErrInvalidRepositoryName, "Expected error invalid repo name") ep, repo, err := ResolveRepositoryName("fooo/bar") if err != nil { diff --git a/docs/service.go b/docs/service.go index 89a4baa7..d2775e3c 100644 --- a/docs/service.go +++ b/docs/service.go @@ -1,7 +1,7 @@ package registry import ( - "github.com/dotcloud/docker/engine" + "github.com/docker/docker/engine" ) // Service exposes registry capabilities in the standard Engine From 775ca3caa33ca2976aef1a8e9abf5a9dd25075d7 Mon Sep 17 00:00:00 2001 From: unclejack Date: Mon, 28 Jul 2014 18:01:21 +0300 Subject: [PATCH 136/375] move resumablerequestreader to pkg Docker-DCO-1.1-Signed-off-by: Cristian Staretu (github: unclejack) --- docs/registry.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/docs/registry.go b/docs/registry.go index 567cd9f7..9563c3b2 100644 --- a/docs/registry.go +++ b/docs/registry.go @@ -24,6 +24,7 @@ import ( "time" "github.com/docker/docker/dockerversion" + "github.com/docker/docker/pkg/httputils" "github.com/docker/docker/utils" ) @@ -423,7 +424,7 @@ func (r *Registry) GetRemoteImageLayer(imgID, registry string, token []string, i if res.Header.Get("Accept-Ranges") == "bytes" && imgSize > 0 { utils.Debugf("server supports resume") - return utils.ResumableRequestReaderWithInitialResponse(client, req, 5, imgSize, res), nil + return httputils.ResumableRequestReaderWithInitialResponse(client, req, 5, imgSize, res), nil } utils.Debugf("server doesn't support resume") return res.Body, nil From 052128c4fc159a7b5f1927a523e7c2826e71fe8f Mon Sep 17 00:00:00 2001 From: Erik Hollensbe Date: Mon, 28 Jul 2014 17:23:38 -0700 Subject: [PATCH 137/375] Move parsing functions to pkg/parsers and the specific kernel handling functions to pkg/parsers/kernel, and parsing filters to pkg/parsers/filter. Adjust imports and package references. Docker-DCO-1.1-Signed-off-by: Erik Hollensbe (github: erikh) --- docs/registry.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/docs/registry.go b/docs/registry.go index 9563c3b2..0d4f2b3c 100644 --- a/docs/registry.go +++ b/docs/registry.go @@ -25,6 +25,7 @@ import ( "github.com/docker/docker/dockerversion" "github.com/docker/docker/pkg/httputils" + "github.com/docker/docker/pkg/parsers/kernel" "github.com/docker/docker/utils" ) @@ -956,7 +957,7 @@ func HTTPRequestFactory(metaHeaders map[string][]string) *utils.HTTPRequestFacto httpVersion = append(httpVersion, &simpleVersionInfo{"docker", dockerversion.VERSION}) httpVersion = append(httpVersion, &simpleVersionInfo{"go", runtime.Version()}) httpVersion = append(httpVersion, &simpleVersionInfo{"git-commit", dockerversion.GITCOMMIT}) - if kernelVersion, err := utils.GetKernelVersion(); err == nil { + if kernelVersion, err := kernel.GetKernelVersion(); err == nil { httpVersion = append(httpVersion, &simpleVersionInfo{"kernel", kernelVersion.String()}) } httpVersion = append(httpVersion, &simpleVersionInfo{"os", runtime.GOOS}) From 7f2dca77d4cb830a321dd2e16d0d01121da29321 Mon Sep 17 00:00:00 2001 From: Erik Hollensbe Date: Wed, 30 Jul 2014 06:42:12 -0700 Subject: [PATCH 138/375] utils/tarsum* -> pkg/tarsum Docker-DCO-1.1-Signed-off-by: Erik Hollensbe (github: erikh) --- docs/registry.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/docs/registry.go b/docs/registry.go index 0d4f2b3c..106a5181 100644 --- a/docs/registry.go +++ b/docs/registry.go @@ -26,6 +26,7 @@ import ( "github.com/docker/docker/dockerversion" "github.com/docker/docker/pkg/httputils" "github.com/docker/docker/pkg/parsers/kernel" + "github.com/docker/docker/pkg/tarsum" "github.com/docker/docker/utils" ) @@ -638,7 +639,7 @@ func (r *Registry) PushImageLayerRegistry(imgID string, layer io.Reader, registr utils.Debugf("[registry] Calling PUT %s", registry+"images/"+imgID+"/layer") - tarsumLayer := &utils.TarSum{Reader: layer} + tarsumLayer := &tarsum.TarSum{Reader: layer} h := sha256.New() h.Write(jsonRaw) h.Write([]byte{'\n'}) From 47261aa8cf7aadda67c2d554ee7985f0852d663e Mon Sep 17 00:00:00 2001 From: Erik Hollensbe Date: Wed, 30 Jul 2014 09:28:42 -0700 Subject: [PATCH 139/375] Remove CheckSum from utils; replace with a TeeReader Docker-DCO-1.1-Signed-off-by: Erik Hollensbe (github: erikh) --- docs/registry.go | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/docs/registry.go b/docs/registry.go index 0d4f2b3c..9035ce90 100644 --- a/docs/registry.go +++ b/docs/registry.go @@ -6,6 +6,7 @@ import ( _ "crypto/sha512" "crypto/tls" "crypto/x509" + "encoding/hex" "encoding/json" "errors" "fmt" @@ -642,7 +643,7 @@ func (r *Registry) PushImageLayerRegistry(imgID string, layer io.Reader, registr h := sha256.New() h.Write(jsonRaw) h.Write([]byte{'\n'}) - checksumLayer := &utils.CheckSum{Reader: tarsumLayer, Hash: h} + checksumLayer := io.TeeReader(tarsumLayer, h) req, err := r.reqFactory.NewRequest("PUT", registry+"images/"+imgID+"/layer", checksumLayer) if err != nil { @@ -671,7 +672,7 @@ func (r *Registry) PushImageLayerRegistry(imgID string, layer io.Reader, registr return "", "", utils.NewHTTPRequestError(fmt.Sprintf("Received HTTP code %d while uploading layer: %s", res.StatusCode, errBody), res) } - checksumPayload = "sha256:" + checksumLayer.Sum() + checksumPayload = "sha256:" + hex.EncodeToString(h.Sum(nil)) return tarsumLayer.Sum(jsonRaw), checksumPayload, nil } From d768343cbe275f34be77d71a2c7c22da256d63dd Mon Sep 17 00:00:00 2001 From: Daniel Menet Date: Sat, 9 Aug 2014 09:16:54 +0200 Subject: [PATCH 140/375] Enable `docker search` on private docker registry. The cli interface works similar to other registry related commands: docker search foo ... searches for foo on the official hub docker search localhost:5000/foo ... does the same for the private reg at localhost:5000 Signed-off-by: Daniel Menet --- docs/service.go | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/docs/service.go b/docs/service.go index d2775e3c..8c2c0cc7 100644 --- a/docs/service.go +++ b/docs/service.go @@ -82,7 +82,11 @@ func (s *Service) Search(job *engine.Job) engine.Status { job.GetenvJson("authConfig", authConfig) job.GetenvJson("metaHeaders", metaHeaders) - r, err := NewRegistry(authConfig, HTTPRequestFactory(metaHeaders), IndexServerAddress(), true) + hostname, term, err := ResolveRepositoryName(term) + if err != nil { + return job.Error(err) + } + r, err := NewRegistry(authConfig, HTTPRequestFactory(metaHeaders), hostname, true) if err != nil { return job.Error(err) } From 94c52da6c0934c6b8e0423c2764ac9cd26edda40 Mon Sep 17 00:00:00 2001 From: Daniel Menet Date: Sun, 10 Aug 2014 11:48:34 +0200 Subject: [PATCH 141/375] Expand hostname before passing it to NewRegistry() Signed-off-by: Daniel Menet --- docs/service.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/docs/service.go b/docs/service.go index 8c2c0cc7..96fc3f48 100644 --- a/docs/service.go +++ b/docs/service.go @@ -86,6 +86,10 @@ func (s *Service) Search(job *engine.Job) engine.Status { if err != nil { return job.Error(err) } + hostname, err = ExpandAndVerifyRegistryUrl(hostname) + if err != nil { + return job.Error(err) + } r, err := NewRegistry(authConfig, HTTPRequestFactory(metaHeaders), hostname, true) if err != nil { return job.Error(err) From 7ef3a5bc73e68b0638aa5794d52d20416fa00fde Mon Sep 17 00:00:00 2001 From: Vincent Batts Date: Thu, 7 Aug 2014 10:43:06 -0400 Subject: [PATCH 142/375] registry.Registry -> registry.Session renaming this struct to more clearly be session, as that is what it handles. Splitting out files for easier readability. Signed-off-by: Vincent Batts --- docs/httpfactory.go | 46 +++ docs/registry.go | 672 ------------------------------------------ docs/registry_test.go | 26 +- docs/service.go | 2 +- docs/session.go | 611 ++++++++++++++++++++++++++++++++++++++ docs/types.go | 33 +++ 6 files changed, 704 insertions(+), 686 deletions(-) create mode 100644 docs/httpfactory.go create mode 100644 docs/session.go create mode 100644 docs/types.go diff --git a/docs/httpfactory.go b/docs/httpfactory.go new file mode 100644 index 00000000..4c784360 --- /dev/null +++ b/docs/httpfactory.go @@ -0,0 +1,46 @@ +package registry + +import ( + "runtime" + + "github.com/docker/docker/dockerversion" + "github.com/docker/docker/pkg/parsers/kernel" + "github.com/docker/docker/utils" +) + +func HTTPRequestFactory(metaHeaders map[string][]string) *utils.HTTPRequestFactory { + // FIXME: this replicates the 'info' job. + httpVersion := make([]utils.VersionInfo, 0, 4) + httpVersion = append(httpVersion, &simpleVersionInfo{"docker", dockerversion.VERSION}) + httpVersion = append(httpVersion, &simpleVersionInfo{"go", runtime.Version()}) + httpVersion = append(httpVersion, &simpleVersionInfo{"git-commit", dockerversion.GITCOMMIT}) + if kernelVersion, err := kernel.GetKernelVersion(); err == nil { + httpVersion = append(httpVersion, &simpleVersionInfo{"kernel", kernelVersion.String()}) + } + httpVersion = append(httpVersion, &simpleVersionInfo{"os", runtime.GOOS}) + httpVersion = append(httpVersion, &simpleVersionInfo{"arch", runtime.GOARCH}) + ud := utils.NewHTTPUserAgentDecorator(httpVersion...) + md := &utils.HTTPMetaHeadersDecorator{ + Headers: metaHeaders, + } + factory := utils.NewHTTPRequestFactory(ud, md) + return factory +} + +// simpleVersionInfo is a simple implementation of +// the interface VersionInfo, which is used +// to provide version information for some product, +// component, etc. It stores the product name and the version +// in string and returns them on calls to Name() and Version(). +type simpleVersionInfo struct { + name string + version string +} + +func (v *simpleVersionInfo) Name() string { + return v.name +} + +func (v *simpleVersionInfo) Version() string { + return v.version +} diff --git a/docs/registry.go b/docs/registry.go index a590bb5f..14b8f6d5 100644 --- a/docs/registry.go +++ b/docs/registry.go @@ -1,33 +1,20 @@ package registry import ( - "bytes" - "crypto/sha256" - _ "crypto/sha512" "crypto/tls" "crypto/x509" - "encoding/hex" "encoding/json" "errors" "fmt" - "io" "io/ioutil" "net" "net/http" - "net/http/cookiejar" - "net/url" "os" "path" "regexp" - "runtime" - "strconv" "strings" "time" - "github.com/docker/docker/dockerversion" - "github.com/docker/docker/pkg/httputils" - "github.com/docker/docker/pkg/parsers/kernel" - "github.com/docker/docker/pkg/tarsum" "github.com/docker/docker/utils" ) @@ -297,595 +284,6 @@ func ExpandAndVerifyRegistryUrl(hostname string) (string, error) { return endpoint, nil } -func setTokenAuth(req *http.Request, token []string) { - if req.Header.Get("Authorization") == "" { // Don't override - req.Header.Set("Authorization", "Token "+strings.Join(token, ",")) - } -} - -func (r *Registry) doRequest(req *http.Request) (*http.Response, *http.Client, error) { - return doRequest(req, r.jar, r.timeout) -} - -// Retrieve the history of a given image from the Registry. -// Return a list of the parent's json (requested image included) -func (r *Registry) GetRemoteHistory(imgID, registry string, token []string) ([]string, error) { - req, err := r.reqFactory.NewRequest("GET", registry+"images/"+imgID+"/ancestry", nil) - if err != nil { - return nil, err - } - setTokenAuth(req, token) - res, _, err := r.doRequest(req) - if err != nil { - return nil, err - } - defer res.Body.Close() - if res.StatusCode != 200 { - if res.StatusCode == 401 { - return nil, errLoginRequired - } - return nil, utils.NewHTTPRequestError(fmt.Sprintf("Server error: %d trying to fetch remote history for %s", res.StatusCode, imgID), res) - } - - jsonString, err := ioutil.ReadAll(res.Body) - if err != nil { - return nil, fmt.Errorf("Error while reading the http response: %s", err) - } - - utils.Debugf("Ancestry: %s", jsonString) - history := new([]string) - if err := json.Unmarshal(jsonString, history); err != nil { - return nil, err - } - return *history, nil -} - -// Check if an image exists in the Registry -// TODO: This method should return the errors instead of masking them and returning false -func (r *Registry) LookupRemoteImage(imgID, registry string, token []string) bool { - - req, err := r.reqFactory.NewRequest("GET", registry+"images/"+imgID+"/json", nil) - if err != nil { - utils.Errorf("Error in LookupRemoteImage %s", err) - return false - } - setTokenAuth(req, token) - res, _, err := r.doRequest(req) - if err != nil { - utils.Errorf("Error in LookupRemoteImage %s", err) - return false - } - res.Body.Close() - return res.StatusCode == 200 -} - -// Retrieve an image from the Registry. -func (r *Registry) GetRemoteImageJSON(imgID, registry string, token []string) ([]byte, int, error) { - // Get the JSON - req, err := r.reqFactory.NewRequest("GET", registry+"images/"+imgID+"/json", nil) - if err != nil { - return nil, -1, fmt.Errorf("Failed to download json: %s", err) - } - setTokenAuth(req, token) - res, _, err := r.doRequest(req) - if err != nil { - return nil, -1, fmt.Errorf("Failed to download json: %s", err) - } - defer res.Body.Close() - if res.StatusCode != 200 { - return nil, -1, utils.NewHTTPRequestError(fmt.Sprintf("HTTP code %d", res.StatusCode), res) - } - - // if the size header is not present, then set it to '-1' - imageSize := -1 - if hdr := res.Header.Get("X-Docker-Size"); hdr != "" { - imageSize, err = strconv.Atoi(hdr) - if err != nil { - return nil, -1, err - } - } - - jsonString, err := ioutil.ReadAll(res.Body) - if err != nil { - return nil, -1, fmt.Errorf("Failed to parse downloaded json: %s (%s)", err, jsonString) - } - return jsonString, imageSize, nil -} - -func (r *Registry) GetRemoteImageLayer(imgID, registry string, token []string, imgSize int64) (io.ReadCloser, error) { - var ( - retries = 5 - client *http.Client - res *http.Response - imageURL = fmt.Sprintf("%simages/%s/layer", registry, imgID) - ) - - req, err := r.reqFactory.NewRequest("GET", imageURL, nil) - if err != nil { - return nil, fmt.Errorf("Error while getting from the server: %s\n", err) - } - setTokenAuth(req, token) - for i := 1; i <= retries; i++ { - res, client, err = r.doRequest(req) - if err != nil { - res.Body.Close() - if i == retries { - return nil, fmt.Errorf("Server error: Status %d while fetching image layer (%s)", - res.StatusCode, imgID) - } - time.Sleep(time.Duration(i) * 5 * time.Second) - continue - } - break - } - - if res.StatusCode != 200 { - res.Body.Close() - return nil, fmt.Errorf("Server error: Status %d while fetching image layer (%s)", - res.StatusCode, imgID) - } - - if res.Header.Get("Accept-Ranges") == "bytes" && imgSize > 0 { - utils.Debugf("server supports resume") - return httputils.ResumableRequestReaderWithInitialResponse(client, req, 5, imgSize, res), nil - } - utils.Debugf("server doesn't support resume") - return res.Body, nil -} - -func (r *Registry) GetRemoteTags(registries []string, repository string, token []string) (map[string]string, error) { - if strings.Count(repository, "/") == 0 { - // This will be removed once the Registry supports auto-resolution on - // the "library" namespace - repository = "library/" + repository - } - for _, host := range registries { - endpoint := fmt.Sprintf("%srepositories/%s/tags", host, repository) - req, err := r.reqFactory.NewRequest("GET", endpoint, nil) - - if err != nil { - return nil, err - } - setTokenAuth(req, token) - res, _, err := r.doRequest(req) - if err != nil { - return nil, err - } - - utils.Debugf("Got status code %d from %s", res.StatusCode, endpoint) - defer res.Body.Close() - - if res.StatusCode != 200 && res.StatusCode != 404 { - continue - } else if res.StatusCode == 404 { - return nil, fmt.Errorf("Repository not found") - } - - result := make(map[string]string) - rawJSON, err := ioutil.ReadAll(res.Body) - if err != nil { - return nil, err - } - if err := json.Unmarshal(rawJSON, &result); err != nil { - return nil, err - } - return result, nil - } - return nil, fmt.Errorf("Could not reach any registry endpoint") -} - -func buildEndpointsList(headers []string, indexEp string) ([]string, error) { - var endpoints []string - parsedUrl, err := url.Parse(indexEp) - if err != nil { - return nil, err - } - var urlScheme = parsedUrl.Scheme - // The Registry's URL scheme has to match the Index' - for _, ep := range headers { - epList := strings.Split(ep, ",") - for _, epListElement := range epList { - endpoints = append( - endpoints, - fmt.Sprintf("%s://%s/v1/", urlScheme, strings.TrimSpace(epListElement))) - } - } - return endpoints, nil -} - -func (r *Registry) GetRepositoryData(remote string) (*RepositoryData, error) { - indexEp := r.indexEndpoint - repositoryTarget := fmt.Sprintf("%srepositories/%s/images", indexEp, remote) - - utils.Debugf("[registry] Calling GET %s", repositoryTarget) - - req, err := r.reqFactory.NewRequest("GET", repositoryTarget, nil) - if err != nil { - return nil, err - } - if r.authConfig != nil && len(r.authConfig.Username) > 0 { - req.SetBasicAuth(r.authConfig.Username, r.authConfig.Password) - } - req.Header.Set("X-Docker-Token", "true") - - res, _, err := r.doRequest(req) - if err != nil { - return nil, err - } - defer res.Body.Close() - if res.StatusCode == 401 { - return nil, errLoginRequired - } - // TODO: Right now we're ignoring checksums in the response body. - // In the future, we need to use them to check image validity. - if res.StatusCode != 200 { - return nil, utils.NewHTTPRequestError(fmt.Sprintf("HTTP code: %d", res.StatusCode), res) - } - - var tokens []string - if res.Header.Get("X-Docker-Token") != "" { - tokens = res.Header["X-Docker-Token"] - } - - var endpoints []string - if res.Header.Get("X-Docker-Endpoints") != "" { - endpoints, err = buildEndpointsList(res.Header["X-Docker-Endpoints"], indexEp) - if err != nil { - return nil, err - } - } else { - // Assume the endpoint is on the same host - u, err := url.Parse(indexEp) - if err != nil { - return nil, err - } - endpoints = append(endpoints, fmt.Sprintf("%s://%s/v1/", u.Scheme, req.URL.Host)) - } - - checksumsJSON, err := ioutil.ReadAll(res.Body) - if err != nil { - return nil, err - } - remoteChecksums := []*ImgData{} - if err := json.Unmarshal(checksumsJSON, &remoteChecksums); err != nil { - return nil, err - } - - // Forge a better object from the retrieved data - imgsData := make(map[string]*ImgData) - for _, elem := range remoteChecksums { - imgsData[elem.ID] = elem - } - - return &RepositoryData{ - ImgList: imgsData, - Endpoints: endpoints, - Tokens: tokens, - }, nil -} - -func (r *Registry) PushImageChecksumRegistry(imgData *ImgData, registry string, token []string) error { - - utils.Debugf("[registry] Calling PUT %s", registry+"images/"+imgData.ID+"/checksum") - - req, err := r.reqFactory.NewRequest("PUT", registry+"images/"+imgData.ID+"/checksum", nil) - if err != nil { - return err - } - setTokenAuth(req, token) - req.Header.Set("X-Docker-Checksum", imgData.Checksum) - req.Header.Set("X-Docker-Checksum-Payload", imgData.ChecksumPayload) - - res, _, err := r.doRequest(req) - if err != nil { - return fmt.Errorf("Failed to upload metadata: %s", err) - } - defer res.Body.Close() - if len(res.Cookies()) > 0 { - r.jar.SetCookies(req.URL, res.Cookies()) - } - if res.StatusCode != 200 { - errBody, err := ioutil.ReadAll(res.Body) - if err != nil { - return fmt.Errorf("HTTP code %d while uploading metadata and error when trying to parse response body: %s", res.StatusCode, err) - } - var jsonBody map[string]string - if err := json.Unmarshal(errBody, &jsonBody); err != nil { - errBody = []byte(err.Error()) - } else if jsonBody["error"] == "Image already exists" { - return ErrAlreadyExists - } - return fmt.Errorf("HTTP code %d while uploading metadata: %s", res.StatusCode, errBody) - } - return nil -} - -// Push a local image to the registry -func (r *Registry) PushImageJSONRegistry(imgData *ImgData, jsonRaw []byte, registry string, token []string) error { - - utils.Debugf("[registry] Calling PUT %s", registry+"images/"+imgData.ID+"/json") - - req, err := r.reqFactory.NewRequest("PUT", registry+"images/"+imgData.ID+"/json", bytes.NewReader(jsonRaw)) - if err != nil { - return err - } - req.Header.Add("Content-type", "application/json") - setTokenAuth(req, token) - - res, _, err := r.doRequest(req) - if err != nil { - return fmt.Errorf("Failed to upload metadata: %s", err) - } - defer res.Body.Close() - if res.StatusCode == 401 && strings.HasPrefix(registry, "http://") { - return utils.NewHTTPRequestError("HTTP code 401, Docker will not send auth headers over HTTP.", res) - } - if res.StatusCode != 200 { - errBody, err := ioutil.ReadAll(res.Body) - if err != nil { - return utils.NewHTTPRequestError(fmt.Sprintf("HTTP code %d while uploading metadata and error when trying to parse response body: %s", res.StatusCode, err), res) - } - var jsonBody map[string]string - if err := json.Unmarshal(errBody, &jsonBody); err != nil { - errBody = []byte(err.Error()) - } else if jsonBody["error"] == "Image already exists" { - return ErrAlreadyExists - } - return utils.NewHTTPRequestError(fmt.Sprintf("HTTP code %d while uploading metadata: %s", res.StatusCode, errBody), res) - } - return nil -} - -func (r *Registry) PushImageLayerRegistry(imgID string, layer io.Reader, registry string, token []string, jsonRaw []byte) (checksum string, checksumPayload string, err error) { - - utils.Debugf("[registry] Calling PUT %s", registry+"images/"+imgID+"/layer") - - tarsumLayer := &tarsum.TarSum{Reader: layer} - h := sha256.New() - h.Write(jsonRaw) - h.Write([]byte{'\n'}) - checksumLayer := io.TeeReader(tarsumLayer, h) - - req, err := r.reqFactory.NewRequest("PUT", registry+"images/"+imgID+"/layer", checksumLayer) - if err != nil { - return "", "", err - } - req.Header.Add("Content-Type", "application/octet-stream") - req.ContentLength = -1 - req.TransferEncoding = []string{"chunked"} - setTokenAuth(req, token) - res, _, err := r.doRequest(req) - if err != nil { - return "", "", fmt.Errorf("Failed to upload layer: %s", err) - } - if rc, ok := layer.(io.Closer); ok { - if err := rc.Close(); err != nil { - return "", "", err - } - } - defer res.Body.Close() - - if res.StatusCode != 200 { - errBody, err := ioutil.ReadAll(res.Body) - if err != nil { - return "", "", utils.NewHTTPRequestError(fmt.Sprintf("HTTP code %d while uploading metadata and error when trying to parse response body: %s", res.StatusCode, err), res) - } - return "", "", utils.NewHTTPRequestError(fmt.Sprintf("Received HTTP code %d while uploading layer: %s", res.StatusCode, errBody), res) - } - - checksumPayload = "sha256:" + hex.EncodeToString(h.Sum(nil)) - return tarsumLayer.Sum(jsonRaw), checksumPayload, nil -} - -// push a tag on the registry. -// Remote has the format '/ -func (r *Registry) PushRegistryTag(remote, revision, tag, registry string, token []string) error { - // "jsonify" the string - revision = "\"" + revision + "\"" - path := fmt.Sprintf("repositories/%s/tags/%s", remote, tag) - - req, err := r.reqFactory.NewRequest("PUT", registry+path, strings.NewReader(revision)) - if err != nil { - return err - } - req.Header.Add("Content-type", "application/json") - setTokenAuth(req, token) - req.ContentLength = int64(len(revision)) - res, _, err := r.doRequest(req) - if err != nil { - return err - } - res.Body.Close() - if res.StatusCode != 200 && res.StatusCode != 201 { - return utils.NewHTTPRequestError(fmt.Sprintf("Internal server error: %d trying to push tag %s on %s", res.StatusCode, tag, remote), res) - } - return nil -} - -func (r *Registry) PushImageJSONIndex(remote string, imgList []*ImgData, validate bool, regs []string) (*RepositoryData, error) { - cleanImgList := []*ImgData{} - indexEp := r.indexEndpoint - - if validate { - for _, elem := range imgList { - if elem.Checksum != "" { - cleanImgList = append(cleanImgList, elem) - } - } - } else { - cleanImgList = imgList - } - - imgListJSON, err := json.Marshal(cleanImgList) - if err != nil { - return nil, err - } - var suffix string - if validate { - suffix = "images" - } - u := fmt.Sprintf("%srepositories/%s/%s", indexEp, remote, suffix) - utils.Debugf("[registry] PUT %s", u) - utils.Debugf("Image list pushed to index:\n%s", imgListJSON) - req, err := r.reqFactory.NewRequest("PUT", u, bytes.NewReader(imgListJSON)) - if err != nil { - return nil, err - } - req.Header.Add("Content-type", "application/json") - req.SetBasicAuth(r.authConfig.Username, r.authConfig.Password) - req.ContentLength = int64(len(imgListJSON)) - req.Header.Set("X-Docker-Token", "true") - if validate { - req.Header["X-Docker-Endpoints"] = regs - } - - res, _, err := r.doRequest(req) - if err != nil { - return nil, err - } - defer res.Body.Close() - - // Redirect if necessary - for res.StatusCode >= 300 && res.StatusCode < 400 { - utils.Debugf("Redirected to %s", res.Header.Get("Location")) - req, err = r.reqFactory.NewRequest("PUT", res.Header.Get("Location"), bytes.NewReader(imgListJSON)) - if err != nil { - return nil, err - } - req.SetBasicAuth(r.authConfig.Username, r.authConfig.Password) - req.ContentLength = int64(len(imgListJSON)) - req.Header.Set("X-Docker-Token", "true") - if validate { - req.Header["X-Docker-Endpoints"] = regs - } - res, _, err := r.doRequest(req) - if err != nil { - return nil, err - } - defer res.Body.Close() - } - - var tokens, endpoints []string - if !validate { - if res.StatusCode != 200 && res.StatusCode != 201 { - errBody, err := ioutil.ReadAll(res.Body) - if err != nil { - return nil, err - } - return nil, utils.NewHTTPRequestError(fmt.Sprintf("Error: Status %d trying to push repository %s: %s", res.StatusCode, remote, errBody), res) - } - if res.Header.Get("X-Docker-Token") != "" { - tokens = res.Header["X-Docker-Token"] - utils.Debugf("Auth token: %v", tokens) - } else { - return nil, fmt.Errorf("Index response didn't contain an access token") - } - - if res.Header.Get("X-Docker-Endpoints") != "" { - endpoints, err = buildEndpointsList(res.Header["X-Docker-Endpoints"], indexEp) - if err != nil { - return nil, err - } - } else { - return nil, fmt.Errorf("Index response didn't contain any endpoints") - } - } - if validate { - if res.StatusCode != 204 { - errBody, err := ioutil.ReadAll(res.Body) - if err != nil { - return nil, err - } - return nil, utils.NewHTTPRequestError(fmt.Sprintf("Error: Status %d trying to push checksums %s: %s", res.StatusCode, remote, errBody), res) - } - } - - return &RepositoryData{ - Tokens: tokens, - Endpoints: endpoints, - }, nil -} - -func (r *Registry) SearchRepositories(term string) (*SearchResults, error) { - utils.Debugf("Index server: %s", r.indexEndpoint) - u := r.indexEndpoint + "search?q=" + url.QueryEscape(term) - req, err := r.reqFactory.NewRequest("GET", u, nil) - if err != nil { - return nil, err - } - if r.authConfig != nil && len(r.authConfig.Username) > 0 { - req.SetBasicAuth(r.authConfig.Username, r.authConfig.Password) - } - req.Header.Set("X-Docker-Token", "true") - res, _, err := r.doRequest(req) - if err != nil { - return nil, err - } - defer res.Body.Close() - if res.StatusCode != 200 { - return nil, utils.NewHTTPRequestError(fmt.Sprintf("Unexepected status code %d", res.StatusCode), res) - } - rawData, err := ioutil.ReadAll(res.Body) - if err != nil { - return nil, err - } - result := new(SearchResults) - err = json.Unmarshal(rawData, result) - return result, err -} - -func (r *Registry) GetAuthConfig(withPasswd bool) *AuthConfig { - password := "" - if withPasswd { - password = r.authConfig.Password - } - return &AuthConfig{ - Username: r.authConfig.Username, - Password: password, - Email: r.authConfig.Email, - } -} - -type SearchResult struct { - StarCount int `json:"star_count"` - IsOfficial bool `json:"is_official"` - Name string `json:"name"` - IsTrusted bool `json:"is_trusted"` - Description string `json:"description"` -} - -type SearchResults struct { - Query string `json:"query"` - NumResults int `json:"num_results"` - Results []SearchResult `json:"results"` -} - -type RepositoryData struct { - ImgList map[string]*ImgData - Endpoints []string - Tokens []string -} - -type ImgData struct { - ID string `json:"id"` - Checksum string `json:"checksum,omitempty"` - ChecksumPayload string `json:"-"` - Tag string `json:",omitempty"` -} - -type RegistryInfo struct { - Version string `json:"version"` - Standalone bool `json:"standalone"` -} - -type Registry struct { - authConfig *AuthConfig - reqFactory *utils.HTTPRequestFactory - indexEndpoint string - jar *cookiejar.Jar - timeout TimeoutType -} - func trustedLocation(req *http.Request) bool { var ( trusteds = []string{"docker.com", "docker.io"} @@ -919,73 +317,3 @@ func AddRequiredHeadersToRedirectedRequests(req *http.Request, via []*http.Reque } return nil } - -func NewRegistry(authConfig *AuthConfig, factory *utils.HTTPRequestFactory, indexEndpoint string, timeout bool) (r *Registry, err error) { - r = &Registry{ - authConfig: authConfig, - indexEndpoint: indexEndpoint, - } - - if timeout { - r.timeout = ReceiveTimeout - } - - r.jar, err = cookiejar.New(nil) - if err != nil { - return nil, err - } - - // If we're working with a standalone private registry over HTTPS, send Basic Auth headers - // alongside our requests. - if indexEndpoint != IndexServerAddress() && strings.HasPrefix(indexEndpoint, "https://") { - info, err := pingRegistryEndpoint(indexEndpoint) - if err != nil { - return nil, err - } - if info.Standalone { - utils.Debugf("Endpoint %s is eligible for private registry registry. Enabling decorator.", indexEndpoint) - dec := utils.NewHTTPAuthDecorator(authConfig.Username, authConfig.Password) - factory.AddDecorator(dec) - } - } - - r.reqFactory = factory - return r, nil -} - -func HTTPRequestFactory(metaHeaders map[string][]string) *utils.HTTPRequestFactory { - // FIXME: this replicates the 'info' job. - httpVersion := make([]utils.VersionInfo, 0, 4) - httpVersion = append(httpVersion, &simpleVersionInfo{"docker", dockerversion.VERSION}) - httpVersion = append(httpVersion, &simpleVersionInfo{"go", runtime.Version()}) - httpVersion = append(httpVersion, &simpleVersionInfo{"git-commit", dockerversion.GITCOMMIT}) - if kernelVersion, err := kernel.GetKernelVersion(); err == nil { - httpVersion = append(httpVersion, &simpleVersionInfo{"kernel", kernelVersion.String()}) - } - httpVersion = append(httpVersion, &simpleVersionInfo{"os", runtime.GOOS}) - httpVersion = append(httpVersion, &simpleVersionInfo{"arch", runtime.GOARCH}) - ud := utils.NewHTTPUserAgentDecorator(httpVersion...) - md := &utils.HTTPMetaHeadersDecorator{ - Headers: metaHeaders, - } - factory := utils.NewHTTPRequestFactory(ud, md) - return factory -} - -// simpleVersionInfo is a simple implementation of -// the interface VersionInfo, which is used -// to provide version information for some product, -// component, etc. It stores the product name and the version -// in string and returns them on calls to Name() and Version(). -type simpleVersionInfo struct { - name string - version string -} - -func (v *simpleVersionInfo) Name() string { - return v.name -} - -func (v *simpleVersionInfo) Version() string { - return v.version -} diff --git a/docs/registry_test.go b/docs/registry_test.go index 12dc7a28..303879e8 100644 --- a/docs/registry_test.go +++ b/docs/registry_test.go @@ -16,9 +16,9 @@ var ( REPO = "foo42/bar" ) -func spawnTestRegistry(t *testing.T) *Registry { +func spawnTestRegistrySession(t *testing.T) *Session { authConfig := &AuthConfig{} - r, err := NewRegistry(authConfig, utils.NewHTTPRequestFactory(), makeURL("/v1/"), true) + r, err := NewSession(authConfig, utils.NewHTTPRequestFactory(), makeURL("/v1/"), true) if err != nil { t.Fatal(err) } @@ -34,7 +34,7 @@ func TestPingRegistryEndpoint(t *testing.T) { } func TestGetRemoteHistory(t *testing.T) { - r := spawnTestRegistry(t) + r := spawnTestRegistrySession(t) hist, err := r.GetRemoteHistory(IMAGE_ID, makeURL("/v1/"), TOKEN) if err != nil { t.Fatal(err) @@ -46,7 +46,7 @@ func TestGetRemoteHistory(t *testing.T) { } func TestLookupRemoteImage(t *testing.T) { - r := spawnTestRegistry(t) + r := spawnTestRegistrySession(t) found := r.LookupRemoteImage(IMAGE_ID, makeURL("/v1/"), TOKEN) assertEqual(t, found, true, "Expected remote lookup to succeed") found = r.LookupRemoteImage("abcdef", makeURL("/v1/"), TOKEN) @@ -54,7 +54,7 @@ func TestLookupRemoteImage(t *testing.T) { } func TestGetRemoteImageJSON(t *testing.T) { - r := spawnTestRegistry(t) + r := spawnTestRegistrySession(t) json, size, err := r.GetRemoteImageJSON(IMAGE_ID, makeURL("/v1/"), TOKEN) if err != nil { t.Fatal(err) @@ -71,7 +71,7 @@ func TestGetRemoteImageJSON(t *testing.T) { } func TestGetRemoteImageLayer(t *testing.T) { - r := spawnTestRegistry(t) + r := spawnTestRegistrySession(t) data, err := r.GetRemoteImageLayer(IMAGE_ID, makeURL("/v1/"), TOKEN, 0) if err != nil { t.Fatal(err) @@ -87,7 +87,7 @@ func TestGetRemoteImageLayer(t *testing.T) { } func TestGetRemoteTags(t *testing.T) { - r := spawnTestRegistry(t) + r := spawnTestRegistrySession(t) tags, err := r.GetRemoteTags([]string{makeURL("/v1/")}, REPO, TOKEN) if err != nil { t.Fatal(err) @@ -102,7 +102,7 @@ func TestGetRemoteTags(t *testing.T) { } func TestGetRepositoryData(t *testing.T) { - r := spawnTestRegistry(t) + r := spawnTestRegistrySession(t) parsedUrl, err := url.Parse(makeURL("/v1/")) if err != nil { t.Fatal(err) @@ -123,7 +123,7 @@ func TestGetRepositoryData(t *testing.T) { } func TestPushImageJSONRegistry(t *testing.T) { - r := spawnTestRegistry(t) + r := spawnTestRegistrySession(t) imgData := &ImgData{ ID: "77dbf71da1d00e3fbddc480176eac8994025630c6590d11cfc8fe1209c2a1d20", Checksum: "sha256:1ac330d56e05eef6d438586545ceff7550d3bdcb6b19961f12c5ba714ee1bb37", @@ -136,7 +136,7 @@ func TestPushImageJSONRegistry(t *testing.T) { } func TestPushImageLayerRegistry(t *testing.T) { - r := spawnTestRegistry(t) + r := spawnTestRegistrySession(t) layer := strings.NewReader("") _, _, err := r.PushImageLayerRegistry(IMAGE_ID, layer, makeURL("/v1/"), TOKEN, []byte{}) if err != nil { @@ -171,7 +171,7 @@ func TestResolveRepositoryName(t *testing.T) { } func TestPushRegistryTag(t *testing.T) { - r := spawnTestRegistry(t) + r := spawnTestRegistrySession(t) err := r.PushRegistryTag("foo42/bar", IMAGE_ID, "stable", makeURL("/v1/"), TOKEN) if err != nil { t.Fatal(err) @@ -179,7 +179,7 @@ func TestPushRegistryTag(t *testing.T) { } func TestPushImageJSONIndex(t *testing.T) { - r := spawnTestRegistry(t) + r := spawnTestRegistrySession(t) imgData := []*ImgData{ { ID: "77dbf71da1d00e3fbddc480176eac8994025630c6590d11cfc8fe1209c2a1d20", @@ -207,7 +207,7 @@ func TestPushImageJSONIndex(t *testing.T) { } func TestSearchRepositories(t *testing.T) { - r := spawnTestRegistry(t) + r := spawnTestRegistrySession(t) results, err := r.SearchRepositories("fakequery") if err != nil { t.Fatal(err) diff --git a/docs/service.go b/docs/service.go index d2775e3c..29afd163 100644 --- a/docs/service.go +++ b/docs/service.go @@ -82,7 +82,7 @@ func (s *Service) Search(job *engine.Job) engine.Status { job.GetenvJson("authConfig", authConfig) job.GetenvJson("metaHeaders", metaHeaders) - r, err := NewRegistry(authConfig, HTTPRequestFactory(metaHeaders), IndexServerAddress(), true) + r, err := NewSession(authConfig, HTTPRequestFactory(metaHeaders), IndexServerAddress(), true) if err != nil { return job.Error(err) } diff --git a/docs/session.go b/docs/session.go new file mode 100644 index 00000000..e60fbeb7 --- /dev/null +++ b/docs/session.go @@ -0,0 +1,611 @@ +package registry + +import ( + "bytes" + "crypto/sha256" + _ "crypto/sha512" + "encoding/hex" + "encoding/json" + "fmt" + "io" + "io/ioutil" + "net/http" + "net/http/cookiejar" + "net/url" + "strconv" + "strings" + "time" + + "github.com/docker/docker/pkg/httputils" + "github.com/docker/docker/pkg/tarsum" + "github.com/docker/docker/utils" +) + +type Session struct { + authConfig *AuthConfig + reqFactory *utils.HTTPRequestFactory + indexEndpoint string + jar *cookiejar.Jar + timeout TimeoutType +} + +func NewSession(authConfig *AuthConfig, factory *utils.HTTPRequestFactory, indexEndpoint string, timeout bool) (r *Session, err error) { + r = &Session{ + authConfig: authConfig, + indexEndpoint: indexEndpoint, + } + + if timeout { + r.timeout = ReceiveTimeout + } + + r.jar, err = cookiejar.New(nil) + if err != nil { + return nil, err + } + + // If we're working with a standalone private registry over HTTPS, send Basic Auth headers + // alongside our requests. + if indexEndpoint != IndexServerAddress() && strings.HasPrefix(indexEndpoint, "https://") { + info, err := pingRegistryEndpoint(indexEndpoint) + if err != nil { + return nil, err + } + if info.Standalone { + utils.Debugf("Endpoint %s is eligible for private registry registry. Enabling decorator.", indexEndpoint) + dec := utils.NewHTTPAuthDecorator(authConfig.Username, authConfig.Password) + factory.AddDecorator(dec) + } + } + + r.reqFactory = factory + return r, nil +} + +func (r *Session) doRequest(req *http.Request) (*http.Response, *http.Client, error) { + return doRequest(req, r.jar, r.timeout) +} + +// Retrieve the history of a given image from the Registry. +// Return a list of the parent's json (requested image included) +func (r *Session) GetRemoteHistory(imgID, registry string, token []string) ([]string, error) { + req, err := r.reqFactory.NewRequest("GET", registry+"images/"+imgID+"/ancestry", nil) + if err != nil { + return nil, err + } + setTokenAuth(req, token) + res, _, err := r.doRequest(req) + if err != nil { + return nil, err + } + defer res.Body.Close() + if res.StatusCode != 200 { + if res.StatusCode == 401 { + return nil, errLoginRequired + } + return nil, utils.NewHTTPRequestError(fmt.Sprintf("Server error: %d trying to fetch remote history for %s", res.StatusCode, imgID), res) + } + + jsonString, err := ioutil.ReadAll(res.Body) + if err != nil { + return nil, fmt.Errorf("Error while reading the http response: %s", err) + } + + utils.Debugf("Ancestry: %s", jsonString) + history := new([]string) + if err := json.Unmarshal(jsonString, history); err != nil { + return nil, err + } + return *history, nil +} + +// Check if an image exists in the Registry +// TODO: This method should return the errors instead of masking them and returning false +func (r *Session) LookupRemoteImage(imgID, registry string, token []string) bool { + + req, err := r.reqFactory.NewRequest("GET", registry+"images/"+imgID+"/json", nil) + if err != nil { + utils.Errorf("Error in LookupRemoteImage %s", err) + return false + } + setTokenAuth(req, token) + res, _, err := r.doRequest(req) + if err != nil { + utils.Errorf("Error in LookupRemoteImage %s", err) + return false + } + res.Body.Close() + return res.StatusCode == 200 +} + +// Retrieve an image from the Registry. +func (r *Session) GetRemoteImageJSON(imgID, registry string, token []string) ([]byte, int, error) { + // Get the JSON + req, err := r.reqFactory.NewRequest("GET", registry+"images/"+imgID+"/json", nil) + if err != nil { + return nil, -1, fmt.Errorf("Failed to download json: %s", err) + } + setTokenAuth(req, token) + res, _, err := r.doRequest(req) + if err != nil { + return nil, -1, fmt.Errorf("Failed to download json: %s", err) + } + defer res.Body.Close() + if res.StatusCode != 200 { + return nil, -1, utils.NewHTTPRequestError(fmt.Sprintf("HTTP code %d", res.StatusCode), res) + } + // if the size header is not present, then set it to '-1' + imageSize := -1 + if hdr := res.Header.Get("X-Docker-Size"); hdr != "" { + imageSize, err = strconv.Atoi(hdr) + if err != nil { + return nil, -1, err + } + } + + jsonString, err := ioutil.ReadAll(res.Body) + if err != nil { + return nil, -1, fmt.Errorf("Failed to parse downloaded json: %s (%s)", err, jsonString) + } + return jsonString, imageSize, nil +} + +func (r *Session) GetRemoteImageLayer(imgID, registry string, token []string, imgSize int64) (io.ReadCloser, error) { + var ( + retries = 5 + client *http.Client + res *http.Response + imageURL = fmt.Sprintf("%simages/%s/layer", registry, imgID) + ) + + req, err := r.reqFactory.NewRequest("GET", imageURL, nil) + if err != nil { + return nil, fmt.Errorf("Error while getting from the server: %s\n", err) + } + setTokenAuth(req, token) + for i := 1; i <= retries; i++ { + res, client, err = r.doRequest(req) + if err != nil { + res.Body.Close() + if i == retries { + return nil, fmt.Errorf("Server error: Status %d while fetching image layer (%s)", + res.StatusCode, imgID) + } + time.Sleep(time.Duration(i) * 5 * time.Second) + continue + } + break + } + + if res.StatusCode != 200 { + res.Body.Close() + return nil, fmt.Errorf("Server error: Status %d while fetching image layer (%s)", + res.StatusCode, imgID) + } + + if res.Header.Get("Accept-Ranges") == "bytes" && imgSize > 0 { + utils.Debugf("server supports resume") + return httputils.ResumableRequestReaderWithInitialResponse(client, req, 5, imgSize, res), nil + } + utils.Debugf("server doesn't support resume") + return res.Body, nil +} + +func (r *Session) GetRemoteTags(registries []string, repository string, token []string) (map[string]string, error) { + if strings.Count(repository, "/") == 0 { + // This will be removed once the Registry supports auto-resolution on + // the "library" namespace + repository = "library/" + repository + } + for _, host := range registries { + endpoint := fmt.Sprintf("%srepositories/%s/tags", host, repository) + req, err := r.reqFactory.NewRequest("GET", endpoint, nil) + + if err != nil { + return nil, err + } + setTokenAuth(req, token) + res, _, err := r.doRequest(req) + if err != nil { + return nil, err + } + + utils.Debugf("Got status code %d from %s", res.StatusCode, endpoint) + defer res.Body.Close() + + if res.StatusCode != 200 && res.StatusCode != 404 { + continue + } else if res.StatusCode == 404 { + return nil, fmt.Errorf("Repository not found") + } + + result := make(map[string]string) + rawJSON, err := ioutil.ReadAll(res.Body) + if err != nil { + return nil, err + } + if err := json.Unmarshal(rawJSON, &result); err != nil { + return nil, err + } + return result, nil + } + return nil, fmt.Errorf("Could not reach any registry endpoint") +} + +func buildEndpointsList(headers []string, indexEp string) ([]string, error) { + var endpoints []string + parsedUrl, err := url.Parse(indexEp) + if err != nil { + return nil, err + } + var urlScheme = parsedUrl.Scheme + // The Registry's URL scheme has to match the Index' + for _, ep := range headers { + epList := strings.Split(ep, ",") + for _, epListElement := range epList { + endpoints = append( + endpoints, + fmt.Sprintf("%s://%s/v1/", urlScheme, strings.TrimSpace(epListElement))) + } + } + return endpoints, nil +} + +func (r *Session) GetRepositoryData(remote string) (*RepositoryData, error) { + indexEp := r.indexEndpoint + repositoryTarget := fmt.Sprintf("%srepositories/%s/images", indexEp, remote) + + utils.Debugf("[registry] Calling GET %s", repositoryTarget) + + req, err := r.reqFactory.NewRequest("GET", repositoryTarget, nil) + if err != nil { + return nil, err + } + if r.authConfig != nil && len(r.authConfig.Username) > 0 { + req.SetBasicAuth(r.authConfig.Username, r.authConfig.Password) + } + req.Header.Set("X-Docker-Token", "true") + + res, _, err := r.doRequest(req) + if err != nil { + return nil, err + } + defer res.Body.Close() + if res.StatusCode == 401 { + return nil, errLoginRequired + } + // TODO: Right now we're ignoring checksums in the response body. + // In the future, we need to use them to check image validity. + if res.StatusCode != 200 { + return nil, utils.NewHTTPRequestError(fmt.Sprintf("HTTP code: %d", res.StatusCode), res) + } + + var tokens []string + if res.Header.Get("X-Docker-Token") != "" { + tokens = res.Header["X-Docker-Token"] + } + + var endpoints []string + if res.Header.Get("X-Docker-Endpoints") != "" { + endpoints, err = buildEndpointsList(res.Header["X-Docker-Endpoints"], indexEp) + if err != nil { + return nil, err + } + } else { + // Assume the endpoint is on the same host + u, err := url.Parse(indexEp) + if err != nil { + return nil, err + } + endpoints = append(endpoints, fmt.Sprintf("%s://%s/v1/", u.Scheme, req.URL.Host)) + } + + checksumsJSON, err := ioutil.ReadAll(res.Body) + if err != nil { + return nil, err + } + remoteChecksums := []*ImgData{} + if err := json.Unmarshal(checksumsJSON, &remoteChecksums); err != nil { + return nil, err + } + + // Forge a better object from the retrieved data + imgsData := make(map[string]*ImgData) + for _, elem := range remoteChecksums { + imgsData[elem.ID] = elem + } + + return &RepositoryData{ + ImgList: imgsData, + Endpoints: endpoints, + Tokens: tokens, + }, nil +} + +func (r *Session) PushImageChecksumRegistry(imgData *ImgData, registry string, token []string) error { + + utils.Debugf("[registry] Calling PUT %s", registry+"images/"+imgData.ID+"/checksum") + + req, err := r.reqFactory.NewRequest("PUT", registry+"images/"+imgData.ID+"/checksum", nil) + if err != nil { + return err + } + setTokenAuth(req, token) + req.Header.Set("X-Docker-Checksum", imgData.Checksum) + req.Header.Set("X-Docker-Checksum-Payload", imgData.ChecksumPayload) + + res, _, err := r.doRequest(req) + if err != nil { + return fmt.Errorf("Failed to upload metadata: %s", err) + } + defer res.Body.Close() + if len(res.Cookies()) > 0 { + r.jar.SetCookies(req.URL, res.Cookies()) + } + if res.StatusCode != 200 { + errBody, err := ioutil.ReadAll(res.Body) + if err != nil { + return fmt.Errorf("HTTP code %d while uploading metadata and error when trying to parse response body: %s", res.StatusCode, err) + } + var jsonBody map[string]string + if err := json.Unmarshal(errBody, &jsonBody); err != nil { + errBody = []byte(err.Error()) + } else if jsonBody["error"] == "Image already exists" { + return ErrAlreadyExists + } + return fmt.Errorf("HTTP code %d while uploading metadata: %s", res.StatusCode, errBody) + } + return nil +} + +// Push a local image to the registry +func (r *Session) PushImageJSONRegistry(imgData *ImgData, jsonRaw []byte, registry string, token []string) error { + + utils.Debugf("[registry] Calling PUT %s", registry+"images/"+imgData.ID+"/json") + + req, err := r.reqFactory.NewRequest("PUT", registry+"images/"+imgData.ID+"/json", bytes.NewReader(jsonRaw)) + if err != nil { + return err + } + req.Header.Add("Content-type", "application/json") + setTokenAuth(req, token) + + res, _, err := r.doRequest(req) + if err != nil { + return fmt.Errorf("Failed to upload metadata: %s", err) + } + defer res.Body.Close() + if res.StatusCode == 401 && strings.HasPrefix(registry, "http://") { + return utils.NewHTTPRequestError("HTTP code 401, Docker will not send auth headers over HTTP.", res) + } + if res.StatusCode != 200 { + errBody, err := ioutil.ReadAll(res.Body) + if err != nil { + return utils.NewHTTPRequestError(fmt.Sprintf("HTTP code %d while uploading metadata and error when trying to parse response body: %s", res.StatusCode, err), res) + } + var jsonBody map[string]string + if err := json.Unmarshal(errBody, &jsonBody); err != nil { + errBody = []byte(err.Error()) + } else if jsonBody["error"] == "Image already exists" { + return ErrAlreadyExists + } + return utils.NewHTTPRequestError(fmt.Sprintf("HTTP code %d while uploading metadata: %s", res.StatusCode, errBody), res) + } + return nil +} + +func (r *Session) PushImageLayerRegistry(imgID string, layer io.Reader, registry string, token []string, jsonRaw []byte) (checksum string, checksumPayload string, err error) { + + utils.Debugf("[registry] Calling PUT %s", registry+"images/"+imgID+"/layer") + + tarsumLayer := &tarsum.TarSum{Reader: layer} + h := sha256.New() + h.Write(jsonRaw) + h.Write([]byte{'\n'}) + checksumLayer := io.TeeReader(tarsumLayer, h) + + req, err := r.reqFactory.NewRequest("PUT", registry+"images/"+imgID+"/layer", checksumLayer) + if err != nil { + return "", "", err + } + req.Header.Add("Content-Type", "application/octet-stream") + req.ContentLength = -1 + req.TransferEncoding = []string{"chunked"} + setTokenAuth(req, token) + res, _, err := r.doRequest(req) + if err != nil { + return "", "", fmt.Errorf("Failed to upload layer: %s", err) + } + if rc, ok := layer.(io.Closer); ok { + if err := rc.Close(); err != nil { + return "", "", err + } + } + defer res.Body.Close() + + if res.StatusCode != 200 { + errBody, err := ioutil.ReadAll(res.Body) + if err != nil { + return "", "", utils.NewHTTPRequestError(fmt.Sprintf("HTTP code %d while uploading metadata and error when trying to parse response body: %s", res.StatusCode, err), res) + } + return "", "", utils.NewHTTPRequestError(fmt.Sprintf("Received HTTP code %d while uploading layer: %s", res.StatusCode, errBody), res) + } + + checksumPayload = "sha256:" + hex.EncodeToString(h.Sum(nil)) + return tarsumLayer.Sum(jsonRaw), checksumPayload, nil +} + +// push a tag on the registry. +// Remote has the format '/ +func (r *Session) PushRegistryTag(remote, revision, tag, registry string, token []string) error { + // "jsonify" the string + revision = "\"" + revision + "\"" + path := fmt.Sprintf("repositories/%s/tags/%s", remote, tag) + + req, err := r.reqFactory.NewRequest("PUT", registry+path, strings.NewReader(revision)) + if err != nil { + return err + } + req.Header.Add("Content-type", "application/json") + setTokenAuth(req, token) + req.ContentLength = int64(len(revision)) + res, _, err := r.doRequest(req) + if err != nil { + return err + } + res.Body.Close() + if res.StatusCode != 200 && res.StatusCode != 201 { + return utils.NewHTTPRequestError(fmt.Sprintf("Internal server error: %d trying to push tag %s on %s", res.StatusCode, tag, remote), res) + } + return nil +} + +func (r *Session) PushImageJSONIndex(remote string, imgList []*ImgData, validate bool, regs []string) (*RepositoryData, error) { + cleanImgList := []*ImgData{} + indexEp := r.indexEndpoint + + if validate { + for _, elem := range imgList { + if elem.Checksum != "" { + cleanImgList = append(cleanImgList, elem) + } + } + } else { + cleanImgList = imgList + } + + imgListJSON, err := json.Marshal(cleanImgList) + if err != nil { + return nil, err + } + var suffix string + if validate { + suffix = "images" + } + u := fmt.Sprintf("%srepositories/%s/%s", indexEp, remote, suffix) + utils.Debugf("[registry] PUT %s", u) + utils.Debugf("Image list pushed to index:\n%s", imgListJSON) + req, err := r.reqFactory.NewRequest("PUT", u, bytes.NewReader(imgListJSON)) + if err != nil { + return nil, err + } + req.Header.Add("Content-type", "application/json") + req.SetBasicAuth(r.authConfig.Username, r.authConfig.Password) + req.ContentLength = int64(len(imgListJSON)) + req.Header.Set("X-Docker-Token", "true") + if validate { + req.Header["X-Docker-Endpoints"] = regs + } + + res, _, err := r.doRequest(req) + if err != nil { + return nil, err + } + defer res.Body.Close() + + // Redirect if necessary + for res.StatusCode >= 300 && res.StatusCode < 400 { + utils.Debugf("Redirected to %s", res.Header.Get("Location")) + req, err = r.reqFactory.NewRequest("PUT", res.Header.Get("Location"), bytes.NewReader(imgListJSON)) + if err != nil { + return nil, err + } + req.SetBasicAuth(r.authConfig.Username, r.authConfig.Password) + req.ContentLength = int64(len(imgListJSON)) + req.Header.Set("X-Docker-Token", "true") + if validate { + req.Header["X-Docker-Endpoints"] = regs + } + res, _, err := r.doRequest(req) + if err != nil { + return nil, err + } + defer res.Body.Close() + } + + var tokens, endpoints []string + if !validate { + if res.StatusCode != 200 && res.StatusCode != 201 { + errBody, err := ioutil.ReadAll(res.Body) + if err != nil { + return nil, err + } + return nil, utils.NewHTTPRequestError(fmt.Sprintf("Error: Status %d trying to push repository %s: %s", res.StatusCode, remote, errBody), res) + } + if res.Header.Get("X-Docker-Token") != "" { + tokens = res.Header["X-Docker-Token"] + utils.Debugf("Auth token: %v", tokens) + } else { + return nil, fmt.Errorf("Index response didn't contain an access token") + } + + if res.Header.Get("X-Docker-Endpoints") != "" { + endpoints, err = buildEndpointsList(res.Header["X-Docker-Endpoints"], indexEp) + if err != nil { + return nil, err + } + } else { + return nil, fmt.Errorf("Index response didn't contain any endpoints") + } + } + if validate { + if res.StatusCode != 204 { + errBody, err := ioutil.ReadAll(res.Body) + if err != nil { + return nil, err + } + return nil, utils.NewHTTPRequestError(fmt.Sprintf("Error: Status %d trying to push checksums %s: %s", res.StatusCode, remote, errBody), res) + } + } + + return &RepositoryData{ + Tokens: tokens, + Endpoints: endpoints, + }, nil +} + +func (r *Session) SearchRepositories(term string) (*SearchResults, error) { + utils.Debugf("Index server: %s", r.indexEndpoint) + u := r.indexEndpoint + "search?q=" + url.QueryEscape(term) + req, err := r.reqFactory.NewRequest("GET", u, nil) + if err != nil { + return nil, err + } + if r.authConfig != nil && len(r.authConfig.Username) > 0 { + req.SetBasicAuth(r.authConfig.Username, r.authConfig.Password) + } + req.Header.Set("X-Docker-Token", "true") + res, _, err := r.doRequest(req) + if err != nil { + return nil, err + } + defer res.Body.Close() + if res.StatusCode != 200 { + return nil, utils.NewHTTPRequestError(fmt.Sprintf("Unexepected status code %d", res.StatusCode), res) + } + rawData, err := ioutil.ReadAll(res.Body) + if err != nil { + return nil, err + } + result := new(SearchResults) + err = json.Unmarshal(rawData, result) + return result, err +} + +func (r *Session) GetAuthConfig(withPasswd bool) *AuthConfig { + password := "" + if withPasswd { + password = r.authConfig.Password + } + return &AuthConfig{ + Username: r.authConfig.Username, + Password: password, + Email: r.authConfig.Email, + } +} + +func setTokenAuth(req *http.Request, token []string) { + if req.Header.Get("Authorization") == "" { // Don't override + req.Header.Set("Authorization", "Token "+strings.Join(token, ",")) + } +} diff --git a/docs/types.go b/docs/types.go new file mode 100644 index 00000000..70d55e42 --- /dev/null +++ b/docs/types.go @@ -0,0 +1,33 @@ +package registry + +type SearchResult struct { + StarCount int `json:"star_count"` + IsOfficial bool `json:"is_official"` + Name string `json:"name"` + IsTrusted bool `json:"is_trusted"` + Description string `json:"description"` +} + +type SearchResults struct { + Query string `json:"query"` + NumResults int `json:"num_results"` + Results []SearchResult `json:"results"` +} + +type RepositoryData struct { + ImgList map[string]*ImgData + Endpoints []string + Tokens []string +} + +type ImgData struct { + ID string `json:"id"` + Checksum string `json:"checksum,omitempty"` + ChecksumPayload string `json:"-"` + Tag string `json:",omitempty"` +} + +type RegistryInfo struct { + Version string `json:"version"` + Standalone bool `json:"standalone"` +} From 2a7cf96c8fbf7c18b96f20fd89a45e1dd6732f6f Mon Sep 17 00:00:00 2001 From: Josiah Kiehl Date: Thu, 24 Jul 2014 13:37:44 -0700 Subject: [PATCH 143/375] Extract log utils into pkg/log Docker-DCO-1.1-Signed-off-by: Josiah Kiehl (github: capoferro) --- docs/registry.go | 13 +++++++------ docs/registry_mock_test.go | 8 +++++--- 2 files changed, 12 insertions(+), 9 deletions(-) diff --git a/docs/registry.go b/docs/registry.go index 14b8f6d5..9c76aca9 100644 --- a/docs/registry.go +++ b/docs/registry.go @@ -15,6 +15,7 @@ import ( "strings" "time" + "github.com/docker/docker/pkg/log" "github.com/docker/docker/utils" ) @@ -186,17 +187,17 @@ func pingRegistryEndpoint(endpoint string) (RegistryInfo, error) { Standalone: true, } if err := json.Unmarshal(jsonString, &info); err != nil { - utils.Debugf("Error unmarshalling the _ping RegistryInfo: %s", err) + log.Debugf("Error unmarshalling the _ping RegistryInfo: %s", err) // don't stop here. Just assume sane defaults } if hdr := resp.Header.Get("X-Docker-Registry-Version"); hdr != "" { - utils.Debugf("Registry version header: '%s'", hdr) + log.Debugf("Registry version header: '%s'", hdr) info.Version = hdr } - utils.Debugf("RegistryInfo.Version: %q", info.Version) + log.Debugf("RegistryInfo.Version: %q", info.Version) standalone := resp.Header.Get("X-Docker-Registry-Standalone") - utils.Debugf("Registry standalone header: '%s'", standalone) + log.Debugf("Registry standalone header: '%s'", standalone) // Accepted values are "true" (case-insensitive) and "1". if strings.EqualFold(standalone, "true") || standalone == "1" { info.Standalone = true @@ -204,7 +205,7 @@ func pingRegistryEndpoint(endpoint string) (RegistryInfo, error) { // there is a header set, and it is not "true" or "1", so assume fails info.Standalone = false } - utils.Debugf("RegistryInfo.Standalone: %q", info.Standalone) + log.Debugf("RegistryInfo.Standalone: %q", info.Standalone) return info, nil } @@ -274,7 +275,7 @@ func ExpandAndVerifyRegistryUrl(hostname string) (string, error) { } endpoint := fmt.Sprintf("https://%s/v1/", hostname) if _, err := pingRegistryEndpoint(endpoint); err != nil { - utils.Debugf("Registry %s does not work (%s), falling back to http", endpoint, err) + log.Debugf("Registry %s does not work (%s), falling back to http", endpoint, err) endpoint = fmt.Sprintf("http://%s/v1/", hostname) if _, err = pingRegistryEndpoint(endpoint); err != nil { //TODO: triggering highland build can be done there without "failing" diff --git a/docs/registry_mock_test.go b/docs/registry_mock_test.go index 1a622228..2b4cd9de 100644 --- a/docs/registry_mock_test.go +++ b/docs/registry_mock_test.go @@ -3,8 +3,6 @@ package registry import ( "encoding/json" "fmt" - "github.com/docker/docker/utils" - "github.com/gorilla/mux" "io" "io/ioutil" "net/http" @@ -14,6 +12,10 @@ import ( "strings" "testing" "time" + + "github.com/gorilla/mux" + + "github.com/docker/docker/pkg/log" ) var ( @@ -96,7 +98,7 @@ func init() { func handlerAccessLog(handler http.Handler) http.Handler { logHandler := func(w http.ResponseWriter, r *http.Request) { - utils.Debugf("%s \"%s %s\"", r.RemoteAddr, r.Method, r.URL) + log.Debugf("%s \"%s %s\"", r.RemoteAddr, r.Method, r.URL) handler.ServeHTTP(w, r) } return http.HandlerFunc(logHandler) From 94ff3f3e4d08c1da67b54b176ad2df96fcf21fc1 Mon Sep 17 00:00:00 2001 From: Erik Hollensbe Date: Wed, 13 Aug 2014 15:13:21 -0700 Subject: [PATCH 144/375] move utils.Fataler to pkg/log.Fataler Docker-DCO-1.1-Signed-off-by: Erik Hollensbe (github: erikh) --- docs/session.go | 33 +++++++++++++++++---------------- 1 file changed, 17 insertions(+), 16 deletions(-) diff --git a/docs/session.go b/docs/session.go index e60fbeb7..82b931f2 100644 --- a/docs/session.go +++ b/docs/session.go @@ -17,6 +17,7 @@ import ( "time" "github.com/docker/docker/pkg/httputils" + "github.com/docker/docker/pkg/log" "github.com/docker/docker/pkg/tarsum" "github.com/docker/docker/utils" ) @@ -52,7 +53,7 @@ func NewSession(authConfig *AuthConfig, factory *utils.HTTPRequestFactory, index return nil, err } if info.Standalone { - utils.Debugf("Endpoint %s is eligible for private registry registry. Enabling decorator.", indexEndpoint) + log.Debugf("Endpoint %s is eligible for private registry registry. Enabling decorator.", indexEndpoint) dec := utils.NewHTTPAuthDecorator(authConfig.Username, authConfig.Password) factory.AddDecorator(dec) } @@ -91,7 +92,7 @@ func (r *Session) GetRemoteHistory(imgID, registry string, token []string) ([]st return nil, fmt.Errorf("Error while reading the http response: %s", err) } - utils.Debugf("Ancestry: %s", jsonString) + log.Debugf("Ancestry: %s", jsonString) history := new([]string) if err := json.Unmarshal(jsonString, history); err != nil { return nil, err @@ -105,13 +106,13 @@ func (r *Session) LookupRemoteImage(imgID, registry string, token []string) bool req, err := r.reqFactory.NewRequest("GET", registry+"images/"+imgID+"/json", nil) if err != nil { - utils.Errorf("Error in LookupRemoteImage %s", err) + log.Errorf("Error in LookupRemoteImage %s", err) return false } setTokenAuth(req, token) res, _, err := r.doRequest(req) if err != nil { - utils.Errorf("Error in LookupRemoteImage %s", err) + log.Errorf("Error in LookupRemoteImage %s", err) return false } res.Body.Close() @@ -184,10 +185,10 @@ func (r *Session) GetRemoteImageLayer(imgID, registry string, token []string, im } if res.Header.Get("Accept-Ranges") == "bytes" && imgSize > 0 { - utils.Debugf("server supports resume") + log.Debugf("server supports resume") return httputils.ResumableRequestReaderWithInitialResponse(client, req, 5, imgSize, res), nil } - utils.Debugf("server doesn't support resume") + log.Debugf("server doesn't support resume") return res.Body, nil } @@ -210,7 +211,7 @@ func (r *Session) GetRemoteTags(registries []string, repository string, token [] return nil, err } - utils.Debugf("Got status code %d from %s", res.StatusCode, endpoint) + log.Debugf("Got status code %d from %s", res.StatusCode, endpoint) defer res.Body.Close() if res.StatusCode != 200 && res.StatusCode != 404 { @@ -255,7 +256,7 @@ func (r *Session) GetRepositoryData(remote string) (*RepositoryData, error) { indexEp := r.indexEndpoint repositoryTarget := fmt.Sprintf("%srepositories/%s/images", indexEp, remote) - utils.Debugf("[registry] Calling GET %s", repositoryTarget) + log.Debugf("[registry] Calling GET %s", repositoryTarget) req, err := r.reqFactory.NewRequest("GET", repositoryTarget, nil) if err != nil { @@ -324,7 +325,7 @@ func (r *Session) GetRepositoryData(remote string) (*RepositoryData, error) { func (r *Session) PushImageChecksumRegistry(imgData *ImgData, registry string, token []string) error { - utils.Debugf("[registry] Calling PUT %s", registry+"images/"+imgData.ID+"/checksum") + log.Debugf("[registry] Calling PUT %s", registry+"images/"+imgData.ID+"/checksum") req, err := r.reqFactory.NewRequest("PUT", registry+"images/"+imgData.ID+"/checksum", nil) if err != nil { @@ -361,7 +362,7 @@ func (r *Session) PushImageChecksumRegistry(imgData *ImgData, registry string, t // Push a local image to the registry func (r *Session) PushImageJSONRegistry(imgData *ImgData, jsonRaw []byte, registry string, token []string) error { - utils.Debugf("[registry] Calling PUT %s", registry+"images/"+imgData.ID+"/json") + log.Debugf("[registry] Calling PUT %s", registry+"images/"+imgData.ID+"/json") req, err := r.reqFactory.NewRequest("PUT", registry+"images/"+imgData.ID+"/json", bytes.NewReader(jsonRaw)) if err != nil { @@ -396,7 +397,7 @@ func (r *Session) PushImageJSONRegistry(imgData *ImgData, jsonRaw []byte, regist func (r *Session) PushImageLayerRegistry(imgID string, layer io.Reader, registry string, token []string, jsonRaw []byte) (checksum string, checksumPayload string, err error) { - utils.Debugf("[registry] Calling PUT %s", registry+"images/"+imgID+"/layer") + log.Debugf("[registry] Calling PUT %s", registry+"images/"+imgID+"/layer") tarsumLayer := &tarsum.TarSum{Reader: layer} h := sha256.New() @@ -483,8 +484,8 @@ func (r *Session) PushImageJSONIndex(remote string, imgList []*ImgData, validate suffix = "images" } u := fmt.Sprintf("%srepositories/%s/%s", indexEp, remote, suffix) - utils.Debugf("[registry] PUT %s", u) - utils.Debugf("Image list pushed to index:\n%s", imgListJSON) + log.Debugf("[registry] PUT %s", u) + log.Debugf("Image list pushed to index:\n%s", imgListJSON) req, err := r.reqFactory.NewRequest("PUT", u, bytes.NewReader(imgListJSON)) if err != nil { return nil, err @@ -505,7 +506,7 @@ func (r *Session) PushImageJSONIndex(remote string, imgList []*ImgData, validate // Redirect if necessary for res.StatusCode >= 300 && res.StatusCode < 400 { - utils.Debugf("Redirected to %s", res.Header.Get("Location")) + log.Debugf("Redirected to %s", res.Header.Get("Location")) req, err = r.reqFactory.NewRequest("PUT", res.Header.Get("Location"), bytes.NewReader(imgListJSON)) if err != nil { return nil, err @@ -534,7 +535,7 @@ func (r *Session) PushImageJSONIndex(remote string, imgList []*ImgData, validate } if res.Header.Get("X-Docker-Token") != "" { tokens = res.Header["X-Docker-Token"] - utils.Debugf("Auth token: %v", tokens) + log.Debugf("Auth token: %v", tokens) } else { return nil, fmt.Errorf("Index response didn't contain an access token") } @@ -565,7 +566,7 @@ func (r *Session) PushImageJSONIndex(remote string, imgList []*ImgData, validate } func (r *Session) SearchRepositories(term string) (*SearchResults, error) { - utils.Debugf("Index server: %s", r.indexEndpoint) + log.Debugf("Index server: %s", r.indexEndpoint) u := r.indexEndpoint + "search?q=" + url.QueryEscape(term) req, err := r.reqFactory.NewRequest("GET", u, nil) if err != nil { From 744919be3d5dde9abe48ef242c4134e8b4cd1898 Mon Sep 17 00:00:00 2001 From: Daniel Menet Date: Sat, 9 Aug 2014 09:16:54 +0200 Subject: [PATCH 145/375] Enable `docker search` on private docker registry. The cli interface works similar to other registry related commands: docker search foo ... searches for foo on the official hub docker search localhost:5000/foo ... does the same for the private reg at localhost:5000 Signed-off-by: Daniel Menet --- docs/service.go | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/docs/service.go b/docs/service.go index 29afd163..2d493f7e 100644 --- a/docs/service.go +++ b/docs/service.go @@ -82,6 +82,14 @@ func (s *Service) Search(job *engine.Job) engine.Status { job.GetenvJson("authConfig", authConfig) job.GetenvJson("metaHeaders", metaHeaders) + hostname, term, err := ResolveRepositoryName(term) + if err != nil { + return job.Error(err) + } + hostname, err = ExpandAndVerifyRegistryUrl(hostname) + if err != nil { + return job.Error(err) + } r, err := NewSession(authConfig, HTTPRequestFactory(metaHeaders), IndexServerAddress(), true) if err != nil { return job.Error(err) From 283fba482103906eacfd1068a202c5e97b45f818 Mon Sep 17 00:00:00 2001 From: Daniel Menet Date: Sun, 10 Aug 2014 11:48:34 +0200 Subject: [PATCH 146/375] Expand hostname before passing it to NewRegistry() Signed-off-by: Daniel Menet --- docs/service.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/service.go b/docs/service.go index 2d493f7e..0e6f1bda 100644 --- a/docs/service.go +++ b/docs/service.go @@ -90,7 +90,7 @@ func (s *Service) Search(job *engine.Job) engine.Status { if err != nil { return job.Error(err) } - r, err := NewSession(authConfig, HTTPRequestFactory(metaHeaders), IndexServerAddress(), true) + r, err := NewSession(authConfig, HTTPRequestFactory(metaHeaders), hostname, true) if err != nil { return job.Error(err) } From 4d8f45a94d89c23d4b98d47318353e705494b534 Mon Sep 17 00:00:00 2001 From: Erik Hollensbe Date: Mon, 25 Aug 2014 10:29:38 -0700 Subject: [PATCH 147/375] fix return values in registry mock service Docker-DCO-1.1-Signed-off-by: Erik Hollensbe (github: erikh) --- docs/registry_mock_test.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/docs/registry_mock_test.go b/docs/registry_mock_test.go index 2b4cd9de..8851dcbe 100644 --- a/docs/registry_mock_test.go +++ b/docs/registry_mock_test.go @@ -236,6 +236,7 @@ func handlerGetDeleteTags(w http.ResponseWriter, r *http.Request) { tags, exists := testRepositories[repositoryName] if !exists { apiError(w, "Repository not found", 404) + return } if r.Method == "DELETE" { delete(testRepositories, repositoryName) @@ -255,10 +256,12 @@ func handlerGetTag(w http.ResponseWriter, r *http.Request) { tags, exists := testRepositories[repositoryName] if !exists { apiError(w, "Repository not found", 404) + return } tag, exists := tags[tagName] if !exists { apiError(w, "Tag not found", 404) + return } writeResponse(w, tag, 200) } From 27e0ec3d584ed32280b9384b1ec8103a91571700 Mon Sep 17 00:00:00 2001 From: Alexandr Morozov Date: Mon, 25 Aug 2014 20:50:18 +0400 Subject: [PATCH 148/375] Style fixes for registry/registry.go Signed-off-by: Alexandr Morozov --- docs/registry.go | 51 +++++++++++++++++++----------------------------- 1 file changed, 20 insertions(+), 31 deletions(-) diff --git a/docs/registry.go b/docs/registry.go index 9c76aca9..e2c9794c 100644 --- a/docs/registry.go +++ b/docs/registry.go @@ -105,22 +105,20 @@ func doRequest(req *http.Request, jar http.CookieJar, timeout TimeoutType) (*htt data, err := ioutil.ReadFile(path.Join(hostDir, f.Name())) if err != nil { return nil, nil, err - } else { - pool.AppendCertsFromPEM(data) } + pool.AppendCertsFromPEM(data) } if strings.HasSuffix(f.Name(), ".cert") { certName := f.Name() keyName := certName[:len(certName)-5] + ".key" if !hasFile(fs, keyName) { return nil, nil, fmt.Errorf("Missing key %s for certificate %s", keyName, certName) - } else { - cert, err := tls.LoadX509KeyPair(path.Join(hostDir, certName), path.Join(hostDir, keyName)) - if err != nil { - return nil, nil, err - } - certs = append(certs, &cert) } + cert, err := tls.LoadX509KeyPair(path.Join(hostDir, certName), path.Join(hostDir, keyName)) + if err != nil { + return nil, nil, err + } + certs = append(certs, &cert) } if strings.HasSuffix(f.Name(), ".key") { keyName := f.Name() @@ -138,19 +136,13 @@ func doRequest(req *http.Request, jar http.CookieJar, timeout TimeoutType) (*htt return nil, nil, err } return res, client, nil - } else { - for i, cert := range certs { - client := newClient(jar, pool, cert, timeout) - res, err := client.Do(req) - if i == len(certs)-1 { - // If this is the last cert, always return the result - return res, client, err - } else { - // Otherwise, continue to next cert if 403 or 5xx - if err == nil && res.StatusCode != 403 && !(res.StatusCode >= 500 && res.StatusCode < 600) { - return res, client, err - } - } + } + for i, cert := range certs { + client := newClient(jar, pool, cert, timeout) + res, err := client.Do(req) + // If this is the last cert, otherwise, continue to next cert if 403 or 5xx + if i == len(certs)-1 || err == nil && res.StatusCode != 403 && res.StatusCode < 500 { + return res, client, err } } @@ -198,10 +190,7 @@ func pingRegistryEndpoint(endpoint string) (RegistryInfo, error) { standalone := resp.Header.Get("X-Docker-Registry-Standalone") log.Debugf("Registry standalone header: '%s'", standalone) - // Accepted values are "true" (case-insensitive) and "1". - if strings.EqualFold(standalone, "true") || standalone == "1" { - info.Standalone = true - } else if len(standalone) > 0 { + if !strings.EqualFold(standalone, "true") && standalone != "1" && len(standalone) > 0 { // there is a header set, and it is not "true" or "1", so assume fails info.Standalone = false } @@ -306,12 +295,12 @@ func AddRequiredHeadersToRedirectedRequests(req *http.Request, via []*http.Reque if via != nil && via[0] != nil { if trustedLocation(req) && trustedLocation(via[0]) { req.Header = via[0].Header - } else { - for k, v := range via[0].Header { - if k != "Authorization" { - for _, vv := range v { - req.Header.Add(k, vv) - } + return nil + } + for k, v := range via[0].Header { + if k != "Authorization" { + for _, vv := range v { + req.Header.Add(k, vv) } } } From 307e253d3330218cba4f40fc9b2d01ef5fcaecae Mon Sep 17 00:00:00 2001 From: Eric Windisch Date: Sun, 17 Aug 2014 20:50:15 -0400 Subject: [PATCH 149/375] Restrict repository names from matching hexadecimal strings To avoid conflicting with layer IDs, repository names must not be tagged with names that collide with hexadecimal strings. Signed-off-by: Eric Windisch --- docs/registry.go | 5 +++++ docs/registry_test.go | 8 ++++++++ 2 files changed, 13 insertions(+) diff --git a/docs/registry.go b/docs/registry.go index 9c76aca9..4233d1f8 100644 --- a/docs/registry.go +++ b/docs/registry.go @@ -23,6 +23,7 @@ var ( ErrAlreadyExists = errors.New("Image already exists") ErrInvalidRepositoryName = errors.New("Invalid repository name (ex: \"registry.domain.tld/myrepos\")") errLoginRequired = errors.New("Authentication is required.") + validHex = regexp.MustCompile(`^([a-f0-9]{64})$`) ) type TimeoutType uint32 @@ -218,6 +219,10 @@ func validateRepositoryName(repositoryName string) error { if len(nameParts) < 2 { namespace = "library" name = nameParts[0] + + if validHex.MatchString(name) { + return fmt.Errorf("Invalid repository name (%s), cannot specify 64-byte hexadecimal strings", name) + } } else { namespace = nameParts[0] name = nameParts[1] diff --git a/docs/registry_test.go b/docs/registry_test.go index 303879e8..9f4f1230 100644 --- a/docs/registry_test.go +++ b/docs/registry_test.go @@ -224,6 +224,10 @@ func TestValidRepositoryName(t *testing.T) { if err := validateRepositoryName("docker/docker"); err != nil { t.Fatal(err) } + // Support 64-byte non-hexadecimal names (hexadecimal names are forbidden) + if err := validateRepositoryName("thisisthesongthatneverendsitgoesonandonandonthisisthesongthatnev"); err != nil { + t.Fatal(err) + } if err := validateRepositoryName("docker/Docker"); err == nil { t.Log("Repository name should be invalid") t.Fail() @@ -232,6 +236,10 @@ func TestValidRepositoryName(t *testing.T) { t.Log("Repository name should be invalid") t.Fail() } + if err := validateRepositoryName("1a3f5e7d9c1b3a5f7e9d1c3b5a7f9e1d3c5b7a9f1e3d5d7c9b1a3f5e7d9c1b3a"); err == nil { + t.Log("Repository name should be invalid, 64-byte hexadecimal names forbidden") + t.Fail() + } } func TestTrustedLocation(t *testing.T) { From 2c78019539192273693460dc33af8b0b76bf5071 Mon Sep 17 00:00:00 2001 From: unclejack Date: Wed, 3 Sep 2014 17:26:56 +0300 Subject: [PATCH 150/375] registry/session: fix panic in GetRemoteImageLayer Docker-DCO-1.1-Signed-off-by: Cristian Staretu (github: unclejack) --- docs/session.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/docs/session.go b/docs/session.go index 82b931f2..a8ade705 100644 --- a/docs/session.go +++ b/docs/session.go @@ -167,7 +167,9 @@ func (r *Session) GetRemoteImageLayer(imgID, registry string, token []string, im for i := 1; i <= retries; i++ { res, client, err = r.doRequest(req) if err != nil { - res.Body.Close() + if res.Body != nil { + res.Body.Close() + } if i == retries { return nil, fmt.Errorf("Server error: Status %d while fetching image layer (%s)", res.StatusCode, imgID) From eaf57e8f559aec9d73cb1d6d3ba203ffa184b1a3 Mon Sep 17 00:00:00 2001 From: Arthur Gautier Date: Wed, 3 Sep 2014 15:21:06 +0200 Subject: [PATCH 151/375] Fix SEGFAULT if dns resolv error Per registry.doRequest, res and client might be nil in case of error For example, dns resolution errors, /etc/docker/certs.d perms, failed loading of x509 cert ... This will make res.StatusCode and res.Body SEGFAULT. Signed-off-by: Arthur Gautier --- docs/session.go | 20 +++++++++++++------- 1 file changed, 13 insertions(+), 7 deletions(-) diff --git a/docs/session.go b/docs/session.go index a8ade705..c78e559b 100644 --- a/docs/session.go +++ b/docs/session.go @@ -153,10 +153,11 @@ func (r *Session) GetRemoteImageJSON(imgID, registry string, token []string) ([] func (r *Session) GetRemoteImageLayer(imgID, registry string, token []string, imgSize int64) (io.ReadCloser, error) { var ( - retries = 5 - client *http.Client - res *http.Response - imageURL = fmt.Sprintf("%simages/%s/layer", registry, imgID) + retries = 5 + statusCode = 0 + client *http.Client + res *http.Response + imageURL = fmt.Sprintf("%simages/%s/layer", registry, imgID) ) req, err := r.reqFactory.NewRequest("GET", imageURL, nil) @@ -165,14 +166,19 @@ func (r *Session) GetRemoteImageLayer(imgID, registry string, token []string, im } setTokenAuth(req, token) for i := 1; i <= retries; i++ { + statusCode = 0 res, client, err = r.doRequest(req) if err != nil { - if res.Body != nil { - res.Body.Close() + log.Debugf("Error contacting registry: %s", err) + if res != nil { + if res.Body != nil { + res.Body.Close() + } + statusCode = res.StatusCode } if i == retries { return nil, fmt.Errorf("Server error: Status %d while fetching image layer (%s)", - res.StatusCode, imgID) + statusCode, imgID) } time.Sleep(time.Duration(i) * 5 * time.Second) continue From 898bcf0f5d2cce3ac8e5aedcb6362743bcb0c27b Mon Sep 17 00:00:00 2001 From: Vincent Batts Date: Thu, 21 Aug 2014 16:12:52 -0400 Subject: [PATCH 152/375] TarSum: versioning This introduces Versions for TarSum checksums. Fixes: https://github.com/docker/docker/issues/7526 It preserves current functionality and abstracts the interface for future flexibility of hashing algorithms. As a POC, the VersionDev Tarsum does not include the mtime in the checksum calculation, and would solve https://github.com/docker/docker/issues/7387 though this is not a settled Version is subject to change until a version number is assigned. Signed-off-by: Vincent Batts --- docs/session.go | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/docs/session.go b/docs/session.go index c78e559b..58263ef6 100644 --- a/docs/session.go +++ b/docs/session.go @@ -407,7 +407,10 @@ func (r *Session) PushImageLayerRegistry(imgID string, layer io.Reader, registry log.Debugf("[registry] Calling PUT %s", registry+"images/"+imgID+"/layer") - tarsumLayer := &tarsum.TarSum{Reader: layer} + tarsumLayer, err := tarsum.NewTarSum(layer, false, tarsum.Version0) + if err != nil { + return "", "", err + } h := sha256.New() h.Write(jsonRaw) h.Write([]byte{'\n'}) From b7da79fd14bfc82d4b902c2ca935e1f0244d1775 Mon Sep 17 00:00:00 2001 From: Phil Estes Date: Mon, 15 Sep 2014 23:30:10 -0400 Subject: [PATCH 153/375] Refactor all pre-compiled regexp to package level vars Addresses #8057 Docker-DCO-1.1-Signed-off-by: Phil Estes --- docs/registry.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/registry.go b/docs/registry.go index c773bd57..a2e1fbdd 100644 --- a/docs/registry.go +++ b/docs/registry.go @@ -24,6 +24,8 @@ var ( ErrInvalidRepositoryName = errors.New("Invalid repository name (ex: \"registry.domain.tld/myrepos\")") errLoginRequired = errors.New("Authentication is required.") validHex = regexp.MustCompile(`^([a-f0-9]{64})$`) + validNamespace = regexp.MustCompile(`^([a-z0-9_]{4,30})$`) + validRepo = regexp.MustCompile(`^([a-z0-9-_.]+)$`) ) type TimeoutType uint32 @@ -216,11 +218,9 @@ func validateRepositoryName(repositoryName string) error { namespace = nameParts[0] name = nameParts[1] } - validNamespace := regexp.MustCompile(`^([a-z0-9_]{4,30})$`) if !validNamespace.MatchString(namespace) { return fmt.Errorf("Invalid namespace name (%s), only [a-z0-9_] are allowed, size between 4 and 30", namespace) } - validRepo := regexp.MustCompile(`^([a-z0-9-_.]+)$`) if !validRepo.MatchString(name) { return fmt.Errorf("Invalid repository name (%s), only [a-z0-9-_.] are allowed", name) } From 48b43c26459d6c8e033a92d47ec8cb81019df14c Mon Sep 17 00:00:00 2001 From: Tibor Vass Date: Tue, 23 Sep 2014 19:18:09 -0400 Subject: [PATCH 154/375] Replace get.docker.io -> get.docker.com and test.docker.io -> test.docker.com Signed-off-by: Tibor Vass --- docs/registry_test.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/registry_test.go b/docs/registry_test.go index 9f4f1230..8a95221d 100644 --- a/docs/registry_test.go +++ b/docs/registry_test.go @@ -243,14 +243,14 @@ func TestValidRepositoryName(t *testing.T) { } func TestTrustedLocation(t *testing.T) { - for _, url := range []string{"http://example.com", "https://example.com:7777", "http://docker.io", "http://test.docker.io", "https://fakedocker.com"} { + for _, url := range []string{"http://example.com", "https://example.com:7777", "http://docker.io", "http://test.docker.com", "https://fakedocker.com"} { req, _ := http.NewRequest("GET", url, nil) if trustedLocation(req) == true { t.Fatalf("'%s' shouldn't be detected as a trusted location", url) } } - for _, url := range []string{"https://docker.io", "https://test.docker.io:80"} { + for _, url := range []string{"https://docker.io", "https://test.docker.com:80"} { req, _ := http.NewRequest("GET", url, nil) if trustedLocation(req) == false { t.Fatalf("'%s' should be detected as a trusted location", url) From d629bebce242acf85e59f4bdc6b12c0960493e7a Mon Sep 17 00:00:00 2001 From: Vincent Batts Date: Tue, 26 Aug 2014 16:21:04 -0700 Subject: [PATCH 155/375] registry: getting Endpoint ironned out Signed-off-by: Vincent Batts --- docs/endpoint.go | 129 ++++++++++++++++++++++++++++++++++++++++++ docs/registry.go | 78 ------------------------- docs/registry_test.go | 14 ++++- docs/service.go | 11 ++-- docs/session.go | 30 ++++------ docs/types.go | 18 ++++++ 6 files changed, 177 insertions(+), 103 deletions(-) create mode 100644 docs/endpoint.go diff --git a/docs/endpoint.go b/docs/endpoint.go new file mode 100644 index 00000000..12df9e0c --- /dev/null +++ b/docs/endpoint.go @@ -0,0 +1,129 @@ +package registry + +import ( + "encoding/json" + "errors" + "fmt" + "io/ioutil" + "net/http" + "net/url" + "strings" + + "github.com/docker/docker/pkg/log" +) + +// scans string for api version in the URL path. returns the trimmed hostname, if version found, string and API version. +func scanForApiVersion(hostname string) (string, APIVersion) { + var ( + chunks []string + apiVersionStr string + ) + if strings.HasSuffix(hostname, "/") { + chunks = strings.Split(hostname[:len(hostname)-1], "/") + apiVersionStr = chunks[len(chunks)-1] + } else { + chunks = strings.Split(hostname, "/") + apiVersionStr = chunks[len(chunks)-1] + } + for k, v := range apiVersions { + if apiVersionStr == v { + hostname = strings.Join(chunks[:len(chunks)-1], "/") + return hostname, k + } + } + return hostname, DefaultAPIVersion +} + +func NewEndpoint(hostname string) (*Endpoint, error) { + var ( + endpoint Endpoint + trimmedHostname string + err error + ) + if !strings.HasPrefix(hostname, "http") { + hostname = "https://" + hostname + } + trimmedHostname, endpoint.Version = scanForApiVersion(hostname) + endpoint.URL, err = url.Parse(trimmedHostname) + if err != nil { + return nil, err + } + + endpoint.URL.Scheme = "https" + if _, err := endpoint.Ping(); err != nil { + log.Debugf("Registry %s does not work (%s), falling back to http", endpoint, err) + // TODO: Check if http fallback is enabled + endpoint.URL.Scheme = "http" + if _, err = endpoint.Ping(); err != nil { + return nil, errors.New("Invalid Registry endpoint: " + err.Error()) + } + } + + return &endpoint, nil +} + +type Endpoint struct { + URL *url.URL + Version APIVersion +} + +// Get the formated URL for the root of this registry Endpoint +func (e Endpoint) String() string { + return fmt.Sprintf("%s/v%d/", e.URL.String(), e.Version) +} + +func (e Endpoint) VersionString(version APIVersion) string { + return fmt.Sprintf("%s/v%d/", e.URL.String(), version) +} + +func (e Endpoint) Ping() (RegistryInfo, error) { + if e.String() == IndexServerAddress() { + // Skip the check, we now this one is valid + // (and we never want to fallback to http in case of error) + return RegistryInfo{Standalone: false}, nil + } + + req, err := http.NewRequest("GET", e.String()+"_ping", nil) + if err != nil { + return RegistryInfo{Standalone: false}, err + } + + resp, _, err := doRequest(req, nil, ConnectTimeout) + if err != nil { + return RegistryInfo{Standalone: false}, err + } + + defer resp.Body.Close() + + jsonString, err := ioutil.ReadAll(resp.Body) + if err != nil { + return RegistryInfo{Standalone: false}, fmt.Errorf("Error while reading the http response: %s", err) + } + + // If the header is absent, we assume true for compatibility with earlier + // versions of the registry. default to true + info := RegistryInfo{ + Standalone: true, + } + if err := json.Unmarshal(jsonString, &info); err != nil { + log.Debugf("Error unmarshalling the _ping RegistryInfo: %s", err) + // don't stop here. Just assume sane defaults + } + if hdr := resp.Header.Get("X-Docker-Registry-Version"); hdr != "" { + log.Debugf("Registry version header: '%s'", hdr) + info.Version = hdr + } + log.Debugf("RegistryInfo.Version: %q", info.Version) + + standalone := resp.Header.Get("X-Docker-Registry-Standalone") + log.Debugf("Registry standalone header: '%s'", standalone) + // Accepted values are "true" (case-insensitive) and "1". + if strings.EqualFold(standalone, "true") || standalone == "1" { + info.Standalone = true + } else if len(standalone) > 0 { + // there is a header set, and it is not "true" or "1", so assume fails + info.Standalone = false + } + log.Debugf("RegistryInfo.Standalone: %q", info.Standalone) + return info, nil +} diff --git a/docs/registry.go b/docs/registry.go index a2e1fbdd..203dfa6f 100644 --- a/docs/registry.go +++ b/docs/registry.go @@ -3,7 +3,6 @@ package registry import ( "crypto/tls" "crypto/x509" - "encoding/json" "errors" "fmt" "io/ioutil" @@ -15,7 +14,6 @@ import ( "strings" "time" - "github.com/docker/docker/pkg/log" "github.com/docker/docker/utils" ) @@ -152,55 +150,6 @@ func doRequest(req *http.Request, jar http.CookieJar, timeout TimeoutType) (*htt return nil, nil, nil } -func pingRegistryEndpoint(endpoint string) (RegistryInfo, error) { - if endpoint == IndexServerAddress() { - // Skip the check, we now this one is valid - // (and we never want to fallback to http in case of error) - return RegistryInfo{Standalone: false}, nil - } - - req, err := http.NewRequest("GET", endpoint+"_ping", nil) - if err != nil { - return RegistryInfo{Standalone: false}, err - } - - resp, _, err := doRequest(req, nil, ConnectTimeout) - if err != nil { - return RegistryInfo{Standalone: false}, err - } - - defer resp.Body.Close() - - jsonString, err := ioutil.ReadAll(resp.Body) - if err != nil { - return RegistryInfo{Standalone: false}, fmt.Errorf("Error while reading the http response: %s", err) - } - - // If the header is absent, we assume true for compatibility with earlier - // versions of the registry. default to true - info := RegistryInfo{ - Standalone: true, - } - if err := json.Unmarshal(jsonString, &info); err != nil { - log.Debugf("Error unmarshalling the _ping RegistryInfo: %s", err) - // don't stop here. Just assume sane defaults - } - if hdr := resp.Header.Get("X-Docker-Registry-Version"); hdr != "" { - log.Debugf("Registry version header: '%s'", hdr) - info.Version = hdr - } - log.Debugf("RegistryInfo.Version: %q", info.Version) - - standalone := resp.Header.Get("X-Docker-Registry-Standalone") - log.Debugf("Registry standalone header: '%s'", standalone) - if !strings.EqualFold(standalone, "true") && standalone != "1" && len(standalone) > 0 { - // there is a header set, and it is not "true" or "1", so assume fails - info.Standalone = false - } - log.Debugf("RegistryInfo.Standalone: %q", info.Standalone) - return info, nil -} - func validateRepositoryName(repositoryName string) error { var ( namespace string @@ -252,33 +201,6 @@ func ResolveRepositoryName(reposName string) (string, string, error) { return hostname, reposName, nil } -// this method expands the registry name as used in the prefix of a repo -// to a full url. if it already is a url, there will be no change. -// The registry is pinged to test if it http or https -func ExpandAndVerifyRegistryUrl(hostname string) (string, error) { - if strings.HasPrefix(hostname, "http:") || strings.HasPrefix(hostname, "https:") { - // if there is no slash after https:// (8 characters) then we have no path in the url - if strings.LastIndex(hostname, "/") < 9 { - // there is no path given. Expand with default path - hostname = hostname + "/v1/" - } - if _, err := pingRegistryEndpoint(hostname); err != nil { - return "", errors.New("Invalid Registry endpoint: " + err.Error()) - } - return hostname, nil - } - endpoint := fmt.Sprintf("https://%s/v1/", hostname) - if _, err := pingRegistryEndpoint(endpoint); err != nil { - log.Debugf("Registry %s does not work (%s), falling back to http", endpoint, err) - endpoint = fmt.Sprintf("http://%s/v1/", hostname) - if _, err = pingRegistryEndpoint(endpoint); err != nil { - //TODO: triggering highland build can be done there without "failing" - return "", errors.New("Invalid Registry endpoint: " + err.Error()) - } - } - return endpoint, nil -} - func trustedLocation(req *http.Request) bool { var ( trusteds = []string{"docker.com", "docker.io"} diff --git a/docs/registry_test.go b/docs/registry_test.go index 8a95221d..ab417812 100644 --- a/docs/registry_test.go +++ b/docs/registry_test.go @@ -18,7 +18,11 @@ var ( func spawnTestRegistrySession(t *testing.T) *Session { authConfig := &AuthConfig{} - r, err := NewSession(authConfig, utils.NewHTTPRequestFactory(), makeURL("/v1/"), true) + endpoint, err := NewEndpoint(makeURL("/v1/")) + if err != nil { + t.Fatal(err) + } + r, err := NewSession(authConfig, utils.NewHTTPRequestFactory(), endpoint, true) if err != nil { t.Fatal(err) } @@ -26,7 +30,11 @@ func spawnTestRegistrySession(t *testing.T) *Session { } func TestPingRegistryEndpoint(t *testing.T) { - regInfo, err := pingRegistryEndpoint(makeURL("/v1/")) + ep, err := NewEndpoint(makeURL("/v1/")) + if err != nil { + t.Fatal(err) + } + regInfo, err := ep.Ping() if err != nil { t.Fatal(err) } @@ -197,7 +205,7 @@ func TestPushImageJSONIndex(t *testing.T) { if repoData == nil { t.Fatal("Expected RepositoryData object") } - repoData, err = r.PushImageJSONIndex("foo42/bar", imgData, true, []string{r.indexEndpoint}) + repoData, err = r.PushImageJSONIndex("foo42/bar", imgData, true, []string{r.indexEndpoint.String()}) if err != nil { t.Fatal(err) } diff --git a/docs/service.go b/docs/service.go index 0e6f1bda..f7b35300 100644 --- a/docs/service.go +++ b/docs/service.go @@ -40,11 +40,14 @@ func (s *Service) Auth(job *engine.Job) engine.Status { job.GetenvJson("authConfig", authConfig) // TODO: this is only done here because auth and registry need to be merged into one pkg if addr := authConfig.ServerAddress; addr != "" && addr != IndexServerAddress() { - addr, err = ExpandAndVerifyRegistryUrl(addr) + endpoint, err := NewEndpoint(addr) if err != nil { return job.Error(err) } - authConfig.ServerAddress = addr + if _, err := endpoint.Ping(); err != nil { + return job.Error(err) + } + authConfig.ServerAddress = endpoint.String() } status, err := Login(authConfig, HTTPRequestFactory(nil)) if err != nil { @@ -86,11 +89,11 @@ func (s *Service) Search(job *engine.Job) engine.Status { if err != nil { return job.Error(err) } - hostname, err = ExpandAndVerifyRegistryUrl(hostname) + endpoint, err := NewEndpoint(hostname) if err != nil { return job.Error(err) } - r, err := NewSession(authConfig, HTTPRequestFactory(metaHeaders), hostname, true) + r, err := NewSession(authConfig, HTTPRequestFactory(metaHeaders), endpoint, true) if err != nil { return job.Error(err) } diff --git a/docs/session.go b/docs/session.go index 58263ef6..48626308 100644 --- a/docs/session.go +++ b/docs/session.go @@ -25,15 +25,15 @@ import ( type Session struct { authConfig *AuthConfig reqFactory *utils.HTTPRequestFactory - indexEndpoint string + indexEndpoint *Endpoint jar *cookiejar.Jar timeout TimeoutType } -func NewSession(authConfig *AuthConfig, factory *utils.HTTPRequestFactory, indexEndpoint string, timeout bool) (r *Session, err error) { +func NewSession(authConfig *AuthConfig, factory *utils.HTTPRequestFactory, endpoint *Endpoint, timeout bool) (r *Session, err error) { r = &Session{ authConfig: authConfig, - indexEndpoint: indexEndpoint, + indexEndpoint: endpoint, } if timeout { @@ -47,13 +47,13 @@ func NewSession(authConfig *AuthConfig, factory *utils.HTTPRequestFactory, index // If we're working with a standalone private registry over HTTPS, send Basic Auth headers // alongside our requests. - if indexEndpoint != IndexServerAddress() && strings.HasPrefix(indexEndpoint, "https://") { - info, err := pingRegistryEndpoint(indexEndpoint) + if r.indexEndpoint.String() != IndexServerAddress() && r.indexEndpoint.URL.Scheme == "https" { + info, err := r.indexEndpoint.Ping() if err != nil { return nil, err } if info.Standalone { - log.Debugf("Endpoint %s is eligible for private registry registry. Enabling decorator.", indexEndpoint) + log.Debugf("Endpoint %s is eligible for private registry registry. Enabling decorator.", r.indexEndpoint.String()) dec := utils.NewHTTPAuthDecorator(authConfig.Username, authConfig.Password) factory.AddDecorator(dec) } @@ -261,8 +261,7 @@ func buildEndpointsList(headers []string, indexEp string) ([]string, error) { } func (r *Session) GetRepositoryData(remote string) (*RepositoryData, error) { - indexEp := r.indexEndpoint - repositoryTarget := fmt.Sprintf("%srepositories/%s/images", indexEp, remote) + repositoryTarget := fmt.Sprintf("%srepositories/%s/images", r.indexEndpoint.String(), remote) log.Debugf("[registry] Calling GET %s", repositoryTarget) @@ -296,17 +295,13 @@ func (r *Session) GetRepositoryData(remote string) (*RepositoryData, error) { var endpoints []string if res.Header.Get("X-Docker-Endpoints") != "" { - endpoints, err = buildEndpointsList(res.Header["X-Docker-Endpoints"], indexEp) + endpoints, err = buildEndpointsList(res.Header["X-Docker-Endpoints"], r.indexEndpoint.String()) if err != nil { return nil, err } } else { // Assume the endpoint is on the same host - u, err := url.Parse(indexEp) - if err != nil { - return nil, err - } - endpoints = append(endpoints, fmt.Sprintf("%s://%s/v1/", u.Scheme, req.URL.Host)) + endpoints = append(endpoints, fmt.Sprintf("%s://%s/v1/", r.indexEndpoint.URL.Scheme, req.URL.Host)) } checksumsJSON, err := ioutil.ReadAll(res.Body) @@ -474,7 +469,6 @@ func (r *Session) PushRegistryTag(remote, revision, tag, registry string, token func (r *Session) PushImageJSONIndex(remote string, imgList []*ImgData, validate bool, regs []string) (*RepositoryData, error) { cleanImgList := []*ImgData{} - indexEp := r.indexEndpoint if validate { for _, elem := range imgList { @@ -494,7 +488,7 @@ func (r *Session) PushImageJSONIndex(remote string, imgList []*ImgData, validate if validate { suffix = "images" } - u := fmt.Sprintf("%srepositories/%s/%s", indexEp, remote, suffix) + u := fmt.Sprintf("%srepositories/%s/%s", r.indexEndpoint.String(), remote, suffix) log.Debugf("[registry] PUT %s", u) log.Debugf("Image list pushed to index:\n%s", imgListJSON) req, err := r.reqFactory.NewRequest("PUT", u, bytes.NewReader(imgListJSON)) @@ -552,7 +546,7 @@ func (r *Session) PushImageJSONIndex(remote string, imgList []*ImgData, validate } if res.Header.Get("X-Docker-Endpoints") != "" { - endpoints, err = buildEndpointsList(res.Header["X-Docker-Endpoints"], indexEp) + endpoints, err = buildEndpointsList(res.Header["X-Docker-Endpoints"], r.indexEndpoint.String()) if err != nil { return nil, err } @@ -578,7 +572,7 @@ func (r *Session) PushImageJSONIndex(remote string, imgList []*ImgData, validate func (r *Session) SearchRepositories(term string) (*SearchResults, error) { log.Debugf("Index server: %s", r.indexEndpoint) - u := r.indexEndpoint + "search?q=" + url.QueryEscape(term) + u := r.indexEndpoint.String() + "search?q=" + url.QueryEscape(term) req, err := r.reqFactory.NewRequest("GET", u, nil) if err != nil { return nil, err diff --git a/docs/types.go b/docs/types.go index 70d55e42..3db236da 100644 --- a/docs/types.go +++ b/docs/types.go @@ -31,3 +31,21 @@ type RegistryInfo struct { Version string `json:"version"` Standalone bool `json:"standalone"` } + +type APIVersion int + +func (av APIVersion) String() string { + return apiVersions[av] +} + +var DefaultAPIVersion APIVersion = APIVersion1 +var apiVersions = map[APIVersion]string{ + 1: "v1", + 2: "v2", +} + +const ( + _ = iota + APIVersion1 = iota + APIVersion2 +) From b7f7b0a2c992558c22d23254390f8a4223160541 Mon Sep 17 00:00:00 2001 From: Derek McGowan Date: Wed, 1 Oct 2014 18:26:06 -0700 Subject: [PATCH 156/375] Add provenance pull flow for official images Add support for pulling signed images from a version 2 registry. Only official images within the library namespace will be pull from the new registry and check the build signature. Signed-off-by: Derek McGowan (github: dmcgowan) --- docs/registry.go | 1 + docs/registry_mock_test.go | 6 + docs/session.go | 12 +- docs/session_v2.go | 386 +++++++++++++++++++++++++++++++++++++ docs/types.go | 12 +- 5 files changed, 409 insertions(+), 8 deletions(-) create mode 100644 docs/session_v2.go diff --git a/docs/registry.go b/docs/registry.go index 203dfa6f..fd74b751 100644 --- a/docs/registry.go +++ b/docs/registry.go @@ -20,6 +20,7 @@ import ( var ( ErrAlreadyExists = errors.New("Image already exists") ErrInvalidRepositoryName = errors.New("Invalid repository name (ex: \"registry.domain.tld/myrepos\")") + ErrDoesNotExist = errors.New("Image does not exist") errLoginRequired = errors.New("Authentication is required.") validHex = regexp.MustCompile(`^([a-f0-9]{64})$`) validNamespace = regexp.MustCompile(`^([a-z0-9_]{4,30})$`) diff --git a/docs/registry_mock_test.go b/docs/registry_mock_test.go index 8851dcbe..379dc78f 100644 --- a/docs/registry_mock_test.go +++ b/docs/registry_mock_test.go @@ -83,6 +83,8 @@ var ( func init() { r := mux.NewRouter() + + // /v1/ r.HandleFunc("/v1/_ping", handlerGetPing).Methods("GET") r.HandleFunc("/v1/images/{image_id:[^/]+}/{action:json|layer|ancestry}", handlerGetImage).Methods("GET") r.HandleFunc("/v1/images/{image_id:[^/]+}/{action:json|layer|checksum}", handlerPutImage).Methods("PUT") @@ -93,6 +95,10 @@ func init() { r.HandleFunc("/v1/repositories/{repository:.+}{action:/images|/}", handlerImages).Methods("GET", "PUT", "DELETE") r.HandleFunc("/v1/repositories/{repository:.+}/auth", handlerAuth).Methods("PUT") r.HandleFunc("/v1/search", handlerSearch).Methods("GET") + + // /v2/ + r.HandleFunc("/v2/version", handlerGetPing).Methods("GET") + testHttpServer = httptest.NewServer(handlerAccessLog(r)) } diff --git a/docs/session.go b/docs/session.go index 48626308..5067b8d5 100644 --- a/docs/session.go +++ b/docs/session.go @@ -47,7 +47,7 @@ func NewSession(authConfig *AuthConfig, factory *utils.HTTPRequestFactory, endpo // If we're working with a standalone private registry over HTTPS, send Basic Auth headers // alongside our requests. - if r.indexEndpoint.String() != IndexServerAddress() && r.indexEndpoint.URL.Scheme == "https" { + if r.indexEndpoint.VersionString(1) != IndexServerAddress() && r.indexEndpoint.URL.Scheme == "https" { info, err := r.indexEndpoint.Ping() if err != nil { return nil, err @@ -261,7 +261,7 @@ func buildEndpointsList(headers []string, indexEp string) ([]string, error) { } func (r *Session) GetRepositoryData(remote string) (*RepositoryData, error) { - repositoryTarget := fmt.Sprintf("%srepositories/%s/images", r.indexEndpoint.String(), remote) + repositoryTarget := fmt.Sprintf("%srepositories/%s/images", r.indexEndpoint.VersionString(1), remote) log.Debugf("[registry] Calling GET %s", repositoryTarget) @@ -295,7 +295,7 @@ func (r *Session) GetRepositoryData(remote string) (*RepositoryData, error) { var endpoints []string if res.Header.Get("X-Docker-Endpoints") != "" { - endpoints, err = buildEndpointsList(res.Header["X-Docker-Endpoints"], r.indexEndpoint.String()) + endpoints, err = buildEndpointsList(res.Header["X-Docker-Endpoints"], r.indexEndpoint.VersionString(1)) if err != nil { return nil, err } @@ -488,7 +488,7 @@ func (r *Session) PushImageJSONIndex(remote string, imgList []*ImgData, validate if validate { suffix = "images" } - u := fmt.Sprintf("%srepositories/%s/%s", r.indexEndpoint.String(), remote, suffix) + u := fmt.Sprintf("%srepositories/%s/%s", r.indexEndpoint.VersionString(1), remote, suffix) log.Debugf("[registry] PUT %s", u) log.Debugf("Image list pushed to index:\n%s", imgListJSON) req, err := r.reqFactory.NewRequest("PUT", u, bytes.NewReader(imgListJSON)) @@ -546,7 +546,7 @@ func (r *Session) PushImageJSONIndex(remote string, imgList []*ImgData, validate } if res.Header.Get("X-Docker-Endpoints") != "" { - endpoints, err = buildEndpointsList(res.Header["X-Docker-Endpoints"], r.indexEndpoint.String()) + endpoints, err = buildEndpointsList(res.Header["X-Docker-Endpoints"], r.indexEndpoint.VersionString(1)) if err != nil { return nil, err } @@ -572,7 +572,7 @@ func (r *Session) PushImageJSONIndex(remote string, imgList []*ImgData, validate func (r *Session) SearchRepositories(term string) (*SearchResults, error) { log.Debugf("Index server: %s", r.indexEndpoint) - u := r.indexEndpoint.String() + "search?q=" + url.QueryEscape(term) + u := r.indexEndpoint.VersionString(1) + "search?q=" + url.QueryEscape(term) req, err := r.reqFactory.NewRequest("GET", u, nil) if err != nil { return nil, err diff --git a/docs/session_v2.go b/docs/session_v2.go new file mode 100644 index 00000000..2e0de49b --- /dev/null +++ b/docs/session_v2.go @@ -0,0 +1,386 @@ +package registry + +import ( + "encoding/json" + "fmt" + "io" + "io/ioutil" + "net/url" + "strconv" + + "github.com/docker/docker/pkg/log" + "github.com/docker/docker/utils" + "github.com/gorilla/mux" +) + +func newV2RegistryRouter() *mux.Router { + router := mux.NewRouter() + + v2Router := router.PathPrefix("/v2/").Subrouter() + + // Version Info + v2Router.Path("/version").Name("version") + + // Image Manifests + v2Router.Path("/manifest/{imagename:[a-z0-9-._/]+}/{tagname:[a-zA-Z0-9-._]+}").Name("manifests") + + // List Image Tags + v2Router.Path("/tags/{imagename:[a-z0-9-._/]+}").Name("tags") + + // Download a blob + v2Router.Path("/blob/{imagename:[a-z0-9-._/]+}/{sumtype:[a-z0-9_+-]+}/{sum:[a-fA-F0-9]{4,}}").Name("downloadBlob") + + // Upload a blob + v2Router.Path("/blob/{imagename:[a-z0-9-._/]+}/{sumtype:[a-z0-9_+-]+}").Name("uploadBlob") + + // Mounting a blob in an image + v2Router.Path("/mountblob/{imagename:[a-z0-9-._/]+}/{sumtype:[a-z0-9_+-]+}/{sum:[a-fA-F0-9]{4,}}").Name("mountBlob") + + return router +} + +// APIVersion2 /v2/ +var v2HTTPRoutes = newV2RegistryRouter() + +func getV2URL(e *Endpoint, routeName string, vars map[string]string) (*url.URL, error) { + route := v2HTTPRoutes.Get(routeName) + if route == nil { + return nil, fmt.Errorf("unknown regisry v2 route name: %q", routeName) + } + + varReplace := make([]string, 0, len(vars)*2) + for key, val := range vars { + varReplace = append(varReplace, key, val) + } + + routePath, err := route.URLPath(varReplace...) + if err != nil { + return nil, fmt.Errorf("unable to make registry route %q with vars %v: %s", routeName, vars, err) + } + + return &url.URL{ + Scheme: e.URL.Scheme, + Host: e.URL.Host, + Path: routePath.Path, + }, nil +} + +// V2 Provenance POC + +func (r *Session) GetV2Version(token []string) (*RegistryInfo, error) { + routeURL, err := getV2URL(r.indexEndpoint, "version", nil) + if err != nil { + return nil, err + } + + method := "GET" + log.Debugf("[registry] Calling %q %s", method, routeURL.String()) + + req, err := r.reqFactory.NewRequest(method, routeURL.String(), nil) + if err != nil { + return nil, err + } + setTokenAuth(req, token) + res, _, err := r.doRequest(req) + if err != nil { + return nil, err + } + defer res.Body.Close() + if res.StatusCode != 200 { + return nil, utils.NewHTTPRequestError(fmt.Sprintf("Server error: %d fetching Version", res.StatusCode), res) + } + + decoder := json.NewDecoder(res.Body) + versionInfo := new(RegistryInfo) + + err = decoder.Decode(versionInfo) + if err != nil { + return nil, fmt.Errorf("unable to decode GetV2Version JSON response: %s", err) + } + + return versionInfo, nil +} + +// +// 1) Check if TarSum of each layer exists /v2/ +// 1.a) if 200, continue +// 1.b) if 300, then push the +// 1.c) if anything else, err +// 2) PUT the created/signed manifest +// +func (r *Session) GetV2ImageManifest(imageName, tagName string, token []string) ([]byte, error) { + vars := map[string]string{ + "imagename": imageName, + "tagname": tagName, + } + + routeURL, err := getV2URL(r.indexEndpoint, "manifests", vars) + if err != nil { + return nil, err + } + + method := "GET" + log.Debugf("[registry] Calling %q %s", method, routeURL.String()) + + req, err := r.reqFactory.NewRequest(method, routeURL.String(), nil) + if err != nil { + return nil, err + } + setTokenAuth(req, token) + res, _, err := r.doRequest(req) + if err != nil { + return nil, err + } + defer res.Body.Close() + if res.StatusCode != 200 { + if res.StatusCode == 401 { + return nil, errLoginRequired + } else if res.StatusCode == 404 { + return nil, ErrDoesNotExist + } + return nil, utils.NewHTTPRequestError(fmt.Sprintf("Server error: %d trying to fetch for %s:%s", res.StatusCode, imageName, tagName), res) + } + + buf, err := ioutil.ReadAll(res.Body) + if err != nil { + return nil, fmt.Errorf("Error while reading the http response: %s", err) + } + return buf, nil +} + +// - Succeeded to mount for this image scope +// - Failed with no error (So continue to Push the Blob) +// - Failed with error +func (r *Session) PostV2ImageMountBlob(imageName, sumType, sum string, token []string) (bool, error) { + vars := map[string]string{ + "imagename": imageName, + "sumtype": sumType, + "sum": sum, + } + + routeURL, err := getV2URL(r.indexEndpoint, "mountBlob", vars) + if err != nil { + return false, err + } + + method := "POST" + log.Debugf("[registry] Calling %q %s", method, routeURL.String()) + + req, err := r.reqFactory.NewRequest(method, routeURL.String(), nil) + if err != nil { + return false, err + } + setTokenAuth(req, token) + res, _, err := r.doRequest(req) + if err != nil { + return false, err + } + res.Body.Close() // close early, since we're not needing a body on this call .. yet? + switch res.StatusCode { + case 200: + // return something indicating no push needed + return true, nil + case 300: + // return something indicating blob push needed + return false, nil + } + return false, fmt.Errorf("Failed to mount %q - %s:%s : %d", imageName, sumType, sum, res.StatusCode) +} + +func (r *Session) GetV2ImageBlob(imageName, sumType, sum string, blobWrtr io.Writer, token []string) error { + vars := map[string]string{ + "imagename": imageName, + "sumtype": sumType, + "sum": sum, + } + + routeURL, err := getV2URL(r.indexEndpoint, "downloadBlob", vars) + if err != nil { + return err + } + + method := "GET" + log.Debugf("[registry] Calling %q %s", method, routeURL.String()) + req, err := r.reqFactory.NewRequest(method, routeURL.String(), nil) + if err != nil { + return err + } + setTokenAuth(req, token) + res, _, err := r.doRequest(req) + if err != nil { + return err + } + defer res.Body.Close() + if res.StatusCode != 200 { + if res.StatusCode == 401 { + return errLoginRequired + } + return utils.NewHTTPRequestError(fmt.Sprintf("Server error: %d trying to pull %s blob", res.StatusCode, imageName), res) + } + + _, err = io.Copy(blobWrtr, res.Body) + return err +} + +func (r *Session) GetV2ImageBlobReader(imageName, sumType, sum string, token []string) (io.ReadCloser, int64, error) { + vars := map[string]string{ + "imagename": imageName, + "sumtype": sumType, + "sum": sum, + } + + routeURL, err := getV2URL(r.indexEndpoint, "downloadBlob", vars) + if err != nil { + return nil, 0, err + } + + method := "GET" + log.Debugf("[registry] Calling %q %s", method, routeURL.String()) + req, err := r.reqFactory.NewRequest(method, routeURL.String(), nil) + if err != nil { + return nil, 0, err + } + setTokenAuth(req, token) + res, _, err := r.doRequest(req) + if err != nil { + return nil, 0, err + } + if res.StatusCode != 200 { + if res.StatusCode == 401 { + return nil, 0, errLoginRequired + } + return nil, 0, utils.NewHTTPRequestError(fmt.Sprintf("Server error: %d trying to pull %s blob", res.StatusCode, imageName), res) + } + lenStr := res.Header.Get("Content-Length") + l, err := strconv.ParseInt(lenStr, 10, 64) + if err != nil { + return nil, 0, err + } + + return res.Body, l, err +} + +// Push the image to the server for storage. +// 'layer' is an uncompressed reader of the blob to be pushed. +// The server will generate it's own checksum calculation. +func (r *Session) PutV2ImageBlob(imageName, sumType string, blobRdr io.Reader, token []string) (serverChecksum string, err error) { + vars := map[string]string{ + "imagename": imageName, + "sumtype": sumType, + } + + routeURL, err := getV2URL(r.indexEndpoint, "uploadBlob", vars) + if err != nil { + return "", err + } + + method := "PUT" + log.Debugf("[registry] Calling %q %s", method, routeURL.String()) + req, err := r.reqFactory.NewRequest(method, routeURL.String(), blobRdr) + if err != nil { + return "", err + } + setTokenAuth(req, token) + res, _, err := r.doRequest(req) + if err != nil { + return "", err + } + defer res.Body.Close() + if res.StatusCode != 201 { + if res.StatusCode == 401 { + return "", errLoginRequired + } + return "", utils.NewHTTPRequestError(fmt.Sprintf("Server error: %d trying to push %s blob", res.StatusCode, imageName), res) + } + + type sumReturn struct { + Checksum string `json:"checksum"` + } + + decoder := json.NewDecoder(res.Body) + var sumInfo sumReturn + + err = decoder.Decode(&sumInfo) + if err != nil { + return "", fmt.Errorf("unable to decode PutV2ImageBlob JSON response: %s", err) + } + + // XXX this is a json struct from the registry, with its checksum + return sumInfo.Checksum, nil +} + +// Finally Push the (signed) manifest of the blobs we've just pushed +func (r *Session) PutV2ImageManifest(imageName, tagName string, manifestRdr io.Reader, token []string) error { + vars := map[string]string{ + "imagename": imageName, + "tagname": tagName, + } + + routeURL, err := getV2URL(r.indexEndpoint, "manifests", vars) + if err != nil { + return err + } + + method := "PUT" + log.Debugf("[registry] Calling %q %s", method, routeURL.String()) + req, err := r.reqFactory.NewRequest(method, routeURL.String(), manifestRdr) + if err != nil { + return err + } + setTokenAuth(req, token) + res, _, err := r.doRequest(req) + if err != nil { + return err + } + res.Body.Close() + if res.StatusCode != 201 { + if res.StatusCode == 401 { + return errLoginRequired + } + return utils.NewHTTPRequestError(fmt.Sprintf("Server error: %d trying to push %s:%s manifest", res.StatusCode, imageName, tagName), res) + } + + return nil +} + +// Given a repository name, returns a json array of string tags +func (r *Session) GetV2RemoteTags(imageName string, token []string) ([]string, error) { + vars := map[string]string{ + "imagename": imageName, + } + + routeURL, err := getV2URL(r.indexEndpoint, "tags", vars) + if err != nil { + return nil, err + } + + method := "GET" + log.Debugf("[registry] Calling %q %s", method, routeURL.String()) + + req, err := r.reqFactory.NewRequest(method, routeURL.String(), nil) + if err != nil { + return nil, err + } + setTokenAuth(req, token) + res, _, err := r.doRequest(req) + if err != nil { + return nil, err + } + defer res.Body.Close() + if res.StatusCode != 200 { + if res.StatusCode == 401 { + return nil, errLoginRequired + } else if res.StatusCode == 404 { + return nil, ErrDoesNotExist + } + return nil, utils.NewHTTPRequestError(fmt.Sprintf("Server error: %d trying to fetch for %s", res.StatusCode, imageName), res) + } + + decoder := json.NewDecoder(res.Body) + var tags []string + err = decoder.Decode(&tags) + if err != nil { + return nil, fmt.Errorf("Error while decoding the http response: %s", err) + } + return tags, nil +} diff --git a/docs/types.go b/docs/types.go index 3db236da..2ba5af0d 100644 --- a/docs/types.go +++ b/docs/types.go @@ -32,6 +32,15 @@ type RegistryInfo struct { Standalone bool `json:"standalone"` } +type ManifestData struct { + Name string `json:"name"` + Tag string `json:"tag"` + Architecture string `json:"architecture"` + BlobSums []string `json:"blobSums"` + History []string `json:"history"` + SchemaVersion int `json:"schemaVersion"` +} + type APIVersion int func (av APIVersion) String() string { @@ -45,7 +54,6 @@ var apiVersions = map[APIVersion]string{ } const ( - _ = iota - APIVersion1 = iota + APIVersion1 = iota + 1 APIVersion2 ) From c47aa21c35255af07ef39308492fbfd485509713 Mon Sep 17 00:00:00 2001 From: Derek McGowan Date: Thu, 2 Oct 2014 17:41:57 -0700 Subject: [PATCH 157/375] Add comment for permission and fix wrong format variable Signed-off-by: Derek McGowan (github: dmcgowan) --- docs/endpoint.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/endpoint.go b/docs/endpoint.go index 12df9e0c..5313a807 100644 --- a/docs/endpoint.go +++ b/docs/endpoint.go @@ -124,6 +124,6 @@ func (e Endpoint) Ping() (RegistryInfo, error) { // there is a header set, and it is not "true" or "1", so assume fails info.Standalone = false } - log.Debugf("RegistryInfo.Standalone: %q", info.Standalone) + log.Debugf("RegistryInfo.Standalone: %t", info.Standalone) return info, nil } From 7bfdb6d495506c95768177fde302d19763a39932 Mon Sep 17 00:00:00 2001 From: unclejack Date: Mon, 6 Oct 2014 22:34:39 +0300 Subject: [PATCH 158/375] registry: lint Docker-DCO-1.1-Signed-off-by: Cristian Staretu (github: unclejack) --- docs/auth.go | 7 +++---- docs/endpoint.go | 4 ++-- docs/registry_mock_test.go | 22 ++++++++++---------- docs/registry_test.go | 41 ++++++++++++++++++++------------------ docs/session.go | 5 +++-- 5 files changed, 41 insertions(+), 38 deletions(-) diff --git a/docs/auth.go b/docs/auth.go index 906a37dd..ba370f4b 100644 --- a/docs/auth.go +++ b/docs/auth.go @@ -224,12 +224,11 @@ func Login(authConfig *AuthConfig, factory *utils.HTTPRequestFactory) (string, e return "", fmt.Errorf("Login: Account is not Active. Please check your e-mail for a confirmation link.") } return "", fmt.Errorf("Login: Account is not Active. Please see the documentation of the registry %s for instructions how to activate it.", serverAddress) - } else { - return "", fmt.Errorf("Login: %s (Code: %d; Headers: %s)", body, resp.StatusCode, resp.Header) } - } else { - return "", fmt.Errorf("Registration: %s", reqBody) + return "", fmt.Errorf("Login: %s (Code: %d; Headers: %s)", body, resp.StatusCode, resp.Header) } + return "", fmt.Errorf("Registration: %s", reqBody) + } else if reqStatusCode == 401 { // This case would happen with private registries where /v1/users is // protected, so people can use `docker login` as an auth check. diff --git a/docs/endpoint.go b/docs/endpoint.go index 5313a807..58311d32 100644 --- a/docs/endpoint.go +++ b/docs/endpoint.go @@ -13,7 +13,7 @@ import ( ) // scans string for api version in the URL path. returns the trimmed hostname, if version found, string and API version. -func scanForApiVersion(hostname string) (string, APIVersion) { +func scanForAPIVersion(hostname string) (string, APIVersion) { var ( chunks []string apiVersionStr string @@ -43,7 +43,7 @@ func NewEndpoint(hostname string) (*Endpoint, error) { if !strings.HasPrefix(hostname, "http") { hostname = "https://" + hostname } - trimmedHostname, endpoint.Version = scanForApiVersion(hostname) + trimmedHostname, endpoint.Version = scanForAPIVersion(hostname) endpoint.URL, err = url.Parse(trimmedHostname) if err != nil { return nil, err diff --git a/docs/registry_mock_test.go b/docs/registry_mock_test.go index 379dc78f..967d8b26 100644 --- a/docs/registry_mock_test.go +++ b/docs/registry_mock_test.go @@ -19,7 +19,7 @@ import ( ) var ( - testHttpServer *httptest.Server + testHTTPServer *httptest.Server testLayers = map[string]map[string]string{ "77dbf71da1d00e3fbddc480176eac8994025630c6590d11cfc8fe1209c2a1d20": { "json": `{"id":"77dbf71da1d00e3fbddc480176eac8994025630c6590d11cfc8fe1209c2a1d20", @@ -99,7 +99,7 @@ func init() { // /v2/ r.HandleFunc("/v2/version", handlerGetPing).Methods("GET") - testHttpServer = httptest.NewServer(handlerAccessLog(r)) + testHTTPServer = httptest.NewServer(handlerAccessLog(r)) } func handlerAccessLog(handler http.Handler) http.Handler { @@ -111,7 +111,7 @@ func handlerAccessLog(handler http.Handler) http.Handler { } func makeURL(req string) string { - return testHttpServer.URL + req + return testHTTPServer.URL + req } func writeHeaders(w http.ResponseWriter) { @@ -198,8 +198,8 @@ func handlerGetImage(w http.ResponseWriter, r *http.Request) { return } writeHeaders(w) - layer_size := len(layer["layer"]) - w.Header().Add("X-Docker-Size", strconv.Itoa(layer_size)) + layerSize := len(layer["layer"]) + w.Header().Add("X-Docker-Size", strconv.Itoa(layerSize)) io.WriteString(w, layer[vars["action"]]) } @@ -208,16 +208,16 @@ func handlerPutImage(w http.ResponseWriter, r *http.Request) { return } vars := mux.Vars(r) - image_id := vars["image_id"] + imageID := vars["image_id"] action := vars["action"] - layer, exists := testLayers[image_id] + layer, exists := testLayers[imageID] if !exists { if action != "json" { http.NotFound(w, r) return } layer = make(map[string]string) - testLayers[image_id] = layer + testLayers[imageID] = layer } if checksum := r.Header.Get("X-Docker-Checksum"); checksum != "" { if checksum != layer["checksum_simple"] && checksum != layer["checksum_tarsum"] { @@ -301,7 +301,7 @@ func handlerUsers(w http.ResponseWriter, r *http.Request) { } func handlerImages(w http.ResponseWriter, r *http.Request) { - u, _ := url.Parse(testHttpServer.URL) + u, _ := url.Parse(testHTTPServer.URL) w.Header().Add("X-Docker-Endpoints", fmt.Sprintf("%s , %s ", u.Host, "test.example.com")) w.Header().Add("X-Docker-Token", fmt.Sprintf("FAKE-SESSION-%d", time.Now().UnixNano())) if r.Method == "PUT" { @@ -317,9 +317,9 @@ func handlerImages(w http.ResponseWriter, r *http.Request) { return } images := []map[string]string{} - for image_id, layer := range testLayers { + for imageID, layer := range testLayers { image := make(map[string]string) - image["id"] = image_id + image["id"] = imageID image["checksum"] = layer["checksum_tarsum"] image["Tag"] = "latest" images = append(images, image) diff --git a/docs/registry_test.go b/docs/registry_test.go index ab417812..fdf714e8 100644 --- a/docs/registry_test.go +++ b/docs/registry_test.go @@ -11,9 +11,12 @@ import ( ) var ( - IMAGE_ID = "42d718c941f5c532ac049bf0b0ab53f0062f09a03afd4aa4a02c098e46032b9d" - TOKEN = []string{"fake-token"} - REPO = "foo42/bar" + token = []string{"fake-token"} +) + +const ( + imageID = "42d718c941f5c532ac049bf0b0ab53f0062f09a03afd4aa4a02c098e46032b9d" + REPO = "foo42/bar" ) func spawnTestRegistrySession(t *testing.T) *Session { @@ -43,27 +46,27 @@ func TestPingRegistryEndpoint(t *testing.T) { func TestGetRemoteHistory(t *testing.T) { r := spawnTestRegistrySession(t) - hist, err := r.GetRemoteHistory(IMAGE_ID, makeURL("/v1/"), TOKEN) + hist, err := r.GetRemoteHistory(imageID, makeURL("/v1/"), token) if err != nil { t.Fatal(err) } assertEqual(t, len(hist), 2, "Expected 2 images in history") - assertEqual(t, hist[0], IMAGE_ID, "Expected "+IMAGE_ID+"as first ancestry") + assertEqual(t, hist[0], imageID, "Expected "+imageID+"as first ancestry") assertEqual(t, hist[1], "77dbf71da1d00e3fbddc480176eac8994025630c6590d11cfc8fe1209c2a1d20", "Unexpected second ancestry") } func TestLookupRemoteImage(t *testing.T) { r := spawnTestRegistrySession(t) - found := r.LookupRemoteImage(IMAGE_ID, makeURL("/v1/"), TOKEN) + found := r.LookupRemoteImage(imageID, makeURL("/v1/"), token) assertEqual(t, found, true, "Expected remote lookup to succeed") - found = r.LookupRemoteImage("abcdef", makeURL("/v1/"), TOKEN) + found = r.LookupRemoteImage("abcdef", makeURL("/v1/"), token) assertEqual(t, found, false, "Expected remote lookup to fail") } func TestGetRemoteImageJSON(t *testing.T) { r := spawnTestRegistrySession(t) - json, size, err := r.GetRemoteImageJSON(IMAGE_ID, makeURL("/v1/"), TOKEN) + json, size, err := r.GetRemoteImageJSON(imageID, makeURL("/v1/"), token) if err != nil { t.Fatal(err) } @@ -72,7 +75,7 @@ func TestGetRemoteImageJSON(t *testing.T) { t.Fatal("Expected non-empty json") } - _, _, err = r.GetRemoteImageJSON("abcdef", makeURL("/v1/"), TOKEN) + _, _, err = r.GetRemoteImageJSON("abcdef", makeURL("/v1/"), token) if err == nil { t.Fatal("Expected image not found error") } @@ -80,7 +83,7 @@ func TestGetRemoteImageJSON(t *testing.T) { func TestGetRemoteImageLayer(t *testing.T) { r := spawnTestRegistrySession(t) - data, err := r.GetRemoteImageLayer(IMAGE_ID, makeURL("/v1/"), TOKEN, 0) + data, err := r.GetRemoteImageLayer(imageID, makeURL("/v1/"), token, 0) if err != nil { t.Fatal(err) } @@ -88,7 +91,7 @@ func TestGetRemoteImageLayer(t *testing.T) { t.Fatal("Expected non-nil data result") } - _, err = r.GetRemoteImageLayer("abcdef", makeURL("/v1/"), TOKEN, 0) + _, err = r.GetRemoteImageLayer("abcdef", makeURL("/v1/"), token, 0) if err == nil { t.Fatal("Expected image not found error") } @@ -96,14 +99,14 @@ func TestGetRemoteImageLayer(t *testing.T) { func TestGetRemoteTags(t *testing.T) { r := spawnTestRegistrySession(t) - tags, err := r.GetRemoteTags([]string{makeURL("/v1/")}, REPO, TOKEN) + tags, err := r.GetRemoteTags([]string{makeURL("/v1/")}, REPO, token) if err != nil { t.Fatal(err) } assertEqual(t, len(tags), 1, "Expected one tag") - assertEqual(t, tags["latest"], IMAGE_ID, "Expected tag latest to map to "+IMAGE_ID) + assertEqual(t, tags["latest"], imageID, "Expected tag latest to map to "+imageID) - _, err = r.GetRemoteTags([]string{makeURL("/v1/")}, "foo42/baz", TOKEN) + _, err = r.GetRemoteTags([]string{makeURL("/v1/")}, "foo42/baz", token) if err == nil { t.Fatal("Expected error when fetching tags for bogus repo") } @@ -111,11 +114,11 @@ func TestGetRemoteTags(t *testing.T) { func TestGetRepositoryData(t *testing.T) { r := spawnTestRegistrySession(t) - parsedUrl, err := url.Parse(makeURL("/v1/")) + parsedURL, err := url.Parse(makeURL("/v1/")) if err != nil { t.Fatal(err) } - host := "http://" + parsedUrl.Host + "/v1/" + host := "http://" + parsedURL.Host + "/v1/" data, err := r.GetRepositoryData("foo42/bar") if err != nil { t.Fatal(err) @@ -137,7 +140,7 @@ func TestPushImageJSONRegistry(t *testing.T) { Checksum: "sha256:1ac330d56e05eef6d438586545ceff7550d3bdcb6b19961f12c5ba714ee1bb37", } - err := r.PushImageJSONRegistry(imgData, []byte{0x42, 0xdf, 0x0}, makeURL("/v1/"), TOKEN) + err := r.PushImageJSONRegistry(imgData, []byte{0x42, 0xdf, 0x0}, makeURL("/v1/"), token) if err != nil { t.Fatal(err) } @@ -146,7 +149,7 @@ func TestPushImageJSONRegistry(t *testing.T) { func TestPushImageLayerRegistry(t *testing.T) { r := spawnTestRegistrySession(t) layer := strings.NewReader("") - _, _, err := r.PushImageLayerRegistry(IMAGE_ID, layer, makeURL("/v1/"), TOKEN, []byte{}) + _, _, err := r.PushImageLayerRegistry(imageID, layer, makeURL("/v1/"), token, []byte{}) if err != nil { t.Fatal(err) } @@ -180,7 +183,7 @@ func TestResolveRepositoryName(t *testing.T) { func TestPushRegistryTag(t *testing.T) { r := spawnTestRegistrySession(t) - err := r.PushRegistryTag("foo42/bar", IMAGE_ID, "stable", makeURL("/v1/"), TOKEN) + err := r.PushRegistryTag("foo42/bar", imageID, "stable", makeURL("/v1/"), token) if err != nil { t.Fatal(err) } diff --git a/docs/session.go b/docs/session.go index 5067b8d5..ff0be343 100644 --- a/docs/session.go +++ b/docs/session.go @@ -3,6 +3,7 @@ package registry import ( "bytes" "crypto/sha256" + // this is required for some certificates _ "crypto/sha512" "encoding/hex" "encoding/json" @@ -243,11 +244,11 @@ func (r *Session) GetRemoteTags(registries []string, repository string, token [] func buildEndpointsList(headers []string, indexEp string) ([]string, error) { var endpoints []string - parsedUrl, err := url.Parse(indexEp) + parsedURL, err := url.Parse(indexEp) if err != nil { return nil, err } - var urlScheme = parsedUrl.Scheme + var urlScheme = parsedURL.Scheme // The Registry's URL scheme has to match the Index' for _, ep := range headers { epList := strings.Split(ep, ",") From f290f446329fcf1c56c5705353b340050ef8a8c5 Mon Sep 17 00:00:00 2001 From: Derek McGowan Date: Wed, 8 Oct 2014 14:03:39 -0700 Subject: [PATCH 159/375] Use direct registry url Signed-off-by: Derek McGowan (github: dmcgowan) --- docs/auth.go | 13 ++++++++----- docs/session_v2.go | 8 ++++++-- 2 files changed, 14 insertions(+), 7 deletions(-) diff --git a/docs/auth.go b/docs/auth.go index 906a37dd..7c0709a4 100644 --- a/docs/auth.go +++ b/docs/auth.go @@ -14,13 +14,16 @@ import ( "github.com/docker/docker/utils" ) -// Where we store the config file -const CONFIGFILE = ".dockercfg" +const ( + // Where we store the config file + CONFIGFILE = ".dockercfg" -// Only used for user auth + account creation -const INDEXSERVER = "https://index.docker.io/v1/" + // Only used for user auth + account creation + INDEXSERVER = "https://index.docker.io/v1/" + REGISTRYSERVER = "https://registry-1.docker.io/v1/" -//const INDEXSERVER = "https://registry-stage.hub.docker.com/v1/" + // INDEXSERVER = "https://registry-stage.hub.docker.com/v1/" +) var ( ErrConfigFileMissing = errors.New("The Auth config file is missing") diff --git a/docs/session_v2.go b/docs/session_v2.go index 2e0de49b..c63cf710 100644 --- a/docs/session_v2.go +++ b/docs/session_v2.go @@ -57,10 +57,14 @@ func getV2URL(e *Endpoint, routeName string, vars map[string]string) (*url.URL, if err != nil { return nil, fmt.Errorf("unable to make registry route %q with vars %v: %s", routeName, vars, err) } + u, err := url.Parse(REGISTRYSERVER) + if err != nil { + return nil, fmt.Errorf("invalid registry url: %s", err) + } return &url.URL{ - Scheme: e.URL.Scheme, - Host: e.URL.Host, + Scheme: u.Scheme, + Host: u.Host, Path: routePath.Path, }, nil } From 1538e42d56b9ea95b4818e4acc95c2d905241ef6 Mon Sep 17 00:00:00 2001 From: Derek McGowan Date: Thu, 9 Oct 2014 17:34:34 -0700 Subject: [PATCH 160/375] Update manifest format to rename blobsums and use arrays of dictionaries Signed-off-by: Derek McGowan (github: dmcgowan) --- docs/types.go | 20 ++++++++++++++------ 1 file changed, 14 insertions(+), 6 deletions(-) diff --git a/docs/types.go b/docs/types.go index 2ba5af0d..3b429f19 100644 --- a/docs/types.go +++ b/docs/types.go @@ -32,13 +32,21 @@ type RegistryInfo struct { Standalone bool `json:"standalone"` } +type FSLayer struct { + BlobSum string `json:"blobSum"` +} + +type ManifestHistory struct { + V1Compatibility string `json:"v1Compatibility"` +} + type ManifestData struct { - Name string `json:"name"` - Tag string `json:"tag"` - Architecture string `json:"architecture"` - BlobSums []string `json:"blobSums"` - History []string `json:"history"` - SchemaVersion int `json:"schemaVersion"` + Name string `json:"name"` + Tag string `json:"tag"` + Architecture string `json:"architecture"` + FSLayers []*FSLayer `json:"fsLayers"` + History []*ManifestHistory `json:"history"` + SchemaVersion int `json:"schemaVersion"` } type APIVersion int From 479ed10e614456ecc9f4214495de31b2a7089a50 Mon Sep 17 00:00:00 2001 From: Derek McGowan Date: Thu, 9 Oct 2014 17:31:54 -0700 Subject: [PATCH 161/375] Support tarsum dev version to fix issue with mtime Signed-off-by: Derek McGowan (github: dmcgowan) --- docs/session_v2.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/session_v2.go b/docs/session_v2.go index c63cf710..c0bc19b3 100644 --- a/docs/session_v2.go +++ b/docs/session_v2.go @@ -28,13 +28,13 @@ func newV2RegistryRouter() *mux.Router { v2Router.Path("/tags/{imagename:[a-z0-9-._/]+}").Name("tags") // Download a blob - v2Router.Path("/blob/{imagename:[a-z0-9-._/]+}/{sumtype:[a-z0-9_+-]+}/{sum:[a-fA-F0-9]{4,}}").Name("downloadBlob") + v2Router.Path("/blob/{imagename:[a-z0-9-._/]+}/{sumtype:[a-z0-9._+-]+}/{sum:[a-fA-F0-9]{4,}}").Name("downloadBlob") // Upload a blob - v2Router.Path("/blob/{imagename:[a-z0-9-._/]+}/{sumtype:[a-z0-9_+-]+}").Name("uploadBlob") + v2Router.Path("/blob/{imagename:[a-z0-9-._/]+}/{sumtype:[a-z0-9._+-]+}").Name("uploadBlob") // Mounting a blob in an image - v2Router.Path("/mountblob/{imagename:[a-z0-9-._/]+}/{sumtype:[a-z0-9_+-]+}/{sum:[a-fA-F0-9]{4,}}").Name("mountBlob") + v2Router.Path("/mountblob/{imagename:[a-z0-9-._/]+}/{sumtype:[a-z0-9._+-]+}/{sum:[a-fA-F0-9]{4,}}").Name("mountBlob") return router } From 20867c3b1ffe5e4bca669295ad2d76b53f2ca672 Mon Sep 17 00:00:00 2001 From: "Daniel, Dao Quang Minh" Date: Wed, 15 Oct 2014 22:39:51 -0400 Subject: [PATCH 162/375] Avoid fallback to SSL protocols < TLS1.0 Signed-off-by: Tibor Vass Docker-DCO-1.1-Signed-off-by: Daniel, Dao Quang Minh (github: dqminh) --- docs/registry.go | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/docs/registry.go b/docs/registry.go index fd74b751..0c648a94 100644 --- a/docs/registry.go +++ b/docs/registry.go @@ -36,7 +36,11 @@ const ( ) func newClient(jar http.CookieJar, roots *x509.CertPool, cert *tls.Certificate, timeout TimeoutType) *http.Client { - tlsConfig := tls.Config{RootCAs: roots} + tlsConfig := tls.Config{ + RootCAs: roots, + // Avoid fallback to SSL protocols < TLS1.0 + MinVersion: tls.VersionTLS10, + } if cert != nil { tlsConfig.Certificates = append(tlsConfig.Certificates, *cert) From 3a6fe4c5c9bc8119c5468b3d1fa8b7046d87ca8f Mon Sep 17 00:00:00 2001 From: Dan Walsh Date: Tue, 14 Oct 2014 09:19:45 -0400 Subject: [PATCH 163/375] On Red Hat Registry Servers we return 404 on certification errors. We do this to prevent leakage of information, we don't want people to be able to probe for existing content. According to RFC 2616, "This status code (404) is commonly used when the server does not wish to reveal exactly why the request has been refused, or when no other response i is applicable." https://www.ietf.org/rfc/rfc2616.txt 10.4.4 403 Forbidden The server understood the request, but is refusing to fulfill it. Authorization will not help and the request SHOULD NOT be repeated. If the request method was not HEAD and the server wishes to make public why the request has not been fulfilled, it SHOULD describe the reason for the refusal in the entity. If the server does not wish to make this information available to the client, the status code 404 (Not Found) can be used instead. 10.4.5 404 Not Found The server has not found anything matching the Request-URI. No indication is given of whether the condition is temporary or permanent. The 410 (Gone) status code SHOULD be used if the server knows, through some internally configurable mechanism, that an old resource is permanently unavailable and has no forwarding address. This status code is commonly used when the server does not wish to reveal exactly why the request has been refused, or when no other response is applicable. When docker is running through its certificates, it should continue trying with a new certificate even if it gets back a 404 error code. Docker-DCO-1.1-Signed-off-by: Dan Walsh (github: rhatdan) --- docs/registry.go | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/docs/registry.go b/docs/registry.go index 0c648a94..d1315ed4 100644 --- a/docs/registry.go +++ b/docs/registry.go @@ -147,7 +147,10 @@ func doRequest(req *http.Request, jar http.CookieJar, timeout TimeoutType) (*htt client := newClient(jar, pool, cert, timeout) res, err := client.Do(req) // If this is the last cert, otherwise, continue to next cert if 403 or 5xx - if i == len(certs)-1 || err == nil && res.StatusCode != 403 && res.StatusCode < 500 { + if i == len(certs)-1 || err == nil && + res.StatusCode != 403 && + res.StatusCode != 404 && + res.StatusCode < 500 { return res, client, err } } From 8b1c40732aeca2c993fdb53febf5596701c869f2 Mon Sep 17 00:00:00 2001 From: unclejack Date: Sat, 16 Aug 2014 13:27:04 +0300 Subject: [PATCH 164/375] make http usage for registry explicit Docker-DCO-1.1-Signed-off-by: Cristian Staretu (github: unclejack) Conflicts: daemon/config.go daemon/daemon.go graph/pull.go graph/push.go graph/tags.go registry/registry.go registry/service.go --- docs/registry.go | 49 ++++++++++++++++++++++++++++++++++++++++++++++++ docs/service.go | 2 +- 2 files changed, 50 insertions(+), 1 deletion(-) diff --git a/docs/registry.go b/docs/registry.go index fd74b751..8f4ae6fa 100644 --- a/docs/registry.go +++ b/docs/registry.go @@ -202,6 +202,55 @@ func ResolveRepositoryName(reposName string) (string, string, error) { return hostname, reposName, nil } +// this method expands the registry name as used in the prefix of a repo +// to a full url. if it already is a url, there will be no change. +func ExpandAndVerifyRegistryUrl(hostname string, secure bool) (endpoint string, err error) { + if strings.HasPrefix(hostname, "http:") || strings.HasPrefix(hostname, "https:") { + // if there is no slash after https:// (8 characters) then we have no path in the url + if strings.LastIndex(hostname, "/") < 9 { + // there is no path given. Expand with default path + hostname = hostname + "/v1/" + } + if _, err := pingRegistryEndpoint(hostname); err != nil { + return "", errors.New("Invalid Registry endpoint: " + err.Error()) + } + return hostname, nil + } + + // use HTTPS if secure, otherwise use HTTP + if secure { + endpoint = fmt.Sprintf("https://%s/v1/", hostname) + } else { + endpoint = fmt.Sprintf("http://%s/v1/", hostname) + } + _, err = pingRegistryEndpoint(endpoint) + if err != nil { + //TODO: triggering highland build can be done there without "failing" + err = fmt.Errorf("Invalid registry endpoint '%s': %s ", endpoint, err) + if secure { + err = fmt.Errorf("%s. If this private registry supports only HTTP, please add `--insecure-registry %s` to the daemon's arguments.", err, hostname) + } + return "", err + } + return endpoint, nil +} + +// this method verifies if the provided hostname is part of the list of +// insecure registries and returns false if HTTP should be used +func IsSecure(hostname string, insecureRegistries []string) (secure bool) { + secure = true + for _, h := range insecureRegistries { + if hostname == h { + secure = false + break + } + } + if hostname == IndexServerAddress() { + secure = true + } + return +} + func trustedLocation(req *http.Request) bool { var ( trusteds = []string{"docker.com", "docker.io"} diff --git a/docs/service.go b/docs/service.go index f7b35300..334e7c2e 100644 --- a/docs/service.go +++ b/docs/service.go @@ -40,7 +40,7 @@ func (s *Service) Auth(job *engine.Job) engine.Status { job.GetenvJson("authConfig", authConfig) // TODO: this is only done here because auth and registry need to be merged into one pkg if addr := authConfig.ServerAddress; addr != "" && addr != IndexServerAddress() { - endpoint, err := NewEndpoint(addr) + endpoint, err := NewEndpoint(addr, true) if err != nil { return job.Error(err) } From 2b9798fa190ac8aef2a6f7630fb07f116bad6289 Mon Sep 17 00:00:00 2001 From: Michael Crosby Date: Tue, 19 Aug 2014 11:54:42 -0700 Subject: [PATCH 165/375] Refactor IsSecure change Fix issue with restoring the tag store and setting static configuration from the daemon. i.e. the field on the TagStore struct must be made internal or the json.Unmarshal in restore will overwrite the insecure registries to be an empty struct. Signed-off-by: Michael Crosby Conflicts: graph/pull.go graph/push.go graph/tags.go --- docs/registry.go | 44 +++++++++++++++++++------------------------- 1 file changed, 19 insertions(+), 25 deletions(-) diff --git a/docs/registry.go b/docs/registry.go index 8f4ae6fa..bcbce401 100644 --- a/docs/registry.go +++ b/docs/registry.go @@ -204,51 +204,45 @@ func ResolveRepositoryName(reposName string) (string, string, error) { // this method expands the registry name as used in the prefix of a repo // to a full url. if it already is a url, there will be no change. -func ExpandAndVerifyRegistryUrl(hostname string, secure bool) (endpoint string, err error) { - if strings.HasPrefix(hostname, "http:") || strings.HasPrefix(hostname, "https:") { - // if there is no slash after https:// (8 characters) then we have no path in the url - if strings.LastIndex(hostname, "/") < 9 { - // there is no path given. Expand with default path - hostname = hostname + "/v1/" - } - if _, err := pingRegistryEndpoint(hostname); err != nil { - return "", errors.New("Invalid Registry endpoint: " + err.Error()) - } +func ExpandAndVerifyRegistryUrl(hostname string, secure bool) (string, error) { + if hostname == IndexServerAddress() { return hostname, nil } - // use HTTPS if secure, otherwise use HTTP + endpoint := fmt.Sprintf("http://%s/v1/", hostname) + if secure { endpoint = fmt.Sprintf("https://%s/v1/", hostname) - } else { - endpoint = fmt.Sprintf("http://%s/v1/", hostname) } - _, err = pingRegistryEndpoint(endpoint) - if err != nil { + + if _, oerr := pingRegistryEndpoint(endpoint); oerr != nil { //TODO: triggering highland build can be done there without "failing" - err = fmt.Errorf("Invalid registry endpoint '%s': %s ", endpoint, err) + err := fmt.Errorf("Invalid registry endpoint '%s': %s ", endpoint, oerr) + if secure { - err = fmt.Errorf("%s. If this private registry supports only HTTP, please add `--insecure-registry %s` to the daemon's arguments.", err, hostname) + err = fmt.Errorf("%s. If this private registry supports only HTTP, please add `--insecure-registry %s` to the daemon's arguments.", oerr, hostname) } + return "", err } + return endpoint, nil } // this method verifies if the provided hostname is part of the list of // insecure registries and returns false if HTTP should be used -func IsSecure(hostname string, insecureRegistries []string) (secure bool) { - secure = true +func IsSecure(hostname string, insecureRegistries []string) bool { + if hostname == IndexServerAddress() { + return true + } + for _, h := range insecureRegistries { if hostname == h { - secure = false - break + return false } } - if hostname == IndexServerAddress() { - secure = true - } - return + + return true } func trustedLocation(req *http.Request) bool { From 27ddc260e215e97c3d4f8b3f787a40dfe60e1241 Mon Sep 17 00:00:00 2001 From: Michael Crosby Date: Wed, 20 Aug 2014 08:31:24 -0700 Subject: [PATCH 166/375] Don't hard code true for auth job Signed-off-by: Michael Crosby Conflicts: registry/service.go --- docs/service.go | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/docs/service.go b/docs/service.go index 334e7c2e..890837ca 100644 --- a/docs/service.go +++ b/docs/service.go @@ -13,12 +13,15 @@ import ( // 'pull': Download images from any registry (TODO) // 'push': Upload images to any registry (TODO) type Service struct { + insecureRegistries []string } // NewService returns a new instance of Service ready to be // installed no an engine. -func NewService() *Service { - return &Service{} +func NewService(insecureRegistries []string) *Service { + return &Service{ + insecureRegistries: insecureRegistries, + } } // Install installs registry capabilities to eng. @@ -32,15 +35,12 @@ func (s *Service) Install(eng *engine.Engine) error { // and returns OK if authentication was sucessful. // It can be used to verify the validity of a client's credentials. func (s *Service) Auth(job *engine.Job) engine.Status { - var ( - err error - authConfig = &AuthConfig{} - ) + var authConfig = new(AuthConfig) job.GetenvJson("authConfig", authConfig) - // TODO: this is only done here because auth and registry need to be merged into one pkg + if addr := authConfig.ServerAddress; addr != "" && addr != IndexServerAddress() { - endpoint, err := NewEndpoint(addr, true) + endpoint, err := NewEndpoint(addr, IsSecure(addr, s.insecureRegistries)) if err != nil { return job.Error(err) } @@ -49,11 +49,11 @@ func (s *Service) Auth(job *engine.Job) engine.Status { } authConfig.ServerAddress = endpoint.String() } - status, err := Login(authConfig, HTTPRequestFactory(nil)) - if err != nil { + + if _, err := Login(authConfig, HTTPRequestFactory(nil)); err != nil { return job.Error(err) } - job.Printf("%s\n", status) + return engine.StatusOK } From 798fd3c7646559ec410b7147583c8bc959355716 Mon Sep 17 00:00:00 2001 From: Tibor Vass Date: Fri, 10 Oct 2014 23:22:12 -0400 Subject: [PATCH 167/375] Do not verify certificate when using --insecure-registry on an HTTPS registry Signed-off-by: Tibor Vass Conflicts: registry/registry.go registry/registry_test.go registry/service.go registry/session.go --- docs/endpoint.go | 47 +++++++++++--- docs/registry.go | 143 +++++++++++++++++------------------------- docs/registry_test.go | 4 +- docs/service.go | 5 +- docs/session.go | 2 +- 5 files changed, 101 insertions(+), 100 deletions(-) diff --git a/docs/endpoint.go b/docs/endpoint.go index 5313a807..6dd4e1f6 100644 --- a/docs/endpoint.go +++ b/docs/endpoint.go @@ -2,7 +2,6 @@ package registry import ( "encoding/json" - "errors" "fmt" "io/ioutil" "net/http" @@ -34,9 +33,9 @@ func scanForApiVersion(hostname string) (string, APIVersion) { return hostname, DefaultAPIVersion } -func NewEndpoint(hostname string) (*Endpoint, error) { +func NewEndpoint(hostname string, secure bool) (*Endpoint, error) { var ( - endpoint Endpoint + endpoint = Endpoint{secure: secure} trimmedHostname string err error ) @@ -49,14 +48,27 @@ func NewEndpoint(hostname string) (*Endpoint, error) { return nil, err } + // Try HTTPS ping to registry endpoint.URL.Scheme = "https" if _, err := endpoint.Ping(); err != nil { - log.Debugf("Registry %s does not work (%s), falling back to http", endpoint, err) - // TODO: Check if http fallback is enabled - endpoint.URL.Scheme = "http" - if _, err = endpoint.Ping(); err != nil { - return nil, errors.New("Invalid Registry endpoint: " + err.Error()) + + //TODO: triggering highland build can be done there without "failing" + + if secure { + // If registry is secure and HTTPS failed, show user the error and tell them about `--insecure-registry` + // in case that's what they need. DO NOT accept unknown CA certificates, and DO NOT fallback to HTTP. + return nil, fmt.Errorf("Invalid registry endpoint %s: %v. If this private registry supports only HTTP or HTTPS with an unknown CA certificate, please add `--insecure-registry %s` to the daemon's arguments. In the case of HTTPS, if you have access to the registry's CA certificate, no need for the flag; simply place the CA certificate at /etc/docker/certs.d/%s/ca.crt", endpoint, err, endpoint.URL.Host, endpoint.URL.Host) } + + // If registry is insecure and HTTPS failed, fallback to HTTP. + log.Debugf("Error from registry %q marked as insecure: %v. Insecurely falling back to HTTP", endpoint, err) + endpoint.URL.Scheme = "http" + _, err2 := endpoint.Ping() + if err2 == nil { + return &endpoint, nil + } + + return nil, fmt.Errorf("Invalid registry endpoint %q. HTTPS attempt: %v. HTTP attempt: %v", endpoint, err, err2) } return &endpoint, nil @@ -65,6 +77,7 @@ func NewEndpoint(hostname string) (*Endpoint, error) { type Endpoint struct { URL *url.URL Version APIVersion + secure bool } // Get the formated URL for the root of this registry Endpoint @@ -88,7 +101,7 @@ func (e Endpoint) Ping() (RegistryInfo, error) { return RegistryInfo{Standalone: false}, err } - resp, _, err := doRequest(req, nil, ConnectTimeout) + resp, _, err := doRequest(req, nil, ConnectTimeout, e.secure) if err != nil { return RegistryInfo{Standalone: false}, err } @@ -127,3 +140,19 @@ func (e Endpoint) Ping() (RegistryInfo, error) { log.Debugf("RegistryInfo.Standalone: %t", info.Standalone) return info, nil } + +// IsSecure returns false if the provided hostname is part of the list of insecure registries. +// Insecure registries accept HTTP and/or accept HTTPS with certificates from unknown CAs. +func IsSecure(hostname string, insecureRegistries []string) bool { + if hostname == IndexServerAddress() { + return true + } + + for _, h := range insecureRegistries { + if hostname == h { + return false + } + } + + return true +} diff --git a/docs/registry.go b/docs/registry.go index bcbce401..15fed1b8 100644 --- a/docs/registry.go +++ b/docs/registry.go @@ -14,6 +14,7 @@ import ( "strings" "time" + "github.com/docker/docker/pkg/log" "github.com/docker/docker/utils" ) @@ -35,13 +36,17 @@ const ( ConnectTimeout ) -func newClient(jar http.CookieJar, roots *x509.CertPool, cert *tls.Certificate, timeout TimeoutType) *http.Client { +func newClient(jar http.CookieJar, roots *x509.CertPool, cert *tls.Certificate, timeout TimeoutType, secure bool) *http.Client { tlsConfig := tls.Config{RootCAs: roots} if cert != nil { tlsConfig.Certificates = append(tlsConfig.Certificates, *cert) } + if !secure { + tlsConfig.InsecureSkipVerify = true + } + httpTransport := &http.Transport{ DisableKeepAlives: true, Proxy: http.ProxyFromEnvironment, @@ -78,69 +83,76 @@ func newClient(jar http.CookieJar, roots *x509.CertPool, cert *tls.Certificate, } } -func doRequest(req *http.Request, jar http.CookieJar, timeout TimeoutType) (*http.Response, *http.Client, error) { - hasFile := func(files []os.FileInfo, name string) bool { - for _, f := range files { - if f.Name() == name { - return true - } - } - return false - } - - hostDir := path.Join("/etc/docker/certs.d", req.URL.Host) - fs, err := ioutil.ReadDir(hostDir) - if err != nil && !os.IsNotExist(err) { - return nil, nil, err - } - +func doRequest(req *http.Request, jar http.CookieJar, timeout TimeoutType, secure bool) (*http.Response, *http.Client, error) { var ( pool *x509.CertPool certs []*tls.Certificate ) - for _, f := range fs { - if strings.HasSuffix(f.Name(), ".crt") { - if pool == nil { - pool = x509.NewCertPool() + if secure && req.URL.Scheme == "https" { + hasFile := func(files []os.FileInfo, name string) bool { + for _, f := range files { + if f.Name() == name { + return true + } } - data, err := ioutil.ReadFile(path.Join(hostDir, f.Name())) - if err != nil { - return nil, nil, err - } - pool.AppendCertsFromPEM(data) + return false } - if strings.HasSuffix(f.Name(), ".cert") { - certName := f.Name() - keyName := certName[:len(certName)-5] + ".key" - if !hasFile(fs, keyName) { - return nil, nil, fmt.Errorf("Missing key %s for certificate %s", keyName, certName) - } - cert, err := tls.LoadX509KeyPair(path.Join(hostDir, certName), path.Join(hostDir, keyName)) - if err != nil { - return nil, nil, err - } - certs = append(certs, &cert) + + hostDir := path.Join("/etc/docker/certs.d", req.URL.Host) + log.Debugf("hostDir: %s", hostDir) + fs, err := ioutil.ReadDir(hostDir) + if err != nil && !os.IsNotExist(err) { + return nil, nil, err } - if strings.HasSuffix(f.Name(), ".key") { - keyName := f.Name() - certName := keyName[:len(keyName)-4] + ".cert" - if !hasFile(fs, certName) { - return nil, nil, fmt.Errorf("Missing certificate %s for key %s", certName, keyName) + + for _, f := range fs { + if strings.HasSuffix(f.Name(), ".crt") { + if pool == nil { + pool = x509.NewCertPool() + } + log.Debugf("crt: %s", hostDir+"/"+f.Name()) + data, err := ioutil.ReadFile(path.Join(hostDir, f.Name())) + if err != nil { + return nil, nil, err + } + pool.AppendCertsFromPEM(data) + } + if strings.HasSuffix(f.Name(), ".cert") { + certName := f.Name() + keyName := certName[:len(certName)-5] + ".key" + log.Debugf("cert: %s", hostDir+"/"+f.Name()) + if !hasFile(fs, keyName) { + return nil, nil, fmt.Errorf("Missing key %s for certificate %s", keyName, certName) + } + cert, err := tls.LoadX509KeyPair(path.Join(hostDir, certName), path.Join(hostDir, keyName)) + if err != nil { + return nil, nil, err + } + certs = append(certs, &cert) + } + if strings.HasSuffix(f.Name(), ".key") { + keyName := f.Name() + certName := keyName[:len(keyName)-4] + ".cert" + log.Debugf("key: %s", hostDir+"/"+f.Name()) + if !hasFile(fs, certName) { + return nil, nil, fmt.Errorf("Missing certificate %s for key %s", certName, keyName) + } } } } if len(certs) == 0 { - client := newClient(jar, pool, nil, timeout) + client := newClient(jar, pool, nil, timeout, secure) res, err := client.Do(req) if err != nil { return nil, nil, err } return res, client, nil } + for i, cert := range certs { - client := newClient(jar, pool, cert, timeout) + client := newClient(jar, pool, cert, timeout, secure) res, err := client.Do(req) // If this is the last cert, otherwise, continue to next cert if 403 or 5xx if i == len(certs)-1 || err == nil && res.StatusCode != 403 && res.StatusCode < 500 { @@ -202,49 +214,6 @@ func ResolveRepositoryName(reposName string) (string, string, error) { return hostname, reposName, nil } -// this method expands the registry name as used in the prefix of a repo -// to a full url. if it already is a url, there will be no change. -func ExpandAndVerifyRegistryUrl(hostname string, secure bool) (string, error) { - if hostname == IndexServerAddress() { - return hostname, nil - } - - endpoint := fmt.Sprintf("http://%s/v1/", hostname) - - if secure { - endpoint = fmt.Sprintf("https://%s/v1/", hostname) - } - - if _, oerr := pingRegistryEndpoint(endpoint); oerr != nil { - //TODO: triggering highland build can be done there without "failing" - err := fmt.Errorf("Invalid registry endpoint '%s': %s ", endpoint, oerr) - - if secure { - err = fmt.Errorf("%s. If this private registry supports only HTTP, please add `--insecure-registry %s` to the daemon's arguments.", oerr, hostname) - } - - return "", err - } - - return endpoint, nil -} - -// this method verifies if the provided hostname is part of the list of -// insecure registries and returns false if HTTP should be used -func IsSecure(hostname string, insecureRegistries []string) bool { - if hostname == IndexServerAddress() { - return true - } - - for _, h := range insecureRegistries { - if hostname == h { - return false - } - } - - return true -} - func trustedLocation(req *http.Request) bool { var ( trusteds = []string{"docker.com", "docker.io"} diff --git a/docs/registry_test.go b/docs/registry_test.go index ab417812..c9a9fc81 100644 --- a/docs/registry_test.go +++ b/docs/registry_test.go @@ -18,7 +18,7 @@ var ( func spawnTestRegistrySession(t *testing.T) *Session { authConfig := &AuthConfig{} - endpoint, err := NewEndpoint(makeURL("/v1/")) + endpoint, err := NewEndpoint(makeURL("/v1/"), false) if err != nil { t.Fatal(err) } @@ -30,7 +30,7 @@ func spawnTestRegistrySession(t *testing.T) *Session { } func TestPingRegistryEndpoint(t *testing.T) { - ep, err := NewEndpoint(makeURL("/v1/")) + ep, err := NewEndpoint(makeURL("/v1/"), false) if err != nil { t.Fatal(err) } diff --git a/docs/service.go b/docs/service.go index 890837ca..32274f40 100644 --- a/docs/service.go +++ b/docs/service.go @@ -89,7 +89,10 @@ func (s *Service) Search(job *engine.Job) engine.Status { if err != nil { return job.Error(err) } - endpoint, err := NewEndpoint(hostname) + + secure := IsSecure(hostname, s.insecureRegistries) + + endpoint, err := NewEndpoint(hostname, secure) if err != nil { return job.Error(err) } diff --git a/docs/session.go b/docs/session.go index 5067b8d5..28959967 100644 --- a/docs/session.go +++ b/docs/session.go @@ -64,7 +64,7 @@ func NewSession(authConfig *AuthConfig, factory *utils.HTTPRequestFactory, endpo } func (r *Session) doRequest(req *http.Request) (*http.Response, *http.Client, error) { - return doRequest(req, r.jar, r.timeout) + return doRequest(req, r.jar, r.timeout, r.indexEndpoint.secure) } // Retrieve the history of a given image from the Registry. From dff06789099b1c515219e5df63a760150515b1d1 Mon Sep 17 00:00:00 2001 From: "Daniel, Dao Quang Minh" Date: Wed, 15 Oct 2014 22:39:51 -0400 Subject: [PATCH 168/375] Avoid fallback to SSL protocols < TLS1.0 Signed-off-by: Tibor Vass Docker-DCO-1.1-Signed-off-by: Daniel, Dao Quang Minh (github: dqminh) Conflicts: registry/registry.go --- docs/registry.go | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/docs/registry.go b/docs/registry.go index 15fed1b8..a03790af 100644 --- a/docs/registry.go +++ b/docs/registry.go @@ -37,7 +37,11 @@ const ( ) func newClient(jar http.CookieJar, roots *x509.CertPool, cert *tls.Certificate, timeout TimeoutType, secure bool) *http.Client { - tlsConfig := tls.Config{RootCAs: roots} + tlsConfig := tls.Config{ + RootCAs: roots, + // Avoid fallback to SSL protocols < TLS1.0 + MinVersion: tls.VersionTLS10, + } if cert != nil { tlsConfig.Certificates = append(tlsConfig.Certificates, *cert) From ef57ab120c604d1d52d402ef58ce0b33a67c0253 Mon Sep 17 00:00:00 2001 From: Gleb M Borisov Date: Tue, 21 Oct 2014 03:45:45 +0400 Subject: [PATCH 169/375] Use dual-stack Dialer when talking to registy Signed-off-by: Gleb M. Borisov --- docs/registry.go | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/docs/registry.go b/docs/registry.go index d1315ed4..0b3ec12b 100644 --- a/docs/registry.go +++ b/docs/registry.go @@ -56,7 +56,9 @@ func newClient(jar http.CookieJar, roots *x509.CertPool, cert *tls.Certificate, case ConnectTimeout: httpTransport.Dial = func(proto string, addr string) (net.Conn, error) { // Set the connect timeout to 5 seconds - conn, err := net.DialTimeout(proto, addr, 5*time.Second) + d := net.Dialer{Timeout: 5 * time.Second, DualStack: true} + + conn, err := d.Dial(proto, addr) if err != nil { return nil, err } @@ -66,7 +68,9 @@ func newClient(jar http.CookieJar, roots *x509.CertPool, cert *tls.Certificate, } case ReceiveTimeout: httpTransport.Dial = func(proto string, addr string) (net.Conn, error) { - conn, err := net.Dial(proto, addr) + d := net.Dialer{DualStack: true} + + conn, err := d.Dial(proto, addr) if err != nil { return nil, err } From bcbb7e0c416f95d35c5709df2049858dffedc358 Mon Sep 17 00:00:00 2001 From: Vincent Batts Date: Fri, 3 Oct 2014 15:46:42 -0400 Subject: [PATCH 170/375] registry/endpoint: make it testable Signed-off-by: Vincent Batts --- docs/endpoint.go | 27 +++++++++++++++++---------- docs/endpoint_test.go | 27 +++++++++++++++++++++++++++ 2 files changed, 44 insertions(+), 10 deletions(-) create mode 100644 docs/endpoint_test.go diff --git a/docs/endpoint.go b/docs/endpoint.go index 58311d32..99f52578 100644 --- a/docs/endpoint.go +++ b/docs/endpoint.go @@ -35,16 +35,7 @@ func scanForAPIVersion(hostname string) (string, APIVersion) { } func NewEndpoint(hostname string) (*Endpoint, error) { - var ( - endpoint Endpoint - trimmedHostname string - err error - ) - if !strings.HasPrefix(hostname, "http") { - hostname = "https://" + hostname - } - trimmedHostname, endpoint.Version = scanForAPIVersion(hostname) - endpoint.URL, err = url.Parse(trimmedHostname) + endpoint, err := newEndpoint(hostname) if err != nil { return nil, err } @@ -59,6 +50,22 @@ func NewEndpoint(hostname string) (*Endpoint, error) { } } + return endpoint, nil +} +func newEndpoint(hostname string) (*Endpoint, error) { + var ( + endpoint Endpoint + trimmedHostname string + err error + ) + if !strings.HasPrefix(hostname, "http") { + hostname = "https://" + hostname + } + trimmedHostname, endpoint.Version = scanForAPIVersion(hostname) + endpoint.URL, err = url.Parse(trimmedHostname) + if err != nil { + return nil, err + } return &endpoint, nil } diff --git a/docs/endpoint_test.go b/docs/endpoint_test.go new file mode 100644 index 00000000..0ec1220d --- /dev/null +++ b/docs/endpoint_test.go @@ -0,0 +1,27 @@ +package registry + +import "testing" + +func TestEndpointParse(t *testing.T) { + testData := []struct { + str string + expected string + }{ + {IndexServerAddress(), IndexServerAddress()}, + {"http://0.0.0.0:5000", "http://0.0.0.0:5000/v1/"}, + {"0.0.0.0:5000", "https://0.0.0.0:5000/v1/"}, + } + for _, td := range testData { + e, err := newEndpoint(td.str) + if err != nil { + t.Errorf("%q: %s", td.str, err) + } + if e == nil { + t.Logf("something's fishy, endpoint for %q is nil", td.str) + continue + } + if e.String() != td.expected { + t.Errorf("expected %q, got %q", td.expected, e.String()) + } + } +} From 32654af8b6e538aaf0a4f4261ced50de1a4fe806 Mon Sep 17 00:00:00 2001 From: Alexandr Morozov Date: Fri, 24 Oct 2014 10:12:35 -0700 Subject: [PATCH 171/375] Use logrus everywhere for logging Fixed #8761 Signed-off-by: Alexandr Morozov --- docs/endpoint.go | 2 +- docs/registry_mock_test.go | 2 +- docs/session.go | 2 +- docs/session_v2.go | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/docs/endpoint.go b/docs/endpoint.go index 58311d32..05b5c08b 100644 --- a/docs/endpoint.go +++ b/docs/endpoint.go @@ -9,7 +9,7 @@ import ( "net/url" "strings" - "github.com/docker/docker/pkg/log" + log "github.com/Sirupsen/logrus" ) // scans string for api version in the URL path. returns the trimmed hostname, if version found, string and API version. diff --git a/docs/registry_mock_test.go b/docs/registry_mock_test.go index 967d8b26..02884c62 100644 --- a/docs/registry_mock_test.go +++ b/docs/registry_mock_test.go @@ -15,7 +15,7 @@ import ( "github.com/gorilla/mux" - "github.com/docker/docker/pkg/log" + log "github.com/Sirupsen/logrus" ) var ( diff --git a/docs/session.go b/docs/session.go index ff0be343..de97db3a 100644 --- a/docs/session.go +++ b/docs/session.go @@ -18,7 +18,7 @@ import ( "time" "github.com/docker/docker/pkg/httputils" - "github.com/docker/docker/pkg/log" + log "github.com/Sirupsen/logrus" "github.com/docker/docker/pkg/tarsum" "github.com/docker/docker/utils" ) diff --git a/docs/session_v2.go b/docs/session_v2.go index c0bc19b3..20e9e2ee 100644 --- a/docs/session_v2.go +++ b/docs/session_v2.go @@ -8,7 +8,7 @@ import ( "net/url" "strconv" - "github.com/docker/docker/pkg/log" + log "github.com/Sirupsen/logrus" "github.com/docker/docker/utils" "github.com/gorilla/mux" ) From 0827b71157a1d2afdca5d6fdf3f0b3b44efca826 Mon Sep 17 00:00:00 2001 From: Alexandr Morozov Date: Fri, 24 Oct 2014 15:11:48 -0700 Subject: [PATCH 172/375] Mass gofmt Signed-off-by: Alexandr Morozov --- docs/session.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/session.go b/docs/session.go index de97db3a..0c5f0139 100644 --- a/docs/session.go +++ b/docs/session.go @@ -17,8 +17,8 @@ import ( "strings" "time" - "github.com/docker/docker/pkg/httputils" log "github.com/Sirupsen/logrus" + "github.com/docker/docker/pkg/httputils" "github.com/docker/docker/pkg/tarsum" "github.com/docker/docker/utils" ) From 1a8edd0d55316d28c46bcac559565d9440d5695f Mon Sep 17 00:00:00 2001 From: Igor Dolzhikov Date: Tue, 28 Oct 2014 01:04:36 +0600 Subject: [PATCH 173/375] excluding unused transformation to []byte Signed-off-by: Igor Dolzhikov --- docs/session.go | 18 +++--------------- 1 file changed, 3 insertions(+), 15 deletions(-) diff --git a/docs/session.go b/docs/session.go index 0c5f0139..8dbf1362 100644 --- a/docs/session.go +++ b/docs/session.go @@ -230,11 +230,7 @@ func (r *Session) GetRemoteTags(registries []string, repository string, token [] } result := make(map[string]string) - rawJSON, err := ioutil.ReadAll(res.Body) - if err != nil { - return nil, err - } - if err := json.Unmarshal(rawJSON, &result); err != nil { + if err := json.NewDecoder(res.Body).Decode(&result); err != nil { return nil, err } return result, nil @@ -305,12 +301,8 @@ func (r *Session) GetRepositoryData(remote string) (*RepositoryData, error) { endpoints = append(endpoints, fmt.Sprintf("%s://%s/v1/", r.indexEndpoint.URL.Scheme, req.URL.Host)) } - checksumsJSON, err := ioutil.ReadAll(res.Body) - if err != nil { - return nil, err - } remoteChecksums := []*ImgData{} - if err := json.Unmarshal(checksumsJSON, &remoteChecksums); err != nil { + if err := json.NewDecoder(res.Body).Decode(&remoteChecksums); err != nil { return nil, err } @@ -590,12 +582,8 @@ func (r *Session) SearchRepositories(term string) (*SearchResults, error) { if res.StatusCode != 200 { return nil, utils.NewHTTPRequestError(fmt.Sprintf("Unexepected status code %d", res.StatusCode), res) } - rawData, err := ioutil.ReadAll(res.Body) - if err != nil { - return nil, err - } result := new(SearchResults) - err = json.Unmarshal(rawData, result) + err = json.NewDecoder(res.Body).Decode(result) return result, err } From 22f87eb9bed22ed4aba4de0146bcba2288df1a87 Mon Sep 17 00:00:00 2001 From: Jessica Frazelle Date: Tue, 28 Oct 2014 17:42:03 -0700 Subject: [PATCH 174/375] Fix error on successful login. Docker-DCO-1.1-Signed-off-by: Jessica Frazelle (github: jfrazelle) --- docs/auth.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/auth.go b/docs/auth.go index c9067e7a..1b111795 100644 --- a/docs/auth.go +++ b/docs/auth.go @@ -219,7 +219,7 @@ func Login(authConfig *AuthConfig, factory *utils.HTTPRequestFactory) (string, e return "", err } if resp.StatusCode == 200 { - status = "Login Succeeded" + return "Login Succeeded", nil } else if resp.StatusCode == 401 { return "", fmt.Errorf("Wrong login/password, please try again") } else if resp.StatusCode == 403 { @@ -247,7 +247,7 @@ func Login(authConfig *AuthConfig, factory *utils.HTTPRequestFactory) (string, e return "", err } if resp.StatusCode == 200 { - status = "Login Succeeded" + return "Login Succeeded", nil } else if resp.StatusCode == 401 { return "", fmt.Errorf("Wrong login/password, please try again") } else { From 0481c669c7ee82ddcb6b7ce78f14d5aa562505e5 Mon Sep 17 00:00:00 2001 From: Tibor Vass Date: Tue, 28 Oct 2014 21:20:30 -0400 Subject: [PATCH 175/375] Fix login command Signed-off-by: Tibor Vass --- docs/service.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/docs/service.go b/docs/service.go index 32274f40..7051d934 100644 --- a/docs/service.go +++ b/docs/service.go @@ -50,9 +50,11 @@ func (s *Service) Auth(job *engine.Job) engine.Status { authConfig.ServerAddress = endpoint.String() } - if _, err := Login(authConfig, HTTPRequestFactory(nil)); err != nil { + status, err := Login(authConfig, HTTPRequestFactory(nil)) + if err != nil { return job.Error(err) } + job.Printf("%s\n", status) return engine.StatusOK } From 034c1cfb9dcd6ba1b7a242ebfbabde80858a91fc Mon Sep 17 00:00:00 2001 From: unclejack Date: Sat, 16 Aug 2014 13:27:04 +0300 Subject: [PATCH 176/375] make http usage for registry explicit Docker-DCO-1.1-Signed-off-by: Cristian Staretu (github: unclejack) Conflicts: daemon/config.go daemon/daemon.go graph/pull.go graph/push.go graph/tags.go registry/registry.go registry/service.go --- docs/registry.go | 49 ++++++++++++++++++++++++++++++++++++++++++++++++ docs/service.go | 2 +- 2 files changed, 50 insertions(+), 1 deletion(-) diff --git a/docs/registry.go b/docs/registry.go index 0b3ec12b..8599d368 100644 --- a/docs/registry.go +++ b/docs/registry.go @@ -213,6 +213,55 @@ func ResolveRepositoryName(reposName string) (string, string, error) { return hostname, reposName, nil } +// this method expands the registry name as used in the prefix of a repo +// to a full url. if it already is a url, there will be no change. +func ExpandAndVerifyRegistryUrl(hostname string, secure bool) (endpoint string, err error) { + if strings.HasPrefix(hostname, "http:") || strings.HasPrefix(hostname, "https:") { + // if there is no slash after https:// (8 characters) then we have no path in the url + if strings.LastIndex(hostname, "/") < 9 { + // there is no path given. Expand with default path + hostname = hostname + "/v1/" + } + if _, err := pingRegistryEndpoint(hostname); err != nil { + return "", errors.New("Invalid Registry endpoint: " + err.Error()) + } + return hostname, nil + } + + // use HTTPS if secure, otherwise use HTTP + if secure { + endpoint = fmt.Sprintf("https://%s/v1/", hostname) + } else { + endpoint = fmt.Sprintf("http://%s/v1/", hostname) + } + _, err = pingRegistryEndpoint(endpoint) + if err != nil { + //TODO: triggering highland build can be done there without "failing" + err = fmt.Errorf("Invalid registry endpoint '%s': %s ", endpoint, err) + if secure { + err = fmt.Errorf("%s. If this private registry supports only HTTP, please add `--insecure-registry %s` to the daemon's arguments.", err, hostname) + } + return "", err + } + return endpoint, nil +} + +// this method verifies if the provided hostname is part of the list of +// insecure registries and returns false if HTTP should be used +func IsSecure(hostname string, insecureRegistries []string) (secure bool) { + secure = true + for _, h := range insecureRegistries { + if hostname == h { + secure = false + break + } + } + if hostname == IndexServerAddress() { + secure = true + } + return +} + func trustedLocation(req *http.Request) bool { var ( trusteds = []string{"docker.com", "docker.io"} diff --git a/docs/service.go b/docs/service.go index f7b35300..334e7c2e 100644 --- a/docs/service.go +++ b/docs/service.go @@ -40,7 +40,7 @@ func (s *Service) Auth(job *engine.Job) engine.Status { job.GetenvJson("authConfig", authConfig) // TODO: this is only done here because auth and registry need to be merged into one pkg if addr := authConfig.ServerAddress; addr != "" && addr != IndexServerAddress() { - endpoint, err := NewEndpoint(addr) + endpoint, err := NewEndpoint(addr, true) if err != nil { return job.Error(err) } From 50e11c9d8ea20e950280757d909857cdecda1951 Mon Sep 17 00:00:00 2001 From: Michael Crosby Date: Tue, 19 Aug 2014 11:54:42 -0700 Subject: [PATCH 177/375] Refactor IsSecure change Fix issue with restoring the tag store and setting static configuration from the daemon. i.e. the field on the TagStore struct must be made internal or the json.Unmarshal in restore will overwrite the insecure registries to be an empty struct. Signed-off-by: Michael Crosby Conflicts: graph/pull.go graph/push.go graph/tags.go --- docs/registry.go | 44 +++++++++++++++++++------------------------- 1 file changed, 19 insertions(+), 25 deletions(-) diff --git a/docs/registry.go b/docs/registry.go index 8599d368..78899681 100644 --- a/docs/registry.go +++ b/docs/registry.go @@ -215,51 +215,45 @@ func ResolveRepositoryName(reposName string) (string, string, error) { // this method expands the registry name as used in the prefix of a repo // to a full url. if it already is a url, there will be no change. -func ExpandAndVerifyRegistryUrl(hostname string, secure bool) (endpoint string, err error) { - if strings.HasPrefix(hostname, "http:") || strings.HasPrefix(hostname, "https:") { - // if there is no slash after https:// (8 characters) then we have no path in the url - if strings.LastIndex(hostname, "/") < 9 { - // there is no path given. Expand with default path - hostname = hostname + "/v1/" - } - if _, err := pingRegistryEndpoint(hostname); err != nil { - return "", errors.New("Invalid Registry endpoint: " + err.Error()) - } +func ExpandAndVerifyRegistryUrl(hostname string, secure bool) (string, error) { + if hostname == IndexServerAddress() { return hostname, nil } - // use HTTPS if secure, otherwise use HTTP + endpoint := fmt.Sprintf("http://%s/v1/", hostname) + if secure { endpoint = fmt.Sprintf("https://%s/v1/", hostname) - } else { - endpoint = fmt.Sprintf("http://%s/v1/", hostname) } - _, err = pingRegistryEndpoint(endpoint) - if err != nil { + + if _, oerr := pingRegistryEndpoint(endpoint); oerr != nil { //TODO: triggering highland build can be done there without "failing" - err = fmt.Errorf("Invalid registry endpoint '%s': %s ", endpoint, err) + err := fmt.Errorf("Invalid registry endpoint '%s': %s ", endpoint, oerr) + if secure { - err = fmt.Errorf("%s. If this private registry supports only HTTP, please add `--insecure-registry %s` to the daemon's arguments.", err, hostname) + err = fmt.Errorf("%s. If this private registry supports only HTTP, please add `--insecure-registry %s` to the daemon's arguments.", oerr, hostname) } + return "", err } + return endpoint, nil } // this method verifies if the provided hostname is part of the list of // insecure registries and returns false if HTTP should be used -func IsSecure(hostname string, insecureRegistries []string) (secure bool) { - secure = true +func IsSecure(hostname string, insecureRegistries []string) bool { + if hostname == IndexServerAddress() { + return true + } + for _, h := range insecureRegistries { if hostname == h { - secure = false - break + return false } } - if hostname == IndexServerAddress() { - secure = true - } - return + + return true } func trustedLocation(req *http.Request) bool { From 552c17d618033c0f07a4d063ce0d261db214bcb4 Mon Sep 17 00:00:00 2001 From: Michael Crosby Date: Wed, 20 Aug 2014 08:31:24 -0700 Subject: [PATCH 178/375] Don't hard code true for auth job Signed-off-by: Michael Crosby Conflicts: registry/service.go --- docs/service.go | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/docs/service.go b/docs/service.go index 334e7c2e..890837ca 100644 --- a/docs/service.go +++ b/docs/service.go @@ -13,12 +13,15 @@ import ( // 'pull': Download images from any registry (TODO) // 'push': Upload images to any registry (TODO) type Service struct { + insecureRegistries []string } // NewService returns a new instance of Service ready to be // installed no an engine. -func NewService() *Service { - return &Service{} +func NewService(insecureRegistries []string) *Service { + return &Service{ + insecureRegistries: insecureRegistries, + } } // Install installs registry capabilities to eng. @@ -32,15 +35,12 @@ func (s *Service) Install(eng *engine.Engine) error { // and returns OK if authentication was sucessful. // It can be used to verify the validity of a client's credentials. func (s *Service) Auth(job *engine.Job) engine.Status { - var ( - err error - authConfig = &AuthConfig{} - ) + var authConfig = new(AuthConfig) job.GetenvJson("authConfig", authConfig) - // TODO: this is only done here because auth and registry need to be merged into one pkg + if addr := authConfig.ServerAddress; addr != "" && addr != IndexServerAddress() { - endpoint, err := NewEndpoint(addr, true) + endpoint, err := NewEndpoint(addr, IsSecure(addr, s.insecureRegistries)) if err != nil { return job.Error(err) } @@ -49,11 +49,11 @@ func (s *Service) Auth(job *engine.Job) engine.Status { } authConfig.ServerAddress = endpoint.String() } - status, err := Login(authConfig, HTTPRequestFactory(nil)) - if err != nil { + + if _, err := Login(authConfig, HTTPRequestFactory(nil)); err != nil { return job.Error(err) } - job.Printf("%s\n", status) + return engine.StatusOK } From 1b72e0234eb0ccc1769059fcdf2d89369e5c7963 Mon Sep 17 00:00:00 2001 From: Tibor Vass Date: Fri, 10 Oct 2014 23:22:12 -0400 Subject: [PATCH 179/375] Do not verify certificate when using --insecure-registry on an HTTPS registry Signed-off-by: Tibor Vass Conflicts: registry/registry.go registry/registry_test.go registry/service.go registry/session.go Conflicts: registry/endpoint.go registry/registry.go --- docs/endpoint.go | 51 +++++++++++---- docs/endpoint_test.go | 2 +- docs/registry.go | 143 +++++++++++++++++------------------------- docs/registry_test.go | 4 +- docs/service.go | 5 +- docs/session.go | 2 +- 6 files changed, 104 insertions(+), 103 deletions(-) diff --git a/docs/endpoint.go b/docs/endpoint.go index 639c9970..88dbeafd 100644 --- a/docs/endpoint.go +++ b/docs/endpoint.go @@ -2,7 +2,6 @@ package registry import ( "encoding/json" - "errors" "fmt" "io/ioutil" "net/http" @@ -34,27 +33,40 @@ func scanForAPIVersion(hostname string) (string, APIVersion) { return hostname, DefaultAPIVersion } -func NewEndpoint(hostname string) (*Endpoint, error) { - endpoint, err := newEndpoint(hostname) +func NewEndpoint(hostname string, secure bool) (*Endpoint, error) { + endpoint, err := newEndpoint(hostname, secure) if err != nil { return nil, err } + // Try HTTPS ping to registry endpoint.URL.Scheme = "https" if _, err := endpoint.Ping(); err != nil { - log.Debugf("Registry %s does not work (%s), falling back to http", endpoint, err) - // TODO: Check if http fallback is enabled - endpoint.URL.Scheme = "http" - if _, err = endpoint.Ping(); err != nil { - return nil, errors.New("Invalid Registry endpoint: " + err.Error()) + + //TODO: triggering highland build can be done there without "failing" + + if secure { + // If registry is secure and HTTPS failed, show user the error and tell them about `--insecure-registry` + // in case that's what they need. DO NOT accept unknown CA certificates, and DO NOT fallback to HTTP. + return nil, fmt.Errorf("Invalid registry endpoint %s: %v. If this private registry supports only HTTP or HTTPS with an unknown CA certificate, please add `--insecure-registry %s` to the daemon's arguments. In the case of HTTPS, if you have access to the registry's CA certificate, no need for the flag; simply place the CA certificate at /etc/docker/certs.d/%s/ca.crt", endpoint, err, endpoint.URL.Host, endpoint.URL.Host) } + + // If registry is insecure and HTTPS failed, fallback to HTTP. + log.Debugf("Error from registry %q marked as insecure: %v. Insecurely falling back to HTTP", endpoint, err) + endpoint.URL.Scheme = "http" + _, err2 := endpoint.Ping() + if err2 == nil { + return endpoint, nil + } + + return nil, fmt.Errorf("Invalid registry endpoint %q. HTTPS attempt: %v. HTTP attempt: %v", endpoint, err, err2) } return endpoint, nil } -func newEndpoint(hostname string) (*Endpoint, error) { +func newEndpoint(hostname string, secure bool) (*Endpoint, error) { var ( - endpoint Endpoint + endpoint = Endpoint{secure: secure} trimmedHostname string err error ) @@ -72,6 +84,7 @@ func newEndpoint(hostname string) (*Endpoint, error) { type Endpoint struct { URL *url.URL Version APIVersion + secure bool } // Get the formated URL for the root of this registry Endpoint @@ -95,7 +108,7 @@ func (e Endpoint) Ping() (RegistryInfo, error) { return RegistryInfo{Standalone: false}, err } - resp, _, err := doRequest(req, nil, ConnectTimeout) + resp, _, err := doRequest(req, nil, ConnectTimeout, e.secure) if err != nil { return RegistryInfo{Standalone: false}, err } @@ -134,3 +147,19 @@ func (e Endpoint) Ping() (RegistryInfo, error) { log.Debugf("RegistryInfo.Standalone: %t", info.Standalone) return info, nil } + +// IsSecure returns false if the provided hostname is part of the list of insecure registries. +// Insecure registries accept HTTP and/or accept HTTPS with certificates from unknown CAs. +func IsSecure(hostname string, insecureRegistries []string) bool { + if hostname == IndexServerAddress() { + return true + } + + for _, h := range insecureRegistries { + if hostname == h { + return false + } + } + + return true +} diff --git a/docs/endpoint_test.go b/docs/endpoint_test.go index 0ec1220d..def5e0d7 100644 --- a/docs/endpoint_test.go +++ b/docs/endpoint_test.go @@ -12,7 +12,7 @@ func TestEndpointParse(t *testing.T) { {"0.0.0.0:5000", "https://0.0.0.0:5000/v1/"}, } for _, td := range testData { - e, err := newEndpoint(td.str) + e, err := newEndpoint(td.str, true) if err != nil { t.Errorf("%q: %s", td.str, err) } diff --git a/docs/registry.go b/docs/registry.go index 78899681..8d436374 100644 --- a/docs/registry.go +++ b/docs/registry.go @@ -14,6 +14,7 @@ import ( "strings" "time" + log "github.com/Sirupsen/logrus" "github.com/docker/docker/utils" ) @@ -35,7 +36,7 @@ const ( ConnectTimeout ) -func newClient(jar http.CookieJar, roots *x509.CertPool, cert *tls.Certificate, timeout TimeoutType) *http.Client { +func newClient(jar http.CookieJar, roots *x509.CertPool, cert *tls.Certificate, timeout TimeoutType, secure bool) *http.Client { tlsConfig := tls.Config{ RootCAs: roots, // Avoid fallback to SSL protocols < TLS1.0 @@ -46,6 +47,10 @@ func newClient(jar http.CookieJar, roots *x509.CertPool, cert *tls.Certificate, tlsConfig.Certificates = append(tlsConfig.Certificates, *cert) } + if !secure { + tlsConfig.InsecureSkipVerify = true + } + httpTransport := &http.Transport{ DisableKeepAlives: true, Proxy: http.ProxyFromEnvironment, @@ -86,69 +91,76 @@ func newClient(jar http.CookieJar, roots *x509.CertPool, cert *tls.Certificate, } } -func doRequest(req *http.Request, jar http.CookieJar, timeout TimeoutType) (*http.Response, *http.Client, error) { - hasFile := func(files []os.FileInfo, name string) bool { - for _, f := range files { - if f.Name() == name { - return true - } - } - return false - } - - hostDir := path.Join("/etc/docker/certs.d", req.URL.Host) - fs, err := ioutil.ReadDir(hostDir) - if err != nil && !os.IsNotExist(err) { - return nil, nil, err - } - +func doRequest(req *http.Request, jar http.CookieJar, timeout TimeoutType, secure bool) (*http.Response, *http.Client, error) { var ( pool *x509.CertPool certs []*tls.Certificate ) - for _, f := range fs { - if strings.HasSuffix(f.Name(), ".crt") { - if pool == nil { - pool = x509.NewCertPool() + if secure && req.URL.Scheme == "https" { + hasFile := func(files []os.FileInfo, name string) bool { + for _, f := range files { + if f.Name() == name { + return true + } } - data, err := ioutil.ReadFile(path.Join(hostDir, f.Name())) - if err != nil { - return nil, nil, err - } - pool.AppendCertsFromPEM(data) + return false } - if strings.HasSuffix(f.Name(), ".cert") { - certName := f.Name() - keyName := certName[:len(certName)-5] + ".key" - if !hasFile(fs, keyName) { - return nil, nil, fmt.Errorf("Missing key %s for certificate %s", keyName, certName) - } - cert, err := tls.LoadX509KeyPair(path.Join(hostDir, certName), path.Join(hostDir, keyName)) - if err != nil { - return nil, nil, err - } - certs = append(certs, &cert) + + hostDir := path.Join("/etc/docker/certs.d", req.URL.Host) + log.Debugf("hostDir: %s", hostDir) + fs, err := ioutil.ReadDir(hostDir) + if err != nil && !os.IsNotExist(err) { + return nil, nil, err } - if strings.HasSuffix(f.Name(), ".key") { - keyName := f.Name() - certName := keyName[:len(keyName)-4] + ".cert" - if !hasFile(fs, certName) { - return nil, nil, fmt.Errorf("Missing certificate %s for key %s", certName, keyName) + + for _, f := range fs { + if strings.HasSuffix(f.Name(), ".crt") { + if pool == nil { + pool = x509.NewCertPool() + } + log.Debugf("crt: %s", hostDir+"/"+f.Name()) + data, err := ioutil.ReadFile(path.Join(hostDir, f.Name())) + if err != nil { + return nil, nil, err + } + pool.AppendCertsFromPEM(data) + } + if strings.HasSuffix(f.Name(), ".cert") { + certName := f.Name() + keyName := certName[:len(certName)-5] + ".key" + log.Debugf("cert: %s", hostDir+"/"+f.Name()) + if !hasFile(fs, keyName) { + return nil, nil, fmt.Errorf("Missing key %s for certificate %s", keyName, certName) + } + cert, err := tls.LoadX509KeyPair(path.Join(hostDir, certName), path.Join(hostDir, keyName)) + if err != nil { + return nil, nil, err + } + certs = append(certs, &cert) + } + if strings.HasSuffix(f.Name(), ".key") { + keyName := f.Name() + certName := keyName[:len(keyName)-4] + ".cert" + log.Debugf("key: %s", hostDir+"/"+f.Name()) + if !hasFile(fs, certName) { + return nil, nil, fmt.Errorf("Missing certificate %s for key %s", certName, keyName) + } } } } if len(certs) == 0 { - client := newClient(jar, pool, nil, timeout) + client := newClient(jar, pool, nil, timeout, secure) res, err := client.Do(req) if err != nil { return nil, nil, err } return res, client, nil } + for i, cert := range certs { - client := newClient(jar, pool, cert, timeout) + client := newClient(jar, pool, cert, timeout, secure) res, err := client.Do(req) // If this is the last cert, otherwise, continue to next cert if 403 or 5xx if i == len(certs)-1 || err == nil && @@ -213,49 +225,6 @@ func ResolveRepositoryName(reposName string) (string, string, error) { return hostname, reposName, nil } -// this method expands the registry name as used in the prefix of a repo -// to a full url. if it already is a url, there will be no change. -func ExpandAndVerifyRegistryUrl(hostname string, secure bool) (string, error) { - if hostname == IndexServerAddress() { - return hostname, nil - } - - endpoint := fmt.Sprintf("http://%s/v1/", hostname) - - if secure { - endpoint = fmt.Sprintf("https://%s/v1/", hostname) - } - - if _, oerr := pingRegistryEndpoint(endpoint); oerr != nil { - //TODO: triggering highland build can be done there without "failing" - err := fmt.Errorf("Invalid registry endpoint '%s': %s ", endpoint, oerr) - - if secure { - err = fmt.Errorf("%s. If this private registry supports only HTTP, please add `--insecure-registry %s` to the daemon's arguments.", oerr, hostname) - } - - return "", err - } - - return endpoint, nil -} - -// this method verifies if the provided hostname is part of the list of -// insecure registries and returns false if HTTP should be used -func IsSecure(hostname string, insecureRegistries []string) bool { - if hostname == IndexServerAddress() { - return true - } - - for _, h := range insecureRegistries { - if hostname == h { - return false - } - } - - return true -} - func trustedLocation(req *http.Request) bool { var ( trusteds = []string{"docker.com", "docker.io"} diff --git a/docs/registry_test.go b/docs/registry_test.go index fdf714e8..23aef6c3 100644 --- a/docs/registry_test.go +++ b/docs/registry_test.go @@ -21,7 +21,7 @@ const ( func spawnTestRegistrySession(t *testing.T) *Session { authConfig := &AuthConfig{} - endpoint, err := NewEndpoint(makeURL("/v1/")) + endpoint, err := NewEndpoint(makeURL("/v1/"), false) if err != nil { t.Fatal(err) } @@ -33,7 +33,7 @@ func spawnTestRegistrySession(t *testing.T) *Session { } func TestPingRegistryEndpoint(t *testing.T) { - ep, err := NewEndpoint(makeURL("/v1/")) + ep, err := NewEndpoint(makeURL("/v1/"), false) if err != nil { t.Fatal(err) } diff --git a/docs/service.go b/docs/service.go index 890837ca..32274f40 100644 --- a/docs/service.go +++ b/docs/service.go @@ -89,7 +89,10 @@ func (s *Service) Search(job *engine.Job) engine.Status { if err != nil { return job.Error(err) } - endpoint, err := NewEndpoint(hostname) + + secure := IsSecure(hostname, s.insecureRegistries) + + endpoint, err := NewEndpoint(hostname, secure) if err != nil { return job.Error(err) } diff --git a/docs/session.go b/docs/session.go index 8dbf1362..ba6df358 100644 --- a/docs/session.go +++ b/docs/session.go @@ -65,7 +65,7 @@ func NewSession(authConfig *AuthConfig, factory *utils.HTTPRequestFactory, endpo } func (r *Session) doRequest(req *http.Request) (*http.Response, *http.Client, error) { - return doRequest(req, r.jar, r.timeout) + return doRequest(req, r.jar, r.timeout, r.indexEndpoint.secure) } // Retrieve the history of a given image from the Registry. From 47a494e0fd9d7b2d4aa31ce1bf48ed9992407f56 Mon Sep 17 00:00:00 2001 From: Tibor Vass Date: Tue, 28 Oct 2014 21:20:30 -0400 Subject: [PATCH 180/375] Fix login command Signed-off-by: Tibor Vass --- docs/service.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/docs/service.go b/docs/service.go index 32274f40..7051d934 100644 --- a/docs/service.go +++ b/docs/service.go @@ -50,9 +50,11 @@ func (s *Service) Auth(job *engine.Job) engine.Status { authConfig.ServerAddress = endpoint.String() } - if _, err := Login(authConfig, HTTPRequestFactory(nil)); err != nil { + status, err := Login(authConfig, HTTPRequestFactory(nil)) + if err != nil { return job.Error(err) } + job.Printf("%s\n", status) return engine.StatusOK } From 7dd4199fe8055877979a51426a2ff50e5e04c0f5 Mon Sep 17 00:00:00 2001 From: Vincent Batts Date: Thu, 9 Oct 2014 13:52:30 -0400 Subject: [PATCH 181/375] registry: don't iterate through certs the golang tls.Conn does a fine job of that. http://golang.org/src/pkg/crypto/tls/handshake_client.go?#L334 Signed-off-by: Vincent Batts --- docs/registry.go | 29 ++++++++--------------------- 1 file changed, 8 insertions(+), 21 deletions(-) diff --git a/docs/registry.go b/docs/registry.go index 8d436374..e1d22b09 100644 --- a/docs/registry.go +++ b/docs/registry.go @@ -36,15 +36,12 @@ const ( ConnectTimeout ) -func newClient(jar http.CookieJar, roots *x509.CertPool, cert *tls.Certificate, timeout TimeoutType, secure bool) *http.Client { +func newClient(jar http.CookieJar, roots *x509.CertPool, certs []tls.Certificate, timeout TimeoutType, secure bool) *http.Client { tlsConfig := tls.Config{ RootCAs: roots, // Avoid fallback to SSL protocols < TLS1.0 - MinVersion: tls.VersionTLS10, - } - - if cert != nil { - tlsConfig.Certificates = append(tlsConfig.Certificates, *cert) + MinVersion: tls.VersionTLS10, + Certificates: certs, } if !secure { @@ -94,7 +91,7 @@ func newClient(jar http.CookieJar, roots *x509.CertPool, cert *tls.Certificate, func doRequest(req *http.Request, jar http.CookieJar, timeout TimeoutType, secure bool) (*http.Response, *http.Client, error) { var ( pool *x509.CertPool - certs []*tls.Certificate + certs []tls.Certificate ) if secure && req.URL.Scheme == "https" { @@ -137,7 +134,7 @@ func doRequest(req *http.Request, jar http.CookieJar, timeout TimeoutType, secur if err != nil { return nil, nil, err } - certs = append(certs, &cert) + certs = append(certs, cert) } if strings.HasSuffix(f.Name(), ".key") { keyName := f.Name() @@ -159,19 +156,9 @@ func doRequest(req *http.Request, jar http.CookieJar, timeout TimeoutType, secur return res, client, nil } - for i, cert := range certs { - client := newClient(jar, pool, cert, timeout, secure) - res, err := client.Do(req) - // If this is the last cert, otherwise, continue to next cert if 403 or 5xx - if i == len(certs)-1 || err == nil && - res.StatusCode != 403 && - res.StatusCode != 404 && - res.StatusCode < 500 { - return res, client, err - } - } - - return nil, nil, nil + client := newClient(jar, pool, certs, timeout, secure) + res, err := client.Do(req) + return res, client, err } func validateRepositoryName(repositoryName string) error { From cd246befe28710ed350c94996e3ee887518a9a4b Mon Sep 17 00:00:00 2001 From: Johan Euphrosine Date: Tue, 11 Nov 2014 11:01:49 -0800 Subject: [PATCH 182/375] registry: add tests for IsSecure Signed-off-by: Johan Euphrosine --- docs/registry_test.go | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/docs/registry_test.go b/docs/registry_test.go index 23aef6c3..f7b5168b 100644 --- a/docs/registry_test.go +++ b/docs/registry_test.go @@ -319,3 +319,23 @@ func TestAddRequiredHeadersToRedirectedRequests(t *testing.T) { } } } + +func TestIsSecure(t *testing.T) { + tests := []struct { + addr string + insecureRegistries []string + expected bool + }{ + {"example.com", []string{}, true}, + {"example.com", []string{"example.com"}, false}, + {"localhost", []string{"localhost:5000"}, true}, + {"localhost:5000", []string{"localhost:5000"}, false}, + {"localhost", []string{"example.com"}, true}, + {"127.0.0.1:5000", []string{"127.0.0.1:5000"}, false}, + } + for _, tt := range tests { + if sec := IsSecure(tt.addr, tt.insecureRegistries); sec != tt.expected { + t.Errorf("IsSecure failed for %q %v, expected %v got %v", tt.addr, tt.insecureRegistries, tt.expected, sec) + } + } +} From 8582d04393a05b074c35915f612a69c376564be3 Mon Sep 17 00:00:00 2001 From: Johan Euphrosine Date: Fri, 31 Oct 2014 13:00:49 -0700 Subject: [PATCH 183/375] registry: default --insecure-registry to localhost and 127.0.0.1 Signed-off-by: Johan Euphrosine --- docs/endpoint.go | 12 +++++++++++- docs/registry_test.go | 21 +++++++++++++++++++++ 2 files changed, 32 insertions(+), 1 deletion(-) diff --git a/docs/endpoint.go b/docs/endpoint.go index 88dbeafd..cb96cb4f 100644 --- a/docs/endpoint.go +++ b/docs/endpoint.go @@ -4,6 +4,7 @@ import ( "encoding/json" "fmt" "io/ioutil" + "net" "net/http" "net/url" "strings" @@ -154,7 +155,16 @@ func IsSecure(hostname string, insecureRegistries []string) bool { if hostname == IndexServerAddress() { return true } - + if len(insecureRegistries) == 0 { + host, _, err := net.SplitHostPort(hostname) + if err != nil { + host = hostname + } + if host == "127.0.0.1" || host == "localhost" { + return false + } + return true + } for _, h := range insecureRegistries { if hostname == h { return false diff --git a/docs/registry_test.go b/docs/registry_test.go index f7b5168b..7191acea 100644 --- a/docs/registry_test.go +++ b/docs/registry_test.go @@ -339,3 +339,24 @@ func TestIsSecure(t *testing.T) { } } } + +func TestIsSecure(t *testing.T) { + tests := []struct { + addr string + insecureRegistries []string + expected bool + }{ + {"localhost", []string{}, false}, + {"localhost:5000", []string{}, false}, + {"127.0.0.1", []string{}, false}, + {"localhost", []string{"example.com"}, true}, + {"127.0.0.1", []string{"example.com"}, true}, + {"example.com", []string{}, true}, + {"example.com", []string{"example.com"}, false}, + } + for _, tt := range tests { + if sec := IsSecure(tt.addr, tt.insecureRegistries); sec != tt.expected { + t.Errorf("IsSecure failed for %q %v, expected %v got %v", tt.addr, tt.insecureRegistries, tt.expected, sec) + } + } +} From 524aa8b1a68ae467315ff26ea9128aa3b996a5d8 Mon Sep 17 00:00:00 2001 From: Erik Hollensbe Date: Wed, 12 Nov 2014 09:08:45 -0800 Subject: [PATCH 184/375] registry: always treat 127.0.0.1 as insecure for all cases anytime anywhere Docker-DCO-1.1-Signed-off-by: Erik Hollensbe (github: erikh) --- docs/endpoint.go | 20 +++++++++++++------- docs/registry_test.go | 24 ++++++------------------ 2 files changed, 19 insertions(+), 25 deletions(-) diff --git a/docs/endpoint.go b/docs/endpoint.go index cb96cb4f..0d0749d7 100644 --- a/docs/endpoint.go +++ b/docs/endpoint.go @@ -152,19 +152,25 @@ func (e Endpoint) Ping() (RegistryInfo, error) { // IsSecure returns false if the provided hostname is part of the list of insecure registries. // Insecure registries accept HTTP and/or accept HTTPS with certificates from unknown CAs. func IsSecure(hostname string, insecureRegistries []string) bool { + if hostname == IndexServerAddress() { return true } + + host, _, err := net.SplitHostPort(hostname) + + if err != nil { + host = hostname + } + + if host == "127.0.0.1" || host == "localhost" { + return false + } + if len(insecureRegistries) == 0 { - host, _, err := net.SplitHostPort(hostname) - if err != nil { - host = hostname - } - if host == "127.0.0.1" || host == "localhost" { - return false - } return true } + for _, h := range insecureRegistries { if hostname == h { return false diff --git a/docs/registry_test.go b/docs/registry_test.go index 7191acea..032c9fbf 100644 --- a/docs/registry_test.go +++ b/docs/registry_test.go @@ -328,31 +328,19 @@ func TestIsSecure(t *testing.T) { }{ {"example.com", []string{}, true}, {"example.com", []string{"example.com"}, false}, - {"localhost", []string{"localhost:5000"}, true}, + {"localhost", []string{"localhost:5000"}, false}, {"localhost:5000", []string{"localhost:5000"}, false}, - {"localhost", []string{"example.com"}, true}, + {"localhost", []string{"example.com"}, false}, {"127.0.0.1:5000", []string{"127.0.0.1:5000"}, false}, - } - for _, tt := range tests { - if sec := IsSecure(tt.addr, tt.insecureRegistries); sec != tt.expected { - t.Errorf("IsSecure failed for %q %v, expected %v got %v", tt.addr, tt.insecureRegistries, tt.expected, sec) - } - } -} - -func TestIsSecure(t *testing.T) { - tests := []struct { - addr string - insecureRegistries []string - expected bool - }{ {"localhost", []string{}, false}, {"localhost:5000", []string{}, false}, {"127.0.0.1", []string{}, false}, - {"localhost", []string{"example.com"}, true}, - {"127.0.0.1", []string{"example.com"}, true}, + {"localhost", []string{"example.com"}, false}, + {"127.0.0.1", []string{"example.com"}, false}, {"example.com", []string{}, true}, {"example.com", []string{"example.com"}, false}, + {"127.0.0.1", []string{"example.com"}, false}, + {"127.0.0.1:5000", []string{"example.com"}, false}, } for _, tt := range tests { if sec := IsSecure(tt.addr, tt.insecureRegistries); sec != tt.expected { From 80255ff22489d42040e00e25ee9e0b388ae78ef2 Mon Sep 17 00:00:00 2001 From: Tibor Vass Date: Tue, 11 Nov 2014 17:37:44 -0500 Subject: [PATCH 185/375] registry: refactor registry.IsSecure calls into registry.NewEndpoint Signed-off-by: Tibor Vass --- docs/endpoint.go | 16 +++++++++------- docs/endpoint_test.go | 2 +- docs/registry_test.go | 4 ++-- docs/service.go | 6 ++---- 4 files changed, 14 insertions(+), 14 deletions(-) diff --git a/docs/endpoint.go b/docs/endpoint.go index 0d0749d7..390eec2e 100644 --- a/docs/endpoint.go +++ b/docs/endpoint.go @@ -34,12 +34,15 @@ func scanForAPIVersion(hostname string) (string, APIVersion) { return hostname, DefaultAPIVersion } -func NewEndpoint(hostname string, secure bool) (*Endpoint, error) { - endpoint, err := newEndpoint(hostname, secure) +func NewEndpoint(hostname string, insecureRegistries []string) (*Endpoint, error) { + endpoint, err := newEndpoint(hostname) if err != nil { return nil, err } + secure := isSecure(endpoint.URL.Host, insecureRegistries) + endpoint.secure = secure + // Try HTTPS ping to registry endpoint.URL.Scheme = "https" if _, err := endpoint.Ping(); err != nil { @@ -65,9 +68,9 @@ func NewEndpoint(hostname string, secure bool) (*Endpoint, error) { return endpoint, nil } -func newEndpoint(hostname string, secure bool) (*Endpoint, error) { +func newEndpoint(hostname string) (*Endpoint, error) { var ( - endpoint = Endpoint{secure: secure} + endpoint = Endpoint{secure: true} trimmedHostname string err error ) @@ -149,10 +152,9 @@ func (e Endpoint) Ping() (RegistryInfo, error) { return info, nil } -// IsSecure returns false if the provided hostname is part of the list of insecure registries. +// isSecure returns false if the provided hostname is part of the list of insecure registries. // Insecure registries accept HTTP and/or accept HTTPS with certificates from unknown CAs. -func IsSecure(hostname string, insecureRegistries []string) bool { - +func isSecure(hostname string, insecureRegistries []string) bool { if hostname == IndexServerAddress() { return true } diff --git a/docs/endpoint_test.go b/docs/endpoint_test.go index def5e0d7..0ec1220d 100644 --- a/docs/endpoint_test.go +++ b/docs/endpoint_test.go @@ -12,7 +12,7 @@ func TestEndpointParse(t *testing.T) { {"0.0.0.0:5000", "https://0.0.0.0:5000/v1/"}, } for _, td := range testData { - e, err := newEndpoint(td.str, true) + e, err := newEndpoint(td.str) if err != nil { t.Errorf("%q: %s", td.str, err) } diff --git a/docs/registry_test.go b/docs/registry_test.go index 032c9fbf..8bc6a351 100644 --- a/docs/registry_test.go +++ b/docs/registry_test.go @@ -343,8 +343,8 @@ func TestIsSecure(t *testing.T) { {"127.0.0.1:5000", []string{"example.com"}, false}, } for _, tt := range tests { - if sec := IsSecure(tt.addr, tt.insecureRegistries); sec != tt.expected { - t.Errorf("IsSecure failed for %q %v, expected %v got %v", tt.addr, tt.insecureRegistries, tt.expected, sec) + if sec := isSecure(tt.addr, tt.insecureRegistries); sec != tt.expected { + t.Errorf("isSecure failed for %q %v, expected %v got %v", tt.addr, tt.insecureRegistries, tt.expected, sec) } } } diff --git a/docs/service.go b/docs/service.go index 7051d934..53e8278b 100644 --- a/docs/service.go +++ b/docs/service.go @@ -40,7 +40,7 @@ func (s *Service) Auth(job *engine.Job) engine.Status { job.GetenvJson("authConfig", authConfig) if addr := authConfig.ServerAddress; addr != "" && addr != IndexServerAddress() { - endpoint, err := NewEndpoint(addr, IsSecure(addr, s.insecureRegistries)) + endpoint, err := NewEndpoint(addr, s.insecureRegistries) if err != nil { return job.Error(err) } @@ -92,9 +92,7 @@ func (s *Service) Search(job *engine.Job) engine.Status { return job.Error(err) } - secure := IsSecure(hostname, s.insecureRegistries) - - endpoint, err := NewEndpoint(hostname, secure) + endpoint, err := NewEndpoint(hostname, s.insecureRegistries) if err != nil { return job.Error(err) } From cca910e878f20c842f599377470015d770784d99 Mon Sep 17 00:00:00 2001 From: Tibor Vass Date: Tue, 11 Nov 2014 20:08:59 -0500 Subject: [PATCH 186/375] Put mock registry address in insecureRegistries for unit tests Signed-off-by: Tibor Vass --- docs/registry_mock_test.go | 10 ++++++++-- docs/registry_test.go | 4 ++-- 2 files changed, 10 insertions(+), 4 deletions(-) diff --git a/docs/registry_mock_test.go b/docs/registry_mock_test.go index 02884c62..1c710e21 100644 --- a/docs/registry_mock_test.go +++ b/docs/registry_mock_test.go @@ -19,8 +19,9 @@ import ( ) var ( - testHTTPServer *httptest.Server - testLayers = map[string]map[string]string{ + testHTTPServer *httptest.Server + insecureRegistries []string + testLayers = map[string]map[string]string{ "77dbf71da1d00e3fbddc480176eac8994025630c6590d11cfc8fe1209c2a1d20": { "json": `{"id":"77dbf71da1d00e3fbddc480176eac8994025630c6590d11cfc8fe1209c2a1d20", "comment":"test base image","created":"2013-03-23T12:53:11.10432-07:00", @@ -100,6 +101,11 @@ func init() { r.HandleFunc("/v2/version", handlerGetPing).Methods("GET") testHTTPServer = httptest.NewServer(handlerAccessLog(r)) + URL, err := url.Parse(testHTTPServer.URL) + if err != nil { + panic(err) + } + insecureRegistries = []string{URL.Host} } func handlerAccessLog(handler http.Handler) http.Handler { diff --git a/docs/registry_test.go b/docs/registry_test.go index 8bc6a351..37dedc2a 100644 --- a/docs/registry_test.go +++ b/docs/registry_test.go @@ -21,7 +21,7 @@ const ( func spawnTestRegistrySession(t *testing.T) *Session { authConfig := &AuthConfig{} - endpoint, err := NewEndpoint(makeURL("/v1/"), false) + endpoint, err := NewEndpoint(makeURL("/v1/"), insecureRegistries) if err != nil { t.Fatal(err) } @@ -33,7 +33,7 @@ func spawnTestRegistrySession(t *testing.T) *Session { } func TestPingRegistryEndpoint(t *testing.T) { - ep, err := NewEndpoint(makeURL("/v1/"), false) + ep, err := NewEndpoint(makeURL("/v1/"), insecureRegistries) if err != nil { t.Fatal(err) } From f0920e61bfbbaa692b4a7ff5148e67317ad087b3 Mon Sep 17 00:00:00 2001 From: Tibor Vass Date: Thu, 13 Nov 2014 06:56:36 -0800 Subject: [PATCH 187/375] registry: parse INDEXSERVERADDRESS into a URL for easier check in isSecure Signed-off-by: Tibor Vass --- docs/auth.go | 10 ++++++++++ docs/endpoint.go | 14 ++++++-------- docs/endpoint_test.go | 2 +- docs/registry_test.go | 1 + 4 files changed, 18 insertions(+), 9 deletions(-) diff --git a/docs/auth.go b/docs/auth.go index 1b111795..a22d0b88 100644 --- a/docs/auth.go +++ b/docs/auth.go @@ -7,6 +7,7 @@ import ( "fmt" "io/ioutil" "net/http" + "net/url" "os" "path" "strings" @@ -27,8 +28,17 @@ const ( var ( ErrConfigFileMissing = errors.New("The Auth config file is missing") + IndexServerURL *url.URL ) +func init() { + url, err := url.Parse(INDEXSERVER) + if err != nil { + panic(err) + } + IndexServerURL = url +} + type AuthConfig struct { Username string `json:"username,omitempty"` Password string `json:"password,omitempty"` diff --git a/docs/endpoint.go b/docs/endpoint.go index 390eec2e..bd23c302 100644 --- a/docs/endpoint.go +++ b/docs/endpoint.go @@ -35,21 +35,18 @@ func scanForAPIVersion(hostname string) (string, APIVersion) { } func NewEndpoint(hostname string, insecureRegistries []string) (*Endpoint, error) { - endpoint, err := newEndpoint(hostname) + endpoint, err := newEndpoint(hostname, insecureRegistries) if err != nil { return nil, err } - secure := isSecure(endpoint.URL.Host, insecureRegistries) - endpoint.secure = secure - // Try HTTPS ping to registry endpoint.URL.Scheme = "https" if _, err := endpoint.Ping(); err != nil { //TODO: triggering highland build can be done there without "failing" - if secure { + if endpoint.secure { // If registry is secure and HTTPS failed, show user the error and tell them about `--insecure-registry` // in case that's what they need. DO NOT accept unknown CA certificates, and DO NOT fallback to HTTP. return nil, fmt.Errorf("Invalid registry endpoint %s: %v. If this private registry supports only HTTP or HTTPS with an unknown CA certificate, please add `--insecure-registry %s` to the daemon's arguments. In the case of HTTPS, if you have access to the registry's CA certificate, no need for the flag; simply place the CA certificate at /etc/docker/certs.d/%s/ca.crt", endpoint, err, endpoint.URL.Host, endpoint.URL.Host) @@ -68,9 +65,9 @@ func NewEndpoint(hostname string, insecureRegistries []string) (*Endpoint, error return endpoint, nil } -func newEndpoint(hostname string) (*Endpoint, error) { +func newEndpoint(hostname string, insecureRegistries []string) (*Endpoint, error) { var ( - endpoint = Endpoint{secure: true} + endpoint = Endpoint{} trimmedHostname string err error ) @@ -82,6 +79,7 @@ func newEndpoint(hostname string) (*Endpoint, error) { if err != nil { return nil, err } + endpoint.secure = isSecure(endpoint.URL.Host, insecureRegistries) return &endpoint, nil } @@ -155,7 +153,7 @@ func (e Endpoint) Ping() (RegistryInfo, error) { // isSecure returns false if the provided hostname is part of the list of insecure registries. // Insecure registries accept HTTP and/or accept HTTPS with certificates from unknown CAs. func isSecure(hostname string, insecureRegistries []string) bool { - if hostname == IndexServerAddress() { + if hostname == IndexServerURL.Host { return true } diff --git a/docs/endpoint_test.go b/docs/endpoint_test.go index 0ec1220d..54105ec1 100644 --- a/docs/endpoint_test.go +++ b/docs/endpoint_test.go @@ -12,7 +12,7 @@ func TestEndpointParse(t *testing.T) { {"0.0.0.0:5000", "https://0.0.0.0:5000/v1/"}, } for _, td := range testData { - e, err := newEndpoint(td.str) + e, err := newEndpoint(td.str, insecureRegistries) if err != nil { t.Errorf("%q: %s", td.str, err) } diff --git a/docs/registry_test.go b/docs/registry_test.go index 37dedc2a..3e0950ef 100644 --- a/docs/registry_test.go +++ b/docs/registry_test.go @@ -326,6 +326,7 @@ func TestIsSecure(t *testing.T) { insecureRegistries []string expected bool }{ + {IndexServerURL.Host, nil, true}, {"example.com", []string{}, true}, {"example.com", []string{"example.com"}, false}, {"localhost", []string{"localhost:5000"}, false}, From ae0ebb9d074e8fe05a0bfc7c057a5d0222df5128 Mon Sep 17 00:00:00 2001 From: Tibor Vass Date: Tue, 11 Nov 2014 16:31:15 -0500 Subject: [PATCH 188/375] Add the possibility of specifying a subnet for --insecure-registry Signed-off-by: Tibor Vass --- docs/endpoint.go | 60 +++++++++++++++++++++++++++++--------- docs/registry_mock_test.go | 26 +++++++++++++++++ docs/registry_test.go | 19 ++++++++---- 3 files changed, 85 insertions(+), 20 deletions(-) diff --git a/docs/endpoint.go b/docs/endpoint.go index bd23c302..c485a13d 100644 --- a/docs/endpoint.go +++ b/docs/endpoint.go @@ -12,6 +12,9 @@ import ( log "github.com/Sirupsen/logrus" ) +// for mocking in unit tests +var lookupIP = net.LookupIP + // scans string for api version in the URL path. returns the trimmed hostname, if version found, string and API version. func scanForAPIVersion(hostname string) (string, APIVersion) { var ( @@ -79,7 +82,10 @@ func newEndpoint(hostname string, insecureRegistries []string) (*Endpoint, error if err != nil { return nil, err } - endpoint.secure = isSecure(endpoint.URL.Host, insecureRegistries) + endpoint.secure, err = isSecure(endpoint.URL.Host, insecureRegistries) + if err != nil { + return nil, err + } return &endpoint, nil } @@ -152,30 +158,56 @@ func (e Endpoint) Ping() (RegistryInfo, error) { // isSecure returns false if the provided hostname is part of the list of insecure registries. // Insecure registries accept HTTP and/or accept HTTPS with certificates from unknown CAs. -func isSecure(hostname string, insecureRegistries []string) bool { +// +// The list of insecure registries can contain an element with CIDR notation to specify a whole subnet. +// If the subnet contains one of the IPs of the registry specified by hostname, the latter is considered +// insecure. +// +// hostname should be a URL.Host (`host:port` or `host`) +func isSecure(hostname string, insecureRegistries []string) (bool, error) { if hostname == IndexServerURL.Host { - return true + return true, nil } host, _, err := net.SplitHostPort(hostname) - if err != nil { + // assume hostname is of the form `host` without the port and go on. host = hostname } - - if host == "127.0.0.1" || host == "localhost" { - return false + addrs, err := lookupIP(host) + if err != nil { + ip := net.ParseIP(host) + if ip == nil { + // if resolving `host` fails, error out, since host is to be net.Dial-ed anyway + return true, fmt.Errorf("issecure: could not resolve %q: %v", host, err) + } + addrs = []net.IP{ip} + } + if len(addrs) == 0 { + return true, fmt.Errorf("issecure: could not resolve %q", host) } - if len(insecureRegistries) == 0 { - return true - } + for _, addr := range addrs { + for _, r := range insecureRegistries { + // hostname matches insecure registry + if hostname == r { + return false, nil + } - for _, h := range insecureRegistries { - if hostname == h { - return false + // now assume a CIDR was passed to --insecure-registry + _, ipnet, err := net.ParseCIDR(r) + if err != nil { + // if could not parse it as a CIDR, even after removing + // assume it's not a CIDR and go on with the next candidate + continue + } + + // check if the addr falls in the subnet + if ipnet.Contains(addr) { + return false, nil + } } } - return true + return true, nil } diff --git a/docs/registry_mock_test.go b/docs/registry_mock_test.go index 1c710e21..887d2ef6 100644 --- a/docs/registry_mock_test.go +++ b/docs/registry_mock_test.go @@ -2,9 +2,11 @@ package registry import ( "encoding/json" + "errors" "fmt" "io" "io/ioutil" + "net" "net/http" "net/http/httptest" "net/url" @@ -80,6 +82,11 @@ var ( "latest": "42d718c941f5c532ac049bf0b0ab53f0062f09a03afd4aa4a02c098e46032b9d", }, } + mockHosts = map[string][]net.IP{ + "": {net.ParseIP("0.0.0.0")}, + "localhost": {net.ParseIP("127.0.0.1"), net.ParseIP("::1")}, + "example.com": {net.ParseIP("42.42.42.42")}, + } ) func init() { @@ -106,6 +113,25 @@ func init() { panic(err) } insecureRegistries = []string{URL.Host} + + // override net.LookupIP + lookupIP = func(host string) ([]net.IP, error) { + if host == "127.0.0.1" { + // I believe in future Go versions this will fail, so let's fix it later + return net.LookupIP(host) + } + for h, addrs := range mockHosts { + if host == h { + return addrs, nil + } + for _, addr := range addrs { + if addr.String() == host { + return []net.IP{addr}, nil + } + } + } + return nil, errors.New("lookup: no such host") + } } func handlerAccessLog(handler http.Handler) http.Handler { diff --git a/docs/registry_test.go b/docs/registry_test.go index 3e0950ef..d24a5f57 100644 --- a/docs/registry_test.go +++ b/docs/registry_test.go @@ -333,19 +333,26 @@ func TestIsSecure(t *testing.T) { {"localhost:5000", []string{"localhost:5000"}, false}, {"localhost", []string{"example.com"}, false}, {"127.0.0.1:5000", []string{"127.0.0.1:5000"}, false}, - {"localhost", []string{}, false}, - {"localhost:5000", []string{}, false}, - {"127.0.0.1", []string{}, false}, + {"localhost", nil, false}, + {"localhost:5000", nil, false}, + {"127.0.0.1", nil, false}, {"localhost", []string{"example.com"}, false}, {"127.0.0.1", []string{"example.com"}, false}, - {"example.com", []string{}, true}, + {"example.com", nil, true}, {"example.com", []string{"example.com"}, false}, {"127.0.0.1", []string{"example.com"}, false}, {"127.0.0.1:5000", []string{"example.com"}, false}, + {"example.com:5000", []string{"42.42.0.0/16"}, false}, + {"example.com", []string{"42.42.0.0/16"}, false}, + {"example.com:5000", []string{"42.42.42.42/8"}, false}, + {"127.0.0.1:5000", []string{"127.0.0.0/8"}, false}, + {"42.42.42.42:5000", []string{"42.1.1.1/8"}, false}, } for _, tt := range tests { - if sec := isSecure(tt.addr, tt.insecureRegistries); sec != tt.expected { - t.Errorf("isSecure failed for %q %v, expected %v got %v", tt.addr, tt.insecureRegistries, tt.expected, sec) + // TODO: remove this once we remove localhost insecure by default + insecureRegistries := append(tt.insecureRegistries, "127.0.0.0/8") + if sec, err := isSecure(tt.addr, insecureRegistries); err != nil || sec != tt.expected { + t.Fatalf("isSecure failed for %q %v, expected %v got %v. Error: %v", tt.addr, insecureRegistries, tt.expected, sec, err) } } } From 44d97c1fd016990ef0287625457fa3b16a1ed7df Mon Sep 17 00:00:00 2001 From: Tibor Vass Date: Tue, 11 Nov 2014 17:37:44 -0500 Subject: [PATCH 189/375] registry: refactor registry.IsSecure calls into registry.NewEndpoint Signed-off-by: Tibor Vass Conflicts: registry/endpoint.go registry/endpoint_test.go registry/registry_test.go --- docs/endpoint.go | 38 ++++++++++++++++++++++++-------------- docs/endpoint_test.go | 27 +++++++++++++++++++++++++++ docs/registry_test.go | 29 +++++++++++++++++++++++++++++ docs/service.go | 6 ++---- 4 files changed, 82 insertions(+), 18 deletions(-) create mode 100644 docs/endpoint_test.go diff --git a/docs/endpoint.go b/docs/endpoint.go index 6dd4e1f6..2eac41ce 100644 --- a/docs/endpoint.go +++ b/docs/endpoint.go @@ -33,21 +33,15 @@ func scanForApiVersion(hostname string) (string, APIVersion) { return hostname, DefaultAPIVersion } -func NewEndpoint(hostname string, secure bool) (*Endpoint, error) { - var ( - endpoint = Endpoint{secure: secure} - trimmedHostname string - err error - ) - if !strings.HasPrefix(hostname, "http") { - hostname = "https://" + hostname - } - trimmedHostname, endpoint.Version = scanForApiVersion(hostname) - endpoint.URL, err = url.Parse(trimmedHostname) +func NewEndpoint(hostname string, insecureRegistries []string) (*Endpoint, error) { + endpoint, err := newEndpoint(hostname) if err != nil { return nil, err } + secure := isSecure(endpoint.URL.Host, insecureRegistries) + endpoint.secure = secure + // Try HTTPS ping to registry endpoint.URL.Scheme = "https" if _, err := endpoint.Ping(); err != nil { @@ -65,12 +59,28 @@ func NewEndpoint(hostname string, secure bool) (*Endpoint, error) { endpoint.URL.Scheme = "http" _, err2 := endpoint.Ping() if err2 == nil { - return &endpoint, nil + return endpoint, nil } return nil, fmt.Errorf("Invalid registry endpoint %q. HTTPS attempt: %v. HTTP attempt: %v", endpoint, err, err2) } + return endpoint, nil +} +func newEndpoint(hostname string) (*Endpoint, error) { + var ( + endpoint = Endpoint{secure: true} + trimmedHostname string + err error + ) + if !strings.HasPrefix(hostname, "http") { + hostname = "https://" + hostname + } + trimmedHostname, endpoint.Version = scanForApiVersion(hostname) + endpoint.URL, err = url.Parse(trimmedHostname) + if err != nil { + return nil, err + } return &endpoint, nil } @@ -141,9 +151,9 @@ func (e Endpoint) Ping() (RegistryInfo, error) { return info, nil } -// IsSecure returns false if the provided hostname is part of the list of insecure registries. +// isSecure returns false if the provided hostname is part of the list of insecure registries. // Insecure registries accept HTTP and/or accept HTTPS with certificates from unknown CAs. -func IsSecure(hostname string, insecureRegistries []string) bool { +func isSecure(hostname string, insecureRegistries []string) bool { if hostname == IndexServerAddress() { return true } diff --git a/docs/endpoint_test.go b/docs/endpoint_test.go new file mode 100644 index 00000000..0ec1220d --- /dev/null +++ b/docs/endpoint_test.go @@ -0,0 +1,27 @@ +package registry + +import "testing" + +func TestEndpointParse(t *testing.T) { + testData := []struct { + str string + expected string + }{ + {IndexServerAddress(), IndexServerAddress()}, + {"http://0.0.0.0:5000", "http://0.0.0.0:5000/v1/"}, + {"0.0.0.0:5000", "https://0.0.0.0:5000/v1/"}, + } + for _, td := range testData { + e, err := newEndpoint(td.str) + if err != nil { + t.Errorf("%q: %s", td.str, err) + } + if e == nil { + t.Logf("something's fishy, endpoint for %q is nil", td.str) + continue + } + if e.String() != td.expected { + t.Errorf("expected %q, got %q", td.expected, e.String()) + } + } +} diff --git a/docs/registry_test.go b/docs/registry_test.go index c9a9fc81..7e63ee92 100644 --- a/docs/registry_test.go +++ b/docs/registry_test.go @@ -316,3 +316,32 @@ func TestAddRequiredHeadersToRedirectedRequests(t *testing.T) { } } } + +func TestIsSecure(t *testing.T) { + tests := []struct { + addr string + insecureRegistries []string + expected bool + }{ + {"example.com", []string{}, true}, + {"example.com", []string{"example.com"}, false}, + {"localhost", []string{"localhost:5000"}, false}, + {"localhost:5000", []string{"localhost:5000"}, false}, + {"localhost", []string{"example.com"}, false}, + {"127.0.0.1:5000", []string{"127.0.0.1:5000"}, false}, + {"localhost", []string{}, false}, + {"localhost:5000", []string{}, false}, + {"127.0.0.1", []string{}, false}, + {"localhost", []string{"example.com"}, false}, + {"127.0.0.1", []string{"example.com"}, false}, + {"example.com", []string{}, true}, + {"example.com", []string{"example.com"}, false}, + {"127.0.0.1", []string{"example.com"}, false}, + {"127.0.0.1:5000", []string{"example.com"}, false}, + } + for _, tt := range tests { + if sec := isSecure(tt.addr, tt.insecureRegistries); sec != tt.expected { + t.Errorf("isSecure failed for %q %v, expected %v got %v", tt.addr, tt.insecureRegistries, tt.expected, sec) + } + } +} diff --git a/docs/service.go b/docs/service.go index 7051d934..53e8278b 100644 --- a/docs/service.go +++ b/docs/service.go @@ -40,7 +40,7 @@ func (s *Service) Auth(job *engine.Job) engine.Status { job.GetenvJson("authConfig", authConfig) if addr := authConfig.ServerAddress; addr != "" && addr != IndexServerAddress() { - endpoint, err := NewEndpoint(addr, IsSecure(addr, s.insecureRegistries)) + endpoint, err := NewEndpoint(addr, s.insecureRegistries) if err != nil { return job.Error(err) } @@ -92,9 +92,7 @@ func (s *Service) Search(job *engine.Job) engine.Status { return job.Error(err) } - secure := IsSecure(hostname, s.insecureRegistries) - - endpoint, err := NewEndpoint(hostname, secure) + endpoint, err := NewEndpoint(hostname, s.insecureRegistries) if err != nil { return job.Error(err) } From 8b0e8b66212a3e5cbd0f15973f53fa0154058145 Mon Sep 17 00:00:00 2001 From: Tibor Vass Date: Tue, 11 Nov 2014 20:08:59 -0500 Subject: [PATCH 190/375] Put mock registry address in insecureRegistries for unit tests Signed-off-by: Tibor Vass Conflicts: registry/registry_mock_test.go --- docs/registry_mock_test.go | 16 +++++++++++----- docs/registry_test.go | 4 ++-- 2 files changed, 13 insertions(+), 7 deletions(-) diff --git a/docs/registry_mock_test.go b/docs/registry_mock_test.go index 379dc78f..fc2b46b9 100644 --- a/docs/registry_mock_test.go +++ b/docs/registry_mock_test.go @@ -19,8 +19,9 @@ import ( ) var ( - testHttpServer *httptest.Server - testLayers = map[string]map[string]string{ + testHTTPServer *httptest.Server + insecureRegistries []string + testLayers = map[string]map[string]string{ "77dbf71da1d00e3fbddc480176eac8994025630c6590d11cfc8fe1209c2a1d20": { "json": `{"id":"77dbf71da1d00e3fbddc480176eac8994025630c6590d11cfc8fe1209c2a1d20", "comment":"test base image","created":"2013-03-23T12:53:11.10432-07:00", @@ -99,7 +100,12 @@ func init() { // /v2/ r.HandleFunc("/v2/version", handlerGetPing).Methods("GET") - testHttpServer = httptest.NewServer(handlerAccessLog(r)) + testHTTPServer = httptest.NewServer(handlerAccessLog(r)) + URL, err := url.Parse(testHTTPServer.URL) + if err != nil { + panic(err) + } + insecureRegistries = []string{URL.Host} } func handlerAccessLog(handler http.Handler) http.Handler { @@ -111,7 +117,7 @@ func handlerAccessLog(handler http.Handler) http.Handler { } func makeURL(req string) string { - return testHttpServer.URL + req + return testHTTPServer.URL + req } func writeHeaders(w http.ResponseWriter) { @@ -301,7 +307,7 @@ func handlerUsers(w http.ResponseWriter, r *http.Request) { } func handlerImages(w http.ResponseWriter, r *http.Request) { - u, _ := url.Parse(testHttpServer.URL) + u, _ := url.Parse(testHTTPServer.URL) w.Header().Add("X-Docker-Endpoints", fmt.Sprintf("%s , %s ", u.Host, "test.example.com")) w.Header().Add("X-Docker-Token", fmt.Sprintf("FAKE-SESSION-%d", time.Now().UnixNano())) if r.Method == "PUT" { diff --git a/docs/registry_test.go b/docs/registry_test.go index 7e63ee92..aaba3f0a 100644 --- a/docs/registry_test.go +++ b/docs/registry_test.go @@ -18,7 +18,7 @@ var ( func spawnTestRegistrySession(t *testing.T) *Session { authConfig := &AuthConfig{} - endpoint, err := NewEndpoint(makeURL("/v1/"), false) + endpoint, err := NewEndpoint(makeURL("/v1/"), insecureRegistries) if err != nil { t.Fatal(err) } @@ -30,7 +30,7 @@ func spawnTestRegistrySession(t *testing.T) *Session { } func TestPingRegistryEndpoint(t *testing.T) { - ep, err := NewEndpoint(makeURL("/v1/"), false) + ep, err := NewEndpoint(makeURL("/v1/"), insecureRegistries) if err != nil { t.Fatal(err) } From 8065dad50b9bacdf4a2f5cbc54a10745c81ab341 Mon Sep 17 00:00:00 2001 From: Tibor Vass Date: Thu, 13 Nov 2014 06:56:36 -0800 Subject: [PATCH 191/375] registry: parse INDEXSERVERADDRESS into a URL for easier check in isSecure Signed-off-by: Tibor Vass --- docs/auth.go | 10 ++++++++++ docs/endpoint.go | 14 ++++++-------- docs/endpoint_test.go | 2 +- docs/registry_test.go | 1 + 4 files changed, 18 insertions(+), 9 deletions(-) diff --git a/docs/auth.go b/docs/auth.go index 7c0709a4..dad58c16 100644 --- a/docs/auth.go +++ b/docs/auth.go @@ -7,6 +7,7 @@ import ( "fmt" "io/ioutil" "net/http" + "net/url" "os" "path" "strings" @@ -27,8 +28,17 @@ const ( var ( ErrConfigFileMissing = errors.New("The Auth config file is missing") + IndexServerURL *url.URL ) +func init() { + url, err := url.Parse(INDEXSERVER) + if err != nil { + panic(err) + } + IndexServerURL = url +} + type AuthConfig struct { Username string `json:"username,omitempty"` Password string `json:"password,omitempty"` diff --git a/docs/endpoint.go b/docs/endpoint.go index 2eac41ce..eb0e9a1f 100644 --- a/docs/endpoint.go +++ b/docs/endpoint.go @@ -34,21 +34,18 @@ func scanForApiVersion(hostname string) (string, APIVersion) { } func NewEndpoint(hostname string, insecureRegistries []string) (*Endpoint, error) { - endpoint, err := newEndpoint(hostname) + endpoint, err := newEndpoint(hostname, insecureRegistries) if err != nil { return nil, err } - secure := isSecure(endpoint.URL.Host, insecureRegistries) - endpoint.secure = secure - // Try HTTPS ping to registry endpoint.URL.Scheme = "https" if _, err := endpoint.Ping(); err != nil { //TODO: triggering highland build can be done there without "failing" - if secure { + if endpoint.secure { // If registry is secure and HTTPS failed, show user the error and tell them about `--insecure-registry` // in case that's what they need. DO NOT accept unknown CA certificates, and DO NOT fallback to HTTP. return nil, fmt.Errorf("Invalid registry endpoint %s: %v. If this private registry supports only HTTP or HTTPS with an unknown CA certificate, please add `--insecure-registry %s` to the daemon's arguments. In the case of HTTPS, if you have access to the registry's CA certificate, no need for the flag; simply place the CA certificate at /etc/docker/certs.d/%s/ca.crt", endpoint, err, endpoint.URL.Host, endpoint.URL.Host) @@ -67,9 +64,9 @@ func NewEndpoint(hostname string, insecureRegistries []string) (*Endpoint, error return endpoint, nil } -func newEndpoint(hostname string) (*Endpoint, error) { +func newEndpoint(hostname string, insecureRegistries []string) (*Endpoint, error) { var ( - endpoint = Endpoint{secure: true} + endpoint = Endpoint{} trimmedHostname string err error ) @@ -81,6 +78,7 @@ func newEndpoint(hostname string) (*Endpoint, error) { if err != nil { return nil, err } + endpoint.secure = isSecure(endpoint.URL.Host, insecureRegistries) return &endpoint, nil } @@ -154,7 +152,7 @@ func (e Endpoint) Ping() (RegistryInfo, error) { // isSecure returns false if the provided hostname is part of the list of insecure registries. // Insecure registries accept HTTP and/or accept HTTPS with certificates from unknown CAs. func isSecure(hostname string, insecureRegistries []string) bool { - if hostname == IndexServerAddress() { + if hostname == IndexServerURL.Host { return true } diff --git a/docs/endpoint_test.go b/docs/endpoint_test.go index 0ec1220d..54105ec1 100644 --- a/docs/endpoint_test.go +++ b/docs/endpoint_test.go @@ -12,7 +12,7 @@ func TestEndpointParse(t *testing.T) { {"0.0.0.0:5000", "https://0.0.0.0:5000/v1/"}, } for _, td := range testData { - e, err := newEndpoint(td.str) + e, err := newEndpoint(td.str, insecureRegistries) if err != nil { t.Errorf("%q: %s", td.str, err) } diff --git a/docs/registry_test.go b/docs/registry_test.go index aaba3f0a..dbedefe0 100644 --- a/docs/registry_test.go +++ b/docs/registry_test.go @@ -323,6 +323,7 @@ func TestIsSecure(t *testing.T) { insecureRegistries []string expected bool }{ + {IndexServerURL.Host, nil, true}, {"example.com", []string{}, true}, {"example.com", []string{"example.com"}, false}, {"localhost", []string{"localhost:5000"}, false}, From 6638cd7bc73015ca11a44abd14a6d06e4b6f49e9 Mon Sep 17 00:00:00 2001 From: Tibor Vass Date: Tue, 11 Nov 2014 16:31:15 -0500 Subject: [PATCH 192/375] Add the possibility of specifying a subnet for --insecure-registry Signed-off-by: Tibor Vass Conflicts: registry/endpoint.go --- docs/endpoint.go | 61 +++++++++++++++++++++++++++++++++----- docs/registry_mock_test.go | 26 ++++++++++++++++ docs/registry_test.go | 19 ++++++++---- 3 files changed, 93 insertions(+), 13 deletions(-) diff --git a/docs/endpoint.go b/docs/endpoint.go index eb0e9a1f..d65fd7e8 100644 --- a/docs/endpoint.go +++ b/docs/endpoint.go @@ -4,6 +4,7 @@ import ( "encoding/json" "fmt" "io/ioutil" + "net" "net/http" "net/url" "strings" @@ -11,6 +12,9 @@ import ( "github.com/docker/docker/pkg/log" ) +// for mocking in unit tests +var lookupIP = net.LookupIP + // scans string for api version in the URL path. returns the trimmed hostname, if version found, string and API version. func scanForApiVersion(hostname string) (string, APIVersion) { var ( @@ -78,7 +82,10 @@ func newEndpoint(hostname string, insecureRegistries []string) (*Endpoint, error if err != nil { return nil, err } - endpoint.secure = isSecure(endpoint.URL.Host, insecureRegistries) + endpoint.secure, err = isSecure(endpoint.URL.Host, insecureRegistries) + if err != nil { + return nil, err + } return &endpoint, nil } @@ -151,16 +158,56 @@ func (e Endpoint) Ping() (RegistryInfo, error) { // isSecure returns false if the provided hostname is part of the list of insecure registries. // Insecure registries accept HTTP and/or accept HTTPS with certificates from unknown CAs. -func isSecure(hostname string, insecureRegistries []string) bool { +// +// The list of insecure registries can contain an element with CIDR notation to specify a whole subnet. +// If the subnet contains one of the IPs of the registry specified by hostname, the latter is considered +// insecure. +// +// hostname should be a URL.Host (`host:port` or `host`) +func isSecure(hostname string, insecureRegistries []string) (bool, error) { if hostname == IndexServerURL.Host { - return true + return true, nil } - for _, h := range insecureRegistries { - if hostname == h { - return false + host, _, err := net.SplitHostPort(hostname) + if err != nil { + // assume hostname is of the form `host` without the port and go on. + host = hostname + } + addrs, err := lookupIP(host) + if err != nil { + ip := net.ParseIP(host) + if ip == nil { + // if resolving `host` fails, error out, since host is to be net.Dial-ed anyway + return true, fmt.Errorf("issecure: could not resolve %q: %v", host, err) + } + addrs = []net.IP{ip} + } + if len(addrs) == 0 { + return true, fmt.Errorf("issecure: could not resolve %q", host) + } + + for _, addr := range addrs { + for _, r := range insecureRegistries { + // hostname matches insecure registry + if hostname == r { + return false, nil + } + + // now assume a CIDR was passed to --insecure-registry + _, ipnet, err := net.ParseCIDR(r) + if err != nil { + // if could not parse it as a CIDR, even after removing + // assume it's not a CIDR and go on with the next candidate + continue + } + + // check if the addr falls in the subnet + if ipnet.Contains(addr) { + return false, nil + } } } - return true + return true, nil } diff --git a/docs/registry_mock_test.go b/docs/registry_mock_test.go index fc2b46b9..50724f0f 100644 --- a/docs/registry_mock_test.go +++ b/docs/registry_mock_test.go @@ -2,9 +2,11 @@ package registry import ( "encoding/json" + "errors" "fmt" "io" "io/ioutil" + "net" "net/http" "net/http/httptest" "net/url" @@ -80,6 +82,11 @@ var ( "latest": "42d718c941f5c532ac049bf0b0ab53f0062f09a03afd4aa4a02c098e46032b9d", }, } + mockHosts = map[string][]net.IP{ + "": {net.ParseIP("0.0.0.0")}, + "localhost": {net.ParseIP("127.0.0.1"), net.ParseIP("::1")}, + "example.com": {net.ParseIP("42.42.42.42")}, + } ) func init() { @@ -106,6 +113,25 @@ func init() { panic(err) } insecureRegistries = []string{URL.Host} + + // override net.LookupIP + lookupIP = func(host string) ([]net.IP, error) { + if host == "127.0.0.1" { + // I believe in future Go versions this will fail, so let's fix it later + return net.LookupIP(host) + } + for h, addrs := range mockHosts { + if host == h { + return addrs, nil + } + for _, addr := range addrs { + if addr.String() == host { + return []net.IP{addr}, nil + } + } + } + return nil, errors.New("lookup: no such host") + } } func handlerAccessLog(handler http.Handler) http.Handler { diff --git a/docs/registry_test.go b/docs/registry_test.go index dbedefe0..1ffb44f3 100644 --- a/docs/registry_test.go +++ b/docs/registry_test.go @@ -330,19 +330,26 @@ func TestIsSecure(t *testing.T) { {"localhost:5000", []string{"localhost:5000"}, false}, {"localhost", []string{"example.com"}, false}, {"127.0.0.1:5000", []string{"127.0.0.1:5000"}, false}, - {"localhost", []string{}, false}, - {"localhost:5000", []string{}, false}, - {"127.0.0.1", []string{}, false}, + {"localhost", nil, false}, + {"localhost:5000", nil, false}, + {"127.0.0.1", nil, false}, {"localhost", []string{"example.com"}, false}, {"127.0.0.1", []string{"example.com"}, false}, - {"example.com", []string{}, true}, + {"example.com", nil, true}, {"example.com", []string{"example.com"}, false}, {"127.0.0.1", []string{"example.com"}, false}, {"127.0.0.1:5000", []string{"example.com"}, false}, + {"example.com:5000", []string{"42.42.0.0/16"}, false}, + {"example.com", []string{"42.42.0.0/16"}, false}, + {"example.com:5000", []string{"42.42.42.42/8"}, false}, + {"127.0.0.1:5000", []string{"127.0.0.0/8"}, false}, + {"42.42.42.42:5000", []string{"42.1.1.1/8"}, false}, } for _, tt := range tests { - if sec := isSecure(tt.addr, tt.insecureRegistries); sec != tt.expected { - t.Errorf("isSecure failed for %q %v, expected %v got %v", tt.addr, tt.insecureRegistries, tt.expected, sec) + // TODO: remove this once we remove localhost insecure by default + insecureRegistries := append(tt.insecureRegistries, "127.0.0.0/8") + if sec, err := isSecure(tt.addr, insecureRegistries); err != nil || sec != tt.expected { + t.Fatalf("isSecure failed for %q %v, expected %v got %v. Error: %v", tt.addr, insecureRegistries, tt.expected, sec, err) } } } From df85a0f700b264dafcbfb3fd4f8d3e5d7eb64930 Mon Sep 17 00:00:00 2001 From: Vaidas Jablonskis Date: Sat, 22 Nov 2014 23:21:47 +0000 Subject: [PATCH 193/375] registry: fix ServerAddress setting This ensures that ServerAddress is set, while previously it was getting set after configFile.Configs. Signed-off-by: Vaidas Jablonskis --- docs/auth.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/auth.go b/docs/auth.go index a22d0b88..42760640 100644 --- a/docs/auth.go +++ b/docs/auth.go @@ -126,8 +126,8 @@ func LoadConfig(rootPath string) (*ConfigFile, error) { return &configFile, err } authConfig.Auth = "" - configFile.Configs[k] = authConfig authConfig.ServerAddress = k + configFile.Configs[k] = authConfig } } return &configFile, nil From b62a9ac9895553bd14259e7769026e9f4d2a60d1 Mon Sep 17 00:00:00 2001 From: unclejack Date: Thu, 27 Nov 2014 23:55:03 +0200 Subject: [PATCH 194/375] validate image ID properly & before load Signed-off-by: Cristian Staretu --- docs/registry.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/registry.go b/docs/registry.go index a03790af..e0285a23 100644 --- a/docs/registry.go +++ b/docs/registry.go @@ -23,7 +23,6 @@ var ( ErrInvalidRepositoryName = errors.New("Invalid repository name (ex: \"registry.domain.tld/myrepos\")") ErrDoesNotExist = errors.New("Image does not exist") errLoginRequired = errors.New("Authentication is required.") - validHex = regexp.MustCompile(`^([a-f0-9]{64})$`) validNamespace = regexp.MustCompile(`^([a-z0-9_]{4,30})$`) validRepo = regexp.MustCompile(`^([a-z0-9-_.]+)$`) ) @@ -177,7 +176,8 @@ func validateRepositoryName(repositoryName string) error { namespace = "library" name = nameParts[0] - if validHex.MatchString(name) { + // the repository name must not be a valid image ID + if err := utils.ValidateID(name); err == nil { return fmt.Errorf("Invalid repository name (%s), cannot specify 64-byte hexadecimal strings", name) } } else { From 3911c8b8dc8bcde030a5ad178bf69263b4b42fe8 Mon Sep 17 00:00:00 2001 From: Michael Crosby Date: Wed, 10 Dec 2014 17:37:31 -0800 Subject: [PATCH 195/375] Prevent loop with var overshadowing Incase of a 3xx redirect the var was being overshowed and ever changed causing an infinite loop. Fixes #9480 Signed-off-by: Michael Crosby --- docs/session.go | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/docs/session.go b/docs/session.go index ba6df358..2658ec1a 100644 --- a/docs/session.go +++ b/docs/session.go @@ -505,7 +505,7 @@ func (r *Session) PushImageJSONIndex(remote string, imgList []*ImgData, validate // Redirect if necessary for res.StatusCode >= 300 && res.StatusCode < 400 { log.Debugf("Redirected to %s", res.Header.Get("Location")) - req, err = r.reqFactory.NewRequest("PUT", res.Header.Get("Location"), bytes.NewReader(imgListJSON)) + req, err := r.reqFactory.NewRequest("PUT", res.Header.Get("Location"), bytes.NewReader(imgListJSON)) if err != nil { return nil, err } @@ -515,10 +515,11 @@ func (r *Session) PushImageJSONIndex(remote string, imgList []*ImgData, validate if validate { req.Header["X-Docker-Endpoints"] = regs } - res, _, err := r.doRequest(req) + redirect, _, err := r.doRequest(req) if err != nil { return nil, err } + res = redirect defer res.Body.Close() } From 6ad54e3df6085f905d742168f676620f80b422ef Mon Sep 17 00:00:00 2001 From: Michael Crosby Date: Wed, 10 Dec 2014 18:08:40 -0800 Subject: [PATCH 196/375] Refactor put image function's redirect loop Signed-off-by: Michael Crosby --- docs/session.go | 64 ++++++++++++++++++++++++++----------------------- 1 file changed, 34 insertions(+), 30 deletions(-) diff --git a/docs/session.go b/docs/session.go index 2658ec1a..4b2f5522 100644 --- a/docs/session.go +++ b/docs/session.go @@ -462,7 +462,6 @@ func (r *Session) PushRegistryTag(remote, revision, tag, registry string, token func (r *Session) PushImageJSONIndex(remote string, imgList []*ImgData, validate bool, regs []string) (*RepositoryData, error) { cleanImgList := []*ImgData{} - if validate { for _, elem := range imgList { if elem.Checksum != "" { @@ -484,44 +483,28 @@ func (r *Session) PushImageJSONIndex(remote string, imgList []*ImgData, validate u := fmt.Sprintf("%srepositories/%s/%s", r.indexEndpoint.VersionString(1), remote, suffix) log.Debugf("[registry] PUT %s", u) log.Debugf("Image list pushed to index:\n%s", imgListJSON) - req, err := r.reqFactory.NewRequest("PUT", u, bytes.NewReader(imgListJSON)) - if err != nil { - return nil, err + headers := map[string][]string{ + "Content-type": {"application/json"}, + "X-Docker-Token": {"true"}, } - req.Header.Add("Content-type", "application/json") - req.SetBasicAuth(r.authConfig.Username, r.authConfig.Password) - req.ContentLength = int64(len(imgListJSON)) - req.Header.Set("X-Docker-Token", "true") if validate { - req.Header["X-Docker-Endpoints"] = regs + headers["X-Docker-Endpoints"] = regs } - res, _, err := r.doRequest(req) - if err != nil { - return nil, err - } - defer res.Body.Close() - // Redirect if necessary - for res.StatusCode >= 300 && res.StatusCode < 400 { - log.Debugf("Redirected to %s", res.Header.Get("Location")) - req, err := r.reqFactory.NewRequest("PUT", res.Header.Get("Location"), bytes.NewReader(imgListJSON)) - if err != nil { + var res *http.Response + for { + if res, err = r.putImageRequest(u, headers, imgListJSON); err != nil { return nil, err } - req.SetBasicAuth(r.authConfig.Username, r.authConfig.Password) - req.ContentLength = int64(len(imgListJSON)) - req.Header.Set("X-Docker-Token", "true") - if validate { - req.Header["X-Docker-Endpoints"] = regs + if !shouldRedirect(res) { + break } - redirect, _, err := r.doRequest(req) - if err != nil { - return nil, err - } - res = redirect - defer res.Body.Close() + res.Body.Close() + u = res.Header.Get("Location") + log.Debugf("Redirected to %s", u) } + defer res.Body.Close() var tokens, endpoints []string if !validate { @@ -564,6 +547,27 @@ func (r *Session) PushImageJSONIndex(remote string, imgList []*ImgData, validate }, nil } +func (r *Session) putImageRequest(u string, headers map[string][]string, body []byte) (*http.Response, error) { + req, err := r.reqFactory.NewRequest("PUT", u, bytes.NewReader(body)) + if err != nil { + return nil, err + } + req.SetBasicAuth(r.authConfig.Username, r.authConfig.Password) + req.ContentLength = int64(len(body)) + for k, v := range headers { + req.Header[k] = v + } + response, _, err := r.doRequest(req) + if err != nil { + return nil, err + } + return response, nil +} + +func shouldRedirect(response *http.Response) bool { + return response.StatusCode >= 300 && response.StatusCode < 400 +} + func (r *Session) SearchRepositories(term string) (*SearchResults, error) { log.Debugf("Index server: %s", r.indexEndpoint) u := r.indexEndpoint.VersionString(1) + "search?q=" + url.QueryEscape(term) From fdd4f4f2d14705f172e8ae8f7110662de621cf08 Mon Sep 17 00:00:00 2001 From: unclejack Date: Thu, 27 Nov 2014 23:55:03 +0200 Subject: [PATCH 197/375] validate image ID properly & before load Signed-off-by: Cristian Staretu Conflicts: graph/load.go --- docs/registry.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/registry.go b/docs/registry.go index e1d22b09..d503a63d 100644 --- a/docs/registry.go +++ b/docs/registry.go @@ -23,7 +23,6 @@ var ( ErrInvalidRepositoryName = errors.New("Invalid repository name (ex: \"registry.domain.tld/myrepos\")") ErrDoesNotExist = errors.New("Image does not exist") errLoginRequired = errors.New("Authentication is required.") - validHex = regexp.MustCompile(`^([a-f0-9]{64})$`) validNamespace = regexp.MustCompile(`^([a-z0-9_]{4,30})$`) validRepo = regexp.MustCompile(`^([a-z0-9-_.]+)$`) ) @@ -171,7 +170,8 @@ func validateRepositoryName(repositoryName string) error { namespace = "library" name = nameParts[0] - if validHex.MatchString(name) { + // the repository name must not be a valid image ID + if err := utils.ValidateID(name); err == nil { return fmt.Errorf("Invalid repository name (%s), cannot specify 64-byte hexadecimal strings", name) } } else { From cb4f91608e945a8cb370c1324a193cd36b49aad1 Mon Sep 17 00:00:00 2001 From: Jessica Frazelle Date: Thu, 11 Dec 2014 17:14:53 -0800 Subject: [PATCH 198/375] Fix conflicts. Docker-DCO-1.1-Signed-off-by: Jessica Frazelle (github: jfrazelle) --- docs/registry.go | 4 ---- 1 file changed, 4 deletions(-) diff --git a/docs/registry.go b/docs/registry.go index f3a4a340..d503a63d 100644 --- a/docs/registry.go +++ b/docs/registry.go @@ -47,10 +47,6 @@ func newClient(jar http.CookieJar, roots *x509.CertPool, certs []tls.Certificate tlsConfig.InsecureSkipVerify = true } - if !secure { - tlsConfig.InsecureSkipVerify = true - } - httpTransport := &http.Transport{ DisableKeepAlives: true, Proxy: http.ProxyFromEnvironment, From b11b1e06e944c9b68dfa94357989c9d326e07f97 Mon Sep 17 00:00:00 2001 From: Daehyeok Mun Date: Sun, 16 Nov 2014 22:25:10 +0900 Subject: [PATCH 199/375] Chnage LookupRemoteImage to return error This commit is patch for following comment // TODO: This method should return the errors instead of masking them and returning false Signed-off-by: Daehyeok Mun Signed-off-by: Michael Crosby --- docs/registry_test.go | 9 +++++---- docs/session.go | 15 +++++++-------- 2 files changed, 12 insertions(+), 12 deletions(-) diff --git a/docs/registry_test.go b/docs/registry_test.go index d24a5f57..5fd80da1 100644 --- a/docs/registry_test.go +++ b/docs/registry_test.go @@ -58,10 +58,11 @@ func TestGetRemoteHistory(t *testing.T) { func TestLookupRemoteImage(t *testing.T) { r := spawnTestRegistrySession(t) - found := r.LookupRemoteImage(imageID, makeURL("/v1/"), token) - assertEqual(t, found, true, "Expected remote lookup to succeed") - found = r.LookupRemoteImage("abcdef", makeURL("/v1/"), token) - assertEqual(t, found, false, "Expected remote lookup to fail") + err := r.LookupRemoteImage(imageID, makeURL("/v1/"), token) + assertEqual(t, err, nil, "Expected error of remote lookup to nil") + if err := r.LookupRemoteImage("abcdef", makeURL("/v1/"), token); err == nil { + t.Fatal("Expected error of remote lookup to not nil") + } } func TestGetRemoteImageJSON(t *testing.T) { diff --git a/docs/session.go b/docs/session.go index 4b2f5522..28cf18fb 100644 --- a/docs/session.go +++ b/docs/session.go @@ -102,22 +102,21 @@ func (r *Session) GetRemoteHistory(imgID, registry string, token []string) ([]st } // Check if an image exists in the Registry -// TODO: This method should return the errors instead of masking them and returning false -func (r *Session) LookupRemoteImage(imgID, registry string, token []string) bool { - +func (r *Session) LookupRemoteImage(imgID, registry string, token []string) error { req, err := r.reqFactory.NewRequest("GET", registry+"images/"+imgID+"/json", nil) if err != nil { - log.Errorf("Error in LookupRemoteImage %s", err) - return false + return err } setTokenAuth(req, token) res, _, err := r.doRequest(req) if err != nil { - log.Errorf("Error in LookupRemoteImage %s", err) - return false + return err } res.Body.Close() - return res.StatusCode == 200 + if res.StatusCode != 200 { + return utils.NewHTTPRequestError(fmt.Sprintf("HTTP code %d", res.StatusCode), res) + } + return nil } // Retrieve an image from the Registry. From 807bb5eb186f6aaec6d5c6e9c96ab554d3e92e3e Mon Sep 17 00:00:00 2001 From: Tibor Vass Date: Thu, 18 Dec 2014 19:13:02 -0500 Subject: [PATCH 200/375] registry: add tests for unresolvable domain names in isSecure Signed-off-by: Tibor Vass --- docs/registry_test.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/docs/registry_test.go b/docs/registry_test.go index 5fd80da1..06619aef 100644 --- a/docs/registry_test.go +++ b/docs/registry_test.go @@ -348,6 +348,9 @@ func TestIsSecure(t *testing.T) { {"example.com:5000", []string{"42.42.42.42/8"}, false}, {"127.0.0.1:5000", []string{"127.0.0.0/8"}, false}, {"42.42.42.42:5000", []string{"42.1.1.1/8"}, false}, + {"invalid.domain.com", []string{"42.42.0.0/16"}, true}, + {"invalid.domain.com", []string{"invalid.domain.com"}, false}, + {"invalid.domain.com:5000", []string{"invalid.domain.com"}, false}, } for _, tt := range tests { // TODO: remove this once we remove localhost insecure by default From d1fcbd9028732cb8db82097ab63bd29476ac83ff Mon Sep 17 00:00:00 2001 From: Tibor Vass Date: Thu, 18 Dec 2014 19:13:56 -0500 Subject: [PATCH 201/375] registry: handle unresolvable domain names in isSecure to allow HTTP proxies to work as expected. Fixes #9708 Signed-off-by: Tibor Vass --- docs/endpoint.go | 34 +++++++++++++++++++--------------- 1 file changed, 19 insertions(+), 15 deletions(-) diff --git a/docs/endpoint.go b/docs/endpoint.go index c485a13d..8609486a 100644 --- a/docs/endpoint.go +++ b/docs/endpoint.go @@ -163,7 +163,10 @@ func (e Endpoint) Ping() (RegistryInfo, error) { // If the subnet contains one of the IPs of the registry specified by hostname, the latter is considered // insecure. // -// hostname should be a URL.Host (`host:port` or `host`) +// hostname should be a URL.Host (`host:port` or `host`) where the `host` part can be either a domain name +// or an IP address. If it is a domain name, then it will be resolved in order to check if the IP is contained +// in a subnet. If the resolving is not successful, isSecure will only try to match hostname to any element +// of insecureRegistries. func isSecure(hostname string, insecureRegistries []string) (bool, error) { if hostname == IndexServerURL.Host { return true, nil @@ -177,29 +180,30 @@ func isSecure(hostname string, insecureRegistries []string) (bool, error) { addrs, err := lookupIP(host) if err != nil { ip := net.ParseIP(host) - if ip == nil { - // if resolving `host` fails, error out, since host is to be net.Dial-ed anyway - return true, fmt.Errorf("issecure: could not resolve %q: %v", host, err) + if ip != nil { + addrs = []net.IP{ip} } - addrs = []net.IP{ip} - } - if len(addrs) == 0 { - return true, fmt.Errorf("issecure: could not resolve %q", host) + + // if ip == nil, then `host` is neither an IP nor it could be looked up, + // either because the index is unreachable, or because the index is behind an HTTP proxy. + // So, len(addrs) == 0 and we're not aborting. } - for _, addr := range addrs { - for _, r := range insecureRegistries { + for _, r := range insecureRegistries { + if hostname == r || host == r { // hostname matches insecure registry - if hostname == r { - return false, nil - } + return false, nil + } + + // Try CIDR notation only if addrs has any elements, i.e. if `host`'s IP could be determined. + for _, addr := range addrs { // now assume a CIDR was passed to --insecure-registry _, ipnet, err := net.ParseCIDR(r) if err != nil { - // if could not parse it as a CIDR, even after removing + // if we could not parse it as a CIDR, even after removing // assume it's not a CIDR and go on with the next candidate - continue + break } // check if the addr falls in the subnet From 4170effd5a09115f6791bf34afe6018206b5011a Mon Sep 17 00:00:00 2001 From: Tibor Vass Date: Fri, 19 Dec 2014 16:40:28 -0500 Subject: [PATCH 202/375] registry: remove accidentally added --insecure-registry feature If `--insecure-registry mydomain.com` was specified, it would match a registry at mydomain.com on any port. This was accidentally added in #9735 and is now being reverted. Signed-off-by: Tibor Vass --- docs/endpoint.go | 2 +- docs/registry_test.go | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/docs/endpoint.go b/docs/endpoint.go index 8609486a..019bccfc 100644 --- a/docs/endpoint.go +++ b/docs/endpoint.go @@ -190,7 +190,7 @@ func isSecure(hostname string, insecureRegistries []string) (bool, error) { } for _, r := range insecureRegistries { - if hostname == r || host == r { + if hostname == r { // hostname matches insecure registry return false, nil } diff --git a/docs/registry_test.go b/docs/registry_test.go index 06619aef..52b8b32c 100644 --- a/docs/registry_test.go +++ b/docs/registry_test.go @@ -350,7 +350,8 @@ func TestIsSecure(t *testing.T) { {"42.42.42.42:5000", []string{"42.1.1.1/8"}, false}, {"invalid.domain.com", []string{"42.42.0.0/16"}, true}, {"invalid.domain.com", []string{"invalid.domain.com"}, false}, - {"invalid.domain.com:5000", []string{"invalid.domain.com"}, false}, + {"invalid.domain.com:5000", []string{"invalid.domain.com"}, true}, + {"invalid.domain.com:5000", []string{"invalid.domain.com:5000"}, false}, } for _, tt := range tests { // TODO: remove this once we remove localhost insecure by default From eb9ddb7b863ffcee628e92c039fc9f933233b3c6 Mon Sep 17 00:00:00 2001 From: Matthew Riley Date: Tue, 4 Nov 2014 15:02:06 -0800 Subject: [PATCH 203/375] Allow hyphens in namespaces. Signed-off-by: Matthew Riley --- docs/registry.go | 15 ++++++++--- docs/registry_test.go | 59 ++++++++++++++++++++++++++++++++----------- 2 files changed, 56 insertions(+), 18 deletions(-) diff --git a/docs/registry.go b/docs/registry.go index d503a63d..a1229189 100644 --- a/docs/registry.go +++ b/docs/registry.go @@ -23,7 +23,7 @@ var ( ErrInvalidRepositoryName = errors.New("Invalid repository name (ex: \"registry.domain.tld/myrepos\")") ErrDoesNotExist = errors.New("Image does not exist") errLoginRequired = errors.New("Authentication is required.") - validNamespace = regexp.MustCompile(`^([a-z0-9_]{4,30})$`) + validNamespaceChars = regexp.MustCompile(`^([a-z0-9-_]*)$`) validRepo = regexp.MustCompile(`^([a-z0-9-_.]+)$`) ) @@ -178,8 +178,17 @@ func validateRepositoryName(repositoryName string) error { namespace = nameParts[0] name = nameParts[1] } - if !validNamespace.MatchString(namespace) { - return fmt.Errorf("Invalid namespace name (%s), only [a-z0-9_] are allowed, size between 4 and 30", namespace) + if !validNamespaceChars.MatchString(namespace) { + return fmt.Errorf("Invalid namespace name (%s). Only [a-z0-9-_] are allowed.", namespace) + } + if len(namespace) < 4 || len(namespace) > 30 { + return fmt.Errorf("Invalid namespace name (%s). Cannot be fewer than 4 or more than 30 characters.", namespace) + } + if strings.HasPrefix(namespace, "-") || strings.HasSuffix(namespace, "-") { + return fmt.Errorf("Invalid namespace name (%s). Cannot begin or end with a hyphen.", namespace) + } + if strings.Contains(namespace, "--") { + return fmt.Errorf("Invalid namespace name (%s). Cannot contain consecutive hyphens.", namespace) } if !validRepo.MatchString(name) { return fmt.Errorf("Invalid repository name (%s), only [a-z0-9-_.] are allowed", name) diff --git a/docs/registry_test.go b/docs/registry_test.go index 52b8b32c..c1bb97d6 100644 --- a/docs/registry_test.go +++ b/docs/registry_test.go @@ -233,24 +233,53 @@ func TestSearchRepositories(t *testing.T) { } func TestValidRepositoryName(t *testing.T) { - if err := validateRepositoryName("docker/docker"); err != nil { - t.Fatal(err) + validRepositoryNames := []string{ + // Sanity check. + "docker/docker", + + // Allow 64-character non-hexadecimal names (hexadecimal names are forbidden). + "thisisthesongthatneverendsitgoesonandonandonthisisthesongthatnev", + + // Allow embedded hyphens. + "docker-rules/docker", + + // Allow underscores everywhere (as opposed to hyphens). + "____/____", } - // Support 64-byte non-hexadecimal names (hexadecimal names are forbidden) - if err := validateRepositoryName("thisisthesongthatneverendsitgoesonandonandonthisisthesongthatnev"); err != nil { - t.Fatal(err) + for _, repositoryName := range validRepositoryNames { + if err := validateRepositoryName(repositoryName); err != nil { + t.Errorf("Repository name should be valid: %v. Error: %v", repositoryName, err) + } } - if err := validateRepositoryName("docker/Docker"); err == nil { - t.Log("Repository name should be invalid") - t.Fail() + + invalidRepositoryNames := []string{ + // Disallow capital letters. + "docker/Docker", + + // Only allow one slash. + "docker///docker", + + // Disallow 64-character hexadecimal. + "1a3f5e7d9c1b3a5f7e9d1c3b5a7f9e1d3c5b7a9f1e3d5d7c9b1a3f5e7d9c1b3a", + + // Disallow leading and trailing hyphens in namespace. + "-docker/docker", + "docker-/docker", + "-docker-/docker", + + // Disallow consecutive hyphens. + "dock--er/docker", + + // Namespace too short. + "doc/docker", + + // No repository. + "docker/", } - if err := validateRepositoryName("docker///docker"); err == nil { - t.Log("Repository name should be invalid") - t.Fail() - } - if err := validateRepositoryName("1a3f5e7d9c1b3a5f7e9d1c3b5a7f9e1d3c5b7a9f1e3d5d7c9b1a3f5e7d9c1b3a"); err == nil { - t.Log("Repository name should be invalid, 64-byte hexadecimal names forbidden") - t.Fail() + for _, repositoryName := range invalidRepositoryNames { + if err := validateRepositoryName(repositoryName); err == nil { + t.Errorf("Repository name should be invalid: %v", repositoryName) + } } } From 64b000c3ea01895124f3ce42e52366eb35c12b24 Mon Sep 17 00:00:00 2001 From: Don Kjer Date: Tue, 7 Oct 2014 01:54:52 +0000 Subject: [PATCH 204/375] Deprecating ResolveRepositoryName Passing RepositoryInfo to ResolveAuthConfig, pullRepository, and pushRepository Moving --registry-mirror configuration to registry config Created resolve_repository job Repo names with 'index.docker.io' or 'docker.io' are now synonymous with omitting an index name. Adding test for RepositoryInfo Adding tests for opts.StringSetOpts and registry.ValidateMirror Fixing search term use of repoInfo Adding integration tests for registry mirror configuration Normalizing LookupImage image name to match LocalName parsing rules Normalizing repository LocalName to avoid multiple references to an official image Removing errorOut use in tests Removing TODO comment gofmt changes golint comments cleanup. renaming RegistryOptions => registry.Options, and RegistryServiceConfig => registry.ServiceConfig Splitting out builtins.Registry and registry.NewService calls Stray whitespace cleanup Moving integration tests for Mirrors and InsecureRegistries into TestNewIndexInfo unit test Factoring out ValidateRepositoryName from NewRepositoryInfo Removing unused IndexServerURL Allowing json marshaling of ServiceConfig. Exposing ServiceConfig in /info Switching to CamelCase for json marshaling PR cleanup; removing 'Is' prefix from boolean members. Removing unneeded json tags. Removing non-cleanup related fix for 'localhost:[port]' in splitReposName Merge fixes for gh9735 Fixing integration test Reapplying #9754 Adding comment on config.IndexConfigs use from isSecureIndex Remove unused error return value from isSecureIndex Signed-off-by: Don Kjer Adding back comment in isSecureIndex Signed-off-by: Don Kjer --- docs/auth.go | 33 +-- docs/auth_test.go | 59 ++-- docs/config.go | 126 ++++++++ docs/config_test.go | 49 ++++ docs/endpoint.go | 67 ++--- docs/endpoint_test.go | 2 +- docs/registry.go | 165 +++++++++-- docs/registry_mock_test.go | 97 ++++++- docs/registry_test.go | 577 ++++++++++++++++++++++++++++++++++--- docs/service.go | 119 +++++++- docs/types.go | 41 +++ 11 files changed, 1179 insertions(+), 156 deletions(-) create mode 100644 docs/config.go create mode 100644 docs/config_test.go diff --git a/docs/auth.go b/docs/auth.go index 42760640..8382869b 100644 --- a/docs/auth.go +++ b/docs/auth.go @@ -7,7 +7,6 @@ import ( "fmt" "io/ioutil" "net/http" - "net/url" "os" "path" "strings" @@ -22,23 +21,15 @@ const ( // Only used for user auth + account creation INDEXSERVER = "https://index.docker.io/v1/" REGISTRYSERVER = "https://registry-1.docker.io/v1/" + INDEXNAME = "docker.io" // INDEXSERVER = "https://registry-stage.hub.docker.com/v1/" ) var ( ErrConfigFileMissing = errors.New("The Auth config file is missing") - IndexServerURL *url.URL ) -func init() { - url, err := url.Parse(INDEXSERVER) - if err != nil { - panic(err) - } - IndexServerURL = url -} - type AuthConfig struct { Username string `json:"username,omitempty"` Password string `json:"password,omitempty"` @@ -56,6 +47,10 @@ func IndexServerAddress() string { return INDEXSERVER } +func IndexServerName() string { + return INDEXNAME +} + // create a base64 encoded auth string to store in config func encodeAuth(authConfig *AuthConfig) string { authStr := authConfig.Username + ":" + authConfig.Password @@ -118,6 +113,7 @@ func LoadConfig(rootPath string) (*ConfigFile, error) { } authConfig.Email = origEmail[1] authConfig.ServerAddress = IndexServerAddress() + // *TODO: Switch to using IndexServerName() instead? configFile.Configs[IndexServerAddress()] = authConfig } else { for k, authConfig := range configFile.Configs { @@ -181,7 +177,7 @@ func Login(authConfig *AuthConfig, factory *utils.HTTPRequestFactory) (string, e ) if serverAddress == "" { - serverAddress = IndexServerAddress() + return "", fmt.Errorf("Server Error: Server Address not set.") } loginAgainstOfficialIndex := serverAddress == IndexServerAddress() @@ -213,6 +209,7 @@ func Login(authConfig *AuthConfig, factory *utils.HTTPRequestFactory) (string, e status = "Account created. Please use the confirmation link we sent" + " to your e-mail to activate it." } else { + // *TODO: Use registry configuration to determine what this says, if anything? status = "Account created. Please see the documentation of the registry " + serverAddress + " for instructions how to activate it." } } else if reqStatusCode == 400 { @@ -236,6 +233,7 @@ func Login(authConfig *AuthConfig, factory *utils.HTTPRequestFactory) (string, e if loginAgainstOfficialIndex { return "", fmt.Errorf("Login: Account is not Active. Please check your e-mail for a confirmation link.") } + // *TODO: Use registry configuration to determine what this says, if anything? return "", fmt.Errorf("Login: Account is not Active. Please see the documentation of the registry %s for instructions how to activate it.", serverAddress) } return "", fmt.Errorf("Login: %s (Code: %d; Headers: %s)", body, resp.StatusCode, resp.Header) @@ -271,14 +269,10 @@ func Login(authConfig *AuthConfig, factory *utils.HTTPRequestFactory) (string, e } // this method matches a auth configuration to a server address or a url -func (config *ConfigFile) ResolveAuthConfig(hostname string) AuthConfig { - if hostname == IndexServerAddress() || len(hostname) == 0 { - // default to the index server - return config.Configs[IndexServerAddress()] - } - +func (config *ConfigFile) ResolveAuthConfig(index *IndexInfo) AuthConfig { + configKey := index.GetAuthConfigKey() // First try the happy case - if c, found := config.Configs[hostname]; found { + if c, found := config.Configs[configKey]; found || index.Official { return c } @@ -297,9 +291,8 @@ func (config *ConfigFile) ResolveAuthConfig(hostname string) AuthConfig { // Maybe they have a legacy config file, we will iterate the keys converting // them to the new format and testing - normalizedHostename := convertToHostname(hostname) for registry, config := range config.Configs { - if registryHostname := convertToHostname(registry); registryHostname == normalizedHostename { + if configKey == convertToHostname(registry) { return config } } diff --git a/docs/auth_test.go b/docs/auth_test.go index 3cb1a9ac..22f87994 100644 --- a/docs/auth_test.go +++ b/docs/auth_test.go @@ -81,12 +81,20 @@ func TestResolveAuthConfigIndexServer(t *testing.T) { } defer os.RemoveAll(configFile.rootPath) - for _, registry := range []string{"", IndexServerAddress()} { - resolved := configFile.ResolveAuthConfig(registry) - if resolved != configFile.Configs[IndexServerAddress()] { - t.Fail() - } + indexConfig := configFile.Configs[IndexServerAddress()] + + officialIndex := &IndexInfo{ + Official: true, } + privateIndex := &IndexInfo{ + Official: false, + } + + resolved := configFile.ResolveAuthConfig(officialIndex) + assertEqual(t, resolved, indexConfig, "Expected ResolveAuthConfig to return IndexServerAddress()") + + resolved = configFile.ResolveAuthConfig(privateIndex) + assertNotEqual(t, resolved, indexConfig, "Expected ResolveAuthConfig to not return IndexServerAddress()") } func TestResolveAuthConfigFullURL(t *testing.T) { @@ -106,18 +114,27 @@ func TestResolveAuthConfigFullURL(t *testing.T) { Password: "bar-pass", Email: "bar@example.com", } - configFile.Configs["https://registry.example.com/v1/"] = registryAuth - configFile.Configs["http://localhost:8000/v1/"] = localAuth - configFile.Configs["registry.com"] = registryAuth + officialAuth := AuthConfig{ + Username: "baz-user", + Password: "baz-pass", + Email: "baz@example.com", + } + configFile.Configs[IndexServerAddress()] = officialAuth + + expectedAuths := map[string]AuthConfig{ + "registry.example.com": registryAuth, + "localhost:8000": localAuth, + "registry.com": localAuth, + } validRegistries := map[string][]string{ - "https://registry.example.com/v1/": { + "registry.example.com": { "https://registry.example.com/v1/", "http://registry.example.com/v1/", "registry.example.com", "registry.example.com/v1/", }, - "http://localhost:8000/v1/": { + "localhost:8000": { "https://localhost:8000/v1/", "http://localhost:8000/v1/", "localhost:8000", @@ -132,18 +149,24 @@ func TestResolveAuthConfigFullURL(t *testing.T) { } for configKey, registries := range validRegistries { + configured, ok := expectedAuths[configKey] + if !ok || configured.Email == "" { + t.Fatal() + } + index := &IndexInfo{ + Name: configKey, + } for _, registry := range registries { - var ( - configured AuthConfig - ok bool - ) - resolved := configFile.ResolveAuthConfig(registry) - if configured, ok = configFile.Configs[configKey]; !ok { - t.Fail() - } + configFile.Configs[registry] = configured + resolved := configFile.ResolveAuthConfig(index) if resolved.Email != configured.Email { t.Errorf("%s -> %q != %q\n", registry, resolved.Email, configured.Email) } + delete(configFile.Configs, registry) + resolved = configFile.ResolveAuthConfig(index) + if resolved.Email == configured.Email { + t.Errorf("%s -> %q == %q\n", registry, resolved.Email, configured.Email) + } } } } diff --git a/docs/config.go b/docs/config.go new file mode 100644 index 00000000..bd993edd --- /dev/null +++ b/docs/config.go @@ -0,0 +1,126 @@ +package registry + +import ( + "encoding/json" + "fmt" + "net" + "net/url" + + "github.com/docker/docker/opts" + flag "github.com/docker/docker/pkg/mflag" +) + +// Options holds command line options. +type Options struct { + Mirrors opts.ListOpts + InsecureRegistries opts.ListOpts +} + +// InstallFlags adds command-line options to the top-level flag parser for +// the current process. +func (options *Options) InstallFlags() { + options.Mirrors = opts.NewListOpts(ValidateMirror) + flag.Var(&options.Mirrors, []string{"-registry-mirror"}, "Specify a preferred Docker registry mirror") + options.InsecureRegistries = opts.NewListOpts(ValidateIndexName) + flag.Var(&options.InsecureRegistries, []string{"-insecure-registry"}, "Enable insecure communication with specified registries (no certificate verification for HTTPS and enable HTTP fallback) (e.g., localhost:5000 or 10.20.0.0/16)") +} + +// ValidateMirror validates an HTTP(S) registry mirror +func ValidateMirror(val string) (string, error) { + uri, err := url.Parse(val) + if err != nil { + return "", fmt.Errorf("%s is not a valid URI", val) + } + + if uri.Scheme != "http" && uri.Scheme != "https" { + return "", fmt.Errorf("Unsupported scheme %s", uri.Scheme) + } + + if uri.Path != "" || uri.RawQuery != "" || uri.Fragment != "" { + return "", fmt.Errorf("Unsupported path/query/fragment at end of the URI") + } + + return fmt.Sprintf("%s://%s/v1/", uri.Scheme, uri.Host), nil +} + +// ValidateIndexName validates an index name. +func ValidateIndexName(val string) (string, error) { + // 'index.docker.io' => 'docker.io' + if val == "index."+IndexServerName() { + val = IndexServerName() + } + // *TODO: Check if valid hostname[:port]/ip[:port]? + return val, nil +} + +type netIPNet net.IPNet + +func (ipnet *netIPNet) MarshalJSON() ([]byte, error) { + return json.Marshal((*net.IPNet)(ipnet).String()) +} + +func (ipnet *netIPNet) UnmarshalJSON(b []byte) (err error) { + var ipnet_str string + if err = json.Unmarshal(b, &ipnet_str); err == nil { + var cidr *net.IPNet + if _, cidr, err = net.ParseCIDR(ipnet_str); err == nil { + *ipnet = netIPNet(*cidr) + } + } + return +} + +// ServiceConfig stores daemon registry services configuration. +type ServiceConfig struct { + InsecureRegistryCIDRs []*netIPNet `json:"InsecureRegistryCIDRs"` + IndexConfigs map[string]*IndexInfo `json:"IndexConfigs"` +} + +// NewServiceConfig returns a new instance of ServiceConfig +func NewServiceConfig(options *Options) *ServiceConfig { + if options == nil { + options = &Options{ + Mirrors: opts.NewListOpts(nil), + InsecureRegistries: opts.NewListOpts(nil), + } + } + + // Localhost is by default considered as an insecure registry + // This is a stop-gap for people who are running a private registry on localhost (especially on Boot2docker). + // + // TODO: should we deprecate this once it is easier for people to set up a TLS registry or change + // daemon flags on boot2docker? + options.InsecureRegistries.Set("127.0.0.0/8") + + config := &ServiceConfig{ + InsecureRegistryCIDRs: make([]*netIPNet, 0), + IndexConfigs: make(map[string]*IndexInfo, 0), + } + // Split --insecure-registry into CIDR and registry-specific settings. + for _, r := range options.InsecureRegistries.GetAll() { + // Check if CIDR was passed to --insecure-registry + _, ipnet, err := net.ParseCIDR(r) + if err == nil { + // Valid CIDR. + config.InsecureRegistryCIDRs = append(config.InsecureRegistryCIDRs, (*netIPNet)(ipnet)) + } else { + // Assume `host:port` if not CIDR. + config.IndexConfigs[r] = &IndexInfo{ + Name: r, + Mirrors: make([]string, 0), + Secure: false, + Official: false, + } + } + } + + // Configure public registry. + config.IndexConfigs[IndexServerName()] = &IndexInfo{ + Name: IndexServerName(), + Mirrors: options.Mirrors.GetAll(), + Secure: true, + Official: true, + } + + return config +} diff --git a/docs/config_test.go b/docs/config_test.go new file mode 100644 index 00000000..25578a7f --- /dev/null +++ b/docs/config_test.go @@ -0,0 +1,49 @@ +package registry + +import ( + "testing" +) + +func TestValidateMirror(t *testing.T) { + valid := []string{ + "http://mirror-1.com", + "https://mirror-1.com", + "http://localhost", + "https://localhost", + "http://localhost:5000", + "https://localhost:5000", + "http://127.0.0.1", + "https://127.0.0.1", + "http://127.0.0.1:5000", + "https://127.0.0.1:5000", + } + + invalid := []string{ + "!invalid!://%as%", + "ftp://mirror-1.com", + "http://mirror-1.com/", + "http://mirror-1.com/?q=foo", + "http://mirror-1.com/v1/", + "http://mirror-1.com/v1/?q=foo", + "http://mirror-1.com/v1/?q=foo#frag", + "http://mirror-1.com?q=foo", + "https://mirror-1.com#frag", + "https://mirror-1.com/", + "https://mirror-1.com/#frag", + "https://mirror-1.com/v1/", + "https://mirror-1.com/v1/#", + "https://mirror-1.com?q", + } + + for _, address := range valid { + if ret, err := ValidateMirror(address); err != nil || ret == "" { + t.Errorf("ValidateMirror(`"+address+"`) got %s %s", ret, err) + } + } + + for _, address := range invalid { + if ret, err := ValidateMirror(address); err == nil || ret != "" { + t.Errorf("ValidateMirror(`"+address+"`) got %s %s", ret, err) + } + } +} diff --git a/docs/endpoint.go b/docs/endpoint.go index 019bccfc..86f53744 100644 --- a/docs/endpoint.go +++ b/docs/endpoint.go @@ -37,8 +37,9 @@ func scanForAPIVersion(hostname string) (string, APIVersion) { return hostname, DefaultAPIVersion } -func NewEndpoint(hostname string, insecureRegistries []string) (*Endpoint, error) { - endpoint, err := newEndpoint(hostname, insecureRegistries) +func NewEndpoint(index *IndexInfo) (*Endpoint, error) { + // *TODO: Allow per-registry configuration of endpoints. + endpoint, err := newEndpoint(index.GetAuthConfigKey(), index.Secure) if err != nil { return nil, err } @@ -49,7 +50,7 @@ func NewEndpoint(hostname string, insecureRegistries []string) (*Endpoint, error //TODO: triggering highland build can be done there without "failing" - if endpoint.secure { + if index.Secure { // If registry is secure and HTTPS failed, show user the error and tell them about `--insecure-registry` // in case that's what they need. DO NOT accept unknown CA certificates, and DO NOT fallback to HTTP. return nil, fmt.Errorf("Invalid registry endpoint %s: %v. If this private registry supports only HTTP or HTTPS with an unknown CA certificate, please add `--insecure-registry %s` to the daemon's arguments. In the case of HTTPS, if you have access to the registry's CA certificate, no need for the flag; simply place the CA certificate at /etc/docker/certs.d/%s/ca.crt", endpoint, err, endpoint.URL.Host, endpoint.URL.Host) @@ -68,7 +69,7 @@ func NewEndpoint(hostname string, insecureRegistries []string) (*Endpoint, error return endpoint, nil } -func newEndpoint(hostname string, insecureRegistries []string) (*Endpoint, error) { +func newEndpoint(hostname string, secure bool) (*Endpoint, error) { var ( endpoint = Endpoint{} trimmedHostname string @@ -82,13 +83,14 @@ func newEndpoint(hostname string, insecureRegistries []string) (*Endpoint, error if err != nil { return nil, err } - endpoint.secure, err = isSecure(endpoint.URL.Host, insecureRegistries) - if err != nil { - return nil, err - } + endpoint.secure = secure return &endpoint, nil } +func (repoInfo *RepositoryInfo) GetEndpoint() (*Endpoint, error) { + return NewEndpoint(repoInfo.Index) +} + type Endpoint struct { URL *url.URL Version APIVersion @@ -156,27 +158,30 @@ func (e Endpoint) Ping() (RegistryInfo, error) { return info, nil } -// isSecure returns false if the provided hostname is part of the list of insecure registries. +// isSecureIndex returns false if the provided indexName is part of the list of insecure registries // Insecure registries accept HTTP and/or accept HTTPS with certificates from unknown CAs. // // The list of insecure registries can contain an element with CIDR notation to specify a whole subnet. -// If the subnet contains one of the IPs of the registry specified by hostname, the latter is considered +// If the subnet contains one of the IPs of the registry specified by indexName, the latter is considered // insecure. // -// hostname should be a URL.Host (`host:port` or `host`) where the `host` part can be either a domain name +// indexName should be a URL.Host (`host:port` or `host`) where the `host` part can be either a domain name // or an IP address. If it is a domain name, then it will be resolved in order to check if the IP is contained -// in a subnet. If the resolving is not successful, isSecure will only try to match hostname to any element +// in a subnet. If the resolving is not successful, isSecureIndex will only try to match hostname to any element // of insecureRegistries. -func isSecure(hostname string, insecureRegistries []string) (bool, error) { - if hostname == IndexServerURL.Host { - return true, nil +func (config *ServiceConfig) isSecureIndex(indexName string) bool { + // Check for configured index, first. This is needed in case isSecureIndex + // is called from anything besides NewIndexInfo, in order to honor per-index configurations. + if index, ok := config.IndexConfigs[indexName]; ok { + return index.Secure } - host, _, err := net.SplitHostPort(hostname) + host, _, err := net.SplitHostPort(indexName) if err != nil { - // assume hostname is of the form `host` without the port and go on. - host = hostname + // assume indexName is of the form `host` without the port and go on. + host = indexName } + addrs, err := lookupIP(host) if err != nil { ip := net.ParseIP(host) @@ -189,29 +194,15 @@ func isSecure(hostname string, insecureRegistries []string) (bool, error) { // So, len(addrs) == 0 and we're not aborting. } - for _, r := range insecureRegistries { - if hostname == r { - // hostname matches insecure registry - return false, nil - } - - // Try CIDR notation only if addrs has any elements, i.e. if `host`'s IP could be determined. - for _, addr := range addrs { - - // now assume a CIDR was passed to --insecure-registry - _, ipnet, err := net.ParseCIDR(r) - if err != nil { - // if we could not parse it as a CIDR, even after removing - // assume it's not a CIDR and go on with the next candidate - break - } - + // Try CIDR notation only if addrs has any elements, i.e. if `host`'s IP could be determined. + for _, addr := range addrs { + for _, ipnet := range config.InsecureRegistryCIDRs { // check if the addr falls in the subnet - if ipnet.Contains(addr) { - return false, nil + if (*net.IPNet)(ipnet).Contains(addr) { + return false } } } - return true, nil + return true } diff --git a/docs/endpoint_test.go b/docs/endpoint_test.go index 54105ec1..b691a4fb 100644 --- a/docs/endpoint_test.go +++ b/docs/endpoint_test.go @@ -12,7 +12,7 @@ func TestEndpointParse(t *testing.T) { {"0.0.0.0:5000", "https://0.0.0.0:5000/v1/"}, } for _, td := range testData { - e, err := newEndpoint(td.str, insecureRegistries) + e, err := newEndpoint(td.str, false) if err != nil { t.Errorf("%q: %s", td.str, err) } diff --git a/docs/registry.go b/docs/registry.go index a1229189..de724ee2 100644 --- a/docs/registry.go +++ b/docs/registry.go @@ -25,6 +25,7 @@ var ( errLoginRequired = errors.New("Authentication is required.") validNamespaceChars = regexp.MustCompile(`^([a-z0-9-_]*)$`) validRepo = regexp.MustCompile(`^([a-z0-9-_.]+)$`) + emptyServiceConfig = NewServiceConfig(nil) ) type TimeoutType uint32 @@ -160,12 +161,12 @@ func doRequest(req *http.Request, jar http.CookieJar, timeout TimeoutType, secur return res, client, err } -func validateRepositoryName(repositoryName string) error { +func validateRemoteName(remoteName string) error { var ( namespace string name string ) - nameParts := strings.SplitN(repositoryName, "/", 2) + nameParts := strings.SplitN(remoteName, "/", 2) if len(nameParts) < 2 { namespace = "library" name = nameParts[0] @@ -196,29 +197,147 @@ func validateRepositoryName(repositoryName string) error { return nil } -// Resolves a repository name to a hostname + name -func ResolveRepositoryName(reposName string) (string, string, error) { - if strings.Contains(reposName, "://") { - // It cannot contain a scheme! - return "", "", ErrInvalidRepositoryName - } - nameParts := strings.SplitN(reposName, "/", 2) - if len(nameParts) == 1 || (!strings.Contains(nameParts[0], ".") && !strings.Contains(nameParts[0], ":") && - nameParts[0] != "localhost") { - // This is a Docker Index repos (ex: samalba/hipache or ubuntu) - err := validateRepositoryName(reposName) - return IndexServerAddress(), reposName, err - } - hostname := nameParts[0] - reposName = nameParts[1] - if strings.Contains(hostname, "index.docker.io") { - return "", "", fmt.Errorf("Invalid repository name, try \"%s\" instead", reposName) - } - if err := validateRepositoryName(reposName); err != nil { - return "", "", err +// NewIndexInfo returns IndexInfo configuration from indexName +func NewIndexInfo(config *ServiceConfig, indexName string) (*IndexInfo, error) { + var err error + indexName, err = ValidateIndexName(indexName) + if err != nil { + return nil, err } - return hostname, reposName, nil + // Return any configured index info, first. + if index, ok := config.IndexConfigs[indexName]; ok { + return index, nil + } + + // Construct a non-configured index info. + index := &IndexInfo{ + Name: indexName, + Mirrors: make([]string, 0), + Official: false, + } + index.Secure = config.isSecureIndex(indexName) + return index, nil +} + +func validateNoSchema(reposName string) error { + if strings.Contains(reposName, "://") { + // It cannot contain a scheme! + return ErrInvalidRepositoryName + } + return nil +} + +// splitReposName breaks a reposName into an index name and remote name +func splitReposName(reposName string) (string, string) { + nameParts := strings.SplitN(reposName, "/", 2) + var indexName, remoteName string + if len(nameParts) == 1 || (!strings.Contains(nameParts[0], ".") && + !strings.Contains(nameParts[0], ":") && nameParts[0] != "localhost") { + // This is a Docker Index repos (ex: samalba/hipache or ubuntu) + // 'docker.io' + indexName = IndexServerName() + remoteName = reposName + } else { + indexName = nameParts[0] + remoteName = nameParts[1] + } + return indexName, remoteName +} + +// NewRepositoryInfo validates and breaks down a repository name into a RepositoryInfo +func NewRepositoryInfo(config *ServiceConfig, reposName string) (*RepositoryInfo, error) { + if err := validateNoSchema(reposName); err != nil { + return nil, err + } + + indexName, remoteName := splitReposName(reposName) + if err := validateRemoteName(remoteName); err != nil { + return nil, err + } + + repoInfo := &RepositoryInfo{ + RemoteName: remoteName, + } + + var err error + repoInfo.Index, err = NewIndexInfo(config, indexName) + if err != nil { + return nil, err + } + + if repoInfo.Index.Official { + normalizedName := repoInfo.RemoteName + if strings.HasPrefix(normalizedName, "library/") { + // If pull "library/foo", it's stored locally under "foo" + normalizedName = strings.SplitN(normalizedName, "/", 2)[1] + } + + repoInfo.LocalName = normalizedName + repoInfo.RemoteName = normalizedName + // If the normalized name does not contain a '/' (e.g. "foo") + // then it is an official repo. + if strings.IndexRune(normalizedName, '/') == -1 { + repoInfo.Official = true + // Fix up remote name for official repos. + repoInfo.RemoteName = "library/" + normalizedName + } + + // *TODO: Prefix this with 'docker.io/'. + repoInfo.CanonicalName = repoInfo.LocalName + } else { + // *TODO: Decouple index name from hostname (via registry configuration?) + repoInfo.LocalName = repoInfo.Index.Name + "/" + repoInfo.RemoteName + repoInfo.CanonicalName = repoInfo.LocalName + } + return repoInfo, nil +} + +// ValidateRepositoryName validates a repository name +func ValidateRepositoryName(reposName string) error { + var err error + if err = validateNoSchema(reposName); err != nil { + return err + } + indexName, remoteName := splitReposName(reposName) + if _, err = ValidateIndexName(indexName); err != nil { + return err + } + return validateRemoteName(remoteName) +} + +// ParseRepositoryInfo performs the breakdown of a repository name into a RepositoryInfo, but +// lacks registry configuration. +func ParseRepositoryInfo(reposName string) (*RepositoryInfo, error) { + return NewRepositoryInfo(emptyServiceConfig, reposName) +} + +// NormalizeLocalName transforms a repository name into a normalize LocalName +// Passes through the name without transformation on error (image id, etc) +func NormalizeLocalName(name string) string { + repoInfo, err := ParseRepositoryInfo(name) + if err != nil { + return name + } + return repoInfo.LocalName +} + +// GetAuthConfigKey special-cases using the full index address of the official +// index as the AuthConfig key, and uses the (host)name[:port] for private indexes. +func (index *IndexInfo) GetAuthConfigKey() string { + if index.Official { + return IndexServerAddress() + } + return index.Name +} + +// GetSearchTerm special-cases using local name for official index, and +// remote name for private indexes. +func (repoInfo *RepositoryInfo) GetSearchTerm() string { + if repoInfo.Index.Official { + return repoInfo.LocalName + } + return repoInfo.RemoteName } func trustedLocation(req *http.Request) bool { diff --git a/docs/registry_mock_test.go b/docs/registry_mock_test.go index 887d2ef6..57233d7c 100644 --- a/docs/registry_mock_test.go +++ b/docs/registry_mock_test.go @@ -15,15 +15,16 @@ import ( "testing" "time" + "github.com/docker/docker/opts" "github.com/gorilla/mux" log "github.com/Sirupsen/logrus" ) var ( - testHTTPServer *httptest.Server - insecureRegistries []string - testLayers = map[string]map[string]string{ + testHTTPServer *httptest.Server + testHTTPSServer *httptest.Server + testLayers = map[string]map[string]string{ "77dbf71da1d00e3fbddc480176eac8994025630c6590d11cfc8fe1209c2a1d20": { "json": `{"id":"77dbf71da1d00e3fbddc480176eac8994025630c6590d11cfc8fe1209c2a1d20", "comment":"test base image","created":"2013-03-23T12:53:11.10432-07:00", @@ -86,6 +87,7 @@ var ( "": {net.ParseIP("0.0.0.0")}, "localhost": {net.ParseIP("127.0.0.1"), net.ParseIP("::1")}, "example.com": {net.ParseIP("42.42.42.42")}, + "other.com": {net.ParseIP("43.43.43.43")}, } ) @@ -108,11 +110,7 @@ func init() { r.HandleFunc("/v2/version", handlerGetPing).Methods("GET") testHTTPServer = httptest.NewServer(handlerAccessLog(r)) - URL, err := url.Parse(testHTTPServer.URL) - if err != nil { - panic(err) - } - insecureRegistries = []string{URL.Host} + testHTTPSServer = httptest.NewTLSServer(handlerAccessLog(r)) // override net.LookupIP lookupIP = func(host string) ([]net.IP, error) { @@ -146,6 +144,52 @@ func makeURL(req string) string { return testHTTPServer.URL + req } +func makeHttpsURL(req string) string { + return testHTTPSServer.URL + req +} + +func makeIndex(req string) *IndexInfo { + index := &IndexInfo{ + Name: makeURL(req), + } + return index +} + +func makeHttpsIndex(req string) *IndexInfo { + index := &IndexInfo{ + Name: makeHttpsURL(req), + } + return index +} + +func makePublicIndex() *IndexInfo { + index := &IndexInfo{ + Name: IndexServerAddress(), + Secure: true, + Official: true, + } + return index +} + +func makeServiceConfig(mirrors []string, insecure_registries []string) *ServiceConfig { + options := &Options{ + Mirrors: opts.NewListOpts(nil), + InsecureRegistries: opts.NewListOpts(nil), + } + if mirrors != nil { + for _, mirror := range mirrors { + options.Mirrors.Set(mirror) + } + } + if insecure_registries != nil { + for _, insecure_registries := range insecure_registries { + options.InsecureRegistries.Set(insecure_registries) + } + } + + return NewServiceConfig(options) +} + func writeHeaders(w http.ResponseWriter) { h := w.Header() h.Add("Server", "docker-tests/mock") @@ -193,6 +237,40 @@ func assertEqual(t *testing.T, a interface{}, b interface{}, message string) { t.Fatal(message) } +func assertNotEqual(t *testing.T, a interface{}, b interface{}, message string) { + if a != b { + return + } + if len(message) == 0 { + message = fmt.Sprintf("%v == %v", a, b) + } + t.Fatal(message) +} + +// Similar to assertEqual, but does not stop test +func checkEqual(t *testing.T, a interface{}, b interface{}, messagePrefix string) { + if a == b { + return + } + message := fmt.Sprintf("%v != %v", a, b) + if len(messagePrefix) != 0 { + message = messagePrefix + ": " + message + } + t.Error(message) +} + +// Similar to assertNotEqual, but does not stop test +func checkNotEqual(t *testing.T, a interface{}, b interface{}, messagePrefix string) { + if a != b { + return + } + message := fmt.Sprintf("%v == %v", a, b) + if len(messagePrefix) != 0 { + message = messagePrefix + ": " + message + } + t.Error(message) +} + func requiresAuth(w http.ResponseWriter, r *http.Request) bool { writeCookie := func() { value := fmt.Sprintf("FAKE-SESSION-%d", time.Now().UnixNano()) @@ -271,6 +349,7 @@ func handlerGetDeleteTags(w http.ResponseWriter, r *http.Request) { return } repositoryName := mux.Vars(r)["repository"] + repositoryName = NormalizeLocalName(repositoryName) tags, exists := testRepositories[repositoryName] if !exists { apiError(w, "Repository not found", 404) @@ -290,6 +369,7 @@ func handlerGetTag(w http.ResponseWriter, r *http.Request) { } vars := mux.Vars(r) repositoryName := vars["repository"] + repositoryName = NormalizeLocalName(repositoryName) tagName := vars["tag"] tags, exists := testRepositories[repositoryName] if !exists { @@ -310,6 +390,7 @@ func handlerPutTag(w http.ResponseWriter, r *http.Request) { } vars := mux.Vars(r) repositoryName := vars["repository"] + repositoryName = NormalizeLocalName(repositoryName) tagName := vars["tag"] tags, exists := testRepositories[repositoryName] if !exists { diff --git a/docs/registry_test.go b/docs/registry_test.go index c1bb97d6..511d7eb1 100644 --- a/docs/registry_test.go +++ b/docs/registry_test.go @@ -21,7 +21,7 @@ const ( func spawnTestRegistrySession(t *testing.T) *Session { authConfig := &AuthConfig{} - endpoint, err := NewEndpoint(makeURL("/v1/"), insecureRegistries) + endpoint, err := NewEndpoint(makeIndex("/v1/")) if err != nil { t.Fatal(err) } @@ -32,16 +32,139 @@ func spawnTestRegistrySession(t *testing.T) *Session { return r } +func TestPublicSession(t *testing.T) { + authConfig := &AuthConfig{} + + getSessionDecorators := func(index *IndexInfo) int { + endpoint, err := NewEndpoint(index) + if err != nil { + t.Fatal(err) + } + r, err := NewSession(authConfig, utils.NewHTTPRequestFactory(), endpoint, true) + if err != nil { + t.Fatal(err) + } + return len(r.reqFactory.GetDecorators()) + } + + decorators := getSessionDecorators(makeIndex("/v1/")) + assertEqual(t, decorators, 0, "Expected no decorator on http session") + + decorators = getSessionDecorators(makeHttpsIndex("/v1/")) + assertNotEqual(t, decorators, 0, "Expected decorator on https session") + + decorators = getSessionDecorators(makePublicIndex()) + assertEqual(t, decorators, 0, "Expected no decorator on public session") +} + func TestPingRegistryEndpoint(t *testing.T) { - ep, err := NewEndpoint(makeURL("/v1/"), insecureRegistries) - if err != nil { - t.Fatal(err) + testPing := func(index *IndexInfo, expectedStandalone bool, assertMessage string) { + ep, err := NewEndpoint(index) + if err != nil { + t.Fatal(err) + } + regInfo, err := ep.Ping() + if err != nil { + t.Fatal(err) + } + + assertEqual(t, regInfo.Standalone, expectedStandalone, assertMessage) } - regInfo, err := ep.Ping() - if err != nil { - t.Fatal(err) + + testPing(makeIndex("/v1/"), true, "Expected standalone to be true (default)") + testPing(makeHttpsIndex("/v1/"), true, "Expected standalone to be true (default)") + testPing(makePublicIndex(), false, "Expected standalone to be false for public index") +} + +func TestEndpoint(t *testing.T) { + // Simple wrapper to fail test if err != nil + expandEndpoint := func(index *IndexInfo) *Endpoint { + endpoint, err := NewEndpoint(index) + if err != nil { + t.Fatal(err) + } + return endpoint + } + + assertInsecureIndex := func(index *IndexInfo) { + index.Secure = true + _, err := NewEndpoint(index) + assertNotEqual(t, err, nil, index.Name+": Expected error for insecure index") + assertEqual(t, strings.Contains(err.Error(), "insecure-registry"), true, index.Name+": Expected insecure-registry error for insecure index") + index.Secure = false + } + + assertSecureIndex := func(index *IndexInfo) { + index.Secure = true + _, err := NewEndpoint(index) + assertNotEqual(t, err, nil, index.Name+": Expected cert error for secure index") + assertEqual(t, strings.Contains(err.Error(), "certificate signed by unknown authority"), true, index.Name+": Expected cert error for secure index") + index.Secure = false + } + + index := &IndexInfo{} + index.Name = makeURL("/v1/") + endpoint := expandEndpoint(index) + assertEqual(t, endpoint.String(), index.Name, "Expected endpoint to be "+index.Name) + if endpoint.Version != APIVersion1 { + t.Fatal("Expected endpoint to be v1") + } + assertInsecureIndex(index) + + index.Name = makeURL("") + endpoint = expandEndpoint(index) + assertEqual(t, endpoint.String(), index.Name+"/v1/", index.Name+": Expected endpoint to be "+index.Name+"/v1/") + if endpoint.Version != APIVersion1 { + t.Fatal("Expected endpoint to be v1") + } + assertInsecureIndex(index) + + httpURL := makeURL("") + index.Name = strings.SplitN(httpURL, "://", 2)[1] + endpoint = expandEndpoint(index) + assertEqual(t, endpoint.String(), httpURL+"/v1/", index.Name+": Expected endpoint to be "+httpURL+"/v1/") + if endpoint.Version != APIVersion1 { + t.Fatal("Expected endpoint to be v1") + } + assertInsecureIndex(index) + + index.Name = makeHttpsURL("/v1/") + endpoint = expandEndpoint(index) + assertEqual(t, endpoint.String(), index.Name, "Expected endpoint to be "+index.Name) + if endpoint.Version != APIVersion1 { + t.Fatal("Expected endpoint to be v1") + } + assertSecureIndex(index) + + index.Name = makeHttpsURL("") + endpoint = expandEndpoint(index) + assertEqual(t, endpoint.String(), index.Name+"/v1/", index.Name+": Expected endpoint to be "+index.Name+"/v1/") + if endpoint.Version != APIVersion1 { + t.Fatal("Expected endpoint to be v1") + } + assertSecureIndex(index) + + httpsURL := makeHttpsURL("") + index.Name = strings.SplitN(httpsURL, "://", 2)[1] + endpoint = expandEndpoint(index) + assertEqual(t, endpoint.String(), httpsURL+"/v1/", index.Name+": Expected endpoint to be "+httpsURL+"/v1/") + if endpoint.Version != APIVersion1 { + t.Fatal("Expected endpoint to be v1") + } + assertSecureIndex(index) + + badEndpoints := []string{ + "http://127.0.0.1/v1/", + "https://127.0.0.1/v1/", + "http://127.0.0.1", + "https://127.0.0.1", + "127.0.0.1", + } + for _, address := range badEndpoints { + index.Name = address + _, err := NewEndpoint(index) + checkNotEqual(t, err, nil, "Expected error while expanding bad endpoint") } - assertEqual(t, regInfo.Standalone, true, "Expected standalone to be true (default)") } func TestGetRemoteHistory(t *testing.T) { @@ -156,30 +279,413 @@ func TestPushImageLayerRegistry(t *testing.T) { } } -func TestResolveRepositoryName(t *testing.T) { - _, _, err := ResolveRepositoryName("https://github.com/docker/docker") - assertEqual(t, err, ErrInvalidRepositoryName, "Expected error invalid repo name") - ep, repo, err := ResolveRepositoryName("fooo/bar") - if err != nil { - t.Fatal(err) +func TestValidateRepositoryName(t *testing.T) { + validRepoNames := []string{ + "docker/docker", + "library/debian", + "debian", + "docker.io/docker/docker", + "docker.io/library/debian", + "docker.io/debian", + "index.docker.io/docker/docker", + "index.docker.io/library/debian", + "index.docker.io/debian", + "127.0.0.1:5000/docker/docker", + "127.0.0.1:5000/library/debian", + "127.0.0.1:5000/debian", + "thisisthesongthatneverendsitgoesonandonandonthisisthesongthatnev", + } + invalidRepoNames := []string{ + "https://github.com/docker/docker", + "docker/Docker", + "docker///docker", + "docker.io/docker/Docker", + "docker.io/docker///docker", + "1a3f5e7d9c1b3a5f7e9d1c3b5a7f9e1d3c5b7a9f1e3d5d7c9b1a3f5e7d9c1b3a", + "docker.io/1a3f5e7d9c1b3a5f7e9d1c3b5a7f9e1d3c5b7a9f1e3d5d7c9b1a3f5e7d9c1b3a", } - assertEqual(t, ep, IndexServerAddress(), "Expected endpoint to be index server address") - assertEqual(t, repo, "fooo/bar", "Expected resolved repo to be foo/bar") - u := makeURL("")[7:] - ep, repo, err = ResolveRepositoryName(u + "/private/moonbase") - if err != nil { - t.Fatal(err) + for _, name := range invalidRepoNames { + err := ValidateRepositoryName(name) + assertNotEqual(t, err, nil, "Expected invalid repo name: "+name) } - assertEqual(t, ep, u, "Expected endpoint to be "+u) - assertEqual(t, repo, "private/moonbase", "Expected endpoint to be private/moonbase") - ep, repo, err = ResolveRepositoryName("ubuntu-12.04-base") - if err != nil { - t.Fatal(err) + for _, name := range validRepoNames { + err := ValidateRepositoryName(name) + assertEqual(t, err, nil, "Expected valid repo name: "+name) } - assertEqual(t, ep, IndexServerAddress(), "Expected endpoint to be "+IndexServerAddress()) - assertEqual(t, repo, "ubuntu-12.04-base", "Expected endpoint to be ubuntu-12.04-base") + + err := ValidateRepositoryName(invalidRepoNames[0]) + assertEqual(t, err, ErrInvalidRepositoryName, "Expected ErrInvalidRepositoryName: "+invalidRepoNames[0]) +} + +func TestParseRepositoryInfo(t *testing.T) { + expectedRepoInfos := map[string]RepositoryInfo{ + "fooo/bar": { + Index: &IndexInfo{ + Name: IndexServerName(), + Official: true, + }, + RemoteName: "fooo/bar", + LocalName: "fooo/bar", + CanonicalName: "fooo/bar", + Official: false, + }, + "library/ubuntu": { + Index: &IndexInfo{ + Name: IndexServerName(), + Official: true, + }, + RemoteName: "library/ubuntu", + LocalName: "ubuntu", + CanonicalName: "ubuntu", + Official: true, + }, + "nonlibrary/ubuntu": { + Index: &IndexInfo{ + Name: IndexServerName(), + Official: true, + }, + RemoteName: "nonlibrary/ubuntu", + LocalName: "nonlibrary/ubuntu", + CanonicalName: "nonlibrary/ubuntu", + Official: false, + }, + "ubuntu": { + Index: &IndexInfo{ + Name: IndexServerName(), + Official: true, + }, + RemoteName: "library/ubuntu", + LocalName: "ubuntu", + CanonicalName: "ubuntu", + Official: true, + }, + "other/library": { + Index: &IndexInfo{ + Name: IndexServerName(), + Official: true, + }, + RemoteName: "other/library", + LocalName: "other/library", + CanonicalName: "other/library", + Official: false, + }, + "127.0.0.1:8000/private/moonbase": { + Index: &IndexInfo{ + Name: "127.0.0.1:8000", + Official: false, + }, + RemoteName: "private/moonbase", + LocalName: "127.0.0.1:8000/private/moonbase", + CanonicalName: "127.0.0.1:8000/private/moonbase", + Official: false, + }, + "127.0.0.1:8000/privatebase": { + Index: &IndexInfo{ + Name: "127.0.0.1:8000", + Official: false, + }, + RemoteName: "privatebase", + LocalName: "127.0.0.1:8000/privatebase", + CanonicalName: "127.0.0.1:8000/privatebase", + Official: false, + }, + "localhost:8000/private/moonbase": { + Index: &IndexInfo{ + Name: "localhost:8000", + Official: false, + }, + RemoteName: "private/moonbase", + LocalName: "localhost:8000/private/moonbase", + CanonicalName: "localhost:8000/private/moonbase", + Official: false, + }, + "localhost:8000/privatebase": { + Index: &IndexInfo{ + Name: "localhost:8000", + Official: false, + }, + RemoteName: "privatebase", + LocalName: "localhost:8000/privatebase", + CanonicalName: "localhost:8000/privatebase", + Official: false, + }, + "example.com/private/moonbase": { + Index: &IndexInfo{ + Name: "example.com", + Official: false, + }, + RemoteName: "private/moonbase", + LocalName: "example.com/private/moonbase", + CanonicalName: "example.com/private/moonbase", + Official: false, + }, + "example.com/privatebase": { + Index: &IndexInfo{ + Name: "example.com", + Official: false, + }, + RemoteName: "privatebase", + LocalName: "example.com/privatebase", + CanonicalName: "example.com/privatebase", + Official: false, + }, + "example.com:8000/private/moonbase": { + Index: &IndexInfo{ + Name: "example.com:8000", + Official: false, + }, + RemoteName: "private/moonbase", + LocalName: "example.com:8000/private/moonbase", + CanonicalName: "example.com:8000/private/moonbase", + Official: false, + }, + "example.com:8000/privatebase": { + Index: &IndexInfo{ + Name: "example.com:8000", + Official: false, + }, + RemoteName: "privatebase", + LocalName: "example.com:8000/privatebase", + CanonicalName: "example.com:8000/privatebase", + Official: false, + }, + "localhost/private/moonbase": { + Index: &IndexInfo{ + Name: "localhost", + Official: false, + }, + RemoteName: "private/moonbase", + LocalName: "localhost/private/moonbase", + CanonicalName: "localhost/private/moonbase", + Official: false, + }, + "localhost/privatebase": { + Index: &IndexInfo{ + Name: "localhost", + Official: false, + }, + RemoteName: "privatebase", + LocalName: "localhost/privatebase", + CanonicalName: "localhost/privatebase", + Official: false, + }, + IndexServerName() + "/public/moonbase": { + Index: &IndexInfo{ + Name: IndexServerName(), + Official: true, + }, + RemoteName: "public/moonbase", + LocalName: "public/moonbase", + CanonicalName: "public/moonbase", + Official: false, + }, + "index." + IndexServerName() + "/public/moonbase": { + Index: &IndexInfo{ + Name: IndexServerName(), + Official: true, + }, + RemoteName: "public/moonbase", + LocalName: "public/moonbase", + CanonicalName: "public/moonbase", + Official: false, + }, + IndexServerName() + "/public/moonbase": { + Index: &IndexInfo{ + Name: IndexServerName(), + Official: true, + }, + RemoteName: "public/moonbase", + LocalName: "public/moonbase", + CanonicalName: "public/moonbase", + Official: false, + }, + "ubuntu-12.04-base": { + Index: &IndexInfo{ + Name: IndexServerName(), + Official: true, + }, + RemoteName: "library/ubuntu-12.04-base", + LocalName: "ubuntu-12.04-base", + CanonicalName: "ubuntu-12.04-base", + Official: true, + }, + IndexServerName() + "/ubuntu-12.04-base": { + Index: &IndexInfo{ + Name: IndexServerName(), + Official: true, + }, + RemoteName: "library/ubuntu-12.04-base", + LocalName: "ubuntu-12.04-base", + CanonicalName: "ubuntu-12.04-base", + Official: true, + }, + IndexServerName() + "/ubuntu-12.04-base": { + Index: &IndexInfo{ + Name: IndexServerName(), + Official: true, + }, + RemoteName: "library/ubuntu-12.04-base", + LocalName: "ubuntu-12.04-base", + CanonicalName: "ubuntu-12.04-base", + Official: true, + }, + "index." + IndexServerName() + "/ubuntu-12.04-base": { + Index: &IndexInfo{ + Name: IndexServerName(), + Official: true, + }, + RemoteName: "library/ubuntu-12.04-base", + LocalName: "ubuntu-12.04-base", + CanonicalName: "ubuntu-12.04-base", + Official: true, + }, + } + + for reposName, expectedRepoInfo := range expectedRepoInfos { + repoInfo, err := ParseRepositoryInfo(reposName) + if err != nil { + t.Error(err) + } else { + checkEqual(t, repoInfo.Index.Name, expectedRepoInfo.Index.Name, reposName) + checkEqual(t, repoInfo.RemoteName, expectedRepoInfo.RemoteName, reposName) + checkEqual(t, repoInfo.LocalName, expectedRepoInfo.LocalName, reposName) + checkEqual(t, repoInfo.CanonicalName, expectedRepoInfo.CanonicalName, reposName) + checkEqual(t, repoInfo.Index.Official, expectedRepoInfo.Index.Official, reposName) + checkEqual(t, repoInfo.Official, expectedRepoInfo.Official, reposName) + } + } +} + +func TestNewIndexInfo(t *testing.T) { + testIndexInfo := func(config *ServiceConfig, expectedIndexInfos map[string]*IndexInfo) { + for indexName, expectedIndexInfo := range expectedIndexInfos { + index, err := NewIndexInfo(config, indexName) + if err != nil { + t.Fatal(err) + } else { + checkEqual(t, index.Name, expectedIndexInfo.Name, indexName+" name") + checkEqual(t, index.Official, expectedIndexInfo.Official, indexName+" is official") + checkEqual(t, index.Secure, expectedIndexInfo.Secure, indexName+" is secure") + checkEqual(t, len(index.Mirrors), len(expectedIndexInfo.Mirrors), indexName+" mirrors") + } + } + } + + config := NewServiceConfig(nil) + noMirrors := make([]string, 0) + expectedIndexInfos := map[string]*IndexInfo{ + IndexServerName(): { + Name: IndexServerName(), + Official: true, + Secure: true, + Mirrors: noMirrors, + }, + "index." + IndexServerName(): { + Name: IndexServerName(), + Official: true, + Secure: true, + Mirrors: noMirrors, + }, + "example.com": { + Name: "example.com", + Official: false, + Secure: true, + Mirrors: noMirrors, + }, + "127.0.0.1:5000": { + Name: "127.0.0.1:5000", + Official: false, + Secure: false, + Mirrors: noMirrors, + }, + } + testIndexInfo(config, expectedIndexInfos) + + publicMirrors := []string{"http://mirror1.local", "http://mirror2.local"} + config = makeServiceConfig(publicMirrors, []string{"example.com"}) + + expectedIndexInfos = map[string]*IndexInfo{ + IndexServerName(): { + Name: IndexServerName(), + Official: true, + Secure: true, + Mirrors: publicMirrors, + }, + "index." + IndexServerName(): { + Name: IndexServerName(), + Official: true, + Secure: true, + Mirrors: publicMirrors, + }, + "example.com": { + Name: "example.com", + Official: false, + Secure: false, + Mirrors: noMirrors, + }, + "example.com:5000": { + Name: "example.com:5000", + Official: false, + Secure: true, + Mirrors: noMirrors, + }, + "127.0.0.1": { + Name: "127.0.0.1", + Official: false, + Secure: false, + Mirrors: noMirrors, + }, + "127.0.0.1:5000": { + Name: "127.0.0.1:5000", + Official: false, + Secure: false, + Mirrors: noMirrors, + }, + "other.com": { + Name: "other.com", + Official: false, + Secure: true, + Mirrors: noMirrors, + }, + } + testIndexInfo(config, expectedIndexInfos) + + config = makeServiceConfig(nil, []string{"42.42.0.0/16"}) + expectedIndexInfos = map[string]*IndexInfo{ + "example.com": { + Name: "example.com", + Official: false, + Secure: false, + Mirrors: noMirrors, + }, + "example.com:5000": { + Name: "example.com:5000", + Official: false, + Secure: false, + Mirrors: noMirrors, + }, + "127.0.0.1": { + Name: "127.0.0.1", + Official: false, + Secure: false, + Mirrors: noMirrors, + }, + "127.0.0.1:5000": { + Name: "127.0.0.1:5000", + Official: false, + Secure: false, + Mirrors: noMirrors, + }, + "other.com": { + Name: "other.com", + Official: false, + Secure: true, + Mirrors: noMirrors, + }, + } + testIndexInfo(config, expectedIndexInfos) } func TestPushRegistryTag(t *testing.T) { @@ -232,7 +738,7 @@ func TestSearchRepositories(t *testing.T) { assertEqual(t, results.Results[0].StarCount, 42, "Expected 'fakeimage' a ot hae 42 stars") } -func TestValidRepositoryName(t *testing.T) { +func TestValidRemoteName(t *testing.T) { validRepositoryNames := []string{ // Sanity check. "docker/docker", @@ -247,7 +753,7 @@ func TestValidRepositoryName(t *testing.T) { "____/____", } for _, repositoryName := range validRepositoryNames { - if err := validateRepositoryName(repositoryName); err != nil { + if err := validateRemoteName(repositoryName); err != nil { t.Errorf("Repository name should be valid: %v. Error: %v", repositoryName, err) } } @@ -277,7 +783,7 @@ func TestValidRepositoryName(t *testing.T) { "docker/", } for _, repositoryName := range invalidRepositoryNames { - if err := validateRepositoryName(repositoryName); err == nil { + if err := validateRemoteName(repositoryName); err == nil { t.Errorf("Repository name should be invalid: %v", repositoryName) } } @@ -350,13 +856,13 @@ func TestAddRequiredHeadersToRedirectedRequests(t *testing.T) { } } -func TestIsSecure(t *testing.T) { +func TestIsSecureIndex(t *testing.T) { tests := []struct { addr string insecureRegistries []string expected bool }{ - {IndexServerURL.Host, nil, true}, + {IndexServerName(), nil, true}, {"example.com", []string{}, true}, {"example.com", []string{"example.com"}, false}, {"localhost", []string{"localhost:5000"}, false}, @@ -383,10 +889,9 @@ func TestIsSecure(t *testing.T) { {"invalid.domain.com:5000", []string{"invalid.domain.com:5000"}, false}, } for _, tt := range tests { - // TODO: remove this once we remove localhost insecure by default - insecureRegistries := append(tt.insecureRegistries, "127.0.0.0/8") - if sec, err := isSecure(tt.addr, insecureRegistries); err != nil || sec != tt.expected { - t.Fatalf("isSecure failed for %q %v, expected %v got %v. Error: %v", tt.addr, insecureRegistries, tt.expected, sec, err) + config := makeServiceConfig(nil, tt.insecureRegistries) + if sec := config.isSecureIndex(tt.addr); sec != tt.expected { + t.Errorf("isSecureIndex failed for %q %v, expected %v got %v", tt.addr, tt.insecureRegistries, tt.expected, sec) } } } diff --git a/docs/service.go b/docs/service.go index 53e8278b..310539c4 100644 --- a/docs/service.go +++ b/docs/service.go @@ -13,14 +13,14 @@ import ( // 'pull': Download images from any registry (TODO) // 'push': Upload images to any registry (TODO) type Service struct { - insecureRegistries []string + Config *ServiceConfig } // NewService returns a new instance of Service ready to be // installed no an engine. -func NewService(insecureRegistries []string) *Service { +func NewService(options *Options) *Service { return &Service{ - insecureRegistries: insecureRegistries, + Config: NewServiceConfig(options), } } @@ -28,6 +28,9 @@ func NewService(insecureRegistries []string) *Service { func (s *Service) Install(eng *engine.Engine) error { eng.Register("auth", s.Auth) eng.Register("search", s.Search) + eng.Register("resolve_repository", s.ResolveRepository) + eng.Register("resolve_index", s.ResolveIndex) + eng.Register("registry_config", s.GetRegistryConfig) return nil } @@ -39,15 +42,18 @@ func (s *Service) Auth(job *engine.Job) engine.Status { job.GetenvJson("authConfig", authConfig) - if addr := authConfig.ServerAddress; addr != "" && addr != IndexServerAddress() { - endpoint, err := NewEndpoint(addr, s.insecureRegistries) + if authConfig.ServerAddress != "" { + index, err := ResolveIndexInfo(job, authConfig.ServerAddress) if err != nil { return job.Error(err) } - if _, err := endpoint.Ping(); err != nil { - return job.Error(err) + if !index.Official { + endpoint, err := NewEndpoint(index) + if err != nil { + return job.Error(err) + } + authConfig.ServerAddress = endpoint.String() } - authConfig.ServerAddress = endpoint.String() } status, err := Login(authConfig, HTTPRequestFactory(nil)) @@ -87,12 +93,12 @@ func (s *Service) Search(job *engine.Job) engine.Status { job.GetenvJson("authConfig", authConfig) job.GetenvJson("metaHeaders", metaHeaders) - hostname, term, err := ResolveRepositoryName(term) + repoInfo, err := ResolveRepositoryInfo(job, term) if err != nil { return job.Error(err) } - - endpoint, err := NewEndpoint(hostname, s.insecureRegistries) + // *TODO: Search multiple indexes. + endpoint, err := repoInfo.GetEndpoint() if err != nil { return job.Error(err) } @@ -100,7 +106,7 @@ func (s *Service) Search(job *engine.Job) engine.Status { if err != nil { return job.Error(err) } - results, err := r.SearchRepositories(term) + results, err := r.SearchRepositories(repoInfo.GetSearchTerm()) if err != nil { return job.Error(err) } @@ -116,3 +122,92 @@ func (s *Service) Search(job *engine.Job) engine.Status { } return engine.StatusOK } + +// ResolveRepository splits a repository name into its components +// and configuration of the associated registry. +func (s *Service) ResolveRepository(job *engine.Job) engine.Status { + var ( + reposName = job.Args[0] + ) + + repoInfo, err := NewRepositoryInfo(s.Config, reposName) + if err != nil { + return job.Error(err) + } + + out := engine.Env{} + err = out.SetJson("repository", repoInfo) + if err != nil { + return job.Error(err) + } + out.WriteTo(job.Stdout) + + return engine.StatusOK +} + +// Convenience wrapper for calling resolve_repository Job from a running job. +func ResolveRepositoryInfo(jobContext *engine.Job, reposName string) (*RepositoryInfo, error) { + job := jobContext.Eng.Job("resolve_repository", reposName) + env, err := job.Stdout.AddEnv() + if err != nil { + return nil, err + } + if err := job.Run(); err != nil { + return nil, err + } + info := RepositoryInfo{} + if err := env.GetJson("repository", &info); err != nil { + return nil, err + } + return &info, nil +} + +// ResolveIndex takes indexName and returns index info +func (s *Service) ResolveIndex(job *engine.Job) engine.Status { + var ( + indexName = job.Args[0] + ) + + index, err := NewIndexInfo(s.Config, indexName) + if err != nil { + return job.Error(err) + } + + out := engine.Env{} + err = out.SetJson("index", index) + if err != nil { + return job.Error(err) + } + out.WriteTo(job.Stdout) + + return engine.StatusOK +} + +// Convenience wrapper for calling resolve_index Job from a running job. +func ResolveIndexInfo(jobContext *engine.Job, indexName string) (*IndexInfo, error) { + job := jobContext.Eng.Job("resolve_index", indexName) + env, err := job.Stdout.AddEnv() + if err != nil { + return nil, err + } + if err := job.Run(); err != nil { + return nil, err + } + info := IndexInfo{} + if err := env.GetJson("index", &info); err != nil { + return nil, err + } + return &info, nil +} + +// GetRegistryConfig returns current registry configuration. +func (s *Service) GetRegistryConfig(job *engine.Job) engine.Status { + out := engine.Env{} + err := out.SetJson("config", s.Config) + if err != nil { + return job.Error(err) + } + out.WriteTo(job.Stdout) + + return engine.StatusOK +} diff --git a/docs/types.go b/docs/types.go index 3b429f19..fbbc0e70 100644 --- a/docs/types.go +++ b/docs/types.go @@ -65,3 +65,44 @@ const ( APIVersion1 = iota + 1 APIVersion2 ) + +// RepositoryInfo Examples: +// { +// "Index" : { +// "Name" : "docker.io", +// "Mirrors" : ["https://registry-2.docker.io/v1/", "https://registry-3.docker.io/v1/"], +// "Secure" : true, +// "Official" : true, +// }, +// "RemoteName" : "library/debian", +// "LocalName" : "debian", +// "CanonicalName" : "docker.io/debian" +// "Official" : true, +// } + +// { +// "Index" : { +// "Name" : "127.0.0.1:5000", +// "Mirrors" : [], +// "Secure" : false, +// "Official" : false, +// }, +// "RemoteName" : "user/repo", +// "LocalName" : "127.0.0.1:5000/user/repo", +// "CanonicalName" : "127.0.0.1:5000/user/repo", +// "Official" : false, +// } +type IndexInfo struct { + Name string + Mirrors []string + Secure bool + Official bool +} + +type RepositoryInfo struct { + Index *IndexInfo + RemoteName string + LocalName string + CanonicalName string + Official bool +} From c899a49a95bc05b896b46460735a268678efc1a3 Mon Sep 17 00:00:00 2001 From: Don Kjer Date: Wed, 7 Jan 2015 23:42:01 +0000 Subject: [PATCH 205/375] Moving NewIndexInfo, NewRepositoryInfo and associated helpers into config.go Signed-off-by: Don Kjer --- docs/auth.go | 15 -- docs/config.go | 312 ++++++++++++++++++++++++++++++++++++++---- docs/endpoint.go | 49 ------- docs/registry.go | 190 +------------------------ docs/registry_test.go | 2 +- docs/service.go | 4 +- 6 files changed, 290 insertions(+), 282 deletions(-) diff --git a/docs/auth.go b/docs/auth.go index 8382869b..102078d7 100644 --- a/docs/auth.go +++ b/docs/auth.go @@ -17,13 +17,6 @@ import ( const ( // Where we store the config file CONFIGFILE = ".dockercfg" - - // Only used for user auth + account creation - INDEXSERVER = "https://index.docker.io/v1/" - REGISTRYSERVER = "https://registry-1.docker.io/v1/" - INDEXNAME = "docker.io" - - // INDEXSERVER = "https://registry-stage.hub.docker.com/v1/" ) var ( @@ -43,14 +36,6 @@ type ConfigFile struct { rootPath string } -func IndexServerAddress() string { - return INDEXSERVER -} - -func IndexServerName() string { - return INDEXNAME -} - // create a base64 encoded auth string to store in config func encodeAuth(authConfig *AuthConfig) string { authStr := authConfig.Username + ":" + authConfig.Password diff --git a/docs/config.go b/docs/config.go index bd993edd..b5652b15 100644 --- a/docs/config.go +++ b/docs/config.go @@ -2,12 +2,16 @@ package registry import ( "encoding/json" + "errors" "fmt" "net" "net/url" + "regexp" + "strings" "github.com/docker/docker/opts" flag "github.com/docker/docker/pkg/mflag" + "github.com/docker/docker/utils" ) // Options holds command line options. @@ -16,6 +20,30 @@ type Options struct { InsecureRegistries opts.ListOpts } +const ( + // Only used for user auth + account creation + INDEXSERVER = "https://index.docker.io/v1/" + REGISTRYSERVER = "https://registry-1.docker.io/v1/" + INDEXNAME = "docker.io" + + // INDEXSERVER = "https://registry-stage.hub.docker.com/v1/" +) + +var ( + ErrInvalidRepositoryName = errors.New("Invalid repository name (ex: \"registry.domain.tld/myrepos\")") + emptyServiceConfig = NewServiceConfig(nil) + validNamespaceChars = regexp.MustCompile(`^([a-z0-9-_]*)$`) + validRepo = regexp.MustCompile(`^([a-z0-9-_.]+)$`) +) + +func IndexServerAddress() string { + return INDEXSERVER +} + +func IndexServerName() string { + return INDEXNAME +} + // InstallFlags adds command-line options to the top-level flag parser for // the current process. func (options *Options) InstallFlags() { @@ -25,34 +53,6 @@ func (options *Options) InstallFlags() { flag.Var(&options.InsecureRegistries, []string{"-insecure-registry"}, "Enable insecure communication with specified registries (no certificate verification for HTTPS and enable HTTP fallback) (e.g., localhost:5000 or 10.20.0.0/16)") } -// ValidateMirror validates an HTTP(S) registry mirror -func ValidateMirror(val string) (string, error) { - uri, err := url.Parse(val) - if err != nil { - return "", fmt.Errorf("%s is not a valid URI", val) - } - - if uri.Scheme != "http" && uri.Scheme != "https" { - return "", fmt.Errorf("Unsupported scheme %s", uri.Scheme) - } - - if uri.Path != "" || uri.RawQuery != "" || uri.Fragment != "" { - return "", fmt.Errorf("Unsupported path/query/fragment at end of the URI") - } - - return fmt.Sprintf("%s://%s/v1/", uri.Scheme, uri.Host), nil -} - -// ValidateIndexName validates an index name. -func ValidateIndexName(val string) (string, error) { - // 'index.docker.io' => 'docker.io' - if val == "index."+IndexServerName() { - val = IndexServerName() - } - // *TODO: Check if valid hostname[:port]/ip[:port]? - return val, nil -} - type netIPNet net.IPNet func (ipnet *netIPNet) MarshalJSON() ([]byte, error) { @@ -124,3 +124,259 @@ func NewServiceConfig(options *Options) *ServiceConfig { return config } + +// isSecureIndex returns false if the provided indexName is part of the list of insecure registries +// Insecure registries accept HTTP and/or accept HTTPS with certificates from unknown CAs. +// +// The list of insecure registries can contain an element with CIDR notation to specify a whole subnet. +// If the subnet contains one of the IPs of the registry specified by indexName, the latter is considered +// insecure. +// +// indexName should be a URL.Host (`host:port` or `host`) where the `host` part can be either a domain name +// or an IP address. If it is a domain name, then it will be resolved in order to check if the IP is contained +// in a subnet. If the resolving is not successful, isSecureIndex will only try to match hostname to any element +// of insecureRegistries. +func (config *ServiceConfig) isSecureIndex(indexName string) bool { + // Check for configured index, first. This is needed in case isSecureIndex + // is called from anything besides NewIndexInfo, in order to honor per-index configurations. + if index, ok := config.IndexConfigs[indexName]; ok { + return index.Secure + } + + host, _, err := net.SplitHostPort(indexName) + if err != nil { + // assume indexName is of the form `host` without the port and go on. + host = indexName + } + + addrs, err := lookupIP(host) + if err != nil { + ip := net.ParseIP(host) + if ip != nil { + addrs = []net.IP{ip} + } + + // if ip == nil, then `host` is neither an IP nor it could be looked up, + // either because the index is unreachable, or because the index is behind an HTTP proxy. + // So, len(addrs) == 0 and we're not aborting. + } + + // Try CIDR notation only if addrs has any elements, i.e. if `host`'s IP could be determined. + for _, addr := range addrs { + for _, ipnet := range config.InsecureRegistryCIDRs { + // check if the addr falls in the subnet + if (*net.IPNet)(ipnet).Contains(addr) { + return false + } + } + } + + return true +} + +// ValidateMirror validates an HTTP(S) registry mirror +func ValidateMirror(val string) (string, error) { + uri, err := url.Parse(val) + if err != nil { + return "", fmt.Errorf("%s is not a valid URI", val) + } + + if uri.Scheme != "http" && uri.Scheme != "https" { + return "", fmt.Errorf("Unsupported scheme %s", uri.Scheme) + } + + if uri.Path != "" || uri.RawQuery != "" || uri.Fragment != "" { + return "", fmt.Errorf("Unsupported path/query/fragment at end of the URI") + } + + return fmt.Sprintf("%s://%s/v1/", uri.Scheme, uri.Host), nil +} + +// ValidateIndexName validates an index name. +func ValidateIndexName(val string) (string, error) { + // 'index.docker.io' => 'docker.io' + if val == "index."+IndexServerName() { + val = IndexServerName() + } + // *TODO: Check if valid hostname[:port]/ip[:port]? + return val, nil +} + +func validateRemoteName(remoteName string) error { + var ( + namespace string + name string + ) + nameParts := strings.SplitN(remoteName, "/", 2) + if len(nameParts) < 2 { + namespace = "library" + name = nameParts[0] + + // the repository name must not be a valid image ID + if err := utils.ValidateID(name); err == nil { + return fmt.Errorf("Invalid repository name (%s), cannot specify 64-byte hexadecimal strings", name) + } + } else { + namespace = nameParts[0] + name = nameParts[1] + } + if !validNamespaceChars.MatchString(namespace) { + return fmt.Errorf("Invalid namespace name (%s). Only [a-z0-9-_] are allowed.", namespace) + } + if len(namespace) < 4 || len(namespace) > 30 { + return fmt.Errorf("Invalid namespace name (%s). Cannot be fewer than 4 or more than 30 characters.", namespace) + } + if strings.HasPrefix(namespace, "-") || strings.HasSuffix(namespace, "-") { + return fmt.Errorf("Invalid namespace name (%s). Cannot begin or end with a hyphen.", namespace) + } + if strings.Contains(namespace, "--") { + return fmt.Errorf("Invalid namespace name (%s). Cannot contain consecutive hyphens.", namespace) + } + if !validRepo.MatchString(name) { + return fmt.Errorf("Invalid repository name (%s), only [a-z0-9-_.] are allowed", name) + } + return nil +} + +func validateNoSchema(reposName string) error { + if strings.Contains(reposName, "://") { + // It cannot contain a scheme! + return ErrInvalidRepositoryName + } + return nil +} + +// ValidateRepositoryName validates a repository name +func ValidateRepositoryName(reposName string) error { + var err error + if err = validateNoSchema(reposName); err != nil { + return err + } + indexName, remoteName := splitReposName(reposName) + if _, err = ValidateIndexName(indexName); err != nil { + return err + } + return validateRemoteName(remoteName) +} + +// NewIndexInfo returns IndexInfo configuration from indexName +func (config *ServiceConfig) NewIndexInfo(indexName string) (*IndexInfo, error) { + var err error + indexName, err = ValidateIndexName(indexName) + if err != nil { + return nil, err + } + + // Return any configured index info, first. + if index, ok := config.IndexConfigs[indexName]; ok { + return index, nil + } + + // Construct a non-configured index info. + index := &IndexInfo{ + Name: indexName, + Mirrors: make([]string, 0), + Official: false, + } + index.Secure = config.isSecureIndex(indexName) + return index, nil +} + +// GetAuthConfigKey special-cases using the full index address of the official +// index as the AuthConfig key, and uses the (host)name[:port] for private indexes. +func (index *IndexInfo) GetAuthConfigKey() string { + if index.Official { + return IndexServerAddress() + } + return index.Name +} + +// splitReposName breaks a reposName into an index name and remote name +func splitReposName(reposName string) (string, string) { + nameParts := strings.SplitN(reposName, "/", 2) + var indexName, remoteName string + if len(nameParts) == 1 || (!strings.Contains(nameParts[0], ".") && + !strings.Contains(nameParts[0], ":") && nameParts[0] != "localhost") { + // This is a Docker Index repos (ex: samalba/hipache or ubuntu) + // 'docker.io' + indexName = IndexServerName() + remoteName = reposName + } else { + indexName = nameParts[0] + remoteName = nameParts[1] + } + return indexName, remoteName +} + +// NewRepositoryInfo validates and breaks down a repository name into a RepositoryInfo +func (config *ServiceConfig) NewRepositoryInfo(reposName string) (*RepositoryInfo, error) { + if err := validateNoSchema(reposName); err != nil { + return nil, err + } + + indexName, remoteName := splitReposName(reposName) + if err := validateRemoteName(remoteName); err != nil { + return nil, err + } + + repoInfo := &RepositoryInfo{ + RemoteName: remoteName, + } + + var err error + repoInfo.Index, err = config.NewIndexInfo(indexName) + if err != nil { + return nil, err + } + + if repoInfo.Index.Official { + normalizedName := repoInfo.RemoteName + if strings.HasPrefix(normalizedName, "library/") { + // If pull "library/foo", it's stored locally under "foo" + normalizedName = strings.SplitN(normalizedName, "/", 2)[1] + } + + repoInfo.LocalName = normalizedName + repoInfo.RemoteName = normalizedName + // If the normalized name does not contain a '/' (e.g. "foo") + // then it is an official repo. + if strings.IndexRune(normalizedName, '/') == -1 { + repoInfo.Official = true + // Fix up remote name for official repos. + repoInfo.RemoteName = "library/" + normalizedName + } + + // *TODO: Prefix this with 'docker.io/'. + repoInfo.CanonicalName = repoInfo.LocalName + } else { + // *TODO: Decouple index name from hostname (via registry configuration?) + repoInfo.LocalName = repoInfo.Index.Name + "/" + repoInfo.RemoteName + repoInfo.CanonicalName = repoInfo.LocalName + } + return repoInfo, nil +} + +// GetSearchTerm special-cases using local name for official index, and +// remote name for private indexes. +func (repoInfo *RepositoryInfo) GetSearchTerm() string { + if repoInfo.Index.Official { + return repoInfo.LocalName + } + return repoInfo.RemoteName +} + +// ParseRepositoryInfo performs the breakdown of a repository name into a RepositoryInfo, but +// lacks registry configuration. +func ParseRepositoryInfo(reposName string) (*RepositoryInfo, error) { + return emptyServiceConfig.NewRepositoryInfo(reposName) +} + +// NormalizeLocalName transforms a repository name into a normalize LocalName +// Passes through the name without transformation on error (image id, etc) +func NormalizeLocalName(name string) string { + repoInfo, err := ParseRepositoryInfo(name) + if err != nil { + return name + } + return repoInfo.LocalName +} diff --git a/docs/endpoint.go b/docs/endpoint.go index 86f53744..95680c5e 100644 --- a/docs/endpoint.go +++ b/docs/endpoint.go @@ -157,52 +157,3 @@ func (e Endpoint) Ping() (RegistryInfo, error) { log.Debugf("RegistryInfo.Standalone: %t", info.Standalone) return info, nil } - -// isSecureIndex returns false if the provided indexName is part of the list of insecure registries -// Insecure registries accept HTTP and/or accept HTTPS with certificates from unknown CAs. -// -// The list of insecure registries can contain an element with CIDR notation to specify a whole subnet. -// If the subnet contains one of the IPs of the registry specified by indexName, the latter is considered -// insecure. -// -// indexName should be a URL.Host (`host:port` or `host`) where the `host` part can be either a domain name -// or an IP address. If it is a domain name, then it will be resolved in order to check if the IP is contained -// in a subnet. If the resolving is not successful, isSecureIndex will only try to match hostname to any element -// of insecureRegistries. -func (config *ServiceConfig) isSecureIndex(indexName string) bool { - // Check for configured index, first. This is needed in case isSecureIndex - // is called from anything besides NewIndexInfo, in order to honor per-index configurations. - if index, ok := config.IndexConfigs[indexName]; ok { - return index.Secure - } - - host, _, err := net.SplitHostPort(indexName) - if err != nil { - // assume indexName is of the form `host` without the port and go on. - host = indexName - } - - addrs, err := lookupIP(host) - if err != nil { - ip := net.ParseIP(host) - if ip != nil { - addrs = []net.IP{ip} - } - - // if ip == nil, then `host` is neither an IP nor it could be looked up, - // either because the index is unreachable, or because the index is behind an HTTP proxy. - // So, len(addrs) == 0 and we're not aborting. - } - - // Try CIDR notation only if addrs has any elements, i.e. if `host`'s IP could be determined. - for _, addr := range addrs { - for _, ipnet := range config.InsecureRegistryCIDRs { - // check if the addr falls in the subnet - if (*net.IPNet)(ipnet).Contains(addr) { - return false - } - } - } - - return true -} diff --git a/docs/registry.go b/docs/registry.go index de724ee2..77a78a82 100644 --- a/docs/registry.go +++ b/docs/registry.go @@ -10,7 +10,6 @@ import ( "net/http" "os" "path" - "regexp" "strings" "time" @@ -19,13 +18,9 @@ import ( ) var ( - ErrAlreadyExists = errors.New("Image already exists") - ErrInvalidRepositoryName = errors.New("Invalid repository name (ex: \"registry.domain.tld/myrepos\")") - ErrDoesNotExist = errors.New("Image does not exist") - errLoginRequired = errors.New("Authentication is required.") - validNamespaceChars = regexp.MustCompile(`^([a-z0-9-_]*)$`) - validRepo = regexp.MustCompile(`^([a-z0-9-_.]+)$`) - emptyServiceConfig = NewServiceConfig(nil) + ErrAlreadyExists = errors.New("Image already exists") + ErrDoesNotExist = errors.New("Image does not exist") + errLoginRequired = errors.New("Authentication is required.") ) type TimeoutType uint32 @@ -161,185 +156,6 @@ func doRequest(req *http.Request, jar http.CookieJar, timeout TimeoutType, secur return res, client, err } -func validateRemoteName(remoteName string) error { - var ( - namespace string - name string - ) - nameParts := strings.SplitN(remoteName, "/", 2) - if len(nameParts) < 2 { - namespace = "library" - name = nameParts[0] - - // the repository name must not be a valid image ID - if err := utils.ValidateID(name); err == nil { - return fmt.Errorf("Invalid repository name (%s), cannot specify 64-byte hexadecimal strings", name) - } - } else { - namespace = nameParts[0] - name = nameParts[1] - } - if !validNamespaceChars.MatchString(namespace) { - return fmt.Errorf("Invalid namespace name (%s). Only [a-z0-9-_] are allowed.", namespace) - } - if len(namespace) < 4 || len(namespace) > 30 { - return fmt.Errorf("Invalid namespace name (%s). Cannot be fewer than 4 or more than 30 characters.", namespace) - } - if strings.HasPrefix(namespace, "-") || strings.HasSuffix(namespace, "-") { - return fmt.Errorf("Invalid namespace name (%s). Cannot begin or end with a hyphen.", namespace) - } - if strings.Contains(namespace, "--") { - return fmt.Errorf("Invalid namespace name (%s). Cannot contain consecutive hyphens.", namespace) - } - if !validRepo.MatchString(name) { - return fmt.Errorf("Invalid repository name (%s), only [a-z0-9-_.] are allowed", name) - } - return nil -} - -// NewIndexInfo returns IndexInfo configuration from indexName -func NewIndexInfo(config *ServiceConfig, indexName string) (*IndexInfo, error) { - var err error - indexName, err = ValidateIndexName(indexName) - if err != nil { - return nil, err - } - - // Return any configured index info, first. - if index, ok := config.IndexConfigs[indexName]; ok { - return index, nil - } - - // Construct a non-configured index info. - index := &IndexInfo{ - Name: indexName, - Mirrors: make([]string, 0), - Official: false, - } - index.Secure = config.isSecureIndex(indexName) - return index, nil -} - -func validateNoSchema(reposName string) error { - if strings.Contains(reposName, "://") { - // It cannot contain a scheme! - return ErrInvalidRepositoryName - } - return nil -} - -// splitReposName breaks a reposName into an index name and remote name -func splitReposName(reposName string) (string, string) { - nameParts := strings.SplitN(reposName, "/", 2) - var indexName, remoteName string - if len(nameParts) == 1 || (!strings.Contains(nameParts[0], ".") && - !strings.Contains(nameParts[0], ":") && nameParts[0] != "localhost") { - // This is a Docker Index repos (ex: samalba/hipache or ubuntu) - // 'docker.io' - indexName = IndexServerName() - remoteName = reposName - } else { - indexName = nameParts[0] - remoteName = nameParts[1] - } - return indexName, remoteName -} - -// NewRepositoryInfo validates and breaks down a repository name into a RepositoryInfo -func NewRepositoryInfo(config *ServiceConfig, reposName string) (*RepositoryInfo, error) { - if err := validateNoSchema(reposName); err != nil { - return nil, err - } - - indexName, remoteName := splitReposName(reposName) - if err := validateRemoteName(remoteName); err != nil { - return nil, err - } - - repoInfo := &RepositoryInfo{ - RemoteName: remoteName, - } - - var err error - repoInfo.Index, err = NewIndexInfo(config, indexName) - if err != nil { - return nil, err - } - - if repoInfo.Index.Official { - normalizedName := repoInfo.RemoteName - if strings.HasPrefix(normalizedName, "library/") { - // If pull "library/foo", it's stored locally under "foo" - normalizedName = strings.SplitN(normalizedName, "/", 2)[1] - } - - repoInfo.LocalName = normalizedName - repoInfo.RemoteName = normalizedName - // If the normalized name does not contain a '/' (e.g. "foo") - // then it is an official repo. - if strings.IndexRune(normalizedName, '/') == -1 { - repoInfo.Official = true - // Fix up remote name for official repos. - repoInfo.RemoteName = "library/" + normalizedName - } - - // *TODO: Prefix this with 'docker.io/'. - repoInfo.CanonicalName = repoInfo.LocalName - } else { - // *TODO: Decouple index name from hostname (via registry configuration?) - repoInfo.LocalName = repoInfo.Index.Name + "/" + repoInfo.RemoteName - repoInfo.CanonicalName = repoInfo.LocalName - } - return repoInfo, nil -} - -// ValidateRepositoryName validates a repository name -func ValidateRepositoryName(reposName string) error { - var err error - if err = validateNoSchema(reposName); err != nil { - return err - } - indexName, remoteName := splitReposName(reposName) - if _, err = ValidateIndexName(indexName); err != nil { - return err - } - return validateRemoteName(remoteName) -} - -// ParseRepositoryInfo performs the breakdown of a repository name into a RepositoryInfo, but -// lacks registry configuration. -func ParseRepositoryInfo(reposName string) (*RepositoryInfo, error) { - return NewRepositoryInfo(emptyServiceConfig, reposName) -} - -// NormalizeLocalName transforms a repository name into a normalize LocalName -// Passes through the name without transformation on error (image id, etc) -func NormalizeLocalName(name string) string { - repoInfo, err := ParseRepositoryInfo(name) - if err != nil { - return name - } - return repoInfo.LocalName -} - -// GetAuthConfigKey special-cases using the full index address of the official -// index as the AuthConfig key, and uses the (host)name[:port] for private indexes. -func (index *IndexInfo) GetAuthConfigKey() string { - if index.Official { - return IndexServerAddress() - } - return index.Name -} - -// GetSearchTerm special-cases using local name for official index, and -// remote name for private indexes. -func (repoInfo *RepositoryInfo) GetSearchTerm() string { - if repoInfo.Index.Official { - return repoInfo.LocalName - } - return repoInfo.RemoteName -} - func trustedLocation(req *http.Request) bool { var ( trusteds = []string{"docker.com", "docker.io"} diff --git a/docs/registry_test.go b/docs/registry_test.go index 511d7eb1..6bf31505 100644 --- a/docs/registry_test.go +++ b/docs/registry_test.go @@ -561,7 +561,7 @@ func TestParseRepositoryInfo(t *testing.T) { func TestNewIndexInfo(t *testing.T) { testIndexInfo := func(config *ServiceConfig, expectedIndexInfos map[string]*IndexInfo) { for indexName, expectedIndexInfo := range expectedIndexInfos { - index, err := NewIndexInfo(config, indexName) + index, err := config.NewIndexInfo(indexName) if err != nil { t.Fatal(err) } else { diff --git a/docs/service.go b/docs/service.go index 310539c4..c34e3842 100644 --- a/docs/service.go +++ b/docs/service.go @@ -130,7 +130,7 @@ func (s *Service) ResolveRepository(job *engine.Job) engine.Status { reposName = job.Args[0] ) - repoInfo, err := NewRepositoryInfo(s.Config, reposName) + repoInfo, err := s.Config.NewRepositoryInfo(reposName) if err != nil { return job.Error(err) } @@ -168,7 +168,7 @@ func (s *Service) ResolveIndex(job *engine.Job) engine.Status { indexName = job.Args[0] ) - index, err := NewIndexInfo(s.Config, indexName) + index, err := s.Config.NewIndexInfo(indexName) if err != nil { return job.Error(err) } From 23f9f8c3f4d15e7bbb5d21714227049690642fb6 Mon Sep 17 00:00:00 2001 From: Qiang Huang Date: Fri, 9 Jan 2015 09:06:27 +0800 Subject: [PATCH 206/375] registry: fix minor type Signed-off-by: Qiang Huang --- docs/session.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/session.go b/docs/session.go index 28cf18fb..781a91b1 100644 --- a/docs/session.go +++ b/docs/session.go @@ -584,7 +584,7 @@ func (r *Session) SearchRepositories(term string) (*SearchResults, error) { } defer res.Body.Close() if res.StatusCode != 200 { - return nil, utils.NewHTTPRequestError(fmt.Sprintf("Unexepected status code %d", res.StatusCode), res) + return nil, utils.NewHTTPRequestError(fmt.Sprintf("Unexpected status code %d", res.StatusCode), res) } result := new(SearchResults) err = json.NewDecoder(res.Body).Decode(result) From 1f98347924e32e6d493911539fcf46eabdb2119e Mon Sep 17 00:00:00 2001 From: Alexander Morozov Date: Wed, 14 Jan 2015 14:12:03 -0800 Subject: [PATCH 207/375] Fix format calls as suggested by vet Signed-off-by: Alexander Morozov --- docs/auth_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/auth_test.go b/docs/auth_test.go index 22f87994..9cc299aa 100644 --- a/docs/auth_test.go +++ b/docs/auth_test.go @@ -151,7 +151,7 @@ func TestResolveAuthConfigFullURL(t *testing.T) { for configKey, registries := range validRegistries { configured, ok := expectedAuths[configKey] if !ok || configured.Email == "" { - t.Fatal() + t.Fail() } index := &IndexInfo{ Name: configKey, From 6b400cd63c203065dcf2f73256ec3caee012243b Mon Sep 17 00:00:00 2001 From: Josh Hawn Date: Thu, 11 Dec 2014 17:55:15 -0800 Subject: [PATCH 208/375] Adds support for v2 registry login summary of changes: registry/auth.go - More logging around the login functions - split Login() out to handle different code paths for v1 (unchanged logic) and v2 (does not currently do account creation) - handling for either basic or token based login attempts registry/authchallenge.go - New File - credit to Brian Bland (github: BrianBland) - handles parsing of WWW-Authenticate response headers registry/endpoint.go - EVEN MOAR LOGGING - Many edits throught to make the coad less dense. Sparse code is more readable code. - slit Ping() out to handle different code paths for v1 (unchanged logic) and v2. - Updated Endpoint struct type to include an entry for authorization challenges discovered during ping of a v2 registry. - If registry endpoint version is unknown, v2 code path is first attempted, then fallback to v1 upon failure. registry/service.go - STILL MOAR LOGGING - simplified the logic around starting the 'auth' job. registry/session.go - updated use of a registry.Endpoint struct field. registry/token.go - New File - Handles getting token from the parameters of a token auth challenge. - Modified from function written by Brian Bland (see above credit). registry/types.go - Removed 'DefaultAPIVersion' in lieu of 'APIVersionUnknown = 0'` Docker-DCO-1.1-Signed-off-by: Josh Hawn (github: jlhawn) --- docs/auth.go | 114 +++++++++++++++++++++++++++++- docs/authchallenge.go | 150 +++++++++++++++++++++++++++++++++++++++ docs/endpoint.go | 158 +++++++++++++++++++++++++++++++----------- docs/endpoint_test.go | 6 +- docs/service.go | 42 +++++++---- docs/session.go | 2 +- docs/token.go | 70 +++++++++++++++++++ docs/types.go | 5 +- 8 files changed, 484 insertions(+), 63 deletions(-) create mode 100644 docs/authchallenge.go create mode 100644 docs/token.go diff --git a/docs/auth.go b/docs/auth.go index 102078d7..2044236c 100644 --- a/docs/auth.go +++ b/docs/auth.go @@ -11,6 +11,7 @@ import ( "path" "strings" + log "github.com/Sirupsen/logrus" "github.com/docker/docker/utils" ) @@ -144,8 +145,18 @@ func SaveConfig(configFile *ConfigFile) error { return nil } -// try to register/login to the registry server -func Login(authConfig *AuthConfig, factory *utils.HTTPRequestFactory) (string, error) { +// Login tries to register/login to the registry server. +func Login(authConfig *AuthConfig, registryEndpoint *Endpoint, factory *utils.HTTPRequestFactory) (string, error) { + // Separates the v2 registry login logic from the v1 logic. + if registryEndpoint.Version == APIVersion2 { + return loginV2(authConfig, registryEndpoint, factory) + } + + return loginV1(authConfig, registryEndpoint, factory) +} + +// loginV1 tries to register/login to the v1 registry server. +func loginV1(authConfig *AuthConfig, registryEndpoint *Endpoint, factory *utils.HTTPRequestFactory) (string, error) { var ( status string reqBody []byte @@ -161,6 +172,8 @@ func Login(authConfig *AuthConfig, factory *utils.HTTPRequestFactory) (string, e serverAddress = authConfig.ServerAddress ) + log.Debugf("attempting v1 login to registry endpoint %s", registryEndpoint) + if serverAddress == "" { return "", fmt.Errorf("Server Error: Server Address not set.") } @@ -253,6 +266,103 @@ func Login(authConfig *AuthConfig, factory *utils.HTTPRequestFactory) (string, e return status, nil } +// loginV2 tries to login to the v2 registry server. The given registry endpoint has been +// pinged or setup with a list of authorization challenges. Each of these challenges are +// tried until one of them succeeds. Currently supported challenge schemes are: +// HTTP Basic Authorization +// Token Authorization with a separate token issuing server +// NOTE: the v2 logic does not attempt to create a user account if one doesn't exist. For +// now, users should create their account through other means like directly from a web page +// served by the v2 registry service provider. Whether this will be supported in the future +// is to be determined. +func loginV2(authConfig *AuthConfig, registryEndpoint *Endpoint, factory *utils.HTTPRequestFactory) (string, error) { + log.Debugf("attempting v2 login to registry endpoint %s", registryEndpoint) + + client := &http.Client{ + Transport: &http.Transport{ + DisableKeepAlives: true, + Proxy: http.ProxyFromEnvironment, + }, + CheckRedirect: AddRequiredHeadersToRedirectedRequests, + } + + var ( + err error + allErrors []error + ) + + for _, challenge := range registryEndpoint.AuthChallenges { + log.Debugf("trying %q auth challenge with params %s", challenge.Scheme, challenge.Parameters) + + switch strings.ToLower(challenge.Scheme) { + case "basic": + err = tryV2BasicAuthLogin(authConfig, challenge.Parameters, registryEndpoint, client, factory) + case "bearer": + err = tryV2TokenAuthLogin(authConfig, challenge.Parameters, registryEndpoint, client, factory) + default: + // Unsupported challenge types are explicitly skipped. + err = fmt.Errorf("unsupported auth scheme: %q", challenge.Scheme) + } + + if err == nil { + return "Login Succeeded", nil + } + + log.Debugf("error trying auth challenge %q: %s", challenge.Scheme, err) + + allErrors = append(allErrors, err) + } + + return "", fmt.Errorf("no successful auth challenge for %s - errors: %s", registryEndpoint, allErrors) +} + +func tryV2BasicAuthLogin(authConfig *AuthConfig, params map[string]string, registryEndpoint *Endpoint, client *http.Client, factory *utils.HTTPRequestFactory) error { + req, err := factory.NewRequest("GET", registryEndpoint.Path(""), nil) + if err != nil { + return err + } + + req.SetBasicAuth(authConfig.Username, authConfig.Password) + + resp, err := client.Do(req) + if err != nil { + return err + } + defer resp.Body.Close() + + if resp.StatusCode != http.StatusOK { + return fmt.Errorf("basic auth attempt to %s realm %q failed with status: %d %s", registryEndpoint, params["realm"], resp.StatusCode, http.StatusText(resp.StatusCode)) + } + + return nil +} + +func tryV2TokenAuthLogin(authConfig *AuthConfig, params map[string]string, registryEndpoint *Endpoint, client *http.Client, factory *utils.HTTPRequestFactory) error { + token, err := getToken(authConfig.Username, authConfig.Password, params, registryEndpoint, client, factory) + if err != nil { + return err + } + + req, err := factory.NewRequest("GET", registryEndpoint.Path(""), nil) + if err != nil { + return err + } + + req.Header.Set("Authorization", fmt.Sprintf("Bearer %s", token)) + + resp, err := client.Do(req) + if err != nil { + return err + } + defer resp.Body.Close() + + if resp.StatusCode != http.StatusOK { + return fmt.Errorf("token auth attempt to %s realm %q failed with status: %d %s", registryEndpoint, params["realm"], resp.StatusCode, http.StatusText(resp.StatusCode)) + } + + return nil +} + // this method matches a auth configuration to a server address or a url func (config *ConfigFile) ResolveAuthConfig(index *IndexInfo) AuthConfig { configKey := index.GetAuthConfigKey() diff --git a/docs/authchallenge.go b/docs/authchallenge.go new file mode 100644 index 00000000..e300d82a --- /dev/null +++ b/docs/authchallenge.go @@ -0,0 +1,150 @@ +package registry + +import ( + "net/http" + "strings" +) + +// Octet types from RFC 2616. +type octetType byte + +// AuthorizationChallenge carries information +// from a WWW-Authenticate response header. +type AuthorizationChallenge struct { + Scheme string + Parameters map[string]string +} + +var octetTypes [256]octetType + +const ( + isToken octetType = 1 << iota + isSpace +) + +func init() { + // OCTET = + // CHAR = + // CTL = + // CR = + // LF = + // SP = + // HT = + // <"> = + // CRLF = CR LF + // LWS = [CRLF] 1*( SP | HT ) + // TEXT = + // separators = "(" | ")" | "<" | ">" | "@" | "," | ";" | ":" | "\" | <"> + // | "/" | "[" | "]" | "?" | "=" | "{" | "}" | SP | HT + // token = 1* + // qdtext = > + + for c := 0; c < 256; c++ { + var t octetType + isCtl := c <= 31 || c == 127 + isChar := 0 <= c && c <= 127 + isSeparator := strings.IndexRune(" \t\"(),/:;<=>?@[]\\{}", rune(c)) >= 0 + if strings.IndexRune(" \t\r\n", rune(c)) >= 0 { + t |= isSpace + } + if isChar && !isCtl && !isSeparator { + t |= isToken + } + octetTypes[c] = t + } +} + +func parseAuthHeader(header http.Header) []*AuthorizationChallenge { + var challenges []*AuthorizationChallenge + for _, h := range header[http.CanonicalHeaderKey("WWW-Authenticate")] { + v, p := parseValueAndParams(h) + if v != "" { + challenges = append(challenges, &AuthorizationChallenge{Scheme: v, Parameters: p}) + } + } + return challenges +} + +func parseValueAndParams(header string) (value string, params map[string]string) { + params = make(map[string]string) + value, s := expectToken(header) + if value == "" { + return + } + value = strings.ToLower(value) + s = "," + skipSpace(s) + for strings.HasPrefix(s, ",") { + var pkey string + pkey, s = expectToken(skipSpace(s[1:])) + if pkey == "" { + return + } + if !strings.HasPrefix(s, "=") { + return + } + var pvalue string + pvalue, s = expectTokenOrQuoted(s[1:]) + if pvalue == "" { + return + } + pkey = strings.ToLower(pkey) + params[pkey] = pvalue + s = skipSpace(s) + } + return +} + +func skipSpace(s string) (rest string) { + i := 0 + for ; i < len(s); i++ { + if octetTypes[s[i]]&isSpace == 0 { + break + } + } + return s[i:] +} + +func expectToken(s string) (token, rest string) { + i := 0 + for ; i < len(s); i++ { + if octetTypes[s[i]]&isToken == 0 { + break + } + } + return s[:i], s[i:] +} + +func expectTokenOrQuoted(s string) (value string, rest string) { + if !strings.HasPrefix(s, "\"") { + return expectToken(s) + } + s = s[1:] + for i := 0; i < len(s); i++ { + switch s[i] { + case '"': + return s[:i], s[i+1:] + case '\\': + p := make([]byte, len(s)-1) + j := copy(p, s[:i]) + escape := true + for i = i + i; i < len(s); i++ { + b := s[i] + switch { + case escape: + escape = false + p[j] = b + j++ + case b == '\\': + escape = true + case b == '"': + return string(p[:j]), s[i+1:] + default: + p[j] = b + j++ + } + } + return "", "" + } + } + return "", "" +} diff --git a/docs/endpoint.go b/docs/endpoint.go index 95680c5e..5c5b0520 100644 --- a/docs/endpoint.go +++ b/docs/endpoint.go @@ -15,28 +15,31 @@ import ( // for mocking in unit tests var lookupIP = net.LookupIP -// scans string for api version in the URL path. returns the trimmed hostname, if version found, string and API version. -func scanForAPIVersion(hostname string) (string, APIVersion) { +// scans string for api version in the URL path. returns the trimmed address, if version found, string and API version. +func scanForAPIVersion(address string) (string, APIVersion) { var ( chunks []string apiVersionStr string ) - if strings.HasSuffix(hostname, "/") { - chunks = strings.Split(hostname[:len(hostname)-1], "/") - apiVersionStr = chunks[len(chunks)-1] - } else { - chunks = strings.Split(hostname, "/") - apiVersionStr = chunks[len(chunks)-1] + + if strings.HasSuffix(address, "/") { + address = address[:len(address)-1] } + + chunks = strings.Split(address, "/") + apiVersionStr = chunks[len(chunks)-1] + for k, v := range apiVersions { if apiVersionStr == v { - hostname = strings.Join(chunks[:len(chunks)-1], "/") - return hostname, k + address = strings.Join(chunks[:len(chunks)-1], "/") + return address, k } } - return hostname, DefaultAPIVersion + + return address, APIVersionUnknown } +// NewEndpoint parses the given address to return a registry endpoint. func NewEndpoint(index *IndexInfo) (*Endpoint, error) { // *TODO: Allow per-registry configuration of endpoints. endpoint, err := newEndpoint(index.GetAuthConfigKey(), index.Secure) @@ -44,81 +47,124 @@ func NewEndpoint(index *IndexInfo) (*Endpoint, error) { return nil, err } + log.Debugf("pinging registry endpoint %s", endpoint) + // Try HTTPS ping to registry endpoint.URL.Scheme = "https" if _, err := endpoint.Ping(); err != nil { - - //TODO: triggering highland build can be done there without "failing" - if index.Secure { // If registry is secure and HTTPS failed, show user the error and tell them about `--insecure-registry` // in case that's what they need. DO NOT accept unknown CA certificates, and DO NOT fallback to HTTP. - return nil, fmt.Errorf("Invalid registry endpoint %s: %v. If this private registry supports only HTTP or HTTPS with an unknown CA certificate, please add `--insecure-registry %s` to the daemon's arguments. In the case of HTTPS, if you have access to the registry's CA certificate, no need for the flag; simply place the CA certificate at /etc/docker/certs.d/%s/ca.crt", endpoint, err, endpoint.URL.Host, endpoint.URL.Host) + return nil, fmt.Errorf("invalid registry endpoint %s: %v. If this private registry supports only HTTP or HTTPS with an unknown CA certificate, please add `--insecure-registry %s` to the daemon's arguments. In the case of HTTPS, if you have access to the registry's CA certificate, no need for the flag; simply place the CA certificate at /etc/docker/certs.d/%s/ca.crt", endpoint, err, endpoint.URL.Host, endpoint.URL.Host) } // If registry is insecure and HTTPS failed, fallback to HTTP. log.Debugf("Error from registry %q marked as insecure: %v. Insecurely falling back to HTTP", endpoint, err) endpoint.URL.Scheme = "http" - _, err2 := endpoint.Ping() - if err2 == nil { + + var err2 error + if _, err2 = endpoint.Ping(); err2 == nil { return endpoint, nil } - return nil, fmt.Errorf("Invalid registry endpoint %q. HTTPS attempt: %v. HTTP attempt: %v", endpoint, err, err2) + return nil, fmt.Errorf("invalid registry endpoint %q. HTTPS attempt: %v. HTTP attempt: %v", endpoint, err, err2) } return endpoint, nil } -func newEndpoint(hostname string, secure bool) (*Endpoint, error) { + +func newEndpoint(address string, secure bool) (*Endpoint, error) { var ( - endpoint = Endpoint{} - trimmedHostname string - err error + endpoint = new(Endpoint) + trimmedAddress string + err error ) - if !strings.HasPrefix(hostname, "http") { - hostname = "https://" + hostname + + if !strings.HasPrefix(address, "http") { + address = "https://" + address } - trimmedHostname, endpoint.Version = scanForAPIVersion(hostname) - endpoint.URL, err = url.Parse(trimmedHostname) - if err != nil { + + trimmedAddress, endpoint.Version = scanForAPIVersion(address) + + if endpoint.URL, err = url.Parse(trimmedAddress); err != nil { return nil, err } - endpoint.secure = secure - return &endpoint, nil + endpoint.IsSecure = secure + return endpoint, nil } func (repoInfo *RepositoryInfo) GetEndpoint() (*Endpoint, error) { return NewEndpoint(repoInfo.Index) } +// Endpoint stores basic information about a registry endpoint. type Endpoint struct { - URL *url.URL - Version APIVersion - secure bool + URL *url.URL + Version APIVersion + IsSecure bool + AuthChallenges []*AuthorizationChallenge } // Get the formated URL for the root of this registry Endpoint -func (e Endpoint) String() string { - return fmt.Sprintf("%s/v%d/", e.URL.String(), e.Version) +func (e *Endpoint) String() string { + return fmt.Sprintf("%s/v%d/", e.URL, e.Version) } -func (e Endpoint) VersionString(version APIVersion) string { - return fmt.Sprintf("%s/v%d/", e.URL.String(), version) +// VersionString returns a formatted string of this +// endpoint address using the given API Version. +func (e *Endpoint) VersionString(version APIVersion) string { + return fmt.Sprintf("%s/v%d/", e.URL, version) } -func (e Endpoint) Ping() (RegistryInfo, error) { +// Path returns a formatted string for the URL +// of this endpoint with the given path appended. +func (e *Endpoint) Path(path string) string { + return fmt.Sprintf("%s/v%d/%s", e.URL, e.Version, path) +} + +func (e *Endpoint) Ping() (RegistryInfo, error) { + // The ping logic to use is determined by the registry endpoint version. + switch e.Version { + case APIVersion1: + return e.pingV1() + case APIVersion2: + return e.pingV2() + } + + // APIVersionUnknown + // We should try v2 first... + e.Version = APIVersion2 + regInfo, errV2 := e.pingV2() + if errV2 == nil { + return regInfo, nil + } + + // ... then fallback to v1. + e.Version = APIVersion1 + regInfo, errV1 := e.pingV1() + if errV1 == nil { + return regInfo, nil + } + + e.Version = APIVersionUnknown + return RegistryInfo{}, fmt.Errorf("unable to ping registry endpoint %s\nv2 ping attempt failed with error: %s\n v1 ping attempt failed with error: %s", e, errV2, errV1) +} + +func (e *Endpoint) pingV1() (RegistryInfo, error) { + log.Debugf("attempting v1 ping for registry endpoint %s", e) + if e.String() == IndexServerAddress() { - // Skip the check, we now this one is valid + // Skip the check, we know this one is valid // (and we never want to fallback to http in case of error) return RegistryInfo{Standalone: false}, nil } - req, err := http.NewRequest("GET", e.String()+"_ping", nil) + req, err := http.NewRequest("GET", e.Path("_ping"), nil) if err != nil { return RegistryInfo{Standalone: false}, err } - resp, _, err := doRequest(req, nil, ConnectTimeout, e.secure) + resp, _, err := doRequest(req, nil, ConnectTimeout, e.IsSecure) if err != nil { return RegistryInfo{Standalone: false}, err } @@ -127,7 +173,7 @@ func (e Endpoint) Ping() (RegistryInfo, error) { jsonString, err := ioutil.ReadAll(resp.Body) if err != nil { - return RegistryInfo{Standalone: false}, fmt.Errorf("Error while reading the http response: %s", err) + return RegistryInfo{Standalone: false}, fmt.Errorf("error while reading the http response: %s", err) } // If the header is absent, we assume true for compatibility with earlier @@ -157,3 +203,33 @@ func (e Endpoint) Ping() (RegistryInfo, error) { log.Debugf("RegistryInfo.Standalone: %t", info.Standalone) return info, nil } + +func (e *Endpoint) pingV2() (RegistryInfo, error) { + log.Debugf("attempting v2 ping for registry endpoint %s", e) + + req, err := http.NewRequest("GET", e.Path(""), nil) + if err != nil { + return RegistryInfo{}, err + } + + resp, _, err := doRequest(req, nil, ConnectTimeout, e.IsSecure) + if err != nil { + return RegistryInfo{}, err + } + defer resp.Body.Close() + + if resp.StatusCode == http.StatusOK { + // It would seem that no authentication/authorization is required. + // So we don't need to parse/add any authorization schemes. + return RegistryInfo{Standalone: true}, nil + } + + if resp.StatusCode == http.StatusUnauthorized { + // Parse the WWW-Authenticate Header and store the challenges + // on this endpoint object. + e.AuthChallenges = parseAuthHeader(resp.Header) + return RegistryInfo{}, nil + } + + return RegistryInfo{}, fmt.Errorf("v2 registry endpoint returned status %d: %q", resp.StatusCode, http.StatusText(resp.StatusCode)) +} diff --git a/docs/endpoint_test.go b/docs/endpoint_test.go index b691a4fb..f6489034 100644 --- a/docs/endpoint_test.go +++ b/docs/endpoint_test.go @@ -8,8 +8,10 @@ func TestEndpointParse(t *testing.T) { expected string }{ {IndexServerAddress(), IndexServerAddress()}, - {"http://0.0.0.0:5000", "http://0.0.0.0:5000/v1/"}, - {"0.0.0.0:5000", "https://0.0.0.0:5000/v1/"}, + {"http://0.0.0.0:5000/v1/", "http://0.0.0.0:5000/v1/"}, + {"http://0.0.0.0:5000/v2/", "http://0.0.0.0:5000/v2/"}, + {"http://0.0.0.0:5000", "http://0.0.0.0:5000/v0/"}, + {"0.0.0.0:5000", "https://0.0.0.0:5000/v0/"}, } for _, td := range testData { e, err := newEndpoint(td.str, false) diff --git a/docs/service.go b/docs/service.go index c34e3842..04834022 100644 --- a/docs/service.go +++ b/docs/service.go @@ -1,6 +1,7 @@ package registry import ( + log "github.com/Sirupsen/logrus" "github.com/docker/docker/engine" ) @@ -38,28 +39,39 @@ func (s *Service) Install(eng *engine.Engine) error { // and returns OK if authentication was sucessful. // It can be used to verify the validity of a client's credentials. func (s *Service) Auth(job *engine.Job) engine.Status { - var authConfig = new(AuthConfig) + var ( + authConfig = new(AuthConfig) + endpoint *Endpoint + index *IndexInfo + status string + err error + ) job.GetenvJson("authConfig", authConfig) - if authConfig.ServerAddress != "" { - index, err := ResolveIndexInfo(job, authConfig.ServerAddress) - if err != nil { - return job.Error(err) - } - if !index.Official { - endpoint, err := NewEndpoint(index) - if err != nil { - return job.Error(err) - } - authConfig.ServerAddress = endpoint.String() - } + addr := authConfig.ServerAddress + if addr == "" { + // Use the official registry address if not specified. + addr = IndexServerAddress() } - status, err := Login(authConfig, HTTPRequestFactory(nil)) - if err != nil { + if index, err = ResolveIndexInfo(job, addr); err != nil { return job.Error(err) } + + if endpoint, err = NewEndpoint(index); err != nil { + log.Errorf("unable to get new registry endpoint: %s", err) + return job.Error(err) + } + + authConfig.ServerAddress = endpoint.String() + + if status, err = Login(authConfig, endpoint, HTTPRequestFactory(nil)); err != nil { + log.Errorf("unable to login against registry endpoint %s: %s", endpoint, err) + return job.Error(err) + } + + log.Infof("successful registry login for endpoint %s: %s", endpoint, status) job.Printf("%s\n", status) return engine.StatusOK diff --git a/docs/session.go b/docs/session.go index 781a91b1..b1980e1a 100644 --- a/docs/session.go +++ b/docs/session.go @@ -65,7 +65,7 @@ func NewSession(authConfig *AuthConfig, factory *utils.HTTPRequestFactory, endpo } func (r *Session) doRequest(req *http.Request) (*http.Response, *http.Client, error) { - return doRequest(req, r.jar, r.timeout, r.indexEndpoint.secure) + return doRequest(req, r.jar, r.timeout, r.indexEndpoint.IsSecure) } // Retrieve the history of a given image from the Registry. diff --git a/docs/token.go b/docs/token.go new file mode 100644 index 00000000..0403734f --- /dev/null +++ b/docs/token.go @@ -0,0 +1,70 @@ +package registry + +import ( + "errors" + "fmt" + "net/http" + "net/url" + "strings" + + "github.com/docker/docker/utils" +) + +func getToken(username, password string, params map[string]string, registryEndpoint *Endpoint, client *http.Client, factory *utils.HTTPRequestFactory) (token string, err error) { + realm, ok := params["realm"] + if !ok { + return "", errors.New("no realm specified for token auth challenge") + } + + realmURL, err := url.Parse(realm) + if err != nil { + return "", fmt.Errorf("invalid token auth challenge realm: %s", err) + } + + if realmURL.Scheme == "" { + if registryEndpoint.IsSecure { + realmURL.Scheme = "https" + } else { + realmURL.Scheme = "http" + } + } + + req, err := factory.NewRequest("GET", realmURL.String(), nil) + if err != nil { + return "", err + } + + reqParams := req.URL.Query() + service := params["service"] + scope := params["scope"] + + if service != "" { + reqParams.Add("service", service) + } + + for _, scopeField := range strings.Fields(scope) { + reqParams.Add("scope", scopeField) + } + + reqParams.Add("account", username) + + req.URL.RawQuery = reqParams.Encode() + req.SetBasicAuth(username, password) + + resp, err := client.Do(req) + if err != nil { + return "", err + } + defer resp.Body.Close() + + if !(resp.StatusCode == http.StatusOK || resp.StatusCode == http.StatusNoContent) { + return "", fmt.Errorf("token auth attempt for registry %s: %s request failed with status: %d %s", registryEndpoint, req.URL, resp.StatusCode, http.StatusText(resp.StatusCode)) + } + + token = resp.Header.Get("X-Auth-Token") + if token == "" { + return "", errors.New("token server did not include a token in the response header") + } + + return token, nil +} diff --git a/docs/types.go b/docs/types.go index fbbc0e70..bd0bf8b7 100644 --- a/docs/types.go +++ b/docs/types.go @@ -55,14 +55,15 @@ func (av APIVersion) String() string { return apiVersions[av] } -var DefaultAPIVersion APIVersion = APIVersion1 var apiVersions = map[APIVersion]string{ 1: "v1", 2: "v2", } +// API Version identifiers. const ( - APIVersion1 = iota + 1 + APIVersionUnknown = iota + APIVersion1 APIVersion2 ) From 06d0ef4179ee2d489018adac3800a491891d2336 Mon Sep 17 00:00:00 2001 From: Derek McGowan Date: Tue, 30 Sep 2014 17:03:57 -0700 Subject: [PATCH 209/375] Push flow Signed-off-by: Derek McGowan (github: dmcgowan) --- docs/session_v2.go | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/docs/session_v2.go b/docs/session_v2.go index 20e9e2ee..0498bf70 100644 --- a/docs/session_v2.go +++ b/docs/session_v2.go @@ -267,7 +267,7 @@ func (r *Session) GetV2ImageBlobReader(imageName, sumType, sum string, token []s // Push the image to the server for storage. // 'layer' is an uncompressed reader of the blob to be pushed. // The server will generate it's own checksum calculation. -func (r *Session) PutV2ImageBlob(imageName, sumType string, blobRdr io.Reader, token []string) (serverChecksum string, err error) { +func (r *Session) PutV2ImageBlob(imageName, sumType, sumStr string, blobRdr io.Reader, token []string) (serverChecksum string, err error) { vars := map[string]string{ "imagename": imageName, "sumtype": sumType, @@ -285,6 +285,7 @@ func (r *Session) PutV2ImageBlob(imageName, sumType string, blobRdr io.Reader, t return "", err } setTokenAuth(req, token) + req.Header.Set("X-Tarsum", sumStr) res, _, err := r.doRequest(req) if err != nil { return "", err @@ -309,6 +310,10 @@ func (r *Session) PutV2ImageBlob(imageName, sumType string, blobRdr io.Reader, t return "", fmt.Errorf("unable to decode PutV2ImageBlob JSON response: %s", err) } + if sumInfo.Checksum != sumStr { + return "", fmt.Errorf("failed checksum comparison. serverChecksum: %q, localChecksum: %q", sumInfo.Checksum, sumStr) + } + // XXX this is a json struct from the registry, with its checksum return sumInfo.Checksum, nil } From 24895820bd88f05bd38c041995ea4ca91b88aa35 Mon Sep 17 00:00:00 2001 From: Derek McGowan Date: Fri, 14 Nov 2014 16:22:06 -0800 Subject: [PATCH 210/375] Update push to use mount blob endpoint Using mount blob prevents repushing images which have already been uploaded Signed-off-by: Derek McGowan (github: dmcgowan) --- docs/session_v2.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/session_v2.go b/docs/session_v2.go index 0498bf70..86d0c228 100644 --- a/docs/session_v2.go +++ b/docs/session_v2.go @@ -34,7 +34,7 @@ func newV2RegistryRouter() *mux.Router { v2Router.Path("/blob/{imagename:[a-z0-9-._/]+}/{sumtype:[a-z0-9._+-]+}").Name("uploadBlob") // Mounting a blob in an image - v2Router.Path("/mountblob/{imagename:[a-z0-9-._/]+}/{sumtype:[a-z0-9._+-]+}/{sum:[a-fA-F0-9]{4,}}").Name("mountBlob") + v2Router.Path("/blob/{imagename:[a-z0-9-._/]+}/{sumtype:[a-z0-9._+-]+}/{sum:[a-fA-F0-9]{4,}}").Name("mountBlob") return router } @@ -184,7 +184,7 @@ func (r *Session) PostV2ImageMountBlob(imageName, sumType, sum string, token []s case 200: // return something indicating no push needed return true, nil - case 300: + case 404: // return something indicating blob push needed return false, nil } From e256a0e0bc06aca81812960b7509d0fa76356ac5 Mon Sep 17 00:00:00 2001 From: Josh Hawn Date: Fri, 12 Dec 2014 13:30:12 -0800 Subject: [PATCH 211/375] Update token response handling Registry authorization token is now taken from the response body rather than the repsonse header. Docker-DCO-1.1-Signed-off-by: Josh Hawn (github: jlhawn) --- docs/token.go | 21 ++++++++++++++++----- 1 file changed, 16 insertions(+), 5 deletions(-) diff --git a/docs/token.go b/docs/token.go index 0403734f..25048630 100644 --- a/docs/token.go +++ b/docs/token.go @@ -1,6 +1,7 @@ package registry import ( + "encoding/json" "errors" "fmt" "net/http" @@ -10,6 +11,10 @@ import ( "github.com/docker/docker/utils" ) +type tokenResponse struct { + Token string `json:"token"` +} + func getToken(username, password string, params map[string]string, registryEndpoint *Endpoint, client *http.Client, factory *utils.HTTPRequestFactory) (token string, err error) { realm, ok := params["realm"] if !ok { @@ -57,14 +62,20 @@ func getToken(username, password string, params map[string]string, registryEndpo } defer resp.Body.Close() - if !(resp.StatusCode == http.StatusOK || resp.StatusCode == http.StatusNoContent) { + if resp.StatusCode != http.StatusOK { return "", fmt.Errorf("token auth attempt for registry %s: %s request failed with status: %d %s", registryEndpoint, req.URL, resp.StatusCode, http.StatusText(resp.StatusCode)) } - token = resp.Header.Get("X-Auth-Token") - if token == "" { - return "", errors.New("token server did not include a token in the response header") + decoder := json.NewDecoder(resp.Body) + + tr := new(tokenResponse) + if err = decoder.Decode(tr); err != nil { + return "", fmt.Errorf("unable to decode token response: %s", err) } - return token, nil + if tr.Token == "" { + return "", errors.New("authorization server did not include a token in the response") + } + + return tr.Token, nil } From 2fcad2a10fa5463bdad44b243f5c257455cb9e14 Mon Sep 17 00:00:00 2001 From: Stephen J Day Date: Fri, 12 Dec 2014 11:27:22 -0800 Subject: [PATCH 212/375] Registry V2 HTTP route and error code definitions This package, ported from next-generation docker regsitry, includes route and error definitions. These facilitate compliant V2 client implementation. The portions of the HTTP API that are included in this package are considered to be locked down and should only be changed through a careful change proposal. Descriptor definitions package layout may change without affecting API behavior until the exported Go API is ready to be locked down. When the new registry stabilizes and becomes the master branch, this package can be vendored from the registry. Signed-off-by: Stephen J Day --- docs/v2/descriptors.go | 144 ++++++++++++++++++++++++++++++++ docs/v2/doc.go | 13 +++ docs/v2/errors.go | 185 +++++++++++++++++++++++++++++++++++++++++ docs/v2/errors_test.go | 165 ++++++++++++++++++++++++++++++++++++ docs/v2/routes.go | 69 +++++++++++++++ docs/v2/routes_test.go | 184 ++++++++++++++++++++++++++++++++++++++++ docs/v2/urls.go | 165 ++++++++++++++++++++++++++++++++++++ docs/v2/urls_test.go | 100 ++++++++++++++++++++++ 8 files changed, 1025 insertions(+) create mode 100644 docs/v2/descriptors.go create mode 100644 docs/v2/doc.go create mode 100644 docs/v2/errors.go create mode 100644 docs/v2/errors_test.go create mode 100644 docs/v2/routes.go create mode 100644 docs/v2/routes_test.go create mode 100644 docs/v2/urls.go create mode 100644 docs/v2/urls_test.go diff --git a/docs/v2/descriptors.go b/docs/v2/descriptors.go new file mode 100644 index 00000000..68d18241 --- /dev/null +++ b/docs/v2/descriptors.go @@ -0,0 +1,144 @@ +package v2 + +import "net/http" + +// TODO(stevvooe): Add route descriptors for each named route, along with +// accepted methods, parameters, returned status codes and error codes. + +// ErrorDescriptor provides relevant information about a given error code. +type ErrorDescriptor struct { + // Code is the error code that this descriptor describes. + Code ErrorCode + + // Value provides a unique, string key, often captilized with + // underscores, to identify the error code. This value is used as the + // keyed value when serializing api errors. + Value string + + // Message is a short, human readable decription of the error condition + // included in API responses. + Message string + + // Description provides a complete account of the errors purpose, suitable + // for use in documentation. + Description string + + // HTTPStatusCodes provides a list of status under which this error + // condition may arise. If it is empty, the error condition may be seen + // for any status code. + HTTPStatusCodes []int +} + +// ErrorDescriptors provides a list of HTTP API Error codes that may be +// encountered when interacting with the registry API. +var ErrorDescriptors = []ErrorDescriptor{ + { + Code: ErrorCodeUnknown, + Value: "UNKNOWN", + Message: "unknown error", + Description: `Generic error returned when the error does not have an + API classification.`, + }, + { + Code: ErrorCodeDigestInvalid, + Value: "DIGEST_INVALID", + Message: "provided digest did not match uploaded content", + Description: `When a blob is uploaded, the registry will check that + the content matches the digest provided by the client. The error may + include a detail structure with the key "digest", including the + invalid digest string. This error may also be returned when a manifest + includes an invalid layer digest.`, + HTTPStatusCodes: []int{http.StatusBadRequest, http.StatusNotFound}, + }, + { + Code: ErrorCodeSizeInvalid, + Value: "SIZE_INVALID", + Message: "provided length did not match content length", + Description: `When a layer is uploaded, the provided size will be + checked against the uploaded content. If they do not match, this error + will be returned.`, + HTTPStatusCodes: []int{http.StatusBadRequest}, + }, + { + Code: ErrorCodeNameInvalid, + Value: "NAME_INVALID", + Message: "manifest name did not match URI", + Description: `During a manifest upload, if the name in the manifest + does not match the uri name, this error will be returned.`, + HTTPStatusCodes: []int{http.StatusBadRequest, http.StatusNotFound}, + }, + { + Code: ErrorCodeTagInvalid, + Value: "TAG_INVALID", + Message: "manifest tag did not match URI", + Description: `During a manifest upload, if the tag in the manifest + does not match the uri tag, this error will be returned.`, + HTTPStatusCodes: []int{http.StatusBadRequest, http.StatusNotFound}, + }, + { + Code: ErrorCodeNameUnknown, + Value: "NAME_UNKNOWN", + Message: "repository name not known to registry", + Description: `This is returned if the name used during an operation is + unknown to the registry.`, + HTTPStatusCodes: []int{http.StatusNotFound}, + }, + { + Code: ErrorCodeManifestUnknown, + Value: "MANIFEST_UNKNOWN", + Message: "manifest unknown", + Description: `This error is returned when the manifest, identified by + name and tag is unknown to the repository.`, + HTTPStatusCodes: []int{http.StatusNotFound}, + }, + { + Code: ErrorCodeManifestInvalid, + Value: "MANIFEST_INVALID", + Message: "manifest invalid", + Description: `During upload, manifests undergo several checks ensuring + validity. If those checks fail, this error may be returned, unless a + more specific error is included. The detail will contain information + the failed validation.`, + HTTPStatusCodes: []int{http.StatusBadRequest}, + }, + { + Code: ErrorCodeManifestUnverified, + Value: "MANIFEST_UNVERIFIED", + Message: "manifest failed signature verification", + Description: `During manifest upload, if the manifest fails signature + verification, this error will be returned.`, + HTTPStatusCodes: []int{http.StatusBadRequest}, + }, + { + Code: ErrorCodeBlobUnknown, + Value: "BLOB_UNKNOWN", + Message: "blob unknown to registry", + Description: `This error may be returned when a blob is unknown to the + registry in a specified repository. This can be returned with a + standard get or if a manifest references an unknown layer during + upload.`, + HTTPStatusCodes: []int{http.StatusBadRequest, http.StatusNotFound}, + }, + + { + Code: ErrorCodeBlobUploadUnknown, + Value: "BLOB_UPLOAD_UNKNOWN", + Message: "blob upload unknown to registry", + Description: `If a blob upload has been cancelled or was never + started, this error code may be returned.`, + HTTPStatusCodes: []int{http.StatusNotFound}, + }, +} + +var errorCodeToDescriptors map[ErrorCode]ErrorDescriptor +var idToDescriptors map[string]ErrorDescriptor + +func init() { + errorCodeToDescriptors = make(map[ErrorCode]ErrorDescriptor, len(ErrorDescriptors)) + idToDescriptors = make(map[string]ErrorDescriptor, len(ErrorDescriptors)) + + for _, descriptor := range ErrorDescriptors { + errorCodeToDescriptors[descriptor.Code] = descriptor + idToDescriptors[descriptor.Value] = descriptor + } +} diff --git a/docs/v2/doc.go b/docs/v2/doc.go new file mode 100644 index 00000000..30fe2271 --- /dev/null +++ b/docs/v2/doc.go @@ -0,0 +1,13 @@ +// Package v2 describes routes, urls and the error codes used in the Docker +// Registry JSON HTTP API V2. In addition to declarations, descriptors are +// provided for routes and error codes that can be used for implementation and +// automatically generating documentation. +// +// Definitions here are considered to be locked down for the V2 registry api. +// Any changes must be considered carefully and should not proceed without a +// change proposal. +// +// Currently, while the HTTP API definitions are considered stable, the Go API +// exports are considered unstable. Go API consumers should take care when +// relying on these definitions until this message is deleted. +package v2 diff --git a/docs/v2/errors.go b/docs/v2/errors.go new file mode 100644 index 00000000..8c85d3a9 --- /dev/null +++ b/docs/v2/errors.go @@ -0,0 +1,185 @@ +package v2 + +import ( + "fmt" + "strings" +) + +// ErrorCode represents the error type. The errors are serialized via strings +// and the integer format may change and should *never* be exported. +type ErrorCode int + +const ( + // ErrorCodeUnknown is a catch-all for errors not defined below. + ErrorCodeUnknown ErrorCode = iota + + // ErrorCodeDigestInvalid is returned when uploading a blob if the + // provided digest does not match the blob contents. + ErrorCodeDigestInvalid + + // ErrorCodeSizeInvalid is returned when uploading a blob if the provided + // size does not match the content length. + ErrorCodeSizeInvalid + + // ErrorCodeNameInvalid is returned when the name in the manifest does not + // match the provided name. + ErrorCodeNameInvalid + + // ErrorCodeTagInvalid is returned when the tag in the manifest does not + // match the provided tag. + ErrorCodeTagInvalid + + // ErrorCodeNameUnknown when the repository name is not known. + ErrorCodeNameUnknown + + // ErrorCodeManifestUnknown returned when image manifest is unknown. + ErrorCodeManifestUnknown + + // ErrorCodeManifestInvalid returned when an image manifest is invalid, + // typically during a PUT operation. This error encompasses all errors + // encountered during manifest validation that aren't signature errors. + ErrorCodeManifestInvalid + + // ErrorCodeManifestUnverified is returned when the manifest fails + // signature verfication. + ErrorCodeManifestUnverified + + // ErrorCodeBlobUnknown is returned when a blob is unknown to the + // registry. This can happen when the manifest references a nonexistent + // layer or the result is not found by a blob fetch. + ErrorCodeBlobUnknown + + // ErrorCodeBlobUploadUnknown is returned when an upload is unknown. + ErrorCodeBlobUploadUnknown +) + +// ParseErrorCode attempts to parse the error code string, returning +// ErrorCodeUnknown if the error is not known. +func ParseErrorCode(s string) ErrorCode { + desc, ok := idToDescriptors[s] + + if !ok { + return ErrorCodeUnknown + } + + return desc.Code +} + +// Descriptor returns the descriptor for the error code. +func (ec ErrorCode) Descriptor() ErrorDescriptor { + d, ok := errorCodeToDescriptors[ec] + + if !ok { + return ErrorCodeUnknown.Descriptor() + } + + return d +} + +// String returns the canonical identifier for this error code. +func (ec ErrorCode) String() string { + return ec.Descriptor().Value +} + +// Message returned the human-readable error message for this error code. +func (ec ErrorCode) Message() string { + return ec.Descriptor().Message +} + +// MarshalText encodes the receiver into UTF-8-encoded text and returns the +// result. +func (ec ErrorCode) MarshalText() (text []byte, err error) { + return []byte(ec.String()), nil +} + +// UnmarshalText decodes the form generated by MarshalText. +func (ec *ErrorCode) UnmarshalText(text []byte) error { + desc, ok := idToDescriptors[string(text)] + + if !ok { + desc = ErrorCodeUnknown.Descriptor() + } + + *ec = desc.Code + + return nil +} + +// Error provides a wrapper around ErrorCode with extra Details provided. +type Error struct { + Code ErrorCode `json:"code"` + Message string `json:"message,omitempty"` + Detail interface{} `json:"detail,omitempty"` +} + +// Error returns a human readable representation of the error. +func (e Error) Error() string { + return fmt.Sprintf("%s: %s", + strings.ToLower(strings.Replace(e.Code.String(), "_", " ", -1)), + e.Message) +} + +// Errors provides the envelope for multiple errors and a few sugar methods +// for use within the application. +type Errors struct { + Errors []Error `json:"errors,omitempty"` +} + +// Push pushes an error on to the error stack, with the optional detail +// argument. It is a programming error (ie panic) to push more than one +// detail at a time. +func (errs *Errors) Push(code ErrorCode, details ...interface{}) { + if len(details) > 1 { + panic("please specify zero or one detail items for this error") + } + + var detail interface{} + if len(details) > 0 { + detail = details[0] + } + + if err, ok := detail.(error); ok { + detail = err.Error() + } + + errs.PushErr(Error{ + Code: code, + Message: code.Message(), + Detail: detail, + }) +} + +// PushErr pushes an error interface onto the error stack. +func (errs *Errors) PushErr(err error) { + switch err.(type) { + case Error: + errs.Errors = append(errs.Errors, err.(Error)) + default: + errs.Errors = append(errs.Errors, Error{Message: err.Error()}) + } +} + +func (errs *Errors) Error() string { + switch errs.Len() { + case 0: + return "" + case 1: + return errs.Errors[0].Error() + default: + msg := "errors:\n" + for _, err := range errs.Errors { + msg += err.Error() + "\n" + } + return msg + } +} + +// Clear clears the errors. +func (errs *Errors) Clear() { + errs.Errors = errs.Errors[:0] +} + +// Len returns the current number of errors. +func (errs *Errors) Len() int { + return len(errs.Errors) +} diff --git a/docs/v2/errors_test.go b/docs/v2/errors_test.go new file mode 100644 index 00000000..d2fc091a --- /dev/null +++ b/docs/v2/errors_test.go @@ -0,0 +1,165 @@ +package v2 + +import ( + "encoding/json" + "reflect" + "testing" + + "github.com/docker/docker-registry/digest" +) + +// TestErrorCodes ensures that error code format, mappings and +// marshaling/unmarshaling. round trips are stable. +func TestErrorCodes(t *testing.T) { + for _, desc := range ErrorDescriptors { + if desc.Code.String() != desc.Value { + t.Fatalf("error code string incorrect: %q != %q", desc.Code.String(), desc.Value) + } + + if desc.Code.Message() != desc.Message { + t.Fatalf("incorrect message for error code %v: %q != %q", desc.Code, desc.Code.Message(), desc.Message) + } + + // Serialize the error code using the json library to ensure that we + // get a string and it works round trip. + p, err := json.Marshal(desc.Code) + + if err != nil { + t.Fatalf("error marshaling error code %v: %v", desc.Code, err) + } + + if len(p) <= 0 { + t.Fatalf("expected content in marshaled before for error code %v", desc.Code) + } + + // First, unmarshal to interface and ensure we have a string. + var ecUnspecified interface{} + if err := json.Unmarshal(p, &ecUnspecified); err != nil { + t.Fatalf("error unmarshaling error code %v: %v", desc.Code, err) + } + + if _, ok := ecUnspecified.(string); !ok { + t.Fatalf("expected a string for error code %v on unmarshal got a %T", desc.Code, ecUnspecified) + } + + // Now, unmarshal with the error code type and ensure they are equal + var ecUnmarshaled ErrorCode + if err := json.Unmarshal(p, &ecUnmarshaled); err != nil { + t.Fatalf("error unmarshaling error code %v: %v", desc.Code, err) + } + + if ecUnmarshaled != desc.Code { + t.Fatalf("unexpected error code during error code marshal/unmarshal: %v != %v", ecUnmarshaled, desc.Code) + } + } +} + +// TestErrorsManagement does a quick check of the Errors type to ensure that +// members are properly pushed and marshaled. +func TestErrorsManagement(t *testing.T) { + var errs Errors + + errs.Push(ErrorCodeDigestInvalid) + errs.Push(ErrorCodeBlobUnknown, + map[string]digest.Digest{"digest": "sometestblobsumdoesntmatter"}) + + p, err := json.Marshal(errs) + + if err != nil { + t.Fatalf("error marashaling errors: %v", err) + } + + expectedJSON := "{\"errors\":[{\"code\":\"DIGEST_INVALID\",\"message\":\"provided digest did not match uploaded content\"},{\"code\":\"BLOB_UNKNOWN\",\"message\":\"blob unknown to registry\",\"detail\":{\"digest\":\"sometestblobsumdoesntmatter\"}}]}" + + if string(p) != expectedJSON { + t.Fatalf("unexpected json: %q != %q", string(p), expectedJSON) + } + + errs.Clear() + errs.Push(ErrorCodeUnknown) + expectedJSON = "{\"errors\":[{\"code\":\"UNKNOWN\",\"message\":\"unknown error\"}]}" + p, err = json.Marshal(errs) + + if err != nil { + t.Fatalf("error marashaling errors: %v", err) + } + + if string(p) != expectedJSON { + t.Fatalf("unexpected json: %q != %q", string(p), expectedJSON) + } +} + +// TestMarshalUnmarshal ensures that api errors can round trip through json +// without losing information. +func TestMarshalUnmarshal(t *testing.T) { + + var errors Errors + + for _, testcase := range []struct { + description string + err Error + }{ + { + description: "unknown error", + err: Error{ + + Code: ErrorCodeUnknown, + Message: ErrorCodeUnknown.Descriptor().Message, + }, + }, + { + description: "unknown manifest", + err: Error{ + Code: ErrorCodeManifestUnknown, + Message: ErrorCodeManifestUnknown.Descriptor().Message, + }, + }, + { + description: "unknown manifest", + err: Error{ + Code: ErrorCodeBlobUnknown, + Message: ErrorCodeBlobUnknown.Descriptor().Message, + Detail: map[string]interface{}{"digest": "asdfqwerqwerqwerqwer"}, + }, + }, + } { + fatalf := func(format string, args ...interface{}) { + t.Fatalf(testcase.description+": "+format, args...) + } + + unexpectedErr := func(err error) { + fatalf("unexpected error: %v", err) + } + + p, err := json.Marshal(testcase.err) + if err != nil { + unexpectedErr(err) + } + + var unmarshaled Error + if err := json.Unmarshal(p, &unmarshaled); err != nil { + unexpectedErr(err) + } + + if !reflect.DeepEqual(unmarshaled, testcase.err) { + fatalf("errors not equal after round trip: %#v != %#v", unmarshaled, testcase.err) + } + + // Roll everything up into an error response envelope. + errors.PushErr(testcase.err) + } + + p, err := json.Marshal(errors) + if err != nil { + t.Fatalf("unexpected error marshaling error envelope: %v", err) + } + + var unmarshaled Errors + if err := json.Unmarshal(p, &unmarshaled); err != nil { + t.Fatalf("unexpected error unmarshaling error envelope: %v", err) + } + + if !reflect.DeepEqual(unmarshaled, errors) { + t.Fatalf("errors not equal after round trip: %#v != %#v", unmarshaled, errors) + } +} diff --git a/docs/v2/routes.go b/docs/v2/routes.go new file mode 100644 index 00000000..7ebe61d6 --- /dev/null +++ b/docs/v2/routes.go @@ -0,0 +1,69 @@ +package v2 + +import ( + "github.com/docker/docker-registry/common" + "github.com/gorilla/mux" +) + +// The following are definitions of the name under which all V2 routes are +// registered. These symbols can be used to look up a route based on the name. +const ( + RouteNameBase = "base" + RouteNameManifest = "manifest" + RouteNameTags = "tags" + RouteNameBlob = "blob" + RouteNameBlobUpload = "blob-upload" + RouteNameBlobUploadChunk = "blob-upload-chunk" +) + +var allEndpoints = []string{ + RouteNameManifest, + RouteNameTags, + RouteNameBlob, + RouteNameBlobUpload, + RouteNameBlobUploadChunk, +} + +// Router builds a gorilla router with named routes for the various API +// methods. This can be used directly by both server implementations and +// clients. +func Router() *mux.Router { + router := mux.NewRouter(). + StrictSlash(true) + + // GET /v2/ Check Check that the registry implements API version 2(.1) + router. + Path("/v2/"). + Name(RouteNameBase) + + // GET /v2//manifest/ Image Manifest Fetch the image manifest identified by name and tag. + // PUT /v2//manifest/ Image Manifest Upload the image manifest identified by name and tag. + // DELETE /v2//manifest/ Image Manifest Delete the image identified by name and tag. + router. + Path("/v2/{name:" + common.RepositoryNameRegexp.String() + "}/manifests/{tag:" + common.TagNameRegexp.String() + "}"). + Name(RouteNameManifest) + + // GET /v2//tags/list Tags Fetch the tags under the repository identified by name. + router. + Path("/v2/{name:" + common.RepositoryNameRegexp.String() + "}/tags/list"). + Name(RouteNameTags) + + // GET /v2//blob/ Layer Fetch the blob identified by digest. + router. + Path("/v2/{name:" + common.RepositoryNameRegexp.String() + "}/blobs/{digest:[a-zA-Z0-9-_+.]+:[a-zA-Z0-9-_+.=]+}"). + Name(RouteNameBlob) + + // POST /v2//blob/upload/ Layer Upload Initiate an upload of the layer identified by tarsum. + router. + Path("/v2/{name:" + common.RepositoryNameRegexp.String() + "}/blobs/uploads/"). + Name(RouteNameBlobUpload) + + // GET /v2//blob/upload/ Layer Upload Get the status of the upload identified by tarsum and uuid. + // PUT /v2//blob/upload/ Layer Upload Upload all or a chunk of the upload identified by tarsum and uuid. + // DELETE /v2//blob/upload/ Layer Upload Cancel the upload identified by layer and uuid + router. + Path("/v2/{name:" + common.RepositoryNameRegexp.String() + "}/blobs/uploads/{uuid}"). + Name(RouteNameBlobUploadChunk) + + return router +} diff --git a/docs/v2/routes_test.go b/docs/v2/routes_test.go new file mode 100644 index 00000000..9969ebcc --- /dev/null +++ b/docs/v2/routes_test.go @@ -0,0 +1,184 @@ +package v2 + +import ( + "encoding/json" + "net/http" + "net/http/httptest" + "reflect" + "testing" + + "github.com/gorilla/mux" +) + +type routeTestCase struct { + RequestURI string + Vars map[string]string + RouteName string + StatusCode int +} + +// TestRouter registers a test handler with all the routes and ensures that +// each route returns the expected path variables. Not method verification is +// present. This not meant to be exhaustive but as check to ensure that the +// expected variables are extracted. +// +// This may go away as the application structure comes together. +func TestRouter(t *testing.T) { + + router := Router() + + testHandler := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + testCase := routeTestCase{ + RequestURI: r.RequestURI, + Vars: mux.Vars(r), + RouteName: mux.CurrentRoute(r).GetName(), + } + + enc := json.NewEncoder(w) + + if err := enc.Encode(testCase); err != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) + return + } + }) + + // Startup test server + server := httptest.NewServer(router) + + for _, testcase := range []routeTestCase{ + { + RouteName: RouteNameBase, + RequestURI: "/v2/", + Vars: map[string]string{}, + }, + { + RouteName: RouteNameManifest, + RequestURI: "/v2/foo/bar/manifests/tag", + Vars: map[string]string{ + "name": "foo/bar", + "tag": "tag", + }, + }, + { + RouteName: RouteNameTags, + RequestURI: "/v2/foo/bar/tags/list", + Vars: map[string]string{ + "name": "foo/bar", + }, + }, + { + RouteName: RouteNameBlob, + RequestURI: "/v2/foo/bar/blobs/tarsum.dev+foo:abcdef0919234", + Vars: map[string]string{ + "name": "foo/bar", + "digest": "tarsum.dev+foo:abcdef0919234", + }, + }, + { + RouteName: RouteNameBlob, + RequestURI: "/v2/foo/bar/blobs/sha256:abcdef0919234", + Vars: map[string]string{ + "name": "foo/bar", + "digest": "sha256:abcdef0919234", + }, + }, + { + RouteName: RouteNameBlobUpload, + RequestURI: "/v2/foo/bar/blobs/uploads/", + Vars: map[string]string{ + "name": "foo/bar", + }, + }, + { + RouteName: RouteNameBlobUploadChunk, + RequestURI: "/v2/foo/bar/blobs/uploads/uuid", + Vars: map[string]string{ + "name": "foo/bar", + "uuid": "uuid", + }, + }, + { + RouteName: RouteNameBlobUploadChunk, + RequestURI: "/v2/foo/bar/blobs/uploads/D95306FA-FAD3-4E36-8D41-CF1C93EF8286", + Vars: map[string]string{ + "name": "foo/bar", + "uuid": "D95306FA-FAD3-4E36-8D41-CF1C93EF8286", + }, + }, + { + RouteName: RouteNameBlobUploadChunk, + RequestURI: "/v2/foo/bar/blobs/uploads/RDk1MzA2RkEtRkFEMy00RTM2LThENDEtQ0YxQzkzRUY4Mjg2IA==", + Vars: map[string]string{ + "name": "foo/bar", + "uuid": "RDk1MzA2RkEtRkFEMy00RTM2LThENDEtQ0YxQzkzRUY4Mjg2IA==", + }, + }, + { + // Check ambiguity: ensure we can distinguish between tags for + // "foo/bar/image/image" and image for "foo/bar/image" with tag + // "tags" + RouteName: RouteNameManifest, + RequestURI: "/v2/foo/bar/manifests/manifests/tags", + Vars: map[string]string{ + "name": "foo/bar/manifests", + "tag": "tags", + }, + }, + { + // This case presents an ambiguity between foo/bar with tag="tags" + // and list tags for "foo/bar/manifest" + RouteName: RouteNameTags, + RequestURI: "/v2/foo/bar/manifests/tags/list", + Vars: map[string]string{ + "name": "foo/bar/manifests", + }, + }, + { + RouteName: RouteNameBlobUploadChunk, + RequestURI: "/v2/foo/../../blob/uploads/D95306FA-FAD3-4E36-8D41-CF1C93EF8286", + StatusCode: http.StatusNotFound, + }, + } { + // Register the endpoint + router.GetRoute(testcase.RouteName).Handler(testHandler) + u := server.URL + testcase.RequestURI + + resp, err := http.Get(u) + + if err != nil { + t.Fatalf("error issuing get request: %v", err) + } + + if testcase.StatusCode == 0 { + // Override default, zero-value + testcase.StatusCode = http.StatusOK + } + + if resp.StatusCode != testcase.StatusCode { + t.Fatalf("unexpected status for %s: %v %v", u, resp.Status, resp.StatusCode) + } + + if testcase.StatusCode != http.StatusOK { + // We don't care about json response. + continue + } + + dec := json.NewDecoder(resp.Body) + + var actualRouteInfo routeTestCase + if err := dec.Decode(&actualRouteInfo); err != nil { + t.Fatalf("error reading json response: %v", err) + } + // Needs to be set out of band + actualRouteInfo.StatusCode = resp.StatusCode + + if actualRouteInfo.RouteName != testcase.RouteName { + t.Fatalf("incorrect route %q matched, expected %q", actualRouteInfo.RouteName, testcase.RouteName) + } + + if !reflect.DeepEqual(actualRouteInfo, testcase) { + t.Fatalf("actual does not equal expected: %#v != %#v", actualRouteInfo, testcase) + } + } + +} diff --git a/docs/v2/urls.go b/docs/v2/urls.go new file mode 100644 index 00000000..72f44299 --- /dev/null +++ b/docs/v2/urls.go @@ -0,0 +1,165 @@ +package v2 + +import ( + "net/http" + "net/url" + + "github.com/docker/docker-registry/digest" + "github.com/gorilla/mux" +) + +// URLBuilder creates registry API urls from a single base endpoint. It can be +// used to create urls for use in a registry client or server. +// +// All urls will be created from the given base, including the api version. +// For example, if a root of "/foo/" is provided, urls generated will be fall +// under "/foo/v2/...". Most application will only provide a schema, host and +// port, such as "https://localhost:5000/". +type URLBuilder struct { + root *url.URL // url root (ie http://localhost/) + router *mux.Router +} + +// NewURLBuilder creates a URLBuilder with provided root url object. +func NewURLBuilder(root *url.URL) *URLBuilder { + return &URLBuilder{ + root: root, + router: Router(), + } +} + +// NewURLBuilderFromString workes identically to NewURLBuilder except it takes +// a string argument for the root, returning an error if it is not a valid +// url. +func NewURLBuilderFromString(root string) (*URLBuilder, error) { + u, err := url.Parse(root) + if err != nil { + return nil, err + } + + return NewURLBuilder(u), nil +} + +// NewURLBuilderFromRequest uses information from an *http.Request to +// construct the root url. +func NewURLBuilderFromRequest(r *http.Request) *URLBuilder { + u := &url.URL{ + Scheme: r.URL.Scheme, + Host: r.Host, + } + + return NewURLBuilder(u) +} + +// BuildBaseURL constructs a base url for the API, typically just "/v2/". +func (ub *URLBuilder) BuildBaseURL() (string, error) { + route := ub.cloneRoute(RouteNameBase) + + baseURL, err := route.URL() + if err != nil { + return "", err + } + + return baseURL.String(), nil +} + +// BuildTagsURL constructs a url to list the tags in the named repository. +func (ub *URLBuilder) BuildTagsURL(name string) (string, error) { + route := ub.cloneRoute(RouteNameTags) + + tagsURL, err := route.URL("name", name) + if err != nil { + return "", err + } + + return tagsURL.String(), nil +} + +// BuildManifestURL constructs a url for the manifest identified by name and tag. +func (ub *URLBuilder) BuildManifestURL(name, tag string) (string, error) { + route := ub.cloneRoute(RouteNameManifest) + + manifestURL, err := route.URL("name", name, "tag", tag) + if err != nil { + return "", err + } + + return manifestURL.String(), nil +} + +// BuildBlobURL constructs the url for the blob identified by name and dgst. +func (ub *URLBuilder) BuildBlobURL(name string, dgst digest.Digest) (string, error) { + route := ub.cloneRoute(RouteNameBlob) + + layerURL, err := route.URL("name", name, "digest", dgst.String()) + if err != nil { + return "", err + } + + return layerURL.String(), nil +} + +// BuildBlobUploadURL constructs a url to begin a blob upload in the +// repository identified by name. +func (ub *URLBuilder) BuildBlobUploadURL(name string, values ...url.Values) (string, error) { + route := ub.cloneRoute(RouteNameBlobUpload) + + uploadURL, err := route.URL("name", name) + if err != nil { + return "", err + } + + return appendValuesURL(uploadURL, values...).String(), nil +} + +// BuildBlobUploadChunkURL constructs a url for the upload identified by uuid, +// including any url values. This should generally not be used by clients, as +// this url is provided by server implementations during the blob upload +// process. +func (ub *URLBuilder) BuildBlobUploadChunkURL(name, uuid string, values ...url.Values) (string, error) { + route := ub.cloneRoute(RouteNameBlobUploadChunk) + + uploadURL, err := route.URL("name", name, "uuid", uuid) + if err != nil { + return "", err + } + + return appendValuesURL(uploadURL, values...).String(), nil +} + +// clondedRoute returns a clone of the named route from the router. Routes +// must be cloned to avoid modifying them during url generation. +func (ub *URLBuilder) cloneRoute(name string) *mux.Route { + route := new(mux.Route) + *route = *ub.router.GetRoute(name) // clone the route + + return route. + Schemes(ub.root.Scheme). + Host(ub.root.Host) +} + +// appendValuesURL appends the parameters to the url. +func appendValuesURL(u *url.URL, values ...url.Values) *url.URL { + merged := u.Query() + + for _, v := range values { + for k, vv := range v { + merged[k] = append(merged[k], vv...) + } + } + + u.RawQuery = merged.Encode() + return u +} + +// appendValues appends the parameters to the url. Panics if the string is not +// a url. +func appendValues(u string, values ...url.Values) string { + up, err := url.Parse(u) + + if err != nil { + panic(err) // should never happen + } + + return appendValuesURL(up, values...).String() +} diff --git a/docs/v2/urls_test.go b/docs/v2/urls_test.go new file mode 100644 index 00000000..a9590dba --- /dev/null +++ b/docs/v2/urls_test.go @@ -0,0 +1,100 @@ +package v2 + +import ( + "net/url" + "testing" +) + +type urlBuilderTestCase struct { + description string + expected string + build func() (string, error) +} + +// TestURLBuilder tests the various url building functions, ensuring they are +// returning the expected values. +func TestURLBuilder(t *testing.T) { + + root := "http://localhost:5000/" + urlBuilder, err := NewURLBuilderFromString(root) + if err != nil { + t.Fatalf("unexpected error creating urlbuilder: %v", err) + } + + for _, testcase := range []struct { + description string + expected string + build func() (string, error) + }{ + { + description: "test base url", + expected: "http://localhost:5000/v2/", + build: urlBuilder.BuildBaseURL, + }, + { + description: "test tags url", + expected: "http://localhost:5000/v2/foo/bar/tags/list", + build: func() (string, error) { + return urlBuilder.BuildTagsURL("foo/bar") + }, + }, + { + description: "test manifest url", + expected: "http://localhost:5000/v2/foo/bar/manifests/tag", + build: func() (string, error) { + return urlBuilder.BuildManifestURL("foo/bar", "tag") + }, + }, + { + description: "build blob url", + expected: "http://localhost:5000/v2/foo/bar/blobs/tarsum.v1+sha256:abcdef0123456789", + build: func() (string, error) { + return urlBuilder.BuildBlobURL("foo/bar", "tarsum.v1+sha256:abcdef0123456789") + }, + }, + { + description: "build blob upload url", + expected: "http://localhost:5000/v2/foo/bar/blobs/uploads/", + build: func() (string, error) { + return urlBuilder.BuildBlobUploadURL("foo/bar") + }, + }, + { + description: "build blob upload url with digest and size", + expected: "http://localhost:5000/v2/foo/bar/blobs/uploads/?digest=tarsum.v1%2Bsha256%3Aabcdef0123456789&size=10000", + build: func() (string, error) { + return urlBuilder.BuildBlobUploadURL("foo/bar", url.Values{ + "size": []string{"10000"}, + "digest": []string{"tarsum.v1+sha256:abcdef0123456789"}, + }) + }, + }, + { + description: "build blob upload chunk url", + expected: "http://localhost:5000/v2/foo/bar/blobs/uploads/uuid-part", + build: func() (string, error) { + return urlBuilder.BuildBlobUploadChunkURL("foo/bar", "uuid-part") + }, + }, + { + description: "build blob upload chunk url with digest and size", + expected: "http://localhost:5000/v2/foo/bar/blobs/uploads/uuid-part?digest=tarsum.v1%2Bsha256%3Aabcdef0123456789&size=10000", + build: func() (string, error) { + return urlBuilder.BuildBlobUploadChunkURL("foo/bar", "uuid-part", url.Values{ + "size": []string{"10000"}, + "digest": []string{"tarsum.v1+sha256:abcdef0123456789"}, + }) + }, + }, + } { + u, err := testcase.build() + if err != nil { + t.Fatalf("%s: error building url: %v", testcase.description, err) + } + + if u != testcase.expected { + t.Fatalf("%s: %q != %q", testcase.description, u, testcase.expected) + } + } + +} From ee1e1abb15a46b325595eab68276f30543a68e92 Mon Sep 17 00:00:00 2001 From: Stephen J Day Date: Mon, 15 Dec 2014 12:42:52 -0800 Subject: [PATCH 213/375] Remove dependencies on registry packages Because docker core cannot vendor non-master Go dependencies, we need to remove dependencies on registry package. The definition of digest.Digest has been changed to a string and the regular expressions have been ported from docker-registry/common library. We'll likely change this be dependent on the registry in the future when the API stabilizies and use of the master branch becomes the norm. Signed-off-by: Stephen J Day --- docs/v2/errors_test.go | 4 +--- docs/v2/regexp.go | 19 +++++++++++++++++++ docs/v2/routes.go | 15 ++++++--------- docs/v2/urls.go | 5 ++--- 4 files changed, 28 insertions(+), 15 deletions(-) create mode 100644 docs/v2/regexp.go diff --git a/docs/v2/errors_test.go b/docs/v2/errors_test.go index d2fc091a..4a80cdfe 100644 --- a/docs/v2/errors_test.go +++ b/docs/v2/errors_test.go @@ -4,8 +4,6 @@ import ( "encoding/json" "reflect" "testing" - - "github.com/docker/docker-registry/digest" ) // TestErrorCodes ensures that error code format, mappings and @@ -61,7 +59,7 @@ func TestErrorsManagement(t *testing.T) { errs.Push(ErrorCodeDigestInvalid) errs.Push(ErrorCodeBlobUnknown, - map[string]digest.Digest{"digest": "sometestblobsumdoesntmatter"}) + map[string]string{"digest": "sometestblobsumdoesntmatter"}) p, err := json.Marshal(errs) diff --git a/docs/v2/regexp.go b/docs/v2/regexp.go new file mode 100644 index 00000000..b7e95b9f --- /dev/null +++ b/docs/v2/regexp.go @@ -0,0 +1,19 @@ +package v2 + +import "regexp" + +// This file defines regular expressions for use in route definition. These +// are also defined in the registry code base. Until they are in a common, +// shared location, and exported, they must be repeated here. + +// RepositoryNameComponentRegexp restricts registtry path components names to +// start with at least two letters or numbers, with following parts able to +// separated by one period, dash or underscore. +var RepositoryNameComponentRegexp = regexp.MustCompile(`[a-z0-9]+(?:[._-][a-z0-9]+)*`) + +// RepositoryNameRegexp builds on RepositoryNameComponentRegexp to allow 2 to +// 5 path components, separated by a forward slash. +var RepositoryNameRegexp = regexp.MustCompile(`(?:` + RepositoryNameComponentRegexp.String() + `/){1,4}` + RepositoryNameComponentRegexp.String()) + +// TagNameRegexp matches valid tag names. From docker/docker:graph/tags.go. +var TagNameRegexp = regexp.MustCompile(`[\w][\w.-]{0,127}`) diff --git a/docs/v2/routes.go b/docs/v2/routes.go index 7ebe61d6..08f36e2f 100644 --- a/docs/v2/routes.go +++ b/docs/v2/routes.go @@ -1,9 +1,6 @@ package v2 -import ( - "github.com/docker/docker-registry/common" - "github.com/gorilla/mux" -) +import "github.com/gorilla/mux" // The following are definitions of the name under which all V2 routes are // registered. These symbols can be used to look up a route based on the name. @@ -40,29 +37,29 @@ func Router() *mux.Router { // PUT /v2//manifest/ Image Manifest Upload the image manifest identified by name and tag. // DELETE /v2//manifest/ Image Manifest Delete the image identified by name and tag. router. - Path("/v2/{name:" + common.RepositoryNameRegexp.String() + "}/manifests/{tag:" + common.TagNameRegexp.String() + "}"). + Path("/v2/{name:" + RepositoryNameRegexp.String() + "}/manifests/{tag:" + TagNameRegexp.String() + "}"). Name(RouteNameManifest) // GET /v2//tags/list Tags Fetch the tags under the repository identified by name. router. - Path("/v2/{name:" + common.RepositoryNameRegexp.String() + "}/tags/list"). + Path("/v2/{name:" + RepositoryNameRegexp.String() + "}/tags/list"). Name(RouteNameTags) // GET /v2//blob/ Layer Fetch the blob identified by digest. router. - Path("/v2/{name:" + common.RepositoryNameRegexp.String() + "}/blobs/{digest:[a-zA-Z0-9-_+.]+:[a-zA-Z0-9-_+.=]+}"). + Path("/v2/{name:" + RepositoryNameRegexp.String() + "}/blobs/{digest:[a-zA-Z0-9-_+.]+:[a-zA-Z0-9-_+.=]+}"). Name(RouteNameBlob) // POST /v2//blob/upload/ Layer Upload Initiate an upload of the layer identified by tarsum. router. - Path("/v2/{name:" + common.RepositoryNameRegexp.String() + "}/blobs/uploads/"). + Path("/v2/{name:" + RepositoryNameRegexp.String() + "}/blobs/uploads/"). Name(RouteNameBlobUpload) // GET /v2//blob/upload/ Layer Upload Get the status of the upload identified by tarsum and uuid. // PUT /v2//blob/upload/ Layer Upload Upload all or a chunk of the upload identified by tarsum and uuid. // DELETE /v2//blob/upload/ Layer Upload Cancel the upload identified by layer and uuid router. - Path("/v2/{name:" + common.RepositoryNameRegexp.String() + "}/blobs/uploads/{uuid}"). + Path("/v2/{name:" + RepositoryNameRegexp.String() + "}/blobs/uploads/{uuid}"). Name(RouteNameBlobUploadChunk) return router diff --git a/docs/v2/urls.go b/docs/v2/urls.go index 72f44299..19ef06fa 100644 --- a/docs/v2/urls.go +++ b/docs/v2/urls.go @@ -4,7 +4,6 @@ import ( "net/http" "net/url" - "github.com/docker/docker-registry/digest" "github.com/gorilla/mux" ) @@ -88,10 +87,10 @@ func (ub *URLBuilder) BuildManifestURL(name, tag string) (string, error) { } // BuildBlobURL constructs the url for the blob identified by name and dgst. -func (ub *URLBuilder) BuildBlobURL(name string, dgst digest.Digest) (string, error) { +func (ub *URLBuilder) BuildBlobURL(name string, dgst string) (string, error) { route := ub.cloneRoute(RouteNameBlob) - layerURL, err := route.URL("name", name, "digest", dgst.String()) + layerURL, err := route.URL("name", name, "digest", dgst) if err != nil { return "", err } From 751a1a8dd0e1985eb921e888be3b027b7a6bfadb Mon Sep 17 00:00:00 2001 From: Derek McGowan Date: Tue, 16 Dec 2014 16:57:37 -0800 Subject: [PATCH 214/375] Update push and pull to registry 2.1 specification Signed-off-by: Derek McGowan --- docs/auth.go | 53 ++++++++ docs/session_v2.go | 298 +++++++++++++++------------------------------ 2 files changed, 151 insertions(+), 200 deletions(-) diff --git a/docs/auth.go b/docs/auth.go index 2044236c..b138fb53 100644 --- a/docs/auth.go +++ b/docs/auth.go @@ -37,6 +37,59 @@ type ConfigFile struct { rootPath string } +type RequestAuthorization struct { + Token string + Username string + Password string +} + +func NewRequestAuthorization(authConfig *AuthConfig, registryEndpoint *Endpoint, resource, scope string, actions []string) (*RequestAuthorization, error) { + var auth RequestAuthorization + + client := &http.Client{ + Transport: &http.Transport{ + DisableKeepAlives: true, + Proxy: http.ProxyFromEnvironment, + }, + CheckRedirect: AddRequiredHeadersToRedirectedRequests, + } + factory := HTTPRequestFactory(nil) + + for _, challenge := range registryEndpoint.AuthChallenges { + log.Debugf("Using %q auth challenge with params %s for %s", challenge.Scheme, challenge.Parameters, authConfig.Username) + + switch strings.ToLower(challenge.Scheme) { + case "basic": + auth.Username = authConfig.Username + auth.Password = authConfig.Password + case "bearer": + params := map[string]string{} + for k, v := range challenge.Parameters { + params[k] = v + } + params["scope"] = fmt.Sprintf("%s:%s:%s", resource, scope, strings.Join(actions, ",")) + token, err := getToken(authConfig.Username, authConfig.Password, params, registryEndpoint, client, factory) + if err != nil { + return nil, err + } + + auth.Token = token + default: + log.Infof("Unsupported auth scheme: %q", challenge.Scheme) + } + } + + return &auth, nil +} + +func (auth *RequestAuthorization) Authorize(req *http.Request) { + if auth.Token != "" { + req.Header.Set("Authorization", fmt.Sprintf("Bearer %s", auth.Token)) + } else if auth.Username != "" && auth.Password != "" { + req.SetBasicAuth(auth.Username, auth.Password) + } +} + // create a base64 encoded auth string to store in config func encodeAuth(authConfig *AuthConfig) string { authStr := authConfig.Username + ":" + authConfig.Password diff --git a/docs/session_v2.go b/docs/session_v2.go index 86d0c228..407c5f3a 100644 --- a/docs/session_v2.go +++ b/docs/session_v2.go @@ -9,100 +9,34 @@ import ( "strconv" log "github.com/Sirupsen/logrus" + "github.com/docker/docker/registry/v2" "github.com/docker/docker/utils" - "github.com/gorilla/mux" ) -func newV2RegistryRouter() *mux.Router { - router := mux.NewRouter() +var registryURLBuilder *v2.URLBuilder - v2Router := router.PathPrefix("/v2/").Subrouter() - - // Version Info - v2Router.Path("/version").Name("version") - - // Image Manifests - v2Router.Path("/manifest/{imagename:[a-z0-9-._/]+}/{tagname:[a-zA-Z0-9-._]+}").Name("manifests") - - // List Image Tags - v2Router.Path("/tags/{imagename:[a-z0-9-._/]+}").Name("tags") - - // Download a blob - v2Router.Path("/blob/{imagename:[a-z0-9-._/]+}/{sumtype:[a-z0-9._+-]+}/{sum:[a-fA-F0-9]{4,}}").Name("downloadBlob") - - // Upload a blob - v2Router.Path("/blob/{imagename:[a-z0-9-._/]+}/{sumtype:[a-z0-9._+-]+}").Name("uploadBlob") - - // Mounting a blob in an image - v2Router.Path("/blob/{imagename:[a-z0-9-._/]+}/{sumtype:[a-z0-9._+-]+}/{sum:[a-fA-F0-9]{4,}}").Name("mountBlob") - - return router -} - -// APIVersion2 /v2/ -var v2HTTPRoutes = newV2RegistryRouter() - -func getV2URL(e *Endpoint, routeName string, vars map[string]string) (*url.URL, error) { - route := v2HTTPRoutes.Get(routeName) - if route == nil { - return nil, fmt.Errorf("unknown regisry v2 route name: %q", routeName) - } - - varReplace := make([]string, 0, len(vars)*2) - for key, val := range vars { - varReplace = append(varReplace, key, val) - } - - routePath, err := route.URLPath(varReplace...) - if err != nil { - return nil, fmt.Errorf("unable to make registry route %q with vars %v: %s", routeName, vars, err) - } +func init() { u, err := url.Parse(REGISTRYSERVER) if err != nil { - return nil, fmt.Errorf("invalid registry url: %s", err) + panic(fmt.Errorf("invalid registry url: %s", err)) } - - return &url.URL{ - Scheme: u.Scheme, - Host: u.Host, - Path: routePath.Path, - }, nil + registryURLBuilder = v2.NewURLBuilder(u) } -// V2 Provenance POC +func getV2Builder(e *Endpoint) *v2.URLBuilder { + return registryURLBuilder +} -func (r *Session) GetV2Version(token []string) (*RegistryInfo, error) { - routeURL, err := getV2URL(r.indexEndpoint, "version", nil) - if err != nil { - return nil, err +// GetV2Authorization gets the authorization needed to the given image +// If readonly access is requested, then only the authorization may +// only be used for Get operations. +func (r *Session) GetV2Authorization(imageName string, readOnly bool) (*RequestAuthorization, error) { + scopes := []string{"pull"} + if !readOnly { + scopes = append(scopes, "push") } - method := "GET" - log.Debugf("[registry] Calling %q %s", method, routeURL.String()) - - req, err := r.reqFactory.NewRequest(method, routeURL.String(), nil) - if err != nil { - return nil, err - } - setTokenAuth(req, token) - res, _, err := r.doRequest(req) - if err != nil { - return nil, err - } - defer res.Body.Close() - if res.StatusCode != 200 { - return nil, utils.NewHTTPRequestError(fmt.Sprintf("Server error: %d fetching Version", res.StatusCode), res) - } - - decoder := json.NewDecoder(res.Body) - versionInfo := new(RegistryInfo) - - err = decoder.Decode(versionInfo) - if err != nil { - return nil, fmt.Errorf("unable to decode GetV2Version JSON response: %s", err) - } - - return versionInfo, nil + return NewRequestAuthorization(r.GetAuthConfig(true), r.indexEndpoint, "repository", imageName, scopes) } // @@ -112,25 +46,20 @@ func (r *Session) GetV2Version(token []string) (*RegistryInfo, error) { // 1.c) if anything else, err // 2) PUT the created/signed manifest // -func (r *Session) GetV2ImageManifest(imageName, tagName string, token []string) ([]byte, error) { - vars := map[string]string{ - "imagename": imageName, - "tagname": tagName, - } - - routeURL, err := getV2URL(r.indexEndpoint, "manifests", vars) +func (r *Session) GetV2ImageManifest(imageName, tagName string, auth *RequestAuthorization) ([]byte, error) { + routeURL, err := getV2Builder(r.indexEndpoint).BuildManifestURL(imageName, tagName) if err != nil { return nil, err } method := "GET" - log.Debugf("[registry] Calling %q %s", method, routeURL.String()) + log.Debugf("[registry] Calling %q %s", method, routeURL) - req, err := r.reqFactory.NewRequest(method, routeURL.String(), nil) + req, err := r.reqFactory.NewRequest(method, routeURL, nil) if err != nil { return nil, err } - setTokenAuth(req, token) + auth.Authorize(req) res, _, err := r.doRequest(req) if err != nil { return nil, err @@ -155,26 +84,20 @@ func (r *Session) GetV2ImageManifest(imageName, tagName string, token []string) // - Succeeded to mount for this image scope // - Failed with no error (So continue to Push the Blob) // - Failed with error -func (r *Session) PostV2ImageMountBlob(imageName, sumType, sum string, token []string) (bool, error) { - vars := map[string]string{ - "imagename": imageName, - "sumtype": sumType, - "sum": sum, - } - - routeURL, err := getV2URL(r.indexEndpoint, "mountBlob", vars) +func (r *Session) PostV2ImageMountBlob(imageName, sumType, sum string, auth *RequestAuthorization) (bool, error) { + routeURL, err := getV2Builder(r.indexEndpoint).BuildBlobURL(imageName, sumType+":"+sum) if err != nil { return false, err } - method := "POST" - log.Debugf("[registry] Calling %q %s", method, routeURL.String()) + method := "HEAD" + log.Debugf("[registry] Calling %q %s", method, routeURL) - req, err := r.reqFactory.NewRequest(method, routeURL.String(), nil) + req, err := r.reqFactory.NewRequest(method, routeURL, nil) if err != nil { return false, err } - setTokenAuth(req, token) + auth.Authorize(req) res, _, err := r.doRequest(req) if err != nil { return false, err @@ -191,25 +114,19 @@ func (r *Session) PostV2ImageMountBlob(imageName, sumType, sum string, token []s return false, fmt.Errorf("Failed to mount %q - %s:%s : %d", imageName, sumType, sum, res.StatusCode) } -func (r *Session) GetV2ImageBlob(imageName, sumType, sum string, blobWrtr io.Writer, token []string) error { - vars := map[string]string{ - "imagename": imageName, - "sumtype": sumType, - "sum": sum, - } - - routeURL, err := getV2URL(r.indexEndpoint, "downloadBlob", vars) +func (r *Session) GetV2ImageBlob(imageName, sumType, sum string, blobWrtr io.Writer, auth *RequestAuthorization) error { + routeURL, err := getV2Builder(r.indexEndpoint).BuildBlobURL(imageName, sumType+":"+sum) if err != nil { return err } method := "GET" - log.Debugf("[registry] Calling %q %s", method, routeURL.String()) - req, err := r.reqFactory.NewRequest(method, routeURL.String(), nil) + log.Debugf("[registry] Calling %q %s", method, routeURL) + req, err := r.reqFactory.NewRequest(method, routeURL, nil) if err != nil { return err } - setTokenAuth(req, token) + auth.Authorize(req) res, _, err := r.doRequest(req) if err != nil { return err @@ -226,25 +143,19 @@ func (r *Session) GetV2ImageBlob(imageName, sumType, sum string, blobWrtr io.Wri return err } -func (r *Session) GetV2ImageBlobReader(imageName, sumType, sum string, token []string) (io.ReadCloser, int64, error) { - vars := map[string]string{ - "imagename": imageName, - "sumtype": sumType, - "sum": sum, - } - - routeURL, err := getV2URL(r.indexEndpoint, "downloadBlob", vars) +func (r *Session) GetV2ImageBlobReader(imageName, sumType, sum string, auth *RequestAuthorization) (io.ReadCloser, int64, error) { + routeURL, err := getV2Builder(r.indexEndpoint).BuildBlobURL(imageName, sumType+":"+sum) if err != nil { return nil, 0, err } method := "GET" - log.Debugf("[registry] Calling %q %s", method, routeURL.String()) - req, err := r.reqFactory.NewRequest(method, routeURL.String(), nil) + log.Debugf("[registry] Calling %q %s", method, routeURL) + req, err := r.reqFactory.NewRequest(method, routeURL, nil) if err != nil { return nil, 0, err } - setTokenAuth(req, token) + auth.Authorize(req) res, _, err := r.doRequest(req) if err != nil { return nil, 0, err @@ -267,85 +178,76 @@ func (r *Session) GetV2ImageBlobReader(imageName, sumType, sum string, token []s // Push the image to the server for storage. // 'layer' is an uncompressed reader of the blob to be pushed. // The server will generate it's own checksum calculation. -func (r *Session) PutV2ImageBlob(imageName, sumType, sumStr string, blobRdr io.Reader, token []string) (serverChecksum string, err error) { - vars := map[string]string{ - "imagename": imageName, - "sumtype": sumType, +func (r *Session) PutV2ImageBlob(imageName, sumType, sumStr string, blobRdr io.Reader, auth *RequestAuthorization) error { + routeURL, err := getV2Builder(r.indexEndpoint).BuildBlobUploadURL(imageName) + if err != nil { + return err } - routeURL, err := getV2URL(r.indexEndpoint, "uploadBlob", vars) + log.Debugf("[registry] Calling %q %s", "POST", routeURL) + req, err := r.reqFactory.NewRequest("POST", routeURL, nil) if err != nil { - return "", err + return err } - method := "PUT" - log.Debugf("[registry] Calling %q %s", method, routeURL.String()) - req, err := r.reqFactory.NewRequest(method, routeURL.String(), blobRdr) - if err != nil { - return "", err - } - setTokenAuth(req, token) - req.Header.Set("X-Tarsum", sumStr) + auth.Authorize(req) res, _, err := r.doRequest(req) if err != nil { - return "", err + return err + } + location := res.Header.Get("Location") + + method := "PUT" + log.Debugf("[registry] Calling %q %s", method, location) + req, err = r.reqFactory.NewRequest(method, location, blobRdr) + if err != nil { + return err + } + queryParams := url.Values{} + queryParams.Add("digest", sumType+":"+sumStr) + req.URL.RawQuery = queryParams.Encode() + auth.Authorize(req) + res, _, err = r.doRequest(req) + if err != nil { + return err } defer res.Body.Close() - if res.StatusCode != 201 { - if res.StatusCode == 401 { - return "", errLoginRequired - } - return "", utils.NewHTTPRequestError(fmt.Sprintf("Server error: %d trying to push %s blob", res.StatusCode, imageName), res) - } - type sumReturn struct { - Checksum string `json:"checksum"` - } - - decoder := json.NewDecoder(res.Body) - var sumInfo sumReturn - - err = decoder.Decode(&sumInfo) - if err != nil { - return "", fmt.Errorf("unable to decode PutV2ImageBlob JSON response: %s", err) - } - - if sumInfo.Checksum != sumStr { - return "", fmt.Errorf("failed checksum comparison. serverChecksum: %q, localChecksum: %q", sumInfo.Checksum, sumStr) - } - - // XXX this is a json struct from the registry, with its checksum - return sumInfo.Checksum, nil -} - -// Finally Push the (signed) manifest of the blobs we've just pushed -func (r *Session) PutV2ImageManifest(imageName, tagName string, manifestRdr io.Reader, token []string) error { - vars := map[string]string{ - "imagename": imageName, - "tagname": tagName, - } - - routeURL, err := getV2URL(r.indexEndpoint, "manifests", vars) - if err != nil { - return err - } - - method := "PUT" - log.Debugf("[registry] Calling %q %s", method, routeURL.String()) - req, err := r.reqFactory.NewRequest(method, routeURL.String(), manifestRdr) - if err != nil { - return err - } - setTokenAuth(req, token) - res, _, err := r.doRequest(req) - if err != nil { - return err - } - res.Body.Close() if res.StatusCode != 201 { if res.StatusCode == 401 { return errLoginRequired } + return utils.NewHTTPRequestError(fmt.Sprintf("Server error: %d trying to push %s blob", res.StatusCode, imageName), res) + } + + return nil +} + +// Finally Push the (signed) manifest of the blobs we've just pushed +func (r *Session) PutV2ImageManifest(imageName, tagName string, manifestRdr io.Reader, auth *RequestAuthorization) error { + routeURL, err := getV2Builder(r.indexEndpoint).BuildManifestURL(imageName, tagName) + if err != nil { + return err + } + + method := "PUT" + log.Debugf("[registry] Calling %q %s", method, routeURL) + req, err := r.reqFactory.NewRequest(method, routeURL, manifestRdr) + if err != nil { + return err + } + auth.Authorize(req) + res, _, err := r.doRequest(req) + if err != nil { + return err + } + b, _ := ioutil.ReadAll(res.Body) + res.Body.Close() + if res.StatusCode != 200 { + if res.StatusCode == 401 { + return errLoginRequired + } + log.Debugf("Unexpected response from server: %q %#v", b, res.Header) return utils.NewHTTPRequestError(fmt.Sprintf("Server error: %d trying to push %s:%s manifest", res.StatusCode, imageName, tagName), res) } @@ -353,24 +255,20 @@ func (r *Session) PutV2ImageManifest(imageName, tagName string, manifestRdr io.R } // Given a repository name, returns a json array of string tags -func (r *Session) GetV2RemoteTags(imageName string, token []string) ([]string, error) { - vars := map[string]string{ - "imagename": imageName, - } - - routeURL, err := getV2URL(r.indexEndpoint, "tags", vars) +func (r *Session) GetV2RemoteTags(imageName string, auth *RequestAuthorization) ([]string, error) { + routeURL, err := getV2Builder(r.indexEndpoint).BuildTagsURL(imageName) if err != nil { return nil, err } method := "GET" - log.Debugf("[registry] Calling %q %s", method, routeURL.String()) + log.Debugf("[registry] Calling %q %s", method, routeURL) - req, err := r.reqFactory.NewRequest(method, routeURL.String(), nil) + req, err := r.reqFactory.NewRequest(method, routeURL, nil) if err != nil { return nil, err } - setTokenAuth(req, token) + auth.Authorize(req) res, _, err := r.doRequest(req) if err != nil { return nil, err From 6f36ce3a0183e750adef930ae4a4cb8e7d47683c Mon Sep 17 00:00:00 2001 From: Derek McGowan Date: Fri, 19 Dec 2014 14:44:18 -0800 Subject: [PATCH 215/375] Allow private V2 registry endpoints Signed-off-by: Derek McGowan --- docs/config.go | 2 +- docs/endpoint.go | 2 ++ docs/session_v2.go | 32 +++++++++++++++++++------------- 3 files changed, 22 insertions(+), 14 deletions(-) diff --git a/docs/config.go b/docs/config.go index b5652b15..4d13aaea 100644 --- a/docs/config.go +++ b/docs/config.go @@ -23,7 +23,7 @@ type Options struct { const ( // Only used for user auth + account creation INDEXSERVER = "https://index.docker.io/v1/" - REGISTRYSERVER = "https://registry-1.docker.io/v1/" + REGISTRYSERVER = "https://registry-1.docker.io/v2/" INDEXNAME = "docker.io" // INDEXSERVER = "https://registry-stage.hub.docker.com/v1/" diff --git a/docs/endpoint.go b/docs/endpoint.go index 5c5b0520..9a783f1f 100644 --- a/docs/endpoint.go +++ b/docs/endpoint.go @@ -10,6 +10,7 @@ import ( "strings" log "github.com/Sirupsen/logrus" + "github.com/docker/docker/registry/v2" ) // for mocking in unit tests @@ -103,6 +104,7 @@ type Endpoint struct { Version APIVersion IsSecure bool AuthChallenges []*AuthorizationChallenge + URLBuilder *v2.URLBuilder } // Get the formated URL for the root of this registry Endpoint diff --git a/docs/session_v2.go b/docs/session_v2.go index 407c5f3a..2304a613 100644 --- a/docs/session_v2.go +++ b/docs/session_v2.go @@ -13,30 +13,36 @@ import ( "github.com/docker/docker/utils" ) -var registryURLBuilder *v2.URLBuilder - -func init() { - u, err := url.Parse(REGISTRYSERVER) - if err != nil { - panic(fmt.Errorf("invalid registry url: %s", err)) - } - registryURLBuilder = v2.NewURLBuilder(u) -} - func getV2Builder(e *Endpoint) *v2.URLBuilder { - return registryURLBuilder + if e.URLBuilder == nil { + e.URLBuilder = v2.NewURLBuilder(e.URL) + } + return e.URLBuilder } // GetV2Authorization gets the authorization needed to the given image // If readonly access is requested, then only the authorization may // only be used for Get operations. -func (r *Session) GetV2Authorization(imageName string, readOnly bool) (*RequestAuthorization, error) { +func (r *Session) GetV2Authorization(imageName string, readOnly bool) (auth *RequestAuthorization, err error) { scopes := []string{"pull"} if !readOnly { scopes = append(scopes, "push") } - return NewRequestAuthorization(r.GetAuthConfig(true), r.indexEndpoint, "repository", imageName, scopes) + var registry *Endpoint + if r.indexEndpoint.URL.Host == IndexServerURL.Host { + registry, err = NewEndpoint(REGISTRYSERVER, nil) + if err != nil { + return + } + } else { + registry = r.indexEndpoint + } + registry.URLBuilder = v2.NewURLBuilder(registry.URL) + r.indexEndpoint = registry + + log.Debugf("Getting authorization for %s %s", imageName, scopes) + return NewRequestAuthorization(r.GetAuthConfig(true), registry, "repository", imageName, scopes) } // From 22c7328529311bfa6c38b67df6156658e2a2f411 Mon Sep 17 00:00:00 2001 From: Derek McGowan Date: Fri, 19 Dec 2014 16:14:04 -0800 Subject: [PATCH 216/375] Get token on each request Signed-off-by: Derek McGowan --- docs/auth.go | 60 ++++++++++++++++++++++++++++------------------ docs/session_v2.go | 34 +++++++++++++++++++------- 2 files changed, 62 insertions(+), 32 deletions(-) diff --git a/docs/auth.go b/docs/auth.go index b138fb53..1e1c7ddb 100644 --- a/docs/auth.go +++ b/docs/auth.go @@ -38,56 +38,70 @@ type ConfigFile struct { } type RequestAuthorization struct { - Token string - Username string - Password string + authConfig *AuthConfig + registryEndpoint *Endpoint + resource string + scope string + actions []string } -func NewRequestAuthorization(authConfig *AuthConfig, registryEndpoint *Endpoint, resource, scope string, actions []string) (*RequestAuthorization, error) { - var auth RequestAuthorization +func NewRequestAuthorization(authConfig *AuthConfig, registryEndpoint *Endpoint, resource, scope string, actions []string) *RequestAuthorization { + return &RequestAuthorization{ + authConfig: authConfig, + registryEndpoint: registryEndpoint, + resource: resource, + scope: scope, + actions: actions, + } +} +func (auth *RequestAuthorization) getToken() (string, error) { + // TODO check if already has token and before expiration client := &http.Client{ Transport: &http.Transport{ DisableKeepAlives: true, - Proxy: http.ProxyFromEnvironment, - }, + Proxy: http.ProxyFromEnvironment}, CheckRedirect: AddRequiredHeadersToRedirectedRequests, } factory := HTTPRequestFactory(nil) - for _, challenge := range registryEndpoint.AuthChallenges { - log.Debugf("Using %q auth challenge with params %s for %s", challenge.Scheme, challenge.Parameters, authConfig.Username) - + for _, challenge := range auth.registryEndpoint.AuthChallenges { switch strings.ToLower(challenge.Scheme) { case "basic": - auth.Username = authConfig.Username - auth.Password = authConfig.Password + // no token necessary case "bearer": + log.Debugf("Getting bearer token with %s for %s", challenge.Parameters, auth.authConfig.Username) params := map[string]string{} for k, v := range challenge.Parameters { params[k] = v } - params["scope"] = fmt.Sprintf("%s:%s:%s", resource, scope, strings.Join(actions, ",")) - token, err := getToken(authConfig.Username, authConfig.Password, params, registryEndpoint, client, factory) + params["scope"] = fmt.Sprintf("%s:%s:%s", auth.resource, auth.scope, strings.Join(auth.actions, ",")) + token, err := getToken(auth.authConfig.Username, auth.authConfig.Password, params, auth.registryEndpoint, client, factory) if err != nil { - return nil, err + return "", err } + // TODO cache token and set expiration to one minute from now - auth.Token = token + return token, nil default: log.Infof("Unsupported auth scheme: %q", challenge.Scheme) } } - - return &auth, nil + // TODO no expiration, do not reattempt to get a token + return "", nil } -func (auth *RequestAuthorization) Authorize(req *http.Request) { - if auth.Token != "" { - req.Header.Set("Authorization", fmt.Sprintf("Bearer %s", auth.Token)) - } else if auth.Username != "" && auth.Password != "" { - req.SetBasicAuth(auth.Username, auth.Password) +func (auth *RequestAuthorization) Authorize(req *http.Request) error { + token, err := auth.getToken() + if err != nil { + return err } + if token != "" { + req.Header.Set("Authorization", fmt.Sprintf("Bearer %s", token)) + } else if auth.authConfig.Username != "" && auth.authConfig.Password != "" { + req.SetBasicAuth(auth.authConfig.Username, auth.authConfig.Password) + } + return nil } // create a base64 encoded auth string to store in config diff --git a/docs/session_v2.go b/docs/session_v2.go index 2304a613..491cd2c6 100644 --- a/docs/session_v2.go +++ b/docs/session_v2.go @@ -42,7 +42,7 @@ func (r *Session) GetV2Authorization(imageName string, readOnly bool) (auth *Req r.indexEndpoint = registry log.Debugf("Getting authorization for %s %s", imageName, scopes) - return NewRequestAuthorization(r.GetAuthConfig(true), registry, "repository", imageName, scopes) + return NewRequestAuthorization(r.GetAuthConfig(true), registry, "repository", imageName, scopes), nil } // @@ -65,7 +65,9 @@ func (r *Session) GetV2ImageManifest(imageName, tagName string, auth *RequestAut if err != nil { return nil, err } - auth.Authorize(req) + if err := auth.Authorize(req) { + return nil, err + } res, _, err := r.doRequest(req) if err != nil { return nil, err @@ -103,7 +105,9 @@ func (r *Session) PostV2ImageMountBlob(imageName, sumType, sum string, auth *Req if err != nil { return false, err } - auth.Authorize(req) + if err := auth.Authorize(req) { + return nil, err + } res, _, err := r.doRequest(req) if err != nil { return false, err @@ -132,7 +136,9 @@ func (r *Session) GetV2ImageBlob(imageName, sumType, sum string, blobWrtr io.Wri if err != nil { return err } - auth.Authorize(req) + if err := auth.Authorize(req) { + return nil, err + } res, _, err := r.doRequest(req) if err != nil { return err @@ -161,7 +167,9 @@ func (r *Session) GetV2ImageBlobReader(imageName, sumType, sum string, auth *Req if err != nil { return nil, 0, err } - auth.Authorize(req) + if err := auth.Authorize(req) { + return nil, err + } res, _, err := r.doRequest(req) if err != nil { return nil, 0, err @@ -196,7 +204,9 @@ func (r *Session) PutV2ImageBlob(imageName, sumType, sumStr string, blobRdr io.R return err } - auth.Authorize(req) + if err := auth.Authorize(req) { + return nil, err + } res, _, err := r.doRequest(req) if err != nil { return err @@ -212,7 +222,9 @@ func (r *Session) PutV2ImageBlob(imageName, sumType, sumStr string, blobRdr io.R queryParams := url.Values{} queryParams.Add("digest", sumType+":"+sumStr) req.URL.RawQuery = queryParams.Encode() - auth.Authorize(req) + if err := auth.Authorize(req) { + return nil, err + } res, _, err = r.doRequest(req) if err != nil { return err @@ -242,7 +254,9 @@ func (r *Session) PutV2ImageManifest(imageName, tagName string, manifestRdr io.R if err != nil { return err } - auth.Authorize(req) + if err := auth.Authorize(req) { + return nil, err + } res, _, err := r.doRequest(req) if err != nil { return err @@ -274,7 +288,9 @@ func (r *Session) GetV2RemoteTags(imageName string, auth *RequestAuthorization) if err != nil { return nil, err } - auth.Authorize(req) + if err := auth.Authorize(req) { + return nil, err + } res, _, err := r.doRequest(req) if err != nil { return nil, err From 6f09abd5c97aa1c8efc491b4fb0eb73c73a53a8f Mon Sep 17 00:00:00 2001 From: Stephen J Day Date: Mon, 22 Dec 2014 14:58:08 -0800 Subject: [PATCH 217/375] Correctly check and propagate errors in v2 session Signed-off-by: Stephen J Day --- docs/session_v2.go | 28 ++++++++++++++-------------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/docs/session_v2.go b/docs/session_v2.go index 491cd2c6..411df46e 100644 --- a/docs/session_v2.go +++ b/docs/session_v2.go @@ -65,7 +65,7 @@ func (r *Session) GetV2ImageManifest(imageName, tagName string, auth *RequestAut if err != nil { return nil, err } - if err := auth.Authorize(req) { + if err := auth.Authorize(req); err != nil { return nil, err } res, _, err := r.doRequest(req) @@ -105,8 +105,8 @@ func (r *Session) PostV2ImageMountBlob(imageName, sumType, sum string, auth *Req if err != nil { return false, err } - if err := auth.Authorize(req) { - return nil, err + if err := auth.Authorize(req); err != nil { + return false, err } res, _, err := r.doRequest(req) if err != nil { @@ -136,8 +136,8 @@ func (r *Session) GetV2ImageBlob(imageName, sumType, sum string, blobWrtr io.Wri if err != nil { return err } - if err := auth.Authorize(req) { - return nil, err + if err := auth.Authorize(req); err != nil { + return err } res, _, err := r.doRequest(req) if err != nil { @@ -167,8 +167,8 @@ func (r *Session) GetV2ImageBlobReader(imageName, sumType, sum string, auth *Req if err != nil { return nil, 0, err } - if err := auth.Authorize(req) { - return nil, err + if err := auth.Authorize(req); err != nil { + return nil, 0, err } res, _, err := r.doRequest(req) if err != nil { @@ -204,8 +204,8 @@ func (r *Session) PutV2ImageBlob(imageName, sumType, sumStr string, blobRdr io.R return err } - if err := auth.Authorize(req) { - return nil, err + if err := auth.Authorize(req); err != nil { + return err } res, _, err := r.doRequest(req) if err != nil { @@ -222,8 +222,8 @@ func (r *Session) PutV2ImageBlob(imageName, sumType, sumStr string, blobRdr io.R queryParams := url.Values{} queryParams.Add("digest", sumType+":"+sumStr) req.URL.RawQuery = queryParams.Encode() - if err := auth.Authorize(req) { - return nil, err + if err := auth.Authorize(req); err != nil { + return err } res, _, err = r.doRequest(req) if err != nil { @@ -254,8 +254,8 @@ func (r *Session) PutV2ImageManifest(imageName, tagName string, manifestRdr io.R if err != nil { return err } - if err := auth.Authorize(req) { - return nil, err + if err := auth.Authorize(req); err != nil { + return err } res, _, err := r.doRequest(req) if err != nil { @@ -288,7 +288,7 @@ func (r *Session) GetV2RemoteTags(imageName string, auth *RequestAuthorization) if err != nil { return nil, err } - if err := auth.Authorize(req) { + if err := auth.Authorize(req); err != nil { return nil, err } res, _, err := r.doRequest(req) From 826bde851b3c0962c9075d123e9c3a00d14dc883 Mon Sep 17 00:00:00 2001 From: Josh Hawn Date: Tue, 23 Dec 2014 13:40:06 -0800 Subject: [PATCH 218/375] Add Tarsum Calculation during v2 Pull operation While the v2 pull operation is writing the body of the layer blob to disk it now computes the tarsum checksum of the archive before extracting it to the backend storage driver. If the checksum does not match that from the image manifest an error is raised. Also adds more debug logging to the pull operation and fixes existing test cases which were failing. Adds a reverse lookup constructor to the tarsum package so that you can get a tarsum object using a checksum label. Docker-DCO-1.1-Signed-off-by: Josh Hawn (github: jlhawn) --- docs/endpoint.go | 17 ++++++++++++----- docs/session_v2.go | 8 ++++++-- 2 files changed, 18 insertions(+), 7 deletions(-) diff --git a/docs/endpoint.go b/docs/endpoint.go index 9a783f1f..9ca9ed8b 100644 --- a/docs/endpoint.go +++ b/docs/endpoint.go @@ -47,16 +47,23 @@ func NewEndpoint(index *IndexInfo) (*Endpoint, error) { if err != nil { return nil, err } + if err := validateEndpoint(endpoint); err != nil { + return nil, err + } + return endpoint, nil +} + +func validateEndpoint(endpoint *Endpoint) error { log.Debugf("pinging registry endpoint %s", endpoint) // Try HTTPS ping to registry endpoint.URL.Scheme = "https" if _, err := endpoint.Ping(); err != nil { - if index.Secure { + if endpoint.IsSecure { // If registry is secure and HTTPS failed, show user the error and tell them about `--insecure-registry` // in case that's what they need. DO NOT accept unknown CA certificates, and DO NOT fallback to HTTP. - return nil, fmt.Errorf("invalid registry endpoint %s: %v. If this private registry supports only HTTP or HTTPS with an unknown CA certificate, please add `--insecure-registry %s` to the daemon's arguments. In the case of HTTPS, if you have access to the registry's CA certificate, no need for the flag; simply place the CA certificate at /etc/docker/certs.d/%s/ca.crt", endpoint, err, endpoint.URL.Host, endpoint.URL.Host) + return fmt.Errorf("invalid registry endpoint %s: %v. If this private registry supports only HTTP or HTTPS with an unknown CA certificate, please add `--insecure-registry %s` to the daemon's arguments. In the case of HTTPS, if you have access to the registry's CA certificate, no need for the flag; simply place the CA certificate at /etc/docker/certs.d/%s/ca.crt", endpoint, err, endpoint.URL.Host, endpoint.URL.Host) } // If registry is insecure and HTTPS failed, fallback to HTTP. @@ -65,13 +72,13 @@ func NewEndpoint(index *IndexInfo) (*Endpoint, error) { var err2 error if _, err2 = endpoint.Ping(); err2 == nil { - return endpoint, nil + return nil } - return nil, fmt.Errorf("invalid registry endpoint %q. HTTPS attempt: %v. HTTP attempt: %v", endpoint, err, err2) + return fmt.Errorf("invalid registry endpoint %q. HTTPS attempt: %v. HTTP attempt: %v", endpoint, err, err2) } - return endpoint, nil + return nil } func newEndpoint(address string, secure bool) (*Endpoint, error) { diff --git a/docs/session_v2.go b/docs/session_v2.go index 411df46e..031122dc 100644 --- a/docs/session_v2.go +++ b/docs/session_v2.go @@ -30,8 +30,12 @@ func (r *Session) GetV2Authorization(imageName string, readOnly bool) (auth *Req } var registry *Endpoint - if r.indexEndpoint.URL.Host == IndexServerURL.Host { - registry, err = NewEndpoint(REGISTRYSERVER, nil) + if r.indexEndpoint.String() == IndexServerAddress() { + registry, err = newEndpoint(REGISTRYSERVER, true) + if err != nil { + return + } + err = validateEndpoint(registry) if err != nil { return } From e5744a3bad5de7d92d0fdc3e14f7f0f17466987e Mon Sep 17 00:00:00 2001 From: Derek McGowan Date: Fri, 2 Jan 2015 11:13:11 -0800 Subject: [PATCH 219/375] Refactor from feedback Signed-off-by: Derek McGowan (github: dmcgowan) --- docs/session_v2.go | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/docs/session_v2.go b/docs/session_v2.go index 031122dc..0e03f4a9 100644 --- a/docs/session_v2.go +++ b/docs/session_v2.go @@ -5,7 +5,6 @@ import ( "fmt" "io" "io/ioutil" - "net/url" "strconv" log "github.com/Sirupsen/logrus" @@ -223,7 +222,7 @@ func (r *Session) PutV2ImageBlob(imageName, sumType, sumStr string, blobRdr io.R if err != nil { return err } - queryParams := url.Values{} + queryParams := req.URL.Query() queryParams.Add("digest", sumType+":"+sumStr) req.URL.RawQuery = queryParams.Encode() if err := auth.Authorize(req); err != nil { From 735a112415ee94795dd3594ae978d45ef6e5b36a Mon Sep 17 00:00:00 2001 From: Derek McGowan Date: Wed, 7 Jan 2015 15:55:29 -0800 Subject: [PATCH 220/375] Fix list tags Signed-off-by: Derek McGowan (github: dmcgowan) --- docs/session_v2.go | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/docs/session_v2.go b/docs/session_v2.go index 0e03f4a9..b08f4cf0 100644 --- a/docs/session_v2.go +++ b/docs/session_v2.go @@ -277,6 +277,11 @@ func (r *Session) PutV2ImageManifest(imageName, tagName string, manifestRdr io.R return nil } +type remoteTags struct { + name string + tags []string +} + // Given a repository name, returns a json array of string tags func (r *Session) GetV2RemoteTags(imageName string, auth *RequestAuthorization) ([]string, error) { routeURL, err := getV2Builder(r.indexEndpoint).BuildTagsURL(imageName) @@ -309,10 +314,10 @@ func (r *Session) GetV2RemoteTags(imageName string, auth *RequestAuthorization) } decoder := json.NewDecoder(res.Body) - var tags []string - err = decoder.Decode(&tags) + var remote remoteTags + err = decoder.Decode(&remote) if err != nil { return nil, fmt.Errorf("Error while decoding the http response: %s", err) } - return tags, nil + return remote.tags, nil } From 5bf94a6438b6619aef7d711b438e8fa0323ca88c Mon Sep 17 00:00:00 2001 From: Derek McGowan Date: Wed, 14 Jan 2015 16:46:31 -0800 Subject: [PATCH 221/375] Cleanup v2 session to require endpoint Signed-off-by: Derek McGowan (github: dmcgowan) --- docs/session_v2.go | 76 +++++++++++++++++++++++++--------------------- 1 file changed, 42 insertions(+), 34 deletions(-) diff --git a/docs/session_v2.go b/docs/session_v2.go index b08f4cf0..11b96bd6 100644 --- a/docs/session_v2.go +++ b/docs/session_v2.go @@ -19,33 +19,41 @@ func getV2Builder(e *Endpoint) *v2.URLBuilder { return e.URLBuilder } +func (r *Session) V2RegistryEndpoint(index *IndexInfo) (ep *Endpoint, err error) { + // TODO check if should use Mirror + if index.Official { + ep, err = newEndpoint(REGISTRYSERVER, true) + if err != nil { + return + } + err = validateEndpoint(ep) + if err != nil { + return + } + } else if r.indexEndpoint.String() == index.GetAuthConfigKey() { + ep = r.indexEndpoint + } else { + ep, err = NewEndpoint(index) + if err != nil { + return + } + } + + ep.URLBuilder = v2.NewURLBuilder(ep.URL) + return +} + // GetV2Authorization gets the authorization needed to the given image // If readonly access is requested, then only the authorization may // only be used for Get operations. -func (r *Session) GetV2Authorization(imageName string, readOnly bool) (auth *RequestAuthorization, err error) { +func (r *Session) GetV2Authorization(ep *Endpoint, imageName string, readOnly bool) (auth *RequestAuthorization, err error) { scopes := []string{"pull"} if !readOnly { scopes = append(scopes, "push") } - var registry *Endpoint - if r.indexEndpoint.String() == IndexServerAddress() { - registry, err = newEndpoint(REGISTRYSERVER, true) - if err != nil { - return - } - err = validateEndpoint(registry) - if err != nil { - return - } - } else { - registry = r.indexEndpoint - } - registry.URLBuilder = v2.NewURLBuilder(registry.URL) - r.indexEndpoint = registry - log.Debugf("Getting authorization for %s %s", imageName, scopes) - return NewRequestAuthorization(r.GetAuthConfig(true), registry, "repository", imageName, scopes), nil + return NewRequestAuthorization(r.GetAuthConfig(true), ep, "repository", imageName, scopes), nil } // @@ -55,8 +63,8 @@ func (r *Session) GetV2Authorization(imageName string, readOnly bool) (auth *Req // 1.c) if anything else, err // 2) PUT the created/signed manifest // -func (r *Session) GetV2ImageManifest(imageName, tagName string, auth *RequestAuthorization) ([]byte, error) { - routeURL, err := getV2Builder(r.indexEndpoint).BuildManifestURL(imageName, tagName) +func (r *Session) GetV2ImageManifest(ep *Endpoint, imageName, tagName string, auth *RequestAuthorization) ([]byte, error) { + routeURL, err := getV2Builder(ep).BuildManifestURL(imageName, tagName) if err != nil { return nil, err } @@ -92,11 +100,11 @@ func (r *Session) GetV2ImageManifest(imageName, tagName string, auth *RequestAut return buf, nil } -// - Succeeded to mount for this image scope -// - Failed with no error (So continue to Push the Blob) +// - Succeeded to head image blob (already exists) +// - Failed with no error (continue to Push the Blob) // - Failed with error -func (r *Session) PostV2ImageMountBlob(imageName, sumType, sum string, auth *RequestAuthorization) (bool, error) { - routeURL, err := getV2Builder(r.indexEndpoint).BuildBlobURL(imageName, sumType+":"+sum) +func (r *Session) HeadV2ImageBlob(ep *Endpoint, imageName, sumType, sum string, auth *RequestAuthorization) (bool, error) { + routeURL, err := getV2Builder(ep).BuildBlobURL(imageName, sumType+":"+sum) if err != nil { return false, err } @@ -127,8 +135,8 @@ func (r *Session) PostV2ImageMountBlob(imageName, sumType, sum string, auth *Req return false, fmt.Errorf("Failed to mount %q - %s:%s : %d", imageName, sumType, sum, res.StatusCode) } -func (r *Session) GetV2ImageBlob(imageName, sumType, sum string, blobWrtr io.Writer, auth *RequestAuthorization) error { - routeURL, err := getV2Builder(r.indexEndpoint).BuildBlobURL(imageName, sumType+":"+sum) +func (r *Session) GetV2ImageBlob(ep *Endpoint, imageName, sumType, sum string, blobWrtr io.Writer, auth *RequestAuthorization) error { + routeURL, err := getV2Builder(ep).BuildBlobURL(imageName, sumType+":"+sum) if err != nil { return err } @@ -158,8 +166,8 @@ func (r *Session) GetV2ImageBlob(imageName, sumType, sum string, blobWrtr io.Wri return err } -func (r *Session) GetV2ImageBlobReader(imageName, sumType, sum string, auth *RequestAuthorization) (io.ReadCloser, int64, error) { - routeURL, err := getV2Builder(r.indexEndpoint).BuildBlobURL(imageName, sumType+":"+sum) +func (r *Session) GetV2ImageBlobReader(ep *Endpoint, imageName, sumType, sum string, auth *RequestAuthorization) (io.ReadCloser, int64, error) { + routeURL, err := getV2Builder(ep).BuildBlobURL(imageName, sumType+":"+sum) if err != nil { return nil, 0, err } @@ -195,8 +203,8 @@ func (r *Session) GetV2ImageBlobReader(imageName, sumType, sum string, auth *Req // Push the image to the server for storage. // 'layer' is an uncompressed reader of the blob to be pushed. // The server will generate it's own checksum calculation. -func (r *Session) PutV2ImageBlob(imageName, sumType, sumStr string, blobRdr io.Reader, auth *RequestAuthorization) error { - routeURL, err := getV2Builder(r.indexEndpoint).BuildBlobUploadURL(imageName) +func (r *Session) PutV2ImageBlob(ep *Endpoint, imageName, sumType, sumStr string, blobRdr io.Reader, auth *RequestAuthorization) error { + routeURL, err := getV2Builder(ep).BuildBlobUploadURL(imageName) if err != nil { return err } @@ -245,8 +253,8 @@ func (r *Session) PutV2ImageBlob(imageName, sumType, sumStr string, blobRdr io.R } // Finally Push the (signed) manifest of the blobs we've just pushed -func (r *Session) PutV2ImageManifest(imageName, tagName string, manifestRdr io.Reader, auth *RequestAuthorization) error { - routeURL, err := getV2Builder(r.indexEndpoint).BuildManifestURL(imageName, tagName) +func (r *Session) PutV2ImageManifest(ep *Endpoint, imageName, tagName string, manifestRdr io.Reader, auth *RequestAuthorization) error { + routeURL, err := getV2Builder(ep).BuildManifestURL(imageName, tagName) if err != nil { return err } @@ -283,8 +291,8 @@ type remoteTags struct { } // Given a repository name, returns a json array of string tags -func (r *Session) GetV2RemoteTags(imageName string, auth *RequestAuthorization) ([]string, error) { - routeURL, err := getV2Builder(r.indexEndpoint).BuildTagsURL(imageName) +func (r *Session) GetV2RemoteTags(ep *Endpoint, imageName string, auth *RequestAuthorization) ([]string, error) { + routeURL, err := getV2Builder(ep).BuildTagsURL(imageName) if err != nil { return nil, err } From 9c24fc93adf4bdc666575f1ca2bb530d7a10c8ac Mon Sep 17 00:00:00 2001 From: Derek McGowan Date: Thu, 15 Jan 2015 13:06:52 -0800 Subject: [PATCH 222/375] Add token cache Token cache prevents the need to get a new token for every registry interaction. Since the tokens are short lived, the cache expires after only a minute. Signed-off-by: Derek McGowan (github: dmcgowan) --- docs/auth.go | 23 ++++++++++++++++++++--- 1 file changed, 20 insertions(+), 3 deletions(-) diff --git a/docs/auth.go b/docs/auth.go index 1e1c7ddb..1ce99805 100644 --- a/docs/auth.go +++ b/docs/auth.go @@ -10,6 +10,8 @@ import ( "os" "path" "strings" + "sync" + "time" log "github.com/Sirupsen/logrus" "github.com/docker/docker/utils" @@ -43,6 +45,10 @@ type RequestAuthorization struct { resource string scope string actions []string + + tokenLock sync.Mutex + tokenCache string + tokenExpiration time.Time } func NewRequestAuthorization(authConfig *AuthConfig, registryEndpoint *Endpoint, resource, scope string, actions []string) *RequestAuthorization { @@ -56,7 +62,14 @@ func NewRequestAuthorization(authConfig *AuthConfig, registryEndpoint *Endpoint, } func (auth *RequestAuthorization) getToken() (string, error) { - // TODO check if already has token and before expiration + auth.tokenLock.Lock() + defer auth.tokenLock.Unlock() + now := time.Now() + if now.Before(auth.tokenExpiration) { + log.Debugf("Using cached token for %s", auth.authConfig.Username) + return auth.tokenCache, nil + } + client := &http.Client{ Transport: &http.Transport{ DisableKeepAlives: true, @@ -80,14 +93,18 @@ func (auth *RequestAuthorization) getToken() (string, error) { if err != nil { return "", err } - // TODO cache token and set expiration to one minute from now + auth.tokenCache = token + auth.tokenExpiration = now.Add(time.Minute) return token, nil default: log.Infof("Unsupported auth scheme: %q", challenge.Scheme) } } - // TODO no expiration, do not reattempt to get a token + + // Do not expire cache since there are no challenges which use a token + auth.tokenExpiration = time.Now().Add(time.Hour * 24) + return "", nil } From 4eaf64432145407281e3e25b9ce471df73e01b0a Mon Sep 17 00:00:00 2001 From: Brian Goff Date: Fri, 16 Jan 2015 09:47:32 -0500 Subject: [PATCH 223/375] Make .dockercfg with json.MarshallIndent Fixes #10129 Makes the .dockercfg more human parsable. Also cleaned up the (technically) racey login test. Signed-off-by: Brian Goff --- docs/auth.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/auth.go b/docs/auth.go index 102078d7..9d223f77 100644 --- a/docs/auth.go +++ b/docs/auth.go @@ -133,7 +133,7 @@ func SaveConfig(configFile *ConfigFile) error { configs[k] = authCopy } - b, err := json.Marshal(configs) + b, err := json.MarshalIndent(configs, "", "\t") if err != nil { return err } From 1c7271129b71cb91ccfed7dfbed5857649194e4e Mon Sep 17 00:00:00 2001 From: Josh Hawn Date: Tue, 20 Jan 2015 19:37:21 -0800 Subject: [PATCH 224/375] Resolve ambiguity on registry v2 ping v2 ping now checks for a Docker-Distribution-API-Version header that identifies the endpoint as "registry/2.0" Docker-DCO-1.1-Signed-off-by: Josh Hawn (github: jlhawn) --- docs/endpoint.go | 15 +++++++++++ docs/endpoint_test.go | 63 ++++++++++++++++++++++++++++++++++++++++++- 2 files changed, 77 insertions(+), 1 deletion(-) diff --git a/docs/endpoint.go b/docs/endpoint.go index 9ca9ed8b..72bcce4a 100644 --- a/docs/endpoint.go +++ b/docs/endpoint.go @@ -227,6 +227,21 @@ func (e *Endpoint) pingV2() (RegistryInfo, error) { } defer resp.Body.Close() + // The endpoint may have multiple supported versions. + // Ensure it supports the v2 Registry API. + var supportsV2 bool + + for _, versionName := range resp.Header[http.CanonicalHeaderKey("Docker-Distribution-API-Version")] { + if versionName == "registry/2.0" { + supportsV2 = true + break + } + } + + if !supportsV2 { + return RegistryInfo{}, fmt.Errorf("%s does not appear to be a v2 registry endpoint", e) + } + if resp.StatusCode == http.StatusOK { // It would seem that no authentication/authorization is required. // So we don't need to parse/add any authorization schemes. diff --git a/docs/endpoint_test.go b/docs/endpoint_test.go index f6489034..ef258999 100644 --- a/docs/endpoint_test.go +++ b/docs/endpoint_test.go @@ -1,6 +1,11 @@ package registry -import "testing" +import ( + "net/http" + "net/http/httptest" + "net/url" + "testing" +) func TestEndpointParse(t *testing.T) { testData := []struct { @@ -27,3 +32,59 @@ func TestEndpointParse(t *testing.T) { } } } + +// Ensure that a registry endpoint that responds with a 401 only is determined +// to be a v1 registry unless it includes a valid v2 API header. +func TestValidateEndpointAmbiguousAPIVersion(t *testing.T) { + requireBasicAuthHandler := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Add("WWW-Authenticate", `Basic realm="localhost"`) + w.WriteHeader(http.StatusUnauthorized) + }) + + requireBasicAuthHandlerV2 := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Add("Docker-Distribution-API-Version", "registry/2.0") + requireBasicAuthHandler.ServeHTTP(w, r) + }) + + // Make a test server which should validate as a v1 server. + testServer := httptest.NewServer(requireBasicAuthHandler) + defer testServer.Close() + + testServerURL, err := url.Parse(testServer.URL) + if err != nil { + t.Fatal(err) + } + + testEndpoint := Endpoint{ + URL: testServerURL, + Version: APIVersionUnknown, + } + + if err = validateEndpoint(&testEndpoint); err != nil { + t.Fatal(err) + } + + if testEndpoint.Version != APIVersion1 { + t.Fatalf("expected endpoint to validate to %s, got %s", APIVersion1, testEndpoint.Version) + } + + // Make a test server which should validate as a v2 server. + testServer = httptest.NewServer(requireBasicAuthHandlerV2) + defer testServer.Close() + + testServerURL, err = url.Parse(testServer.URL) + if err != nil { + t.Fatal(err) + } + + testEndpoint.URL = testServerURL + testEndpoint.Version = APIVersionUnknown + + if err = validateEndpoint(&testEndpoint); err != nil { + t.Fatal(err) + } + + if testEndpoint.Version != APIVersion2 { + t.Fatalf("expected endpoint to validate to %s, got %s", APIVersion2, testEndpoint.Version) + } +} From 6a736c20f01735f797f14353b180bc8178bfb752 Mon Sep 17 00:00:00 2001 From: Josh Hawn Date: Wed, 21 Jan 2015 12:11:53 -0800 Subject: [PATCH 225/375] Split API Version header when checking for v2 Since the Docker-Distribution-API-Version header value may contain multiple space delimited versions as well as many instances of the header key, the header value is now split on whitespace characters to iterate over all versions that may be listed in one instance of the header. Docker-DCO-1.1-Signed-off-by: Josh Hawn (github: jlhawn) --- docs/endpoint.go | 11 +++++++---- docs/endpoint_test.go | 4 +++- 2 files changed, 10 insertions(+), 5 deletions(-) diff --git a/docs/endpoint.go b/docs/endpoint.go index 72bcce4a..de9c1f86 100644 --- a/docs/endpoint.go +++ b/docs/endpoint.go @@ -231,10 +231,13 @@ func (e *Endpoint) pingV2() (RegistryInfo, error) { // Ensure it supports the v2 Registry API. var supportsV2 bool - for _, versionName := range resp.Header[http.CanonicalHeaderKey("Docker-Distribution-API-Version")] { - if versionName == "registry/2.0" { - supportsV2 = true - break +HeaderLoop: + for _, supportedVersions := range resp.Header[http.CanonicalHeaderKey("Docker-Distribution-API-Version")] { + for _, versionName := range strings.Fields(supportedVersions) { + if versionName == "registry/2.0" { + supportsV2 = true + break HeaderLoop + } } } diff --git a/docs/endpoint_test.go b/docs/endpoint_test.go index ef258999..00c27b44 100644 --- a/docs/endpoint_test.go +++ b/docs/endpoint_test.go @@ -42,7 +42,9 @@ func TestValidateEndpointAmbiguousAPIVersion(t *testing.T) { }) requireBasicAuthHandlerV2 := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - w.Header().Add("Docker-Distribution-API-Version", "registry/2.0") + // This mock server supports v2.0, v2.1, v42.0, and v100.0 + w.Header().Add("Docker-Distribution-API-Version", "registry/100.0 registry/42.0") + w.Header().Add("Docker-Distribution-API-Version", "registry/2.0 registry/2.1") requireBasicAuthHandler.ServeHTTP(w, r) }) From 41703e2bb7fe2adc32835b150d0ad0c7469dd689 Mon Sep 17 00:00:00 2001 From: Derek McGowan Date: Wed, 21 Jan 2015 09:44:24 -0800 Subject: [PATCH 226/375] Fix write after close on http response Signed-off-by: Derek McGowan (github: dmcgowan) --- docs/session_v2.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/session_v2.go b/docs/session_v2.go index 11b96bd6..fa02bd3e 100644 --- a/docs/session_v2.go +++ b/docs/session_v2.go @@ -226,7 +226,7 @@ func (r *Session) PutV2ImageBlob(ep *Endpoint, imageName, sumType, sumStr string method := "PUT" log.Debugf("[registry] Calling %q %s", method, location) - req, err = r.reqFactory.NewRequest(method, location, blobRdr) + req, err = r.reqFactory.NewRequest(method, location, ioutil.NopCloser(blobRdr)) if err != nil { return err } From d96d4aa9f030a9d499a5f05a77aeaeb4f7d3b904 Mon Sep 17 00:00:00 2001 From: Derek McGowan Date: Mon, 26 Jan 2015 14:00:51 -0800 Subject: [PATCH 227/375] Better error messaging and logging for v2 registry requests Signed-off-by: Derek McGowan (github: dmcgowan) --- docs/session_v2.go | 20 ++++++++++++++------ 1 file changed, 14 insertions(+), 6 deletions(-) diff --git a/docs/session_v2.go b/docs/session_v2.go index fa02bd3e..8bbc9fe9 100644 --- a/docs/session_v2.go +++ b/docs/session_v2.go @@ -132,7 +132,7 @@ func (r *Session) HeadV2ImageBlob(ep *Endpoint, imageName, sumType, sum string, // return something indicating blob push needed return false, nil } - return false, fmt.Errorf("Failed to mount %q - %s:%s : %d", imageName, sumType, sum, res.StatusCode) + return false, utils.NewHTTPRequestError(fmt.Sprintf("Server error: %d trying head request for %s - %s:%s", res.StatusCode, imageName, sumType, sum), res) } func (r *Session) GetV2ImageBlob(ep *Endpoint, imageName, sumType, sum string, blobWrtr io.Writer, auth *RequestAuthorization) error { @@ -189,7 +189,7 @@ func (r *Session) GetV2ImageBlobReader(ep *Endpoint, imageName, sumType, sum str if res.StatusCode == 401 { return nil, 0, errLoginRequired } - return nil, 0, utils.NewHTTPRequestError(fmt.Sprintf("Server error: %d trying to pull %s blob", res.StatusCode, imageName), res) + return nil, 0, utils.NewHTTPRequestError(fmt.Sprintf("Server error: %d trying to pull %s blob - %s:%s", res.StatusCode, imageName, sumType, sum), res) } lenStr := res.Header.Get("Content-Length") l, err := strconv.ParseInt(lenStr, 10, 64) @@ -246,7 +246,12 @@ func (r *Session) PutV2ImageBlob(ep *Endpoint, imageName, sumType, sumStr string if res.StatusCode == 401 { return errLoginRequired } - return utils.NewHTTPRequestError(fmt.Sprintf("Server error: %d trying to push %s blob", res.StatusCode, imageName), res) + errBody, err := ioutil.ReadAll(res.Body) + if err != nil { + return err + } + log.Debugf("Unexpected response from server: %q %#v", errBody, res.Header) + return utils.NewHTTPRequestError(fmt.Sprintf("Server error: %d trying to push %s blob - %s:%s", res.StatusCode, imageName, sumType, sumStr), res) } return nil @@ -272,13 +277,16 @@ func (r *Session) PutV2ImageManifest(ep *Endpoint, imageName, tagName string, ma if err != nil { return err } - b, _ := ioutil.ReadAll(res.Body) - res.Body.Close() + defer res.Body.Close() if res.StatusCode != 200 { if res.StatusCode == 401 { return errLoginRequired } - log.Debugf("Unexpected response from server: %q %#v", b, res.Header) + errBody, err := ioutil.ReadAll(res.Body) + if err != nil { + return err + } + log.Debugf("Unexpected response from server: %q %#v", errBody, res.Header) return utils.NewHTTPRequestError(fmt.Sprintf("Server error: %d trying to push %s:%s manifest", res.StatusCode, imageName, tagName), res) } From 0818476cb1e4487a21ba7d05f53761ec079916d6 Mon Sep 17 00:00:00 2001 From: Stephen J Day Date: Tue, 27 Jan 2015 18:09:53 -0800 Subject: [PATCH 228/375] Open up v2 http status code checks for put and head checks Under certain cases, such as when putting a manifest or check for the existence of a layer, the status code checks in session_v2.go were too narrow for their purpose. In the case of putting a manifest, the handler only cares that an error is not returned. Whether it is a 304 or 202 does not matter, as long as the server reports success. Having the client only accept specific http codes inhibits future protocol evolution. Signed-off-by: Stephen J Day --- docs/session_v2.go | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/docs/session_v2.go b/docs/session_v2.go index 8bbc9fe9..dbef7df1 100644 --- a/docs/session_v2.go +++ b/docs/session_v2.go @@ -124,14 +124,15 @@ func (r *Session) HeadV2ImageBlob(ep *Endpoint, imageName, sumType, sum string, return false, err } res.Body.Close() // close early, since we're not needing a body on this call .. yet? - switch res.StatusCode { - case 200: + switch { + case res.StatusCode >= 200 && res.StatusCode < 400: // return something indicating no push needed return true, nil - case 404: + case res.StatusCode == 404: // return something indicating blob push needed return false, nil } + return false, utils.NewHTTPRequestError(fmt.Sprintf("Server error: %d trying head request for %s - %s:%s", res.StatusCode, imageName, sumType, sum), res) } @@ -278,7 +279,9 @@ func (r *Session) PutV2ImageManifest(ep *Endpoint, imageName, tagName string, ma return err } defer res.Body.Close() - if res.StatusCode != 200 { + + // All 2xx and 3xx responses can be accepted for a put. + if res.StatusCode >= 400 { if res.StatusCode == 401 { return errLoginRequired } From 9dc3529dfe057eeef1da4b111692c20fd27f8a9a Mon Sep 17 00:00:00 2001 From: Derek McGowan Date: Wed, 28 Jan 2015 16:30:00 -0800 Subject: [PATCH 229/375] Add distribution maintainers to maintainers files Signed-off-by: Derek McGowan (github: dmcgowan) --- docs/MAINTAINERS | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/docs/MAINTAINERS b/docs/MAINTAINERS index fdb03ed5..a75e15b4 100644 --- a/docs/MAINTAINERS +++ b/docs/MAINTAINERS @@ -1,5 +1,7 @@ Sam Alba (@samalba) Joffrey Fuhrer (@shin-) -Ken Cochrane (@kencochrane) Vincent Batts (@vbatts) Olivier Gambier (@dmp42) +Josh Hawn (@jlhawn) +Derek McGowan (@dmcgowan) +Stephen Day (@stevvooe) From 5589ce8b8ab11576f7432d758754f65a470dc37c Mon Sep 17 00:00:00 2001 From: Liu Hua Date: Sat, 31 Jan 2015 20:09:07 +0800 Subject: [PATCH 230/375] delete duplicated word in registry/session.go Signed-off-by: Liu Hua --- docs/session.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/session.go b/docs/session.go index b1980e1a..a668dfea 100644 --- a/docs/session.go +++ b/docs/session.go @@ -54,7 +54,7 @@ func NewSession(authConfig *AuthConfig, factory *utils.HTTPRequestFactory, endpo return nil, err } if info.Standalone { - log.Debugf("Endpoint %s is eligible for private registry registry. Enabling decorator.", r.indexEndpoint.String()) + log.Debugf("Endpoint %s is eligible for private registry. Enabling decorator.", r.indexEndpoint.String()) dec := utils.NewHTTPAuthDecorator(authConfig.Username, authConfig.Password) factory.AddDecorator(dec) } From 050337b25798d23b2a8296531d5d492bdaeb1774 Mon Sep 17 00:00:00 2001 From: Josh Hawn Date: Fri, 30 Jan 2015 17:26:00 -0800 Subject: [PATCH 231/375] Handle gorilla/mux route url bug When getting the URL from a v2 registry url builder, it does not honor the scheme from the endpoint object and will cause an https endpoint to return urls starting with http. Docker-DCO-1.1-Signed-off-by: Josh Hawn (github: jlhawn) --- docs/v2/urls.go | 25 +++++++++--- docs/v2/urls_test.go | 93 +++++++++++++++++++++++++------------------- 2 files changed, 73 insertions(+), 45 deletions(-) diff --git a/docs/v2/urls.go b/docs/v2/urls.go index 19ef06fa..d1380b47 100644 --- a/docs/v2/urls.go +++ b/docs/v2/urls.go @@ -128,13 +128,28 @@ func (ub *URLBuilder) BuildBlobUploadChunkURL(name, uuid string, values ...url.V // clondedRoute returns a clone of the named route from the router. Routes // must be cloned to avoid modifying them during url generation. -func (ub *URLBuilder) cloneRoute(name string) *mux.Route { +func (ub *URLBuilder) cloneRoute(name string) clonedRoute { route := new(mux.Route) - *route = *ub.router.GetRoute(name) // clone the route + root := new(url.URL) - return route. - Schemes(ub.root.Scheme). - Host(ub.root.Host) + *route = *ub.router.GetRoute(name) // clone the route + *root = *ub.root + + return clonedRoute{Route: route, root: root} +} + +type clonedRoute struct { + *mux.Route + root *url.URL +} + +func (cr clonedRoute) URL(pairs ...string) (*url.URL, error) { + routeURL, err := cr.Route.URL(pairs...) + if err != nil { + return nil, err + } + + return cr.root.ResolveReference(routeURL), nil } // appendValuesURL appends the parameters to the url. diff --git a/docs/v2/urls_test.go b/docs/v2/urls_test.go index a9590dba..f30c96c0 100644 --- a/docs/v2/urls_test.go +++ b/docs/v2/urls_test.go @@ -6,62 +6,58 @@ import ( ) type urlBuilderTestCase struct { - description string - expected string - build func() (string, error) + description string + expectedPath string + build func() (string, error) } // TestURLBuilder tests the various url building functions, ensuring they are // returning the expected values. func TestURLBuilder(t *testing.T) { + var ( + urlBuilder *URLBuilder + err error + ) - root := "http://localhost:5000/" - urlBuilder, err := NewURLBuilderFromString(root) - if err != nil { - t.Fatalf("unexpected error creating urlbuilder: %v", err) - } - - for _, testcase := range []struct { - description string - expected string - build func() (string, error) - }{ + testCases := []urlBuilderTestCase{ { - description: "test base url", - expected: "http://localhost:5000/v2/", - build: urlBuilder.BuildBaseURL, + description: "test base url", + expectedPath: "/v2/", + build: func() (string, error) { + return urlBuilder.BuildBaseURL() + }, }, { - description: "test tags url", - expected: "http://localhost:5000/v2/foo/bar/tags/list", + description: "test tags url", + expectedPath: "/v2/foo/bar/tags/list", build: func() (string, error) { return urlBuilder.BuildTagsURL("foo/bar") }, }, { - description: "test manifest url", - expected: "http://localhost:5000/v2/foo/bar/manifests/tag", + description: "test manifest url", + expectedPath: "/v2/foo/bar/manifests/tag", build: func() (string, error) { return urlBuilder.BuildManifestURL("foo/bar", "tag") }, }, { - description: "build blob url", - expected: "http://localhost:5000/v2/foo/bar/blobs/tarsum.v1+sha256:abcdef0123456789", + description: "build blob url", + expectedPath: "/v2/foo/bar/blobs/tarsum.v1+sha256:abcdef0123456789", build: func() (string, error) { return urlBuilder.BuildBlobURL("foo/bar", "tarsum.v1+sha256:abcdef0123456789") }, }, { - description: "build blob upload url", - expected: "http://localhost:5000/v2/foo/bar/blobs/uploads/", + description: "build blob upload url", + expectedPath: "/v2/foo/bar/blobs/uploads/", build: func() (string, error) { return urlBuilder.BuildBlobUploadURL("foo/bar") }, }, { - description: "build blob upload url with digest and size", - expected: "http://localhost:5000/v2/foo/bar/blobs/uploads/?digest=tarsum.v1%2Bsha256%3Aabcdef0123456789&size=10000", + description: "build blob upload url with digest and size", + expectedPath: "/v2/foo/bar/blobs/uploads/?digest=tarsum.v1%2Bsha256%3Aabcdef0123456789&size=10000", build: func() (string, error) { return urlBuilder.BuildBlobUploadURL("foo/bar", url.Values{ "size": []string{"10000"}, @@ -70,15 +66,15 @@ func TestURLBuilder(t *testing.T) { }, }, { - description: "build blob upload chunk url", - expected: "http://localhost:5000/v2/foo/bar/blobs/uploads/uuid-part", + description: "build blob upload chunk url", + expectedPath: "/v2/foo/bar/blobs/uploads/uuid-part", build: func() (string, error) { return urlBuilder.BuildBlobUploadChunkURL("foo/bar", "uuid-part") }, }, { - description: "build blob upload chunk url with digest and size", - expected: "http://localhost:5000/v2/foo/bar/blobs/uploads/uuid-part?digest=tarsum.v1%2Bsha256%3Aabcdef0123456789&size=10000", + description: "build blob upload chunk url with digest and size", + expectedPath: "/v2/foo/bar/blobs/uploads/uuid-part?digest=tarsum.v1%2Bsha256%3Aabcdef0123456789&size=10000", build: func() (string, error) { return urlBuilder.BuildBlobUploadChunkURL("foo/bar", "uuid-part", url.Values{ "size": []string{"10000"}, @@ -86,15 +82,32 @@ func TestURLBuilder(t *testing.T) { }) }, }, - } { - u, err := testcase.build() - if err != nil { - t.Fatalf("%s: error building url: %v", testcase.description, err) - } - - if u != testcase.expected { - t.Fatalf("%s: %q != %q", testcase.description, u, testcase.expected) - } } + roots := []string{ + "http://example.com", + "https://example.com", + "http://localhost:5000", + "https://localhost:5443", + } + + for _, root := range roots { + urlBuilder, err = NewURLBuilderFromString(root) + if err != nil { + t.Fatalf("unexpected error creating urlbuilder: %v", err) + } + + for _, testCase := range testCases { + url, err := testCase.build() + if err != nil { + t.Fatalf("%s: error building url: %v", testCase.description, err) + } + + expectedURL := root + testCase.expectedPath + + if url != expectedURL { + t.Fatalf("%s: %q != %q", testCase.description, url, expectedURL) + } + } + } } From 63af81b88366692716e601f770cf2d404543dc9c Mon Sep 17 00:00:00 2001 From: Josh Hawn Date: Fri, 30 Jan 2015 16:11:47 -0800 Subject: [PATCH 232/375] Fix token basic auth header issue When requesting a token, the basic auth header is always being set even if there is no username value. This patch corrects this and does not set the basic auth header if the username is empty. Also fixes an issue where pulling all tags from a v2 registry succeeds when the image does not actually exist on the registry. Docker-DCO-1.1-Signed-off-by: Josh Hawn (github: jlhawn) --- docs/session_v2.go | 2 ++ docs/token.go | 6 ++++-- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/docs/session_v2.go b/docs/session_v2.go index dbef7df1..da5371d8 100644 --- a/docs/session_v2.go +++ b/docs/session_v2.go @@ -128,6 +128,8 @@ func (r *Session) HeadV2ImageBlob(ep *Endpoint, imageName, sumType, sum string, case res.StatusCode >= 200 && res.StatusCode < 400: // return something indicating no push needed return true, nil + case res.StatusCode == 401: + return false, errLoginRequired case res.StatusCode == 404: // return something indicating blob push needed return false, nil diff --git a/docs/token.go b/docs/token.go index 25048630..c79a8ca6 100644 --- a/docs/token.go +++ b/docs/token.go @@ -51,10 +51,12 @@ func getToken(username, password string, params map[string]string, registryEndpo reqParams.Add("scope", scopeField) } - reqParams.Add("account", username) + if username != "" { + reqParams.Add("account", username) + req.SetBasicAuth(username, password) + } req.URL.RawQuery = reqParams.Encode() - req.SetBasicAuth(username, password) resp, err := client.Do(req) if err != nil { From 3790b5d6b457a1d5da9f1de31c2b59bc0074ff05 Mon Sep 17 00:00:00 2001 From: Alexander Morozov Date: Mon, 2 Feb 2015 14:53:20 -0800 Subject: [PATCH 233/375] Fix some go vet errors Signed-off-by: Alexander Morozov --- docs/endpoint_test.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/endpoint_test.go b/docs/endpoint_test.go index 00c27b44..9567ba23 100644 --- a/docs/endpoint_test.go +++ b/docs/endpoint_test.go @@ -67,7 +67,7 @@ func TestValidateEndpointAmbiguousAPIVersion(t *testing.T) { } if testEndpoint.Version != APIVersion1 { - t.Fatalf("expected endpoint to validate to %s, got %s", APIVersion1, testEndpoint.Version) + t.Fatalf("expected endpoint to validate to %d, got %d", APIVersion1, testEndpoint.Version) } // Make a test server which should validate as a v2 server. @@ -87,6 +87,6 @@ func TestValidateEndpointAmbiguousAPIVersion(t *testing.T) { } if testEndpoint.Version != APIVersion2 { - t.Fatalf("expected endpoint to validate to %s, got %s", APIVersion2, testEndpoint.Version) + t.Fatalf("expected endpoint to validate to %d, got %d", APIVersion2, testEndpoint.Version) } } From 92de07cee0818906a5a34671a0e6eefca9ec0b8e Mon Sep 17 00:00:00 2001 From: Doug Davis Date: Tue, 3 Feb 2015 19:51:35 -0800 Subject: [PATCH 234/375] Pretty the help text This modifies the "docker help" text so that it is no wider than 80 chars and each description fits on one line. This will also try to use ~ when possible Added a test to make sure we don't go over 80 chars again. Added a test to make sure we use ~ Applied rules/tests to all docker commands - not just main help text Closes #10214 Signed-off-by: Doug Davis --- docs/config.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/config.go b/docs/config.go index 4d13aaea..3d7e41e3 100644 --- a/docs/config.go +++ b/docs/config.go @@ -48,9 +48,9 @@ func IndexServerName() string { // the current process. func (options *Options) InstallFlags() { options.Mirrors = opts.NewListOpts(ValidateMirror) - flag.Var(&options.Mirrors, []string{"-registry-mirror"}, "Specify a preferred Docker registry mirror") + flag.Var(&options.Mirrors, []string{"-registry-mirror"}, "Preferred Docker registry mirror") options.InsecureRegistries = opts.NewListOpts(ValidateIndexName) - flag.Var(&options.InsecureRegistries, []string{"-insecure-registry"}, "Enable insecure communication with specified registries (no certificate verification for HTTPS and enable HTTP fallback) (e.g., localhost:5000 or 10.20.0.0/16)") + flag.Var(&options.InsecureRegistries, []string{"-insecure-registry"}, "Enable insecure registry communication") } type netIPNet net.IPNet From 2867d39cd9259df5be0d25f295de68ab45c88b24 Mon Sep 17 00:00:00 2001 From: Srini Brahmaroutu Date: Wed, 4 Feb 2015 21:22:38 +0000 Subject: [PATCH 235/375] Removing -X flag option and autogenerated code to create Dockerversion.go functionality Addresses #9207 Signed-off-by: Srini Brahmaroutu --- docs/httpfactory.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/httpfactory.go b/docs/httpfactory.go index 4c784360..a4fea382 100644 --- a/docs/httpfactory.go +++ b/docs/httpfactory.go @@ -3,7 +3,7 @@ package registry import ( "runtime" - "github.com/docker/docker/dockerversion" + "github.com/docker/docker/autogen/dockerversion" "github.com/docker/docker/pkg/parsers/kernel" "github.com/docker/docker/utils" ) From ec7ed3eefde92418ba350a39cfe4e29be0a3ece8 Mon Sep 17 00:00:00 2001 From: Rik Nijessen Date: Wed, 25 Feb 2015 16:59:29 +0100 Subject: [PATCH 236/375] Move TimeoutConn to seperate pkg dir. Fixes #10965 Signed-off-by: Rik Nijessen --- docs/registry.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/registry.go b/docs/registry.go index 77a78a82..ed57ed1a 100644 --- a/docs/registry.go +++ b/docs/registry.go @@ -14,7 +14,7 @@ import ( "time" log "github.com/Sirupsen/logrus" - "github.com/docker/docker/utils" + "github.com/docker/docker/pkg/timeout" ) var ( @@ -71,7 +71,7 @@ func newClient(jar http.CookieJar, roots *x509.CertPool, certs []tls.Certificate if err != nil { return nil, err } - conn = utils.NewTimeoutConn(conn, 1*time.Minute) + conn = timeout.New(conn, 1*time.Minute) return conn, nil } } From d3ad1c3cbb289ca8acb49fa944e0b9c589585ba6 Mon Sep 17 00:00:00 2001 From: Rik Nijessen Date: Wed, 25 Feb 2015 20:52:37 +0100 Subject: [PATCH 237/375] Rename package timeout to timeoutconn. Signed-off-by: Rik Nijessen --- docs/registry.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/registry.go b/docs/registry.go index ed57ed1a..a8bb8331 100644 --- a/docs/registry.go +++ b/docs/registry.go @@ -14,7 +14,7 @@ import ( "time" log "github.com/Sirupsen/logrus" - "github.com/docker/docker/pkg/timeout" + "github.com/docker/docker/pkg/timeoutconn" ) var ( @@ -71,7 +71,7 @@ func newClient(jar http.CookieJar, roots *x509.CertPool, certs []tls.Certificate if err != nil { return nil, err } - conn = timeout.New(conn, 1*time.Minute) + conn = timeoutconn.New(conn, 1*time.Minute) return conn, nil } } From 99401d7290fee5aff529eedb8974ee0b68db092e Mon Sep 17 00:00:00 2001 From: Derek McGowan Date: Wed, 28 Jan 2015 14:44:09 -0800 Subject: [PATCH 238/375] Allow single name component repository names Private registries should support having images pushed with only a single name component (e.g. localhost:5000/myapp). The public registry currently requires two name components, but this is already enforced in the registry code. Signed-off-by: Derek McGowan (github: dmcgowan) --- docs/v2/regexp.go | 4 ++-- docs/v2/routes_test.go | 8 ++++++++ 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/docs/v2/regexp.go b/docs/v2/regexp.go index b7e95b9f..e1e923b9 100644 --- a/docs/v2/regexp.go +++ b/docs/v2/regexp.go @@ -11,9 +11,9 @@ import "regexp" // separated by one period, dash or underscore. var RepositoryNameComponentRegexp = regexp.MustCompile(`[a-z0-9]+(?:[._-][a-z0-9]+)*`) -// RepositoryNameRegexp builds on RepositoryNameComponentRegexp to allow 2 to +// RepositoryNameRegexp builds on RepositoryNameComponentRegexp to allow 1 to // 5 path components, separated by a forward slash. -var RepositoryNameRegexp = regexp.MustCompile(`(?:` + RepositoryNameComponentRegexp.String() + `/){1,4}` + RepositoryNameComponentRegexp.String()) +var RepositoryNameRegexp = regexp.MustCompile(`(?:` + RepositoryNameComponentRegexp.String() + `/){0,4}` + RepositoryNameComponentRegexp.String()) // TagNameRegexp matches valid tag names. From docker/docker:graph/tags.go. var TagNameRegexp = regexp.MustCompile(`[\w][\w.-]{0,127}`) diff --git a/docs/v2/routes_test.go b/docs/v2/routes_test.go index 9969ebcc..7682792e 100644 --- a/docs/v2/routes_test.go +++ b/docs/v2/routes_test.go @@ -51,6 +51,14 @@ func TestRouter(t *testing.T) { RequestURI: "/v2/", Vars: map[string]string{}, }, + { + RouteName: RouteNameManifest, + RequestURI: "/v2/foo/manifests/bar", + Vars: map[string]string{ + "name": "foo", + "tag": "bar", + }, + }, { RouteName: RouteNameManifest, RequestURI: "/v2/foo/bar/manifests/tag", From 7a35d98c5ef35b283b119009ffa2ecb5d5eabe4f Mon Sep 17 00:00:00 2001 From: Arnaud Porterie Date: Fri, 6 Mar 2015 17:39:32 -0800 Subject: [PATCH 239/375] Remove subdirectories MAINTAINERS files Signed-off-by: Arnaud Porterie --- docs/MAINTAINERS | 7 ------- 1 file changed, 7 deletions(-) delete mode 100644 docs/MAINTAINERS diff --git a/docs/MAINTAINERS b/docs/MAINTAINERS deleted file mode 100644 index a75e15b4..00000000 --- a/docs/MAINTAINERS +++ /dev/null @@ -1,7 +0,0 @@ -Sam Alba (@samalba) -Joffrey Fuhrer (@shin-) -Vincent Batts (@vbatts) -Olivier Gambier (@dmp42) -Josh Hawn (@jlhawn) -Derek McGowan (@dmcgowan) -Stephen Day (@stevvooe) From dc4e9c6e90f724ee501a6da1270a013ce31b9292 Mon Sep 17 00:00:00 2001 From: Shishir Mahajan Date: Mon, 2 Mar 2015 16:11:49 -0500 Subject: [PATCH 240/375] Docker Tag command: Relax the restriction on namespace (username) length from 30 to 255 characters. Signed-off-by: Shishir Mahajan --- docs/config.go | 4 ++-- docs/registry_test.go | 12 +++++++++--- 2 files changed, 11 insertions(+), 5 deletions(-) diff --git a/docs/config.go b/docs/config.go index 3d7e41e3..a706f17e 100644 --- a/docs/config.go +++ b/docs/config.go @@ -223,8 +223,8 @@ func validateRemoteName(remoteName string) error { if !validNamespaceChars.MatchString(namespace) { return fmt.Errorf("Invalid namespace name (%s). Only [a-z0-9-_] are allowed.", namespace) } - if len(namespace) < 4 || len(namespace) > 30 { - return fmt.Errorf("Invalid namespace name (%s). Cannot be fewer than 4 or more than 30 characters.", namespace) + if len(namespace) < 2 || len(namespace) > 255 { + return fmt.Errorf("Invalid namespace name (%s). Cannot be fewer than 2 or more than 255 characters.", namespace) } if strings.HasPrefix(namespace, "-") || strings.HasSuffix(namespace, "-") { return fmt.Errorf("Invalid namespace name (%s). Cannot begin or end with a hyphen.", namespace) diff --git a/docs/registry_test.go b/docs/registry_test.go index 6bf31505..d96630d9 100644 --- a/docs/registry_test.go +++ b/docs/registry_test.go @@ -751,6 +751,9 @@ func TestValidRemoteName(t *testing.T) { // Allow underscores everywhere (as opposed to hyphens). "____/____", + + //Username doc and image name docker being tested. + "doc/docker", } for _, repositoryName := range validRepositoryNames { if err := validateRemoteName(repositoryName); err != nil { @@ -776,11 +779,14 @@ func TestValidRemoteName(t *testing.T) { // Disallow consecutive hyphens. "dock--er/docker", - // Namespace too short. - "doc/docker", - // No repository. "docker/", + + //namespace too short + "d/docker", + + //namespace too long + "this_is_not_a_valid_namespace_because_its_lenth_is_greater_than_255_this_is_not_a_valid_namespace_because_its_lenth_is_greater_than_255_this_is_not_a_valid_namespace_because_its_lenth_is_greater_than_255_this_is_not_a_valid_namespace_because_its_lenth_is_greater_than_255/docker", } for _, repositoryName := range invalidRepositoryNames { if err := validateRemoteName(repositoryName); err == nil { From 9879aefa816a3c0ce8e18179c0e555e0c9c4fec8 Mon Sep 17 00:00:00 2001 From: Derek McGowan Date: Mon, 16 Mar 2015 14:18:33 -0700 Subject: [PATCH 241/375] Use request factory for registry ping Currently when the registry ping is sent, it creates the request directly from http.NewRequest instead of from the http request factory. The request factory adds useful header information such as user agent which is needed by the registry. Signed-off-by: Derek McGowan (github: dmcgowan) --- docs/endpoint.go | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/docs/endpoint.go b/docs/endpoint.go index de9c1f86..b1785e4f 100644 --- a/docs/endpoint.go +++ b/docs/endpoint.go @@ -11,6 +11,7 @@ import ( log "github.com/Sirupsen/logrus" "github.com/docker/docker/registry/v2" + "github.com/docker/docker/utils" ) // for mocking in unit tests @@ -133,24 +134,25 @@ func (e *Endpoint) Path(path string) string { func (e *Endpoint) Ping() (RegistryInfo, error) { // The ping logic to use is determined by the registry endpoint version. + factory := HTTPRequestFactory(nil) switch e.Version { case APIVersion1: - return e.pingV1() + return e.pingV1(factory) case APIVersion2: - return e.pingV2() + return e.pingV2(factory) } // APIVersionUnknown // We should try v2 first... e.Version = APIVersion2 - regInfo, errV2 := e.pingV2() + regInfo, errV2 := e.pingV2(factory) if errV2 == nil { return regInfo, nil } // ... then fallback to v1. e.Version = APIVersion1 - regInfo, errV1 := e.pingV1() + regInfo, errV1 := e.pingV1(factory) if errV1 == nil { return regInfo, nil } @@ -159,7 +161,7 @@ func (e *Endpoint) Ping() (RegistryInfo, error) { return RegistryInfo{}, fmt.Errorf("unable to ping registry endpoint %s\nv2 ping attempt failed with error: %s\n v1 ping attempt failed with error: %s", e, errV2, errV1) } -func (e *Endpoint) pingV1() (RegistryInfo, error) { +func (e *Endpoint) pingV1(factory *utils.HTTPRequestFactory) (RegistryInfo, error) { log.Debugf("attempting v1 ping for registry endpoint %s", e) if e.String() == IndexServerAddress() { @@ -168,7 +170,7 @@ func (e *Endpoint) pingV1() (RegistryInfo, error) { return RegistryInfo{Standalone: false}, nil } - req, err := http.NewRequest("GET", e.Path("_ping"), nil) + req, err := factory.NewRequest("GET", e.Path("_ping"), nil) if err != nil { return RegistryInfo{Standalone: false}, err } @@ -213,10 +215,10 @@ func (e *Endpoint) pingV1() (RegistryInfo, error) { return info, nil } -func (e *Endpoint) pingV2() (RegistryInfo, error) { +func (e *Endpoint) pingV2(factory *utils.HTTPRequestFactory) (RegistryInfo, error) { log.Debugf("attempting v2 ping for registry endpoint %s", e) - req, err := http.NewRequest("GET", e.Path(""), nil) + req, err := factory.NewRequest("GET", e.Path(""), nil) if err != nil { return RegistryInfo{}, err } From 1d6ccc1b723e6417f9ca2ad7053dcac0259be3ad Mon Sep 17 00:00:00 2001 From: Derek McGowan Date: Mon, 16 Mar 2015 15:32:47 -0700 Subject: [PATCH 242/375] Quote registry error strings Currently when registry error strings contain new line characters only the last line is displayed to the client. Quote the string to ensure the client can see the entire body value. fixes #11346 Signed-off-by: Derek McGowan (github: dmcgowan) --- docs/session.go | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/docs/session.go b/docs/session.go index a668dfea..a7daeb81 100644 --- a/docs/session.go +++ b/docs/session.go @@ -349,7 +349,7 @@ func (r *Session) PushImageChecksumRegistry(imgData *ImgData, registry string, t } else if jsonBody["error"] == "Image already exists" { return ErrAlreadyExists } - return fmt.Errorf("HTTP code %d while uploading metadata: %s", res.StatusCode, errBody) + return fmt.Errorf("HTTP code %d while uploading metadata: %q", res.StatusCode, errBody) } return nil } @@ -385,7 +385,7 @@ func (r *Session) PushImageJSONRegistry(imgData *ImgData, jsonRaw []byte, regist } else if jsonBody["error"] == "Image already exists" { return ErrAlreadyExists } - return utils.NewHTTPRequestError(fmt.Sprintf("HTTP code %d while uploading metadata: %s", res.StatusCode, errBody), res) + return utils.NewHTTPRequestError(fmt.Sprintf("HTTP code %d while uploading metadata: %q", res.StatusCode, errBody), res) } return nil } @@ -427,7 +427,7 @@ func (r *Session) PushImageLayerRegistry(imgID string, layer io.Reader, registry if err != nil { return "", "", utils.NewHTTPRequestError(fmt.Sprintf("HTTP code %d while uploading metadata and error when trying to parse response body: %s", res.StatusCode, err), res) } - return "", "", utils.NewHTTPRequestError(fmt.Sprintf("Received HTTP code %d while uploading layer: %s", res.StatusCode, errBody), res) + return "", "", utils.NewHTTPRequestError(fmt.Sprintf("Received HTTP code %d while uploading layer: %q", res.StatusCode, errBody), res) } checksumPayload = "sha256:" + hex.EncodeToString(h.Sum(nil)) @@ -512,7 +512,7 @@ func (r *Session) PushImageJSONIndex(remote string, imgList []*ImgData, validate if err != nil { return nil, err } - return nil, utils.NewHTTPRequestError(fmt.Sprintf("Error: Status %d trying to push repository %s: %s", res.StatusCode, remote, errBody), res) + return nil, utils.NewHTTPRequestError(fmt.Sprintf("Error: Status %d trying to push repository %s: %q", res.StatusCode, remote, errBody), res) } if res.Header.Get("X-Docker-Token") != "" { tokens = res.Header["X-Docker-Token"] @@ -536,7 +536,7 @@ func (r *Session) PushImageJSONIndex(remote string, imgList []*ImgData, validate if err != nil { return nil, err } - return nil, utils.NewHTTPRequestError(fmt.Sprintf("Error: Status %d trying to push checksums %s: %s", res.StatusCode, remote, errBody), res) + return nil, utils.NewHTTPRequestError(fmt.Sprintf("Error: Status %d trying to push checksums %s: %q", res.StatusCode, remote, errBody), res) } } From 4b813b38476089abc347dd387e4d576530dd1e23 Mon Sep 17 00:00:00 2001 From: Andy Goldstein Date: Fri, 27 Feb 2015 02:23:50 +0000 Subject: [PATCH 243/375] Add ability to refer to image by name + digest Add ability to refer to an image by repository name and digest using the format repository@digest. Works for pull, push, run, build, and rmi. Signed-off-by: Andy Goldstein --- docs/session_v2.go | 40 +++++++++++++++++++++------------------- docs/v2/regexp.go | 3 +++ docs/v2/routes.go | 8 ++++---- docs/v2/routes_test.go | 12 ++++++------ docs/v2/urls.go | 6 +++--- 5 files changed, 37 insertions(+), 32 deletions(-) diff --git a/docs/session_v2.go b/docs/session_v2.go index da5371d8..c5bee11b 100644 --- a/docs/session_v2.go +++ b/docs/session_v2.go @@ -12,6 +12,8 @@ import ( "github.com/docker/docker/utils" ) +const DockerDigestHeader = "Docker-Content-Digest" + func getV2Builder(e *Endpoint) *v2.URLBuilder { if e.URLBuilder == nil { e.URLBuilder = v2.NewURLBuilder(e.URL) @@ -63,10 +65,10 @@ func (r *Session) GetV2Authorization(ep *Endpoint, imageName string, readOnly bo // 1.c) if anything else, err // 2) PUT the created/signed manifest // -func (r *Session) GetV2ImageManifest(ep *Endpoint, imageName, tagName string, auth *RequestAuthorization) ([]byte, error) { +func (r *Session) GetV2ImageManifest(ep *Endpoint, imageName, tagName string, auth *RequestAuthorization) ([]byte, string, error) { routeURL, err := getV2Builder(ep).BuildManifestURL(imageName, tagName) if err != nil { - return nil, err + return nil, "", err } method := "GET" @@ -74,30 +76,30 @@ func (r *Session) GetV2ImageManifest(ep *Endpoint, imageName, tagName string, au req, err := r.reqFactory.NewRequest(method, routeURL, nil) if err != nil { - return nil, err + return nil, "", err } if err := auth.Authorize(req); err != nil { - return nil, err + return nil, "", err } res, _, err := r.doRequest(req) if err != nil { - return nil, err + return nil, "", err } defer res.Body.Close() if res.StatusCode != 200 { if res.StatusCode == 401 { - return nil, errLoginRequired + return nil, "", errLoginRequired } else if res.StatusCode == 404 { - return nil, ErrDoesNotExist + return nil, "", ErrDoesNotExist } - return nil, utils.NewHTTPRequestError(fmt.Sprintf("Server error: %d trying to fetch for %s:%s", res.StatusCode, imageName, tagName), res) + return nil, "", utils.NewHTTPRequestError(fmt.Sprintf("Server error: %d trying to fetch for %s:%s", res.StatusCode, imageName, tagName), res) } buf, err := ioutil.ReadAll(res.Body) if err != nil { - return nil, fmt.Errorf("Error while reading the http response: %s", err) + return nil, "", fmt.Errorf("Error while reading the http response: %s", err) } - return buf, nil + return buf, res.Header.Get(DockerDigestHeader), nil } // - Succeeded to head image blob (already exists) @@ -261,41 +263,41 @@ func (r *Session) PutV2ImageBlob(ep *Endpoint, imageName, sumType, sumStr string } // Finally Push the (signed) manifest of the blobs we've just pushed -func (r *Session) PutV2ImageManifest(ep *Endpoint, imageName, tagName string, manifestRdr io.Reader, auth *RequestAuthorization) error { +func (r *Session) PutV2ImageManifest(ep *Endpoint, imageName, tagName string, manifestRdr io.Reader, auth *RequestAuthorization) (string, error) { routeURL, err := getV2Builder(ep).BuildManifestURL(imageName, tagName) if err != nil { - return err + return "", err } method := "PUT" log.Debugf("[registry] Calling %q %s", method, routeURL) req, err := r.reqFactory.NewRequest(method, routeURL, manifestRdr) if err != nil { - return err + return "", err } if err := auth.Authorize(req); err != nil { - return err + return "", err } res, _, err := r.doRequest(req) if err != nil { - return err + return "", err } defer res.Body.Close() // All 2xx and 3xx responses can be accepted for a put. if res.StatusCode >= 400 { if res.StatusCode == 401 { - return errLoginRequired + return "", errLoginRequired } errBody, err := ioutil.ReadAll(res.Body) if err != nil { - return err + return "", err } log.Debugf("Unexpected response from server: %q %#v", errBody, res.Header) - return utils.NewHTTPRequestError(fmt.Sprintf("Server error: %d trying to push %s:%s manifest", res.StatusCode, imageName, tagName), res) + return "", utils.NewHTTPRequestError(fmt.Sprintf("Server error: %d trying to push %s:%s manifest", res.StatusCode, imageName, tagName), res) } - return nil + return res.Header.Get(DockerDigestHeader), nil } type remoteTags struct { diff --git a/docs/v2/regexp.go b/docs/v2/regexp.go index e1e923b9..07484dcd 100644 --- a/docs/v2/regexp.go +++ b/docs/v2/regexp.go @@ -17,3 +17,6 @@ var RepositoryNameRegexp = regexp.MustCompile(`(?:` + RepositoryNameComponentReg // TagNameRegexp matches valid tag names. From docker/docker:graph/tags.go. var TagNameRegexp = regexp.MustCompile(`[\w][\w.-]{0,127}`) + +// DigestRegexp matches valid digest types. +var DigestRegexp = regexp.MustCompile(`[a-zA-Z0-9-_+.]+:[a-zA-Z0-9-_+.=]+`) diff --git a/docs/v2/routes.go b/docs/v2/routes.go index 08f36e2f..de0a38fb 100644 --- a/docs/v2/routes.go +++ b/docs/v2/routes.go @@ -33,11 +33,11 @@ func Router() *mux.Router { Path("/v2/"). Name(RouteNameBase) - // GET /v2//manifest/ Image Manifest Fetch the image manifest identified by name and tag. - // PUT /v2//manifest/ Image Manifest Upload the image manifest identified by name and tag. - // DELETE /v2//manifest/ Image Manifest Delete the image identified by name and tag. + // GET /v2//manifest/ Image Manifest Fetch the image manifest identified by name and reference where reference can be a tag or digest. + // PUT /v2//manifest/ Image Manifest Upload the image manifest identified by name and reference where reference can be a tag or digest. + // DELETE /v2//manifest/ Image Manifest Delete the image identified by name and reference where reference can be a tag or digest. router. - Path("/v2/{name:" + RepositoryNameRegexp.String() + "}/manifests/{tag:" + TagNameRegexp.String() + "}"). + Path("/v2/{name:" + RepositoryNameRegexp.String() + "}/manifests/{reference:" + TagNameRegexp.String() + "|" + DigestRegexp.String() + "}"). Name(RouteNameManifest) // GET /v2//tags/list Tags Fetch the tags under the repository identified by name. diff --git a/docs/v2/routes_test.go b/docs/v2/routes_test.go index 7682792e..0191feed 100644 --- a/docs/v2/routes_test.go +++ b/docs/v2/routes_test.go @@ -55,16 +55,16 @@ func TestRouter(t *testing.T) { RouteName: RouteNameManifest, RequestURI: "/v2/foo/manifests/bar", Vars: map[string]string{ - "name": "foo", - "tag": "bar", + "name": "foo", + "reference": "bar", }, }, { RouteName: RouteNameManifest, RequestURI: "/v2/foo/bar/manifests/tag", Vars: map[string]string{ - "name": "foo/bar", - "tag": "tag", + "name": "foo/bar", + "reference": "tag", }, }, { @@ -128,8 +128,8 @@ func TestRouter(t *testing.T) { RouteName: RouteNameManifest, RequestURI: "/v2/foo/bar/manifests/manifests/tags", Vars: map[string]string{ - "name": "foo/bar/manifests", - "tag": "tags", + "name": "foo/bar/manifests", + "reference": "tags", }, }, { diff --git a/docs/v2/urls.go b/docs/v2/urls.go index d1380b47..38fa98af 100644 --- a/docs/v2/urls.go +++ b/docs/v2/urls.go @@ -74,11 +74,11 @@ func (ub *URLBuilder) BuildTagsURL(name string) (string, error) { return tagsURL.String(), nil } -// BuildManifestURL constructs a url for the manifest identified by name and tag. -func (ub *URLBuilder) BuildManifestURL(name, tag string) (string, error) { +// BuildManifestURL constructs a url for the manifest identified by name and reference. +func (ub *URLBuilder) BuildManifestURL(name, reference string) (string, error) { route := ub.cloneRoute(RouteNameManifest) - manifestURL, err := route.URL("name", name, "tag", tag) + manifestURL, err := route.URL("name", name, "reference", reference) if err != nil { return "", err } From 7d4c1d1e979a023db8241785a7dcdfd87a69f8ac Mon Sep 17 00:00:00 2001 From: Zhang Wei Date: Sat, 14 Mar 2015 16:31:35 +0800 Subject: [PATCH 244/375] print detailed error info for docker pull When docker push get response with unknown HTTP status, docker daemon print: "Error: Status XXX trying to push repository XXX: XXX" But when docker pull meets response with unknown status code, it gives: "HTTP code: XXX" This commit helps docker pull print more detailed error info like push does, so push and pull can behave consistently when error happens. Signed-off-by: Zhang Wei --- docs/session.go | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/docs/session.go b/docs/session.go index a7daeb81..470aeab4 100644 --- a/docs/session.go +++ b/docs/session.go @@ -281,7 +281,11 @@ func (r *Session) GetRepositoryData(remote string) (*RepositoryData, error) { // TODO: Right now we're ignoring checksums in the response body. // In the future, we need to use them to check image validity. if res.StatusCode != 200 { - return nil, utils.NewHTTPRequestError(fmt.Sprintf("HTTP code: %d", res.StatusCode), res) + errBody, err := ioutil.ReadAll(res.Body) + if err != nil { + log.Debugf("Error reading response body: %s", err) + } + return nil, utils.NewHTTPRequestError(fmt.Sprintf("Error: Status %d trying to pull repository %s: %q", res.StatusCode, remote, errBody), res) } var tokens []string @@ -510,7 +514,7 @@ func (r *Session) PushImageJSONIndex(remote string, imgList []*ImgData, validate if res.StatusCode != 200 && res.StatusCode != 201 { errBody, err := ioutil.ReadAll(res.Body) if err != nil { - return nil, err + log.Debugf("Error reading response body: %s", err) } return nil, utils.NewHTTPRequestError(fmt.Sprintf("Error: Status %d trying to push repository %s: %q", res.StatusCode, remote, errBody), res) } @@ -534,7 +538,7 @@ func (r *Session) PushImageJSONIndex(remote string, imgList []*ImgData, validate if res.StatusCode != 204 { errBody, err := ioutil.ReadAll(res.Body) if err != nil { - return nil, err + log.Debugf("Error reading response body: %s", err) } return nil, utils.NewHTTPRequestError(fmt.Sprintf("Error: Status %d trying to push checksums %s: %q", res.StatusCode, remote, errBody), res) } From 4bf6791328cb1d47a69918858b3605ba0df7fdc7 Mon Sep 17 00:00:00 2001 From: Derek McGowan Date: Wed, 18 Mar 2015 14:52:49 -0700 Subject: [PATCH 245/375] Update auth client configuration to use proper tls config Currently the http clients used by auth use the default tls config. The config needs to be updated to only support TLS1.0 and newer as well as respect registry insecure configuration. Signed-off-by: Derek McGowan (github: dmcgowan) --- docs/auth.go | 20 +++++++++++++++++++- 1 file changed, 19 insertions(+), 1 deletion(-) diff --git a/docs/auth.go b/docs/auth.go index 3207c87e..bb91c95c 100644 --- a/docs/auth.go +++ b/docs/auth.go @@ -1,6 +1,7 @@ package registry import ( + "crypto/tls" "encoding/base64" "encoding/json" "errors" @@ -70,10 +71,19 @@ func (auth *RequestAuthorization) getToken() (string, error) { return auth.tokenCache, nil } + tlsConfig := tls.Config{ + MinVersion: tls.VersionTLS10, + } + if !auth.registryEndpoint.IsSecure { + tlsConfig.InsecureSkipVerify = true + } + client := &http.Client{ Transport: &http.Transport{ DisableKeepAlives: true, - Proxy: http.ProxyFromEnvironment}, + Proxy: http.ProxyFromEnvironment, + TLSClientConfig: &tlsConfig, + }, CheckRedirect: AddRequiredHeadersToRedirectedRequests, } factory := HTTPRequestFactory(nil) @@ -362,10 +372,18 @@ func loginV1(authConfig *AuthConfig, registryEndpoint *Endpoint, factory *utils. func loginV2(authConfig *AuthConfig, registryEndpoint *Endpoint, factory *utils.HTTPRequestFactory) (string, error) { log.Debugf("attempting v2 login to registry endpoint %s", registryEndpoint) + tlsConfig := tls.Config{ + MinVersion: tls.VersionTLS10, + } + if !registryEndpoint.IsSecure { + tlsConfig.InsecureSkipVerify = true + } + client := &http.Client{ Transport: &http.Transport{ DisableKeepAlives: true, Proxy: http.ProxyFromEnvironment, + TLSClientConfig: &tlsConfig, }, CheckRedirect: AddRequiredHeadersToRedirectedRequests, } From 11d08c30654eceeeb9f0b189447b7f698819c060 Mon Sep 17 00:00:00 2001 From: Josh Hawn Date: Tue, 17 Mar 2015 23:45:30 -0700 Subject: [PATCH 246/375] Add verification of image manifest digests Docker-DCO-1.1-Signed-off-by: Josh Hawn (github: jlhawn) --- docs/session_v2.go | 30 +++++++++++++++++++++++++----- 1 file changed, 25 insertions(+), 5 deletions(-) diff --git a/docs/session_v2.go b/docs/session_v2.go index c5bee11b..ec628ad1 100644 --- a/docs/session_v2.go +++ b/docs/session_v2.go @@ -1,6 +1,7 @@ package registry import ( + "bytes" "encoding/json" "fmt" "io" @@ -8,6 +9,7 @@ import ( "strconv" log "github.com/Sirupsen/logrus" + "github.com/docker/distribution/digest" "github.com/docker/docker/registry/v2" "github.com/docker/docker/utils" ) @@ -95,11 +97,12 @@ func (r *Session) GetV2ImageManifest(ep *Endpoint, imageName, tagName string, au return nil, "", utils.NewHTTPRequestError(fmt.Sprintf("Server error: %d trying to fetch for %s:%s", res.StatusCode, imageName, tagName), res) } - buf, err := ioutil.ReadAll(res.Body) + manifestBytes, err := ioutil.ReadAll(res.Body) if err != nil { return nil, "", fmt.Errorf("Error while reading the http response: %s", err) } - return buf, res.Header.Get(DockerDigestHeader), nil + + return manifestBytes, res.Header.Get(DockerDigestHeader), nil } // - Succeeded to head image blob (already exists) @@ -263,7 +266,7 @@ func (r *Session) PutV2ImageBlob(ep *Endpoint, imageName, sumType, sumStr string } // Finally Push the (signed) manifest of the blobs we've just pushed -func (r *Session) PutV2ImageManifest(ep *Endpoint, imageName, tagName string, manifestRdr io.Reader, auth *RequestAuthorization) (string, error) { +func (r *Session) PutV2ImageManifest(ep *Endpoint, imageName, tagName string, signedManifest, rawManifest []byte, auth *RequestAuthorization) (digest.Digest, error) { routeURL, err := getV2Builder(ep).BuildManifestURL(imageName, tagName) if err != nil { return "", err @@ -271,7 +274,7 @@ func (r *Session) PutV2ImageManifest(ep *Endpoint, imageName, tagName string, ma method := "PUT" log.Debugf("[registry] Calling %q %s", method, routeURL) - req, err := r.reqFactory.NewRequest(method, routeURL, manifestRdr) + req, err := r.reqFactory.NewRequest(method, routeURL, bytes.NewReader(signedManifest)) if err != nil { return "", err } @@ -297,7 +300,24 @@ func (r *Session) PutV2ImageManifest(ep *Endpoint, imageName, tagName string, ma return "", utils.NewHTTPRequestError(fmt.Sprintf("Server error: %d trying to push %s:%s manifest", res.StatusCode, imageName, tagName), res) } - return res.Header.Get(DockerDigestHeader), nil + hdrDigest, err := digest.ParseDigest(res.Header.Get(DockerDigestHeader)) + if err != nil { + return "", fmt.Errorf("invalid manifest digest from registry: %s", err) + } + + dgstVerifier, err := digest.NewDigestVerifier(hdrDigest) + if err != nil { + return "", fmt.Errorf("invalid manifest digest from registry: %s", err) + } + + dgstVerifier.Write(rawManifest) + + if !dgstVerifier.Verified() { + computedDigest, _ := digest.FromBytes(rawManifest) + return "", fmt.Errorf("unable to verify manifest digest: registry has %q, computed %q", hdrDigest, computedDigest) + } + + return hdrDigest, nil } type remoteTags struct { From bcccf35bb2b63035feb988e9d166d806d54d613e Mon Sep 17 00:00:00 2001 From: Josh Hawn Date: Fri, 20 Mar 2015 12:10:06 -0700 Subject: [PATCH 247/375] Separate init blob upload Pushing a v2 image layer has two steps: - POST to get a new upload URL - PUT to that upload URL We were previously not checking the response code of the POST request and the PUT would fail in weird ways. Docker-DCO-1.1-Signed-off-by: Josh Hawn (github: jlhawn) --- docs/session_v2.go | 67 +++++++++++++++++++++++++++++++++------------- 1 file changed, 49 insertions(+), 18 deletions(-) diff --git a/docs/session_v2.go b/docs/session_v2.go index c5bee11b..b10b15e7 100644 --- a/docs/session_v2.go +++ b/docs/session_v2.go @@ -5,6 +5,7 @@ import ( "fmt" "io" "io/ioutil" + "net/http" "strconv" log "github.com/Sirupsen/logrus" @@ -209,29 +210,14 @@ func (r *Session) GetV2ImageBlobReader(ep *Endpoint, imageName, sumType, sum str // 'layer' is an uncompressed reader of the blob to be pushed. // The server will generate it's own checksum calculation. func (r *Session) PutV2ImageBlob(ep *Endpoint, imageName, sumType, sumStr string, blobRdr io.Reader, auth *RequestAuthorization) error { - routeURL, err := getV2Builder(ep).BuildBlobUploadURL(imageName) + location, err := r.initiateBlobUpload(ep, imageName, auth) if err != nil { return err } - log.Debugf("[registry] Calling %q %s", "POST", routeURL) - req, err := r.reqFactory.NewRequest("POST", routeURL, nil) - if err != nil { - return err - } - - if err := auth.Authorize(req); err != nil { - return err - } - res, _, err := r.doRequest(req) - if err != nil { - return err - } - location := res.Header.Get("Location") - method := "PUT" log.Debugf("[registry] Calling %q %s", method, location) - req, err = r.reqFactory.NewRequest(method, location, ioutil.NopCloser(blobRdr)) + req, err := r.reqFactory.NewRequest(method, location, ioutil.NopCloser(blobRdr)) if err != nil { return err } @@ -241,7 +227,7 @@ func (r *Session) PutV2ImageBlob(ep *Endpoint, imageName, sumType, sumStr string if err := auth.Authorize(req); err != nil { return err } - res, _, err = r.doRequest(req) + res, _, err := r.doRequest(req) if err != nil { return err } @@ -262,6 +248,51 @@ func (r *Session) PutV2ImageBlob(ep *Endpoint, imageName, sumType, sumStr string return nil } +// initiateBlobUpload gets the blob upload location for the given image name. +func (r *Session) initiateBlobUpload(ep *Endpoint, imageName string, auth *RequestAuthorization) (location string, err error) { + routeURL, err := getV2Builder(ep).BuildBlobUploadURL(imageName) + if err != nil { + return "", err + } + + log.Debugf("[registry] Calling %q %s", "POST", routeURL) + req, err := r.reqFactory.NewRequest("POST", routeURL, nil) + if err != nil { + return "", err + } + + if err := auth.Authorize(req); err != nil { + return "", err + } + res, _, err := r.doRequest(req) + if err != nil { + return "", err + } + + if res.StatusCode != http.StatusAccepted { + if res.StatusCode == http.StatusUnauthorized { + return "", errLoginRequired + } + if res.StatusCode == http.StatusNotFound { + return "", ErrDoesNotExist + } + + errBody, err := ioutil.ReadAll(res.Body) + if err != nil { + return "", err + } + + log.Debugf("Unexpected response from server: %q %#v", errBody, res.Header) + return "", utils.NewHTTPRequestError(fmt.Sprintf("Server error: unexpected %d response status trying to initiate upload of %s", res.StatusCode, imageName), res) + } + + if location = res.Header.Get("Location"); location == "" { + return "", fmt.Errorf("registry did not return a Location header for resumable blob upload for image %s", imageName) + } + + return +} + // Finally Push the (signed) manifest of the blobs we've just pushed func (r *Session) PutV2ImageManifest(ep *Endpoint, imageName, tagName string, manifestRdr io.Reader, auth *RequestAuthorization) (string, error) { routeURL, err := getV2Builder(ep).BuildManifestURL(imageName, tagName) From 9f5184c1116760716f33ba69345567ac33b2ea94 Mon Sep 17 00:00:00 2001 From: Derek McGowan Date: Sun, 22 Mar 2015 18:15:18 -0700 Subject: [PATCH 248/375] Add check for 404 on get repository data No longer add the body to the error when a 404 is received on get repository data. closes #11510 Signed-off-by: Derek McGowan (github: dmcgowan) --- docs/session.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/docs/session.go b/docs/session.go index 470aeab4..82338252 100644 --- a/docs/session.go +++ b/docs/session.go @@ -280,7 +280,9 @@ func (r *Session) GetRepositoryData(remote string) (*RepositoryData, error) { } // TODO: Right now we're ignoring checksums in the response body. // In the future, we need to use them to check image validity. - if res.StatusCode != 200 { + if res.StatusCode == 404 { + return nil, utils.NewHTTPRequestError(fmt.Sprintf("HTTP code: %d", res.StatusCode), res) + } else if res.StatusCode != 200 { errBody, err := ioutil.ReadAll(res.Body) if err != nil { log.Debugf("Error reading response body: %s", err) From 0e7650f958592a1ab291f5c719a272d3cb1156e7 Mon Sep 17 00:00:00 2001 From: Meaglith Ma Date: Thu, 12 Mar 2015 03:45:01 +0800 Subject: [PATCH 249/375] Fix decode tags value error when call get /v2//tags/list in registry api v2. Signed-off-by: Meaglith Ma --- docs/session_v2.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/session_v2.go b/docs/session_v2.go index 833abeed..ed8ce061 100644 --- a/docs/session_v2.go +++ b/docs/session_v2.go @@ -352,8 +352,8 @@ func (r *Session) PutV2ImageManifest(ep *Endpoint, imageName, tagName string, si } type remoteTags struct { - name string - tags []string + Name string + Tags []string } // Given a repository name, returns a json array of string tags @@ -393,5 +393,5 @@ func (r *Session) GetV2RemoteTags(ep *Endpoint, imageName string, auth *RequestA if err != nil { return nil, fmt.Errorf("Error while decoding the http response: %s", err) } - return remote.tags, nil + return remote.Tags, nil } From 10128f6e8cde28fec9c3179fec8bd6d7cf8e20de Mon Sep 17 00:00:00 2001 From: Derek McGowan Date: Mon, 23 Mar 2015 14:23:47 -0700 Subject: [PATCH 250/375] Add struct tags on v2 remote tags struct Signed-off-by: Derek McGowan (github: dmcgowan) --- docs/session_v2.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/session_v2.go b/docs/session_v2.go index ed8ce061..22f39317 100644 --- a/docs/session_v2.go +++ b/docs/session_v2.go @@ -352,8 +352,8 @@ func (r *Session) PutV2ImageManifest(ep *Endpoint, imageName, tagName string, si } type remoteTags struct { - Name string - Tags []string + Name string `json:"name"` + Tags []string `json:"tags"` } // Given a repository name, returns a json array of string tags From 9c08a436249db722579cc5db672777142f177e34 Mon Sep 17 00:00:00 2001 From: Antonio Murdaca Date: Wed, 25 Mar 2015 08:44:12 +0100 Subject: [PATCH 251/375] Remove engine.Status and replace it with standard go error Signed-off-by: Antonio Murdaca --- docs/service.go | 50 +++++++++++++++++++++++++------------------------ 1 file changed, 26 insertions(+), 24 deletions(-) diff --git a/docs/service.go b/docs/service.go index 04834022..5daacb2b 100644 --- a/docs/service.go +++ b/docs/service.go @@ -1,6 +1,8 @@ package registry import ( + "fmt" + log "github.com/Sirupsen/logrus" "github.com/docker/docker/engine" ) @@ -38,7 +40,7 @@ func (s *Service) Install(eng *engine.Engine) error { // Auth contacts the public registry with the provided credentials, // and returns OK if authentication was sucessful. // It can be used to verify the validity of a client's credentials. -func (s *Service) Auth(job *engine.Job) engine.Status { +func (s *Service) Auth(job *engine.Job) error { var ( authConfig = new(AuthConfig) endpoint *Endpoint @@ -56,25 +58,25 @@ func (s *Service) Auth(job *engine.Job) engine.Status { } if index, err = ResolveIndexInfo(job, addr); err != nil { - return job.Error(err) + return err } if endpoint, err = NewEndpoint(index); err != nil { log.Errorf("unable to get new registry endpoint: %s", err) - return job.Error(err) + return err } authConfig.ServerAddress = endpoint.String() if status, err = Login(authConfig, endpoint, HTTPRequestFactory(nil)); err != nil { log.Errorf("unable to login against registry endpoint %s: %s", endpoint, err) - return job.Error(err) + return err } log.Infof("successful registry login for endpoint %s: %s", endpoint, status) job.Printf("%s\n", status) - return engine.StatusOK + return nil } // Search queries the public registry for images matching the specified @@ -93,9 +95,9 @@ func (s *Service) Auth(job *engine.Job) engine.Status { // Results are sent as a collection of structured messages (using engine.Table). // Each result is sent as a separate message. // Results are ordered by number of stars on the public registry. -func (s *Service) Search(job *engine.Job) engine.Status { +func (s *Service) Search(job *engine.Job) error { if n := len(job.Args); n != 1 { - return job.Errorf("Usage: %s TERM", job.Name) + return fmt.Errorf("Usage: %s TERM", job.Name) } var ( term = job.Args[0] @@ -107,20 +109,20 @@ func (s *Service) Search(job *engine.Job) engine.Status { repoInfo, err := ResolveRepositoryInfo(job, term) if err != nil { - return job.Error(err) + return err } // *TODO: Search multiple indexes. endpoint, err := repoInfo.GetEndpoint() if err != nil { - return job.Error(err) + return err } r, err := NewSession(authConfig, HTTPRequestFactory(metaHeaders), endpoint, true) if err != nil { - return job.Error(err) + return err } results, err := r.SearchRepositories(repoInfo.GetSearchTerm()) if err != nil { - return job.Error(err) + return err } outs := engine.NewTable("star_count", 0) for _, result := range results.Results { @@ -130,31 +132,31 @@ func (s *Service) Search(job *engine.Job) engine.Status { } outs.ReverseSort() if _, err := outs.WriteListTo(job.Stdout); err != nil { - return job.Error(err) + return err } - return engine.StatusOK + return nil } // ResolveRepository splits a repository name into its components // and configuration of the associated registry. -func (s *Service) ResolveRepository(job *engine.Job) engine.Status { +func (s *Service) ResolveRepository(job *engine.Job) error { var ( reposName = job.Args[0] ) repoInfo, err := s.Config.NewRepositoryInfo(reposName) if err != nil { - return job.Error(err) + return err } out := engine.Env{} err = out.SetJson("repository", repoInfo) if err != nil { - return job.Error(err) + return err } out.WriteTo(job.Stdout) - return engine.StatusOK + return nil } // Convenience wrapper for calling resolve_repository Job from a running job. @@ -175,24 +177,24 @@ func ResolveRepositoryInfo(jobContext *engine.Job, reposName string) (*Repositor } // ResolveIndex takes indexName and returns index info -func (s *Service) ResolveIndex(job *engine.Job) engine.Status { +func (s *Service) ResolveIndex(job *engine.Job) error { var ( indexName = job.Args[0] ) index, err := s.Config.NewIndexInfo(indexName) if err != nil { - return job.Error(err) + return err } out := engine.Env{} err = out.SetJson("index", index) if err != nil { - return job.Error(err) + return err } out.WriteTo(job.Stdout) - return engine.StatusOK + return nil } // Convenience wrapper for calling resolve_index Job from a running job. @@ -213,13 +215,13 @@ func ResolveIndexInfo(jobContext *engine.Job, indexName string) (*IndexInfo, err } // GetRegistryConfig returns current registry configuration. -func (s *Service) GetRegistryConfig(job *engine.Job) engine.Status { +func (s *Service) GetRegistryConfig(job *engine.Job) error { out := engine.Env{} err := out.SetJson("config", s.Config) if err != nil { - return job.Error(err) + return err } out.WriteTo(job.Stdout) - return engine.StatusOK + return nil } From eff5278d12d264ff8d80eaba85a6d16786252714 Mon Sep 17 00:00:00 2001 From: Don Kjer Date: Mon, 12 Jan 2015 19:56:01 +0000 Subject: [PATCH 252/375] Fix for issue 9922: private registry search with auth returns 401 Signed-off-by: Don Kjer --- docs/auth.go | 51 +++++++----------------------------------------- docs/endpoint.go | 18 +++++++++++++++++ docs/session.go | 4 ++++ 3 files changed, 29 insertions(+), 44 deletions(-) diff --git a/docs/auth.go b/docs/auth.go index bb91c95c..4baf114c 100644 --- a/docs/auth.go +++ b/docs/auth.go @@ -1,7 +1,6 @@ package registry import ( - "crypto/tls" "encoding/base64" "encoding/json" "errors" @@ -71,21 +70,7 @@ func (auth *RequestAuthorization) getToken() (string, error) { return auth.tokenCache, nil } - tlsConfig := tls.Config{ - MinVersion: tls.VersionTLS10, - } - if !auth.registryEndpoint.IsSecure { - tlsConfig.InsecureSkipVerify = true - } - - client := &http.Client{ - Transport: &http.Transport{ - DisableKeepAlives: true, - Proxy: http.ProxyFromEnvironment, - TLSClientConfig: &tlsConfig, - }, - CheckRedirect: AddRequiredHeadersToRedirectedRequests, - } + client := auth.registryEndpoint.HTTPClient() factory := HTTPRequestFactory(nil) for _, challenge := range auth.registryEndpoint.AuthChallenges { @@ -252,16 +237,10 @@ func Login(authConfig *AuthConfig, registryEndpoint *Endpoint, factory *utils.HT // loginV1 tries to register/login to the v1 registry server. func loginV1(authConfig *AuthConfig, registryEndpoint *Endpoint, factory *utils.HTTPRequestFactory) (string, error) { var ( - status string - reqBody []byte - err error - client = &http.Client{ - Transport: &http.Transport{ - DisableKeepAlives: true, - Proxy: http.ProxyFromEnvironment, - }, - CheckRedirect: AddRequiredHeadersToRedirectedRequests, - } + status string + reqBody []byte + err error + client = registryEndpoint.HTTPClient() reqStatusCode = 0 serverAddress = authConfig.ServerAddress ) @@ -285,7 +264,7 @@ func loginV1(authConfig *AuthConfig, registryEndpoint *Endpoint, factory *utils. // using `bytes.NewReader(jsonBody)` here causes the server to respond with a 411 status. b := strings.NewReader(string(jsonBody)) - req1, err := http.Post(serverAddress+"users/", "application/json; charset=utf-8", b) + req1, err := client.Post(serverAddress+"users/", "application/json; charset=utf-8", b) if err != nil { return "", fmt.Errorf("Server Error: %s", err) } @@ -371,26 +350,10 @@ func loginV1(authConfig *AuthConfig, registryEndpoint *Endpoint, factory *utils. // is to be determined. func loginV2(authConfig *AuthConfig, registryEndpoint *Endpoint, factory *utils.HTTPRequestFactory) (string, error) { log.Debugf("attempting v2 login to registry endpoint %s", registryEndpoint) - - tlsConfig := tls.Config{ - MinVersion: tls.VersionTLS10, - } - if !registryEndpoint.IsSecure { - tlsConfig.InsecureSkipVerify = true - } - - client := &http.Client{ - Transport: &http.Transport{ - DisableKeepAlives: true, - Proxy: http.ProxyFromEnvironment, - TLSClientConfig: &tlsConfig, - }, - CheckRedirect: AddRequiredHeadersToRedirectedRequests, - } - var ( err error allErrors []error + client = registryEndpoint.HTTPClient() ) for _, challenge := range registryEndpoint.AuthChallenges { diff --git a/docs/endpoint.go b/docs/endpoint.go index b1785e4f..59ae4dd5 100644 --- a/docs/endpoint.go +++ b/docs/endpoint.go @@ -1,6 +1,7 @@ package registry import ( + "crypto/tls" "encoding/json" "fmt" "io/ioutil" @@ -262,3 +263,20 @@ HeaderLoop: return RegistryInfo{}, fmt.Errorf("v2 registry endpoint returned status %d: %q", resp.StatusCode, http.StatusText(resp.StatusCode)) } + +func (e *Endpoint) HTTPClient() *http.Client { + tlsConfig := tls.Config{ + MinVersion: tls.VersionTLS10, + } + if !e.IsSecure { + tlsConfig.InsecureSkipVerify = true + } + return &http.Client{ + Transport: &http.Transport{ + DisableKeepAlives: true, + Proxy: http.ProxyFromEnvironment, + TLSClientConfig: &tlsConfig, + }, + CheckRedirect: AddRequiredHeadersToRedirectedRequests, + } +} diff --git a/docs/session.go b/docs/session.go index 82338252..bf04b586 100644 --- a/docs/session.go +++ b/docs/session.go @@ -511,6 +511,10 @@ func (r *Session) PushImageJSONIndex(remote string, imgList []*ImgData, validate } defer res.Body.Close() + if res.StatusCode == 401 { + return nil, errLoginRequired + } + var tokens, endpoints []string if !validate { if res.StatusCode != 200 && res.StatusCode != 201 { From b085d5556e9e69da5321e45730de43f8bb6665bc Mon Sep 17 00:00:00 2001 From: Peter Choi Date: Wed, 25 Mar 2015 19:40:23 -0600 Subject: [PATCH 253/375] Changed snake case naming to camelCase Signed-off-by: Peter Choi --- docs/config.go | 6 +++--- docs/registry_mock_test.go | 8 ++++---- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/docs/config.go b/docs/config.go index a706f17e..3515836d 100644 --- a/docs/config.go +++ b/docs/config.go @@ -60,10 +60,10 @@ func (ipnet *netIPNet) MarshalJSON() ([]byte, error) { } func (ipnet *netIPNet) UnmarshalJSON(b []byte) (err error) { - var ipnet_str string - if err = json.Unmarshal(b, &ipnet_str); err == nil { + var ipnetStr string + if err = json.Unmarshal(b, &ipnetStr); err == nil { var cidr *net.IPNet - if _, cidr, err = net.ParseCIDR(ipnet_str); err == nil { + if _, cidr, err = net.ParseCIDR(ipnetStr); err == nil { *ipnet = netIPNet(*cidr) } } diff --git a/docs/registry_mock_test.go b/docs/registry_mock_test.go index 57233d7c..0d987abc 100644 --- a/docs/registry_mock_test.go +++ b/docs/registry_mock_test.go @@ -171,7 +171,7 @@ func makePublicIndex() *IndexInfo { return index } -func makeServiceConfig(mirrors []string, insecure_registries []string) *ServiceConfig { +func makeServiceConfig(mirrors []string, insecureRegistries []string) *ServiceConfig { options := &Options{ Mirrors: opts.NewListOpts(nil), InsecureRegistries: opts.NewListOpts(nil), @@ -181,9 +181,9 @@ func makeServiceConfig(mirrors []string, insecure_registries []string) *ServiceC options.Mirrors.Set(mirror) } } - if insecure_registries != nil { - for _, insecure_registries := range insecure_registries { - options.InsecureRegistries.Set(insecure_registries) + if insecureRegistries != nil { + for _, insecureRegistries := range insecureRegistries { + options.InsecureRegistries.Set(insecureRegistries) } } From d5045d054baa6ce9c607b52ea01f44720387acc6 Mon Sep 17 00:00:00 2001 From: Antonio Murdaca Date: Thu, 26 Mar 2015 23:22:04 +0100 Subject: [PATCH 254/375] Replace aliased imports of logrus, fixes #11762 Signed-off-by: Antonio Murdaca --- docs/auth.go | 16 ++++++++-------- docs/endpoint.go | 20 ++++++++++---------- docs/registry.go | 10 +++++----- docs/registry_mock_test.go | 6 +++--- docs/service.go | 8 ++++---- docs/session.go | 38 +++++++++++++++++++------------------- docs/session_v2.go | 26 +++++++++++++------------- 7 files changed, 62 insertions(+), 62 deletions(-) diff --git a/docs/auth.go b/docs/auth.go index 4baf114c..eaecc0f2 100644 --- a/docs/auth.go +++ b/docs/auth.go @@ -13,7 +13,7 @@ import ( "sync" "time" - log "github.com/Sirupsen/logrus" + "github.com/Sirupsen/logrus" "github.com/docker/docker/utils" ) @@ -66,7 +66,7 @@ func (auth *RequestAuthorization) getToken() (string, error) { defer auth.tokenLock.Unlock() now := time.Now() if now.Before(auth.tokenExpiration) { - log.Debugf("Using cached token for %s", auth.authConfig.Username) + logrus.Debugf("Using cached token for %s", auth.authConfig.Username) return auth.tokenCache, nil } @@ -78,7 +78,7 @@ func (auth *RequestAuthorization) getToken() (string, error) { case "basic": // no token necessary case "bearer": - log.Debugf("Getting bearer token with %s for %s", challenge.Parameters, auth.authConfig.Username) + logrus.Debugf("Getting bearer token with %s for %s", challenge.Parameters, auth.authConfig.Username) params := map[string]string{} for k, v := range challenge.Parameters { params[k] = v @@ -93,7 +93,7 @@ func (auth *RequestAuthorization) getToken() (string, error) { return token, nil default: - log.Infof("Unsupported auth scheme: %q", challenge.Scheme) + logrus.Infof("Unsupported auth scheme: %q", challenge.Scheme) } } @@ -245,7 +245,7 @@ func loginV1(authConfig *AuthConfig, registryEndpoint *Endpoint, factory *utils. serverAddress = authConfig.ServerAddress ) - log.Debugf("attempting v1 login to registry endpoint %s", registryEndpoint) + logrus.Debugf("attempting v1 login to registry endpoint %s", registryEndpoint) if serverAddress == "" { return "", fmt.Errorf("Server Error: Server Address not set.") @@ -349,7 +349,7 @@ func loginV1(authConfig *AuthConfig, registryEndpoint *Endpoint, factory *utils. // served by the v2 registry service provider. Whether this will be supported in the future // is to be determined. func loginV2(authConfig *AuthConfig, registryEndpoint *Endpoint, factory *utils.HTTPRequestFactory) (string, error) { - log.Debugf("attempting v2 login to registry endpoint %s", registryEndpoint) + logrus.Debugf("attempting v2 login to registry endpoint %s", registryEndpoint) var ( err error allErrors []error @@ -357,7 +357,7 @@ func loginV2(authConfig *AuthConfig, registryEndpoint *Endpoint, factory *utils. ) for _, challenge := range registryEndpoint.AuthChallenges { - log.Debugf("trying %q auth challenge with params %s", challenge.Scheme, challenge.Parameters) + logrus.Debugf("trying %q auth challenge with params %s", challenge.Scheme, challenge.Parameters) switch strings.ToLower(challenge.Scheme) { case "basic": @@ -373,7 +373,7 @@ func loginV2(authConfig *AuthConfig, registryEndpoint *Endpoint, factory *utils. return "Login Succeeded", nil } - log.Debugf("error trying auth challenge %q: %s", challenge.Scheme, err) + logrus.Debugf("error trying auth challenge %q: %s", challenge.Scheme, err) allErrors = append(allErrors, err) } diff --git a/docs/endpoint.go b/docs/endpoint.go index 59ae4dd5..b883d36d 100644 --- a/docs/endpoint.go +++ b/docs/endpoint.go @@ -10,7 +10,7 @@ import ( "net/url" "strings" - log "github.com/Sirupsen/logrus" + "github.com/Sirupsen/logrus" "github.com/docker/docker/registry/v2" "github.com/docker/docker/utils" ) @@ -57,7 +57,7 @@ func NewEndpoint(index *IndexInfo) (*Endpoint, error) { } func validateEndpoint(endpoint *Endpoint) error { - log.Debugf("pinging registry endpoint %s", endpoint) + logrus.Debugf("pinging registry endpoint %s", endpoint) // Try HTTPS ping to registry endpoint.URL.Scheme = "https" @@ -69,7 +69,7 @@ func validateEndpoint(endpoint *Endpoint) error { } // If registry is insecure and HTTPS failed, fallback to HTTP. - log.Debugf("Error from registry %q marked as insecure: %v. Insecurely falling back to HTTP", endpoint, err) + logrus.Debugf("Error from registry %q marked as insecure: %v. Insecurely falling back to HTTP", endpoint, err) endpoint.URL.Scheme = "http" var err2 error @@ -163,7 +163,7 @@ func (e *Endpoint) Ping() (RegistryInfo, error) { } func (e *Endpoint) pingV1(factory *utils.HTTPRequestFactory) (RegistryInfo, error) { - log.Debugf("attempting v1 ping for registry endpoint %s", e) + logrus.Debugf("attempting v1 ping for registry endpoint %s", e) if e.String() == IndexServerAddress() { // Skip the check, we know this one is valid @@ -194,17 +194,17 @@ func (e *Endpoint) pingV1(factory *utils.HTTPRequestFactory) (RegistryInfo, erro Standalone: true, } if err := json.Unmarshal(jsonString, &info); err != nil { - log.Debugf("Error unmarshalling the _ping RegistryInfo: %s", err) + logrus.Debugf("Error unmarshalling the _ping RegistryInfo: %s", err) // don't stop here. Just assume sane defaults } if hdr := resp.Header.Get("X-Docker-Registry-Version"); hdr != "" { - log.Debugf("Registry version header: '%s'", hdr) + logrus.Debugf("Registry version header: '%s'", hdr) info.Version = hdr } - log.Debugf("RegistryInfo.Version: %q", info.Version) + logrus.Debugf("RegistryInfo.Version: %q", info.Version) standalone := resp.Header.Get("X-Docker-Registry-Standalone") - log.Debugf("Registry standalone header: '%s'", standalone) + logrus.Debugf("Registry standalone header: '%s'", standalone) // Accepted values are "true" (case-insensitive) and "1". if strings.EqualFold(standalone, "true") || standalone == "1" { info.Standalone = true @@ -212,12 +212,12 @@ func (e *Endpoint) pingV1(factory *utils.HTTPRequestFactory) (RegistryInfo, erro // there is a header set, and it is not "true" or "1", so assume fails info.Standalone = false } - log.Debugf("RegistryInfo.Standalone: %t", info.Standalone) + logrus.Debugf("RegistryInfo.Standalone: %t", info.Standalone) return info, nil } func (e *Endpoint) pingV2(factory *utils.HTTPRequestFactory) (RegistryInfo, error) { - log.Debugf("attempting v2 ping for registry endpoint %s", e) + logrus.Debugf("attempting v2 ping for registry endpoint %s", e) req, err := factory.NewRequest("GET", e.Path(""), nil) if err != nil { diff --git a/docs/registry.go b/docs/registry.go index a8bb8331..163e2de3 100644 --- a/docs/registry.go +++ b/docs/registry.go @@ -13,7 +13,7 @@ import ( "strings" "time" - log "github.com/Sirupsen/logrus" + "github.com/Sirupsen/logrus" "github.com/docker/docker/pkg/timeoutconn" ) @@ -100,7 +100,7 @@ func doRequest(req *http.Request, jar http.CookieJar, timeout TimeoutType, secur } hostDir := path.Join("/etc/docker/certs.d", req.URL.Host) - log.Debugf("hostDir: %s", hostDir) + logrus.Debugf("hostDir: %s", hostDir) fs, err := ioutil.ReadDir(hostDir) if err != nil && !os.IsNotExist(err) { return nil, nil, err @@ -111,7 +111,7 @@ func doRequest(req *http.Request, jar http.CookieJar, timeout TimeoutType, secur if pool == nil { pool = x509.NewCertPool() } - log.Debugf("crt: %s", hostDir+"/"+f.Name()) + logrus.Debugf("crt: %s", hostDir+"/"+f.Name()) data, err := ioutil.ReadFile(path.Join(hostDir, f.Name())) if err != nil { return nil, nil, err @@ -121,7 +121,7 @@ func doRequest(req *http.Request, jar http.CookieJar, timeout TimeoutType, secur if strings.HasSuffix(f.Name(), ".cert") { certName := f.Name() keyName := certName[:len(certName)-5] + ".key" - log.Debugf("cert: %s", hostDir+"/"+f.Name()) + logrus.Debugf("cert: %s", hostDir+"/"+f.Name()) if !hasFile(fs, keyName) { return nil, nil, fmt.Errorf("Missing key %s for certificate %s", keyName, certName) } @@ -134,7 +134,7 @@ func doRequest(req *http.Request, jar http.CookieJar, timeout TimeoutType, secur if strings.HasSuffix(f.Name(), ".key") { keyName := f.Name() certName := keyName[:len(keyName)-4] + ".cert" - log.Debugf("key: %s", hostDir+"/"+f.Name()) + logrus.Debugf("key: %s", hostDir+"/"+f.Name()) if !hasFile(fs, certName) { return nil, nil, fmt.Errorf("Missing certificate %s for key %s", certName, keyName) } diff --git a/docs/registry_mock_test.go b/docs/registry_mock_test.go index 57233d7c..82818b41 100644 --- a/docs/registry_mock_test.go +++ b/docs/registry_mock_test.go @@ -18,7 +18,7 @@ import ( "github.com/docker/docker/opts" "github.com/gorilla/mux" - log "github.com/Sirupsen/logrus" + "github.com/Sirupsen/logrus" ) var ( @@ -134,7 +134,7 @@ func init() { func handlerAccessLog(handler http.Handler) http.Handler { logHandler := func(w http.ResponseWriter, r *http.Request) { - log.Debugf("%s \"%s %s\"", r.RemoteAddr, r.Method, r.URL) + logrus.Debugf("%s \"%s %s\"", r.RemoteAddr, r.Method, r.URL) handler.ServeHTTP(w, r) } return http.HandlerFunc(logHandler) @@ -467,7 +467,7 @@ func TestPing(t *testing.T) { * WARNING: Don't push on the repos uncommented, it'll block the tests * func TestWait(t *testing.T) { - log.Println("Test HTTP server ready and waiting:", testHttpServer.URL) + logrus.Println("Test HTTP server ready and waiting:", testHttpServer.URL) c := make(chan int) <-c } diff --git a/docs/service.go b/docs/service.go index 5daacb2b..f464faab 100644 --- a/docs/service.go +++ b/docs/service.go @@ -3,7 +3,7 @@ package registry import ( "fmt" - log "github.com/Sirupsen/logrus" + "github.com/Sirupsen/logrus" "github.com/docker/docker/engine" ) @@ -62,18 +62,18 @@ func (s *Service) Auth(job *engine.Job) error { } if endpoint, err = NewEndpoint(index); err != nil { - log.Errorf("unable to get new registry endpoint: %s", err) + logrus.Errorf("unable to get new registry endpoint: %s", err) return err } authConfig.ServerAddress = endpoint.String() if status, err = Login(authConfig, endpoint, HTTPRequestFactory(nil)); err != nil { - log.Errorf("unable to login against registry endpoint %s: %s", endpoint, err) + logrus.Errorf("unable to login against registry endpoint %s: %s", endpoint, err) return err } - log.Infof("successful registry login for endpoint %s: %s", endpoint, status) + logrus.Infof("successful registry login for endpoint %s: %s", endpoint, status) job.Printf("%s\n", status) return nil diff --git a/docs/session.go b/docs/session.go index bf04b586..1d70eff9 100644 --- a/docs/session.go +++ b/docs/session.go @@ -17,7 +17,7 @@ import ( "strings" "time" - log "github.com/Sirupsen/logrus" + "github.com/Sirupsen/logrus" "github.com/docker/docker/pkg/httputils" "github.com/docker/docker/pkg/tarsum" "github.com/docker/docker/utils" @@ -54,7 +54,7 @@ func NewSession(authConfig *AuthConfig, factory *utils.HTTPRequestFactory, endpo return nil, err } if info.Standalone { - log.Debugf("Endpoint %s is eligible for private registry. Enabling decorator.", r.indexEndpoint.String()) + logrus.Debugf("Endpoint %s is eligible for private registry. Enabling decorator.", r.indexEndpoint.String()) dec := utils.NewHTTPAuthDecorator(authConfig.Username, authConfig.Password) factory.AddDecorator(dec) } @@ -93,7 +93,7 @@ func (r *Session) GetRemoteHistory(imgID, registry string, token []string) ([]st return nil, fmt.Errorf("Error while reading the http response: %s", err) } - log.Debugf("Ancestry: %s", jsonString) + logrus.Debugf("Ancestry: %s", jsonString) history := new([]string) if err := json.Unmarshal(jsonString, history); err != nil { return nil, err @@ -169,7 +169,7 @@ func (r *Session) GetRemoteImageLayer(imgID, registry string, token []string, im statusCode = 0 res, client, err = r.doRequest(req) if err != nil { - log.Debugf("Error contacting registry: %s", err) + logrus.Debugf("Error contacting registry: %s", err) if res != nil { if res.Body != nil { res.Body.Close() @@ -193,10 +193,10 @@ func (r *Session) GetRemoteImageLayer(imgID, registry string, token []string, im } if res.Header.Get("Accept-Ranges") == "bytes" && imgSize > 0 { - log.Debugf("server supports resume") + logrus.Debugf("server supports resume") return httputils.ResumableRequestReaderWithInitialResponse(client, req, 5, imgSize, res), nil } - log.Debugf("server doesn't support resume") + logrus.Debugf("server doesn't support resume") return res.Body, nil } @@ -219,7 +219,7 @@ func (r *Session) GetRemoteTags(registries []string, repository string, token [] return nil, err } - log.Debugf("Got status code %d from %s", res.StatusCode, endpoint) + logrus.Debugf("Got status code %d from %s", res.StatusCode, endpoint) defer res.Body.Close() if res.StatusCode != 200 && res.StatusCode != 404 { @@ -259,7 +259,7 @@ func buildEndpointsList(headers []string, indexEp string) ([]string, error) { func (r *Session) GetRepositoryData(remote string) (*RepositoryData, error) { repositoryTarget := fmt.Sprintf("%srepositories/%s/images", r.indexEndpoint.VersionString(1), remote) - log.Debugf("[registry] Calling GET %s", repositoryTarget) + logrus.Debugf("[registry] Calling GET %s", repositoryTarget) req, err := r.reqFactory.NewRequest("GET", repositoryTarget, nil) if err != nil { @@ -285,7 +285,7 @@ func (r *Session) GetRepositoryData(remote string) (*RepositoryData, error) { } else if res.StatusCode != 200 { errBody, err := ioutil.ReadAll(res.Body) if err != nil { - log.Debugf("Error reading response body: %s", err) + logrus.Debugf("Error reading response body: %s", err) } return nil, utils.NewHTTPRequestError(fmt.Sprintf("Error: Status %d trying to pull repository %s: %q", res.StatusCode, remote, errBody), res) } @@ -326,7 +326,7 @@ func (r *Session) GetRepositoryData(remote string) (*RepositoryData, error) { func (r *Session) PushImageChecksumRegistry(imgData *ImgData, registry string, token []string) error { - log.Debugf("[registry] Calling PUT %s", registry+"images/"+imgData.ID+"/checksum") + logrus.Debugf("[registry] Calling PUT %s", registry+"images/"+imgData.ID+"/checksum") req, err := r.reqFactory.NewRequest("PUT", registry+"images/"+imgData.ID+"/checksum", nil) if err != nil { @@ -363,7 +363,7 @@ func (r *Session) PushImageChecksumRegistry(imgData *ImgData, registry string, t // Push a local image to the registry func (r *Session) PushImageJSONRegistry(imgData *ImgData, jsonRaw []byte, registry string, token []string) error { - log.Debugf("[registry] Calling PUT %s", registry+"images/"+imgData.ID+"/json") + logrus.Debugf("[registry] Calling PUT %s", registry+"images/"+imgData.ID+"/json") req, err := r.reqFactory.NewRequest("PUT", registry+"images/"+imgData.ID+"/json", bytes.NewReader(jsonRaw)) if err != nil { @@ -398,7 +398,7 @@ func (r *Session) PushImageJSONRegistry(imgData *ImgData, jsonRaw []byte, regist func (r *Session) PushImageLayerRegistry(imgID string, layer io.Reader, registry string, token []string, jsonRaw []byte) (checksum string, checksumPayload string, err error) { - log.Debugf("[registry] Calling PUT %s", registry+"images/"+imgID+"/layer") + logrus.Debugf("[registry] Calling PUT %s", registry+"images/"+imgID+"/layer") tarsumLayer, err := tarsum.NewTarSum(layer, false, tarsum.Version0) if err != nil { @@ -486,8 +486,8 @@ func (r *Session) PushImageJSONIndex(remote string, imgList []*ImgData, validate suffix = "images" } u := fmt.Sprintf("%srepositories/%s/%s", r.indexEndpoint.VersionString(1), remote, suffix) - log.Debugf("[registry] PUT %s", u) - log.Debugf("Image list pushed to index:\n%s", imgListJSON) + logrus.Debugf("[registry] PUT %s", u) + logrus.Debugf("Image list pushed to index:\n%s", imgListJSON) headers := map[string][]string{ "Content-type": {"application/json"}, "X-Docker-Token": {"true"}, @@ -507,7 +507,7 @@ func (r *Session) PushImageJSONIndex(remote string, imgList []*ImgData, validate } res.Body.Close() u = res.Header.Get("Location") - log.Debugf("Redirected to %s", u) + logrus.Debugf("Redirected to %s", u) } defer res.Body.Close() @@ -520,13 +520,13 @@ func (r *Session) PushImageJSONIndex(remote string, imgList []*ImgData, validate if res.StatusCode != 200 && res.StatusCode != 201 { errBody, err := ioutil.ReadAll(res.Body) if err != nil { - log.Debugf("Error reading response body: %s", err) + logrus.Debugf("Error reading response body: %s", err) } return nil, utils.NewHTTPRequestError(fmt.Sprintf("Error: Status %d trying to push repository %s: %q", res.StatusCode, remote, errBody), res) } if res.Header.Get("X-Docker-Token") != "" { tokens = res.Header["X-Docker-Token"] - log.Debugf("Auth token: %v", tokens) + logrus.Debugf("Auth token: %v", tokens) } else { return nil, fmt.Errorf("Index response didn't contain an access token") } @@ -544,7 +544,7 @@ func (r *Session) PushImageJSONIndex(remote string, imgList []*ImgData, validate if res.StatusCode != 204 { errBody, err := ioutil.ReadAll(res.Body) if err != nil { - log.Debugf("Error reading response body: %s", err) + logrus.Debugf("Error reading response body: %s", err) } return nil, utils.NewHTTPRequestError(fmt.Sprintf("Error: Status %d trying to push checksums %s: %q", res.StatusCode, remote, errBody), res) } @@ -578,7 +578,7 @@ func shouldRedirect(response *http.Response) bool { } func (r *Session) SearchRepositories(term string) (*SearchResults, error) { - log.Debugf("Index server: %s", r.indexEndpoint) + logrus.Debugf("Index server: %s", r.indexEndpoint) u := r.indexEndpoint.VersionString(1) + "search?q=" + url.QueryEscape(term) req, err := r.reqFactory.NewRequest("GET", u, nil) if err != nil { diff --git a/docs/session_v2.go b/docs/session_v2.go index 22f39317..a01c8b9a 100644 --- a/docs/session_v2.go +++ b/docs/session_v2.go @@ -9,7 +9,7 @@ import ( "net/http" "strconv" - log "github.com/Sirupsen/logrus" + "github.com/Sirupsen/logrus" "github.com/docker/distribution/digest" "github.com/docker/docker/registry/v2" "github.com/docker/docker/utils" @@ -57,7 +57,7 @@ func (r *Session) GetV2Authorization(ep *Endpoint, imageName string, readOnly bo scopes = append(scopes, "push") } - log.Debugf("Getting authorization for %s %s", imageName, scopes) + logrus.Debugf("Getting authorization for %s %s", imageName, scopes) return NewRequestAuthorization(r.GetAuthConfig(true), ep, "repository", imageName, scopes), nil } @@ -75,7 +75,7 @@ func (r *Session) GetV2ImageManifest(ep *Endpoint, imageName, tagName string, au } method := "GET" - log.Debugf("[registry] Calling %q %s", method, routeURL) + logrus.Debugf("[registry] Calling %q %s", method, routeURL) req, err := r.reqFactory.NewRequest(method, routeURL, nil) if err != nil { @@ -116,7 +116,7 @@ func (r *Session) HeadV2ImageBlob(ep *Endpoint, imageName, sumType, sum string, } method := "HEAD" - log.Debugf("[registry] Calling %q %s", method, routeURL) + logrus.Debugf("[registry] Calling %q %s", method, routeURL) req, err := r.reqFactory.NewRequest(method, routeURL, nil) if err != nil { @@ -151,7 +151,7 @@ func (r *Session) GetV2ImageBlob(ep *Endpoint, imageName, sumType, sum string, b } method := "GET" - log.Debugf("[registry] Calling %q %s", method, routeURL) + logrus.Debugf("[registry] Calling %q %s", method, routeURL) req, err := r.reqFactory.NewRequest(method, routeURL, nil) if err != nil { return err @@ -182,7 +182,7 @@ func (r *Session) GetV2ImageBlobReader(ep *Endpoint, imageName, sumType, sum str } method := "GET" - log.Debugf("[registry] Calling %q %s", method, routeURL) + logrus.Debugf("[registry] Calling %q %s", method, routeURL) req, err := r.reqFactory.NewRequest(method, routeURL, nil) if err != nil { return nil, 0, err @@ -219,7 +219,7 @@ func (r *Session) PutV2ImageBlob(ep *Endpoint, imageName, sumType, sumStr string } method := "PUT" - log.Debugf("[registry] Calling %q %s", method, location) + logrus.Debugf("[registry] Calling %q %s", method, location) req, err := r.reqFactory.NewRequest(method, location, ioutil.NopCloser(blobRdr)) if err != nil { return err @@ -244,7 +244,7 @@ func (r *Session) PutV2ImageBlob(ep *Endpoint, imageName, sumType, sumStr string if err != nil { return err } - log.Debugf("Unexpected response from server: %q %#v", errBody, res.Header) + logrus.Debugf("Unexpected response from server: %q %#v", errBody, res.Header) return utils.NewHTTPRequestError(fmt.Sprintf("Server error: %d trying to push %s blob - %s:%s", res.StatusCode, imageName, sumType, sumStr), res) } @@ -258,7 +258,7 @@ func (r *Session) initiateBlobUpload(ep *Endpoint, imageName string, auth *Reque return "", err } - log.Debugf("[registry] Calling %q %s", "POST", routeURL) + logrus.Debugf("[registry] Calling %q %s", "POST", routeURL) req, err := r.reqFactory.NewRequest("POST", routeURL, nil) if err != nil { return "", err @@ -285,7 +285,7 @@ func (r *Session) initiateBlobUpload(ep *Endpoint, imageName string, auth *Reque return "", err } - log.Debugf("Unexpected response from server: %q %#v", errBody, res.Header) + logrus.Debugf("Unexpected response from server: %q %#v", errBody, res.Header) return "", utils.NewHTTPRequestError(fmt.Sprintf("Server error: unexpected %d response status trying to initiate upload of %s", res.StatusCode, imageName), res) } @@ -304,7 +304,7 @@ func (r *Session) PutV2ImageManifest(ep *Endpoint, imageName, tagName string, si } method := "PUT" - log.Debugf("[registry] Calling %q %s", method, routeURL) + logrus.Debugf("[registry] Calling %q %s", method, routeURL) req, err := r.reqFactory.NewRequest(method, routeURL, bytes.NewReader(signedManifest)) if err != nil { return "", err @@ -327,7 +327,7 @@ func (r *Session) PutV2ImageManifest(ep *Endpoint, imageName, tagName string, si if err != nil { return "", err } - log.Debugf("Unexpected response from server: %q %#v", errBody, res.Header) + logrus.Debugf("Unexpected response from server: %q %#v", errBody, res.Header) return "", utils.NewHTTPRequestError(fmt.Sprintf("Server error: %d trying to push %s:%s manifest", res.StatusCode, imageName, tagName), res) } @@ -364,7 +364,7 @@ func (r *Session) GetV2RemoteTags(ep *Endpoint, imageName string, auth *RequestA } method := "GET" - log.Debugf("[registry] Calling %q %s", method, routeURL) + logrus.Debugf("[registry] Calling %q %s", method, routeURL) req, err := r.reqFactory.NewRequest(method, routeURL, nil) if err != nil { From 5fa2d814f8e985747b80d6cb4e05eb6dee1d3f12 Mon Sep 17 00:00:00 2001 From: Antonio Murdaca Date: Sun, 29 Mar 2015 15:51:08 +0200 Subject: [PATCH 255/375] Refactor utils/http.go, fixes #11899 Signed-off-by: Antonio Murdaca --- docs/auth.go | 12 ++++++------ docs/endpoint.go | 6 +++--- docs/httpfactory.go | 44 ++++++++++++++----------------------------- docs/registry_test.go | 6 +++--- docs/session.go | 7 ++++--- docs/token.go | 4 ++-- 6 files changed, 32 insertions(+), 47 deletions(-) diff --git a/docs/auth.go b/docs/auth.go index eaecc0f2..2c37f7f6 100644 --- a/docs/auth.go +++ b/docs/auth.go @@ -14,7 +14,7 @@ import ( "time" "github.com/Sirupsen/logrus" - "github.com/docker/docker/utils" + "github.com/docker/docker/pkg/requestdecorator" ) const ( @@ -225,7 +225,7 @@ func SaveConfig(configFile *ConfigFile) error { } // Login tries to register/login to the registry server. -func Login(authConfig *AuthConfig, registryEndpoint *Endpoint, factory *utils.HTTPRequestFactory) (string, error) { +func Login(authConfig *AuthConfig, registryEndpoint *Endpoint, factory *requestdecorator.RequestFactory) (string, error) { // Separates the v2 registry login logic from the v1 logic. if registryEndpoint.Version == APIVersion2 { return loginV2(authConfig, registryEndpoint, factory) @@ -235,7 +235,7 @@ func Login(authConfig *AuthConfig, registryEndpoint *Endpoint, factory *utils.HT } // loginV1 tries to register/login to the v1 registry server. -func loginV1(authConfig *AuthConfig, registryEndpoint *Endpoint, factory *utils.HTTPRequestFactory) (string, error) { +func loginV1(authConfig *AuthConfig, registryEndpoint *Endpoint, factory *requestdecorator.RequestFactory) (string, error) { var ( status string reqBody []byte @@ -348,7 +348,7 @@ func loginV1(authConfig *AuthConfig, registryEndpoint *Endpoint, factory *utils. // now, users should create their account through other means like directly from a web page // served by the v2 registry service provider. Whether this will be supported in the future // is to be determined. -func loginV2(authConfig *AuthConfig, registryEndpoint *Endpoint, factory *utils.HTTPRequestFactory) (string, error) { +func loginV2(authConfig *AuthConfig, registryEndpoint *Endpoint, factory *requestdecorator.RequestFactory) (string, error) { logrus.Debugf("attempting v2 login to registry endpoint %s", registryEndpoint) var ( err error @@ -381,7 +381,7 @@ func loginV2(authConfig *AuthConfig, registryEndpoint *Endpoint, factory *utils. return "", fmt.Errorf("no successful auth challenge for %s - errors: %s", registryEndpoint, allErrors) } -func tryV2BasicAuthLogin(authConfig *AuthConfig, params map[string]string, registryEndpoint *Endpoint, client *http.Client, factory *utils.HTTPRequestFactory) error { +func tryV2BasicAuthLogin(authConfig *AuthConfig, params map[string]string, registryEndpoint *Endpoint, client *http.Client, factory *requestdecorator.RequestFactory) error { req, err := factory.NewRequest("GET", registryEndpoint.Path(""), nil) if err != nil { return err @@ -402,7 +402,7 @@ func tryV2BasicAuthLogin(authConfig *AuthConfig, params map[string]string, regis return nil } -func tryV2TokenAuthLogin(authConfig *AuthConfig, params map[string]string, registryEndpoint *Endpoint, client *http.Client, factory *utils.HTTPRequestFactory) error { +func tryV2TokenAuthLogin(authConfig *AuthConfig, params map[string]string, registryEndpoint *Endpoint, client *http.Client, factory *requestdecorator.RequestFactory) error { token, err := getToken(authConfig.Username, authConfig.Password, params, registryEndpoint, client, factory) if err != nil { return err diff --git a/docs/endpoint.go b/docs/endpoint.go index b883d36d..69a718e1 100644 --- a/docs/endpoint.go +++ b/docs/endpoint.go @@ -11,8 +11,8 @@ import ( "strings" "github.com/Sirupsen/logrus" + "github.com/docker/docker/pkg/requestdecorator" "github.com/docker/docker/registry/v2" - "github.com/docker/docker/utils" ) // for mocking in unit tests @@ -162,7 +162,7 @@ func (e *Endpoint) Ping() (RegistryInfo, error) { return RegistryInfo{}, fmt.Errorf("unable to ping registry endpoint %s\nv2 ping attempt failed with error: %s\n v1 ping attempt failed with error: %s", e, errV2, errV1) } -func (e *Endpoint) pingV1(factory *utils.HTTPRequestFactory) (RegistryInfo, error) { +func (e *Endpoint) pingV1(factory *requestdecorator.RequestFactory) (RegistryInfo, error) { logrus.Debugf("attempting v1 ping for registry endpoint %s", e) if e.String() == IndexServerAddress() { @@ -216,7 +216,7 @@ func (e *Endpoint) pingV1(factory *utils.HTTPRequestFactory) (RegistryInfo, erro return info, nil } -func (e *Endpoint) pingV2(factory *utils.HTTPRequestFactory) (RegistryInfo, error) { +func (e *Endpoint) pingV2(factory *requestdecorator.RequestFactory) (RegistryInfo, error) { logrus.Debugf("attempting v2 ping for registry endpoint %s", e) req, err := factory.NewRequest("GET", e.Path(""), nil) diff --git a/docs/httpfactory.go b/docs/httpfactory.go index a4fea382..f1b89e58 100644 --- a/docs/httpfactory.go +++ b/docs/httpfactory.go @@ -5,42 +5,26 @@ import ( "github.com/docker/docker/autogen/dockerversion" "github.com/docker/docker/pkg/parsers/kernel" - "github.com/docker/docker/utils" + "github.com/docker/docker/pkg/requestdecorator" ) -func HTTPRequestFactory(metaHeaders map[string][]string) *utils.HTTPRequestFactory { +func HTTPRequestFactory(metaHeaders map[string][]string) *requestdecorator.RequestFactory { // FIXME: this replicates the 'info' job. - httpVersion := make([]utils.VersionInfo, 0, 4) - httpVersion = append(httpVersion, &simpleVersionInfo{"docker", dockerversion.VERSION}) - httpVersion = append(httpVersion, &simpleVersionInfo{"go", runtime.Version()}) - httpVersion = append(httpVersion, &simpleVersionInfo{"git-commit", dockerversion.GITCOMMIT}) + httpVersion := make([]requestdecorator.UAVersionInfo, 0, 4) + httpVersion = append(httpVersion, requestdecorator.NewUAVersionInfo("docker", dockerversion.VERSION)) + httpVersion = append(httpVersion, requestdecorator.NewUAVersionInfo("go", runtime.Version())) + httpVersion = append(httpVersion, requestdecorator.NewUAVersionInfo("git-commit", dockerversion.GITCOMMIT)) if kernelVersion, err := kernel.GetKernelVersion(); err == nil { - httpVersion = append(httpVersion, &simpleVersionInfo{"kernel", kernelVersion.String()}) + httpVersion = append(httpVersion, requestdecorator.NewUAVersionInfo("kernel", kernelVersion.String())) } - httpVersion = append(httpVersion, &simpleVersionInfo{"os", runtime.GOOS}) - httpVersion = append(httpVersion, &simpleVersionInfo{"arch", runtime.GOARCH}) - ud := utils.NewHTTPUserAgentDecorator(httpVersion...) - md := &utils.HTTPMetaHeadersDecorator{ + httpVersion = append(httpVersion, requestdecorator.NewUAVersionInfo("os", runtime.GOOS)) + httpVersion = append(httpVersion, requestdecorator.NewUAVersionInfo("arch", runtime.GOARCH)) + uad := &requestdecorator.UserAgentDecorator{ + Versions: httpVersion, + } + mhd := &requestdecorator.MetaHeadersDecorator{ Headers: metaHeaders, } - factory := utils.NewHTTPRequestFactory(ud, md) + factory := requestdecorator.NewRequestFactory(uad, mhd) return factory } - -// simpleVersionInfo is a simple implementation of -// the interface VersionInfo, which is used -// to provide version information for some product, -// component, etc. It stores the product name and the version -// in string and returns them on calls to Name() and Version(). -type simpleVersionInfo struct { - name string - version string -} - -func (v *simpleVersionInfo) Name() string { - return v.name -} - -func (v *simpleVersionInfo) Version() string { - return v.version -} diff --git a/docs/registry_test.go b/docs/registry_test.go index d96630d9..a066de9f 100644 --- a/docs/registry_test.go +++ b/docs/registry_test.go @@ -7,7 +7,7 @@ import ( "strings" "testing" - "github.com/docker/docker/utils" + "github.com/docker/docker/pkg/requestdecorator" ) var ( @@ -25,7 +25,7 @@ func spawnTestRegistrySession(t *testing.T) *Session { if err != nil { t.Fatal(err) } - r, err := NewSession(authConfig, utils.NewHTTPRequestFactory(), endpoint, true) + r, err := NewSession(authConfig, requestdecorator.NewRequestFactory(), endpoint, true) if err != nil { t.Fatal(err) } @@ -40,7 +40,7 @@ func TestPublicSession(t *testing.T) { if err != nil { t.Fatal(err) } - r, err := NewSession(authConfig, utils.NewHTTPRequestFactory(), endpoint, true) + r, err := NewSession(authConfig, requestdecorator.NewRequestFactory(), endpoint, true) if err != nil { t.Fatal(err) } diff --git a/docs/session.go b/docs/session.go index 1d70eff9..4682a507 100644 --- a/docs/session.go +++ b/docs/session.go @@ -19,19 +19,20 @@ import ( "github.com/Sirupsen/logrus" "github.com/docker/docker/pkg/httputils" + "github.com/docker/docker/pkg/requestdecorator" "github.com/docker/docker/pkg/tarsum" "github.com/docker/docker/utils" ) type Session struct { authConfig *AuthConfig - reqFactory *utils.HTTPRequestFactory + reqFactory *requestdecorator.RequestFactory indexEndpoint *Endpoint jar *cookiejar.Jar timeout TimeoutType } -func NewSession(authConfig *AuthConfig, factory *utils.HTTPRequestFactory, endpoint *Endpoint, timeout bool) (r *Session, err error) { +func NewSession(authConfig *AuthConfig, factory *requestdecorator.RequestFactory, endpoint *Endpoint, timeout bool) (r *Session, err error) { r = &Session{ authConfig: authConfig, indexEndpoint: endpoint, @@ -55,7 +56,7 @@ func NewSession(authConfig *AuthConfig, factory *utils.HTTPRequestFactory, endpo } if info.Standalone { logrus.Debugf("Endpoint %s is eligible for private registry. Enabling decorator.", r.indexEndpoint.String()) - dec := utils.NewHTTPAuthDecorator(authConfig.Username, authConfig.Password) + dec := requestdecorator.NewAuthDecorator(authConfig.Username, authConfig.Password) factory.AddDecorator(dec) } } diff --git a/docs/token.go b/docs/token.go index c79a8ca6..b03bd891 100644 --- a/docs/token.go +++ b/docs/token.go @@ -8,14 +8,14 @@ import ( "net/url" "strings" - "github.com/docker/docker/utils" + "github.com/docker/docker/pkg/requestdecorator" ) type tokenResponse struct { Token string `json:"token"` } -func getToken(username, password string, params map[string]string, registryEndpoint *Endpoint, client *http.Client, factory *utils.HTTPRequestFactory) (token string, err error) { +func getToken(username, password string, params map[string]string, registryEndpoint *Endpoint, client *http.Client, factory *requestdecorator.RequestFactory) (token string, err error) { realm, ok := params["realm"] if !ok { return "", errors.New("no realm specified for token auth challenge") From 67e5c940c40c10780d2ed451255a24703f0e4b3f Mon Sep 17 00:00:00 2001 From: Derek McGowan Date: Tue, 31 Mar 2015 15:02:27 -0700 Subject: [PATCH 256/375] Use vendored v2 registry api Update registry package to use the v2 registry api from distribution. Update interfaces to directly take in digests. Signed-off-by: Derek McGowan (github: dmcgowan) --- docs/endpoint.go | 2 +- docs/session_v2.go | 24 +++--- docs/v2/descriptors.go | 144 ------------------------------- docs/v2/doc.go | 13 --- docs/v2/errors.go | 185 --------------------------------------- docs/v2/errors_test.go | 163 ---------------------------------- docs/v2/regexp.go | 22 ----- docs/v2/routes.go | 66 -------------- docs/v2/routes_test.go | 192 ----------------------------------------- docs/v2/urls.go | 179 -------------------------------------- docs/v2/urls_test.go | 113 ------------------------ 11 files changed, 13 insertions(+), 1090 deletions(-) delete mode 100644 docs/v2/descriptors.go delete mode 100644 docs/v2/doc.go delete mode 100644 docs/v2/errors.go delete mode 100644 docs/v2/errors_test.go delete mode 100644 docs/v2/regexp.go delete mode 100644 docs/v2/routes.go delete mode 100644 docs/v2/routes_test.go delete mode 100644 docs/v2/urls.go delete mode 100644 docs/v2/urls_test.go diff --git a/docs/endpoint.go b/docs/endpoint.go index 69a718e1..84b11a98 100644 --- a/docs/endpoint.go +++ b/docs/endpoint.go @@ -11,8 +11,8 @@ import ( "strings" "github.com/Sirupsen/logrus" + "github.com/docker/distribution/registry/api/v2" "github.com/docker/docker/pkg/requestdecorator" - "github.com/docker/docker/registry/v2" ) // for mocking in unit tests diff --git a/docs/session_v2.go b/docs/session_v2.go index a01c8b9a..fb1d18e8 100644 --- a/docs/session_v2.go +++ b/docs/session_v2.go @@ -11,7 +11,7 @@ import ( "github.com/Sirupsen/logrus" "github.com/docker/distribution/digest" - "github.com/docker/docker/registry/v2" + "github.com/docker/distribution/registry/api/v2" "github.com/docker/docker/utils" ) @@ -109,8 +109,8 @@ func (r *Session) GetV2ImageManifest(ep *Endpoint, imageName, tagName string, au // - Succeeded to head image blob (already exists) // - Failed with no error (continue to Push the Blob) // - Failed with error -func (r *Session) HeadV2ImageBlob(ep *Endpoint, imageName, sumType, sum string, auth *RequestAuthorization) (bool, error) { - routeURL, err := getV2Builder(ep).BuildBlobURL(imageName, sumType+":"+sum) +func (r *Session) HeadV2ImageBlob(ep *Endpoint, imageName string, dgst digest.Digest, auth *RequestAuthorization) (bool, error) { + routeURL, err := getV2Builder(ep).BuildBlobURL(imageName, dgst) if err != nil { return false, err } @@ -141,11 +141,11 @@ func (r *Session) HeadV2ImageBlob(ep *Endpoint, imageName, sumType, sum string, return false, nil } - return false, utils.NewHTTPRequestError(fmt.Sprintf("Server error: %d trying head request for %s - %s:%s", res.StatusCode, imageName, sumType, sum), res) + return false, utils.NewHTTPRequestError(fmt.Sprintf("Server error: %d trying head request for %s - %s", res.StatusCode, imageName, dgst), res) } -func (r *Session) GetV2ImageBlob(ep *Endpoint, imageName, sumType, sum string, blobWrtr io.Writer, auth *RequestAuthorization) error { - routeURL, err := getV2Builder(ep).BuildBlobURL(imageName, sumType+":"+sum) +func (r *Session) GetV2ImageBlob(ep *Endpoint, imageName string, dgst digest.Digest, blobWrtr io.Writer, auth *RequestAuthorization) error { + routeURL, err := getV2Builder(ep).BuildBlobURL(imageName, dgst) if err != nil { return err } @@ -175,8 +175,8 @@ func (r *Session) GetV2ImageBlob(ep *Endpoint, imageName, sumType, sum string, b return err } -func (r *Session) GetV2ImageBlobReader(ep *Endpoint, imageName, sumType, sum string, auth *RequestAuthorization) (io.ReadCloser, int64, error) { - routeURL, err := getV2Builder(ep).BuildBlobURL(imageName, sumType+":"+sum) +func (r *Session) GetV2ImageBlobReader(ep *Endpoint, imageName string, dgst digest.Digest, auth *RequestAuthorization) (io.ReadCloser, int64, error) { + routeURL, err := getV2Builder(ep).BuildBlobURL(imageName, dgst) if err != nil { return nil, 0, err } @@ -198,7 +198,7 @@ func (r *Session) GetV2ImageBlobReader(ep *Endpoint, imageName, sumType, sum str if res.StatusCode == 401 { return nil, 0, errLoginRequired } - return nil, 0, utils.NewHTTPRequestError(fmt.Sprintf("Server error: %d trying to pull %s blob - %s:%s", res.StatusCode, imageName, sumType, sum), res) + return nil, 0, utils.NewHTTPRequestError(fmt.Sprintf("Server error: %d trying to pull %s blob - %s", res.StatusCode, imageName, dgst), res) } lenStr := res.Header.Get("Content-Length") l, err := strconv.ParseInt(lenStr, 10, 64) @@ -212,7 +212,7 @@ func (r *Session) GetV2ImageBlobReader(ep *Endpoint, imageName, sumType, sum str // Push the image to the server for storage. // 'layer' is an uncompressed reader of the blob to be pushed. // The server will generate it's own checksum calculation. -func (r *Session) PutV2ImageBlob(ep *Endpoint, imageName, sumType, sumStr string, blobRdr io.Reader, auth *RequestAuthorization) error { +func (r *Session) PutV2ImageBlob(ep *Endpoint, imageName string, dgst digest.Digest, blobRdr io.Reader, auth *RequestAuthorization) error { location, err := r.initiateBlobUpload(ep, imageName, auth) if err != nil { return err @@ -225,7 +225,7 @@ func (r *Session) PutV2ImageBlob(ep *Endpoint, imageName, sumType, sumStr string return err } queryParams := req.URL.Query() - queryParams.Add("digest", sumType+":"+sumStr) + queryParams.Add("digest", dgst.String()) req.URL.RawQuery = queryParams.Encode() if err := auth.Authorize(req); err != nil { return err @@ -245,7 +245,7 @@ func (r *Session) PutV2ImageBlob(ep *Endpoint, imageName, sumType, sumStr string return err } logrus.Debugf("Unexpected response from server: %q %#v", errBody, res.Header) - return utils.NewHTTPRequestError(fmt.Sprintf("Server error: %d trying to push %s blob - %s:%s", res.StatusCode, imageName, sumType, sumStr), res) + return utils.NewHTTPRequestError(fmt.Sprintf("Server error: %d trying to push %s blob - %s", res.StatusCode, imageName, dgst), res) } return nil diff --git a/docs/v2/descriptors.go b/docs/v2/descriptors.go deleted file mode 100644 index 68d18241..00000000 --- a/docs/v2/descriptors.go +++ /dev/null @@ -1,144 +0,0 @@ -package v2 - -import "net/http" - -// TODO(stevvooe): Add route descriptors for each named route, along with -// accepted methods, parameters, returned status codes and error codes. - -// ErrorDescriptor provides relevant information about a given error code. -type ErrorDescriptor struct { - // Code is the error code that this descriptor describes. - Code ErrorCode - - // Value provides a unique, string key, often captilized with - // underscores, to identify the error code. This value is used as the - // keyed value when serializing api errors. - Value string - - // Message is a short, human readable decription of the error condition - // included in API responses. - Message string - - // Description provides a complete account of the errors purpose, suitable - // for use in documentation. - Description string - - // HTTPStatusCodes provides a list of status under which this error - // condition may arise. If it is empty, the error condition may be seen - // for any status code. - HTTPStatusCodes []int -} - -// ErrorDescriptors provides a list of HTTP API Error codes that may be -// encountered when interacting with the registry API. -var ErrorDescriptors = []ErrorDescriptor{ - { - Code: ErrorCodeUnknown, - Value: "UNKNOWN", - Message: "unknown error", - Description: `Generic error returned when the error does not have an - API classification.`, - }, - { - Code: ErrorCodeDigestInvalid, - Value: "DIGEST_INVALID", - Message: "provided digest did not match uploaded content", - Description: `When a blob is uploaded, the registry will check that - the content matches the digest provided by the client. The error may - include a detail structure with the key "digest", including the - invalid digest string. This error may also be returned when a manifest - includes an invalid layer digest.`, - HTTPStatusCodes: []int{http.StatusBadRequest, http.StatusNotFound}, - }, - { - Code: ErrorCodeSizeInvalid, - Value: "SIZE_INVALID", - Message: "provided length did not match content length", - Description: `When a layer is uploaded, the provided size will be - checked against the uploaded content. If they do not match, this error - will be returned.`, - HTTPStatusCodes: []int{http.StatusBadRequest}, - }, - { - Code: ErrorCodeNameInvalid, - Value: "NAME_INVALID", - Message: "manifest name did not match URI", - Description: `During a manifest upload, if the name in the manifest - does not match the uri name, this error will be returned.`, - HTTPStatusCodes: []int{http.StatusBadRequest, http.StatusNotFound}, - }, - { - Code: ErrorCodeTagInvalid, - Value: "TAG_INVALID", - Message: "manifest tag did not match URI", - Description: `During a manifest upload, if the tag in the manifest - does not match the uri tag, this error will be returned.`, - HTTPStatusCodes: []int{http.StatusBadRequest, http.StatusNotFound}, - }, - { - Code: ErrorCodeNameUnknown, - Value: "NAME_UNKNOWN", - Message: "repository name not known to registry", - Description: `This is returned if the name used during an operation is - unknown to the registry.`, - HTTPStatusCodes: []int{http.StatusNotFound}, - }, - { - Code: ErrorCodeManifestUnknown, - Value: "MANIFEST_UNKNOWN", - Message: "manifest unknown", - Description: `This error is returned when the manifest, identified by - name and tag is unknown to the repository.`, - HTTPStatusCodes: []int{http.StatusNotFound}, - }, - { - Code: ErrorCodeManifestInvalid, - Value: "MANIFEST_INVALID", - Message: "manifest invalid", - Description: `During upload, manifests undergo several checks ensuring - validity. If those checks fail, this error may be returned, unless a - more specific error is included. The detail will contain information - the failed validation.`, - HTTPStatusCodes: []int{http.StatusBadRequest}, - }, - { - Code: ErrorCodeManifestUnverified, - Value: "MANIFEST_UNVERIFIED", - Message: "manifest failed signature verification", - Description: `During manifest upload, if the manifest fails signature - verification, this error will be returned.`, - HTTPStatusCodes: []int{http.StatusBadRequest}, - }, - { - Code: ErrorCodeBlobUnknown, - Value: "BLOB_UNKNOWN", - Message: "blob unknown to registry", - Description: `This error may be returned when a blob is unknown to the - registry in a specified repository. This can be returned with a - standard get or if a manifest references an unknown layer during - upload.`, - HTTPStatusCodes: []int{http.StatusBadRequest, http.StatusNotFound}, - }, - - { - Code: ErrorCodeBlobUploadUnknown, - Value: "BLOB_UPLOAD_UNKNOWN", - Message: "blob upload unknown to registry", - Description: `If a blob upload has been cancelled or was never - started, this error code may be returned.`, - HTTPStatusCodes: []int{http.StatusNotFound}, - }, -} - -var errorCodeToDescriptors map[ErrorCode]ErrorDescriptor -var idToDescriptors map[string]ErrorDescriptor - -func init() { - errorCodeToDescriptors = make(map[ErrorCode]ErrorDescriptor, len(ErrorDescriptors)) - idToDescriptors = make(map[string]ErrorDescriptor, len(ErrorDescriptors)) - - for _, descriptor := range ErrorDescriptors { - errorCodeToDescriptors[descriptor.Code] = descriptor - idToDescriptors[descriptor.Value] = descriptor - } -} diff --git a/docs/v2/doc.go b/docs/v2/doc.go deleted file mode 100644 index 30fe2271..00000000 --- a/docs/v2/doc.go +++ /dev/null @@ -1,13 +0,0 @@ -// Package v2 describes routes, urls and the error codes used in the Docker -// Registry JSON HTTP API V2. In addition to declarations, descriptors are -// provided for routes and error codes that can be used for implementation and -// automatically generating documentation. -// -// Definitions here are considered to be locked down for the V2 registry api. -// Any changes must be considered carefully and should not proceed without a -// change proposal. -// -// Currently, while the HTTP API definitions are considered stable, the Go API -// exports are considered unstable. Go API consumers should take care when -// relying on these definitions until this message is deleted. -package v2 diff --git a/docs/v2/errors.go b/docs/v2/errors.go deleted file mode 100644 index 8c85d3a9..00000000 --- a/docs/v2/errors.go +++ /dev/null @@ -1,185 +0,0 @@ -package v2 - -import ( - "fmt" - "strings" -) - -// ErrorCode represents the error type. The errors are serialized via strings -// and the integer format may change and should *never* be exported. -type ErrorCode int - -const ( - // ErrorCodeUnknown is a catch-all for errors not defined below. - ErrorCodeUnknown ErrorCode = iota - - // ErrorCodeDigestInvalid is returned when uploading a blob if the - // provided digest does not match the blob contents. - ErrorCodeDigestInvalid - - // ErrorCodeSizeInvalid is returned when uploading a blob if the provided - // size does not match the content length. - ErrorCodeSizeInvalid - - // ErrorCodeNameInvalid is returned when the name in the manifest does not - // match the provided name. - ErrorCodeNameInvalid - - // ErrorCodeTagInvalid is returned when the tag in the manifest does not - // match the provided tag. - ErrorCodeTagInvalid - - // ErrorCodeNameUnknown when the repository name is not known. - ErrorCodeNameUnknown - - // ErrorCodeManifestUnknown returned when image manifest is unknown. - ErrorCodeManifestUnknown - - // ErrorCodeManifestInvalid returned when an image manifest is invalid, - // typically during a PUT operation. This error encompasses all errors - // encountered during manifest validation that aren't signature errors. - ErrorCodeManifestInvalid - - // ErrorCodeManifestUnverified is returned when the manifest fails - // signature verfication. - ErrorCodeManifestUnverified - - // ErrorCodeBlobUnknown is returned when a blob is unknown to the - // registry. This can happen when the manifest references a nonexistent - // layer or the result is not found by a blob fetch. - ErrorCodeBlobUnknown - - // ErrorCodeBlobUploadUnknown is returned when an upload is unknown. - ErrorCodeBlobUploadUnknown -) - -// ParseErrorCode attempts to parse the error code string, returning -// ErrorCodeUnknown if the error is not known. -func ParseErrorCode(s string) ErrorCode { - desc, ok := idToDescriptors[s] - - if !ok { - return ErrorCodeUnknown - } - - return desc.Code -} - -// Descriptor returns the descriptor for the error code. -func (ec ErrorCode) Descriptor() ErrorDescriptor { - d, ok := errorCodeToDescriptors[ec] - - if !ok { - return ErrorCodeUnknown.Descriptor() - } - - return d -} - -// String returns the canonical identifier for this error code. -func (ec ErrorCode) String() string { - return ec.Descriptor().Value -} - -// Message returned the human-readable error message for this error code. -func (ec ErrorCode) Message() string { - return ec.Descriptor().Message -} - -// MarshalText encodes the receiver into UTF-8-encoded text and returns the -// result. -func (ec ErrorCode) MarshalText() (text []byte, err error) { - return []byte(ec.String()), nil -} - -// UnmarshalText decodes the form generated by MarshalText. -func (ec *ErrorCode) UnmarshalText(text []byte) error { - desc, ok := idToDescriptors[string(text)] - - if !ok { - desc = ErrorCodeUnknown.Descriptor() - } - - *ec = desc.Code - - return nil -} - -// Error provides a wrapper around ErrorCode with extra Details provided. -type Error struct { - Code ErrorCode `json:"code"` - Message string `json:"message,omitempty"` - Detail interface{} `json:"detail,omitempty"` -} - -// Error returns a human readable representation of the error. -func (e Error) Error() string { - return fmt.Sprintf("%s: %s", - strings.ToLower(strings.Replace(e.Code.String(), "_", " ", -1)), - e.Message) -} - -// Errors provides the envelope for multiple errors and a few sugar methods -// for use within the application. -type Errors struct { - Errors []Error `json:"errors,omitempty"` -} - -// Push pushes an error on to the error stack, with the optional detail -// argument. It is a programming error (ie panic) to push more than one -// detail at a time. -func (errs *Errors) Push(code ErrorCode, details ...interface{}) { - if len(details) > 1 { - panic("please specify zero or one detail items for this error") - } - - var detail interface{} - if len(details) > 0 { - detail = details[0] - } - - if err, ok := detail.(error); ok { - detail = err.Error() - } - - errs.PushErr(Error{ - Code: code, - Message: code.Message(), - Detail: detail, - }) -} - -// PushErr pushes an error interface onto the error stack. -func (errs *Errors) PushErr(err error) { - switch err.(type) { - case Error: - errs.Errors = append(errs.Errors, err.(Error)) - default: - errs.Errors = append(errs.Errors, Error{Message: err.Error()}) - } -} - -func (errs *Errors) Error() string { - switch errs.Len() { - case 0: - return "" - case 1: - return errs.Errors[0].Error() - default: - msg := "errors:\n" - for _, err := range errs.Errors { - msg += err.Error() + "\n" - } - return msg - } -} - -// Clear clears the errors. -func (errs *Errors) Clear() { - errs.Errors = errs.Errors[:0] -} - -// Len returns the current number of errors. -func (errs *Errors) Len() int { - return len(errs.Errors) -} diff --git a/docs/v2/errors_test.go b/docs/v2/errors_test.go deleted file mode 100644 index 4a80cdfe..00000000 --- a/docs/v2/errors_test.go +++ /dev/null @@ -1,163 +0,0 @@ -package v2 - -import ( - "encoding/json" - "reflect" - "testing" -) - -// TestErrorCodes ensures that error code format, mappings and -// marshaling/unmarshaling. round trips are stable. -func TestErrorCodes(t *testing.T) { - for _, desc := range ErrorDescriptors { - if desc.Code.String() != desc.Value { - t.Fatalf("error code string incorrect: %q != %q", desc.Code.String(), desc.Value) - } - - if desc.Code.Message() != desc.Message { - t.Fatalf("incorrect message for error code %v: %q != %q", desc.Code, desc.Code.Message(), desc.Message) - } - - // Serialize the error code using the json library to ensure that we - // get a string and it works round trip. - p, err := json.Marshal(desc.Code) - - if err != nil { - t.Fatalf("error marshaling error code %v: %v", desc.Code, err) - } - - if len(p) <= 0 { - t.Fatalf("expected content in marshaled before for error code %v", desc.Code) - } - - // First, unmarshal to interface and ensure we have a string. - var ecUnspecified interface{} - if err := json.Unmarshal(p, &ecUnspecified); err != nil { - t.Fatalf("error unmarshaling error code %v: %v", desc.Code, err) - } - - if _, ok := ecUnspecified.(string); !ok { - t.Fatalf("expected a string for error code %v on unmarshal got a %T", desc.Code, ecUnspecified) - } - - // Now, unmarshal with the error code type and ensure they are equal - var ecUnmarshaled ErrorCode - if err := json.Unmarshal(p, &ecUnmarshaled); err != nil { - t.Fatalf("error unmarshaling error code %v: %v", desc.Code, err) - } - - if ecUnmarshaled != desc.Code { - t.Fatalf("unexpected error code during error code marshal/unmarshal: %v != %v", ecUnmarshaled, desc.Code) - } - } -} - -// TestErrorsManagement does a quick check of the Errors type to ensure that -// members are properly pushed and marshaled. -func TestErrorsManagement(t *testing.T) { - var errs Errors - - errs.Push(ErrorCodeDigestInvalid) - errs.Push(ErrorCodeBlobUnknown, - map[string]string{"digest": "sometestblobsumdoesntmatter"}) - - p, err := json.Marshal(errs) - - if err != nil { - t.Fatalf("error marashaling errors: %v", err) - } - - expectedJSON := "{\"errors\":[{\"code\":\"DIGEST_INVALID\",\"message\":\"provided digest did not match uploaded content\"},{\"code\":\"BLOB_UNKNOWN\",\"message\":\"blob unknown to registry\",\"detail\":{\"digest\":\"sometestblobsumdoesntmatter\"}}]}" - - if string(p) != expectedJSON { - t.Fatalf("unexpected json: %q != %q", string(p), expectedJSON) - } - - errs.Clear() - errs.Push(ErrorCodeUnknown) - expectedJSON = "{\"errors\":[{\"code\":\"UNKNOWN\",\"message\":\"unknown error\"}]}" - p, err = json.Marshal(errs) - - if err != nil { - t.Fatalf("error marashaling errors: %v", err) - } - - if string(p) != expectedJSON { - t.Fatalf("unexpected json: %q != %q", string(p), expectedJSON) - } -} - -// TestMarshalUnmarshal ensures that api errors can round trip through json -// without losing information. -func TestMarshalUnmarshal(t *testing.T) { - - var errors Errors - - for _, testcase := range []struct { - description string - err Error - }{ - { - description: "unknown error", - err: Error{ - - Code: ErrorCodeUnknown, - Message: ErrorCodeUnknown.Descriptor().Message, - }, - }, - { - description: "unknown manifest", - err: Error{ - Code: ErrorCodeManifestUnknown, - Message: ErrorCodeManifestUnknown.Descriptor().Message, - }, - }, - { - description: "unknown manifest", - err: Error{ - Code: ErrorCodeBlobUnknown, - Message: ErrorCodeBlobUnknown.Descriptor().Message, - Detail: map[string]interface{}{"digest": "asdfqwerqwerqwerqwer"}, - }, - }, - } { - fatalf := func(format string, args ...interface{}) { - t.Fatalf(testcase.description+": "+format, args...) - } - - unexpectedErr := func(err error) { - fatalf("unexpected error: %v", err) - } - - p, err := json.Marshal(testcase.err) - if err != nil { - unexpectedErr(err) - } - - var unmarshaled Error - if err := json.Unmarshal(p, &unmarshaled); err != nil { - unexpectedErr(err) - } - - if !reflect.DeepEqual(unmarshaled, testcase.err) { - fatalf("errors not equal after round trip: %#v != %#v", unmarshaled, testcase.err) - } - - // Roll everything up into an error response envelope. - errors.PushErr(testcase.err) - } - - p, err := json.Marshal(errors) - if err != nil { - t.Fatalf("unexpected error marshaling error envelope: %v", err) - } - - var unmarshaled Errors - if err := json.Unmarshal(p, &unmarshaled); err != nil { - t.Fatalf("unexpected error unmarshaling error envelope: %v", err) - } - - if !reflect.DeepEqual(unmarshaled, errors) { - t.Fatalf("errors not equal after round trip: %#v != %#v", unmarshaled, errors) - } -} diff --git a/docs/v2/regexp.go b/docs/v2/regexp.go deleted file mode 100644 index 07484dcd..00000000 --- a/docs/v2/regexp.go +++ /dev/null @@ -1,22 +0,0 @@ -package v2 - -import "regexp" - -// This file defines regular expressions for use in route definition. These -// are also defined in the registry code base. Until they are in a common, -// shared location, and exported, they must be repeated here. - -// RepositoryNameComponentRegexp restricts registtry path components names to -// start with at least two letters or numbers, with following parts able to -// separated by one period, dash or underscore. -var RepositoryNameComponentRegexp = regexp.MustCompile(`[a-z0-9]+(?:[._-][a-z0-9]+)*`) - -// RepositoryNameRegexp builds on RepositoryNameComponentRegexp to allow 1 to -// 5 path components, separated by a forward slash. -var RepositoryNameRegexp = regexp.MustCompile(`(?:` + RepositoryNameComponentRegexp.String() + `/){0,4}` + RepositoryNameComponentRegexp.String()) - -// TagNameRegexp matches valid tag names. From docker/docker:graph/tags.go. -var TagNameRegexp = regexp.MustCompile(`[\w][\w.-]{0,127}`) - -// DigestRegexp matches valid digest types. -var DigestRegexp = regexp.MustCompile(`[a-zA-Z0-9-_+.]+:[a-zA-Z0-9-_+.=]+`) diff --git a/docs/v2/routes.go b/docs/v2/routes.go deleted file mode 100644 index de0a38fb..00000000 --- a/docs/v2/routes.go +++ /dev/null @@ -1,66 +0,0 @@ -package v2 - -import "github.com/gorilla/mux" - -// The following are definitions of the name under which all V2 routes are -// registered. These symbols can be used to look up a route based on the name. -const ( - RouteNameBase = "base" - RouteNameManifest = "manifest" - RouteNameTags = "tags" - RouteNameBlob = "blob" - RouteNameBlobUpload = "blob-upload" - RouteNameBlobUploadChunk = "blob-upload-chunk" -) - -var allEndpoints = []string{ - RouteNameManifest, - RouteNameTags, - RouteNameBlob, - RouteNameBlobUpload, - RouteNameBlobUploadChunk, -} - -// Router builds a gorilla router with named routes for the various API -// methods. This can be used directly by both server implementations and -// clients. -func Router() *mux.Router { - router := mux.NewRouter(). - StrictSlash(true) - - // GET /v2/ Check Check that the registry implements API version 2(.1) - router. - Path("/v2/"). - Name(RouteNameBase) - - // GET /v2//manifest/ Image Manifest Fetch the image manifest identified by name and reference where reference can be a tag or digest. - // PUT /v2//manifest/ Image Manifest Upload the image manifest identified by name and reference where reference can be a tag or digest. - // DELETE /v2//manifest/ Image Manifest Delete the image identified by name and reference where reference can be a tag or digest. - router. - Path("/v2/{name:" + RepositoryNameRegexp.String() + "}/manifests/{reference:" + TagNameRegexp.String() + "|" + DigestRegexp.String() + "}"). - Name(RouteNameManifest) - - // GET /v2//tags/list Tags Fetch the tags under the repository identified by name. - router. - Path("/v2/{name:" + RepositoryNameRegexp.String() + "}/tags/list"). - Name(RouteNameTags) - - // GET /v2//blob/ Layer Fetch the blob identified by digest. - router. - Path("/v2/{name:" + RepositoryNameRegexp.String() + "}/blobs/{digest:[a-zA-Z0-9-_+.]+:[a-zA-Z0-9-_+.=]+}"). - Name(RouteNameBlob) - - // POST /v2//blob/upload/ Layer Upload Initiate an upload of the layer identified by tarsum. - router. - Path("/v2/{name:" + RepositoryNameRegexp.String() + "}/blobs/uploads/"). - Name(RouteNameBlobUpload) - - // GET /v2//blob/upload/ Layer Upload Get the status of the upload identified by tarsum and uuid. - // PUT /v2//blob/upload/ Layer Upload Upload all or a chunk of the upload identified by tarsum and uuid. - // DELETE /v2//blob/upload/ Layer Upload Cancel the upload identified by layer and uuid - router. - Path("/v2/{name:" + RepositoryNameRegexp.String() + "}/blobs/uploads/{uuid}"). - Name(RouteNameBlobUploadChunk) - - return router -} diff --git a/docs/v2/routes_test.go b/docs/v2/routes_test.go deleted file mode 100644 index 0191feed..00000000 --- a/docs/v2/routes_test.go +++ /dev/null @@ -1,192 +0,0 @@ -package v2 - -import ( - "encoding/json" - "net/http" - "net/http/httptest" - "reflect" - "testing" - - "github.com/gorilla/mux" -) - -type routeTestCase struct { - RequestURI string - Vars map[string]string - RouteName string - StatusCode int -} - -// TestRouter registers a test handler with all the routes and ensures that -// each route returns the expected path variables. Not method verification is -// present. This not meant to be exhaustive but as check to ensure that the -// expected variables are extracted. -// -// This may go away as the application structure comes together. -func TestRouter(t *testing.T) { - - router := Router() - - testHandler := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - testCase := routeTestCase{ - RequestURI: r.RequestURI, - Vars: mux.Vars(r), - RouteName: mux.CurrentRoute(r).GetName(), - } - - enc := json.NewEncoder(w) - - if err := enc.Encode(testCase); err != nil { - http.Error(w, err.Error(), http.StatusInternalServerError) - return - } - }) - - // Startup test server - server := httptest.NewServer(router) - - for _, testcase := range []routeTestCase{ - { - RouteName: RouteNameBase, - RequestURI: "/v2/", - Vars: map[string]string{}, - }, - { - RouteName: RouteNameManifest, - RequestURI: "/v2/foo/manifests/bar", - Vars: map[string]string{ - "name": "foo", - "reference": "bar", - }, - }, - { - RouteName: RouteNameManifest, - RequestURI: "/v2/foo/bar/manifests/tag", - Vars: map[string]string{ - "name": "foo/bar", - "reference": "tag", - }, - }, - { - RouteName: RouteNameTags, - RequestURI: "/v2/foo/bar/tags/list", - Vars: map[string]string{ - "name": "foo/bar", - }, - }, - { - RouteName: RouteNameBlob, - RequestURI: "/v2/foo/bar/blobs/tarsum.dev+foo:abcdef0919234", - Vars: map[string]string{ - "name": "foo/bar", - "digest": "tarsum.dev+foo:abcdef0919234", - }, - }, - { - RouteName: RouteNameBlob, - RequestURI: "/v2/foo/bar/blobs/sha256:abcdef0919234", - Vars: map[string]string{ - "name": "foo/bar", - "digest": "sha256:abcdef0919234", - }, - }, - { - RouteName: RouteNameBlobUpload, - RequestURI: "/v2/foo/bar/blobs/uploads/", - Vars: map[string]string{ - "name": "foo/bar", - }, - }, - { - RouteName: RouteNameBlobUploadChunk, - RequestURI: "/v2/foo/bar/blobs/uploads/uuid", - Vars: map[string]string{ - "name": "foo/bar", - "uuid": "uuid", - }, - }, - { - RouteName: RouteNameBlobUploadChunk, - RequestURI: "/v2/foo/bar/blobs/uploads/D95306FA-FAD3-4E36-8D41-CF1C93EF8286", - Vars: map[string]string{ - "name": "foo/bar", - "uuid": "D95306FA-FAD3-4E36-8D41-CF1C93EF8286", - }, - }, - { - RouteName: RouteNameBlobUploadChunk, - RequestURI: "/v2/foo/bar/blobs/uploads/RDk1MzA2RkEtRkFEMy00RTM2LThENDEtQ0YxQzkzRUY4Mjg2IA==", - Vars: map[string]string{ - "name": "foo/bar", - "uuid": "RDk1MzA2RkEtRkFEMy00RTM2LThENDEtQ0YxQzkzRUY4Mjg2IA==", - }, - }, - { - // Check ambiguity: ensure we can distinguish between tags for - // "foo/bar/image/image" and image for "foo/bar/image" with tag - // "tags" - RouteName: RouteNameManifest, - RequestURI: "/v2/foo/bar/manifests/manifests/tags", - Vars: map[string]string{ - "name": "foo/bar/manifests", - "reference": "tags", - }, - }, - { - // This case presents an ambiguity between foo/bar with tag="tags" - // and list tags for "foo/bar/manifest" - RouteName: RouteNameTags, - RequestURI: "/v2/foo/bar/manifests/tags/list", - Vars: map[string]string{ - "name": "foo/bar/manifests", - }, - }, - { - RouteName: RouteNameBlobUploadChunk, - RequestURI: "/v2/foo/../../blob/uploads/D95306FA-FAD3-4E36-8D41-CF1C93EF8286", - StatusCode: http.StatusNotFound, - }, - } { - // Register the endpoint - router.GetRoute(testcase.RouteName).Handler(testHandler) - u := server.URL + testcase.RequestURI - - resp, err := http.Get(u) - - if err != nil { - t.Fatalf("error issuing get request: %v", err) - } - - if testcase.StatusCode == 0 { - // Override default, zero-value - testcase.StatusCode = http.StatusOK - } - - if resp.StatusCode != testcase.StatusCode { - t.Fatalf("unexpected status for %s: %v %v", u, resp.Status, resp.StatusCode) - } - - if testcase.StatusCode != http.StatusOK { - // We don't care about json response. - continue - } - - dec := json.NewDecoder(resp.Body) - - var actualRouteInfo routeTestCase - if err := dec.Decode(&actualRouteInfo); err != nil { - t.Fatalf("error reading json response: %v", err) - } - // Needs to be set out of band - actualRouteInfo.StatusCode = resp.StatusCode - - if actualRouteInfo.RouteName != testcase.RouteName { - t.Fatalf("incorrect route %q matched, expected %q", actualRouteInfo.RouteName, testcase.RouteName) - } - - if !reflect.DeepEqual(actualRouteInfo, testcase) { - t.Fatalf("actual does not equal expected: %#v != %#v", actualRouteInfo, testcase) - } - } - -} diff --git a/docs/v2/urls.go b/docs/v2/urls.go deleted file mode 100644 index 38fa98af..00000000 --- a/docs/v2/urls.go +++ /dev/null @@ -1,179 +0,0 @@ -package v2 - -import ( - "net/http" - "net/url" - - "github.com/gorilla/mux" -) - -// URLBuilder creates registry API urls from a single base endpoint. It can be -// used to create urls for use in a registry client or server. -// -// All urls will be created from the given base, including the api version. -// For example, if a root of "/foo/" is provided, urls generated will be fall -// under "/foo/v2/...". Most application will only provide a schema, host and -// port, such as "https://localhost:5000/". -type URLBuilder struct { - root *url.URL // url root (ie http://localhost/) - router *mux.Router -} - -// NewURLBuilder creates a URLBuilder with provided root url object. -func NewURLBuilder(root *url.URL) *URLBuilder { - return &URLBuilder{ - root: root, - router: Router(), - } -} - -// NewURLBuilderFromString workes identically to NewURLBuilder except it takes -// a string argument for the root, returning an error if it is not a valid -// url. -func NewURLBuilderFromString(root string) (*URLBuilder, error) { - u, err := url.Parse(root) - if err != nil { - return nil, err - } - - return NewURLBuilder(u), nil -} - -// NewURLBuilderFromRequest uses information from an *http.Request to -// construct the root url. -func NewURLBuilderFromRequest(r *http.Request) *URLBuilder { - u := &url.URL{ - Scheme: r.URL.Scheme, - Host: r.Host, - } - - return NewURLBuilder(u) -} - -// BuildBaseURL constructs a base url for the API, typically just "/v2/". -func (ub *URLBuilder) BuildBaseURL() (string, error) { - route := ub.cloneRoute(RouteNameBase) - - baseURL, err := route.URL() - if err != nil { - return "", err - } - - return baseURL.String(), nil -} - -// BuildTagsURL constructs a url to list the tags in the named repository. -func (ub *URLBuilder) BuildTagsURL(name string) (string, error) { - route := ub.cloneRoute(RouteNameTags) - - tagsURL, err := route.URL("name", name) - if err != nil { - return "", err - } - - return tagsURL.String(), nil -} - -// BuildManifestURL constructs a url for the manifest identified by name and reference. -func (ub *URLBuilder) BuildManifestURL(name, reference string) (string, error) { - route := ub.cloneRoute(RouteNameManifest) - - manifestURL, err := route.URL("name", name, "reference", reference) - if err != nil { - return "", err - } - - return manifestURL.String(), nil -} - -// BuildBlobURL constructs the url for the blob identified by name and dgst. -func (ub *URLBuilder) BuildBlobURL(name string, dgst string) (string, error) { - route := ub.cloneRoute(RouteNameBlob) - - layerURL, err := route.URL("name", name, "digest", dgst) - if err != nil { - return "", err - } - - return layerURL.String(), nil -} - -// BuildBlobUploadURL constructs a url to begin a blob upload in the -// repository identified by name. -func (ub *URLBuilder) BuildBlobUploadURL(name string, values ...url.Values) (string, error) { - route := ub.cloneRoute(RouteNameBlobUpload) - - uploadURL, err := route.URL("name", name) - if err != nil { - return "", err - } - - return appendValuesURL(uploadURL, values...).String(), nil -} - -// BuildBlobUploadChunkURL constructs a url for the upload identified by uuid, -// including any url values. This should generally not be used by clients, as -// this url is provided by server implementations during the blob upload -// process. -func (ub *URLBuilder) BuildBlobUploadChunkURL(name, uuid string, values ...url.Values) (string, error) { - route := ub.cloneRoute(RouteNameBlobUploadChunk) - - uploadURL, err := route.URL("name", name, "uuid", uuid) - if err != nil { - return "", err - } - - return appendValuesURL(uploadURL, values...).String(), nil -} - -// clondedRoute returns a clone of the named route from the router. Routes -// must be cloned to avoid modifying them during url generation. -func (ub *URLBuilder) cloneRoute(name string) clonedRoute { - route := new(mux.Route) - root := new(url.URL) - - *route = *ub.router.GetRoute(name) // clone the route - *root = *ub.root - - return clonedRoute{Route: route, root: root} -} - -type clonedRoute struct { - *mux.Route - root *url.URL -} - -func (cr clonedRoute) URL(pairs ...string) (*url.URL, error) { - routeURL, err := cr.Route.URL(pairs...) - if err != nil { - return nil, err - } - - return cr.root.ResolveReference(routeURL), nil -} - -// appendValuesURL appends the parameters to the url. -func appendValuesURL(u *url.URL, values ...url.Values) *url.URL { - merged := u.Query() - - for _, v := range values { - for k, vv := range v { - merged[k] = append(merged[k], vv...) - } - } - - u.RawQuery = merged.Encode() - return u -} - -// appendValues appends the parameters to the url. Panics if the string is not -// a url. -func appendValues(u string, values ...url.Values) string { - up, err := url.Parse(u) - - if err != nil { - panic(err) // should never happen - } - - return appendValuesURL(up, values...).String() -} diff --git a/docs/v2/urls_test.go b/docs/v2/urls_test.go deleted file mode 100644 index f30c96c0..00000000 --- a/docs/v2/urls_test.go +++ /dev/null @@ -1,113 +0,0 @@ -package v2 - -import ( - "net/url" - "testing" -) - -type urlBuilderTestCase struct { - description string - expectedPath string - build func() (string, error) -} - -// TestURLBuilder tests the various url building functions, ensuring they are -// returning the expected values. -func TestURLBuilder(t *testing.T) { - var ( - urlBuilder *URLBuilder - err error - ) - - testCases := []urlBuilderTestCase{ - { - description: "test base url", - expectedPath: "/v2/", - build: func() (string, error) { - return urlBuilder.BuildBaseURL() - }, - }, - { - description: "test tags url", - expectedPath: "/v2/foo/bar/tags/list", - build: func() (string, error) { - return urlBuilder.BuildTagsURL("foo/bar") - }, - }, - { - description: "test manifest url", - expectedPath: "/v2/foo/bar/manifests/tag", - build: func() (string, error) { - return urlBuilder.BuildManifestURL("foo/bar", "tag") - }, - }, - { - description: "build blob url", - expectedPath: "/v2/foo/bar/blobs/tarsum.v1+sha256:abcdef0123456789", - build: func() (string, error) { - return urlBuilder.BuildBlobURL("foo/bar", "tarsum.v1+sha256:abcdef0123456789") - }, - }, - { - description: "build blob upload url", - expectedPath: "/v2/foo/bar/blobs/uploads/", - build: func() (string, error) { - return urlBuilder.BuildBlobUploadURL("foo/bar") - }, - }, - { - description: "build blob upload url with digest and size", - expectedPath: "/v2/foo/bar/blobs/uploads/?digest=tarsum.v1%2Bsha256%3Aabcdef0123456789&size=10000", - build: func() (string, error) { - return urlBuilder.BuildBlobUploadURL("foo/bar", url.Values{ - "size": []string{"10000"}, - "digest": []string{"tarsum.v1+sha256:abcdef0123456789"}, - }) - }, - }, - { - description: "build blob upload chunk url", - expectedPath: "/v2/foo/bar/blobs/uploads/uuid-part", - build: func() (string, error) { - return urlBuilder.BuildBlobUploadChunkURL("foo/bar", "uuid-part") - }, - }, - { - description: "build blob upload chunk url with digest and size", - expectedPath: "/v2/foo/bar/blobs/uploads/uuid-part?digest=tarsum.v1%2Bsha256%3Aabcdef0123456789&size=10000", - build: func() (string, error) { - return urlBuilder.BuildBlobUploadChunkURL("foo/bar", "uuid-part", url.Values{ - "size": []string{"10000"}, - "digest": []string{"tarsum.v1+sha256:abcdef0123456789"}, - }) - }, - }, - } - - roots := []string{ - "http://example.com", - "https://example.com", - "http://localhost:5000", - "https://localhost:5443", - } - - for _, root := range roots { - urlBuilder, err = NewURLBuilderFromString(root) - if err != nil { - t.Fatalf("unexpected error creating urlbuilder: %v", err) - } - - for _, testCase := range testCases { - url, err := testCase.build() - if err != nil { - t.Fatalf("%s: error building url: %v", testCase.description, err) - } - - expectedURL := root + testCase.expectedPath - - if url != expectedURL { - t.Fatalf("%s: %q != %q", testCase.description, url, expectedURL) - } - } - } -} From 638ccff56443bbc11ac0656673196d6a4debead7 Mon Sep 17 00:00:00 2001 From: Michael Crosby Date: Tue, 31 Mar 2015 16:21:37 -0700 Subject: [PATCH 257/375] Remove jobs from registry.Service This makes `registry.Service` a first class type and does not use jobs to interact with this type. Signed-off-by: Michael Crosby --- docs/auth.go | 1 - docs/service.go | 201 +++++------------------------------------------- 2 files changed, 19 insertions(+), 183 deletions(-) diff --git a/docs/auth.go b/docs/auth.go index 2c37f7f6..51b781dd 100644 --- a/docs/auth.go +++ b/docs/auth.go @@ -230,7 +230,6 @@ func Login(authConfig *AuthConfig, registryEndpoint *Endpoint, factory *requestd if registryEndpoint.Version == APIVersion2 { return loginV2(authConfig, registryEndpoint, factory) } - return loginV1(authConfig, registryEndpoint, factory) } diff --git a/docs/service.go b/docs/service.go index f464faab..cf29732f 100644 --- a/docs/service.go +++ b/docs/service.go @@ -1,20 +1,5 @@ package registry -import ( - "fmt" - - "github.com/Sirupsen/logrus" - "github.com/docker/docker/engine" -) - -// Service exposes registry capabilities in the standard Engine -// interface. Once installed, it extends the engine with the -// following calls: -// -// 'auth': Authenticate against the public registry -// 'search': Search for images on the public registry -// 'pull': Download images from any registry (TODO) -// 'push': Upload images to any registry (TODO) type Service struct { Config *ServiceConfig } @@ -27,201 +12,53 @@ func NewService(options *Options) *Service { } } -// Install installs registry capabilities to eng. -func (s *Service) Install(eng *engine.Engine) error { - eng.Register("auth", s.Auth) - eng.Register("search", s.Search) - eng.Register("resolve_repository", s.ResolveRepository) - eng.Register("resolve_index", s.ResolveIndex) - eng.Register("registry_config", s.GetRegistryConfig) - return nil -} - // Auth contacts the public registry with the provided credentials, // and returns OK if authentication was sucessful. // It can be used to verify the validity of a client's credentials. -func (s *Service) Auth(job *engine.Job) error { - var ( - authConfig = new(AuthConfig) - endpoint *Endpoint - index *IndexInfo - status string - err error - ) - - job.GetenvJson("authConfig", authConfig) - +func (s *Service) Auth(authConfig *AuthConfig) (string, error) { addr := authConfig.ServerAddress if addr == "" { // Use the official registry address if not specified. addr = IndexServerAddress() } - - if index, err = ResolveIndexInfo(job, addr); err != nil { - return err + index, err := s.ResolveIndex(addr) + if err != nil { + return "", err } - - if endpoint, err = NewEndpoint(index); err != nil { - logrus.Errorf("unable to get new registry endpoint: %s", err) - return err + endpoint, err := NewEndpoint(index) + if err != nil { + return "", err } - authConfig.ServerAddress = endpoint.String() - - if status, err = Login(authConfig, endpoint, HTTPRequestFactory(nil)); err != nil { - logrus.Errorf("unable to login against registry endpoint %s: %s", endpoint, err) - return err - } - - logrus.Infof("successful registry login for endpoint %s: %s", endpoint, status) - job.Printf("%s\n", status) - - return nil + return Login(authConfig, endpoint, HTTPRequestFactory(nil)) } // Search queries the public registry for images matching the specified // search terms, and returns the results. -// -// Argument syntax: search TERM -// -// Option environment: -// 'authConfig': json-encoded credentials to authenticate against the registry. -// The search extends to images only accessible via the credentials. -// -// 'metaHeaders': extra HTTP headers to include in the request to the registry. -// The headers should be passed as a json-encoded dictionary. -// -// Output: -// Results are sent as a collection of structured messages (using engine.Table). -// Each result is sent as a separate message. -// Results are ordered by number of stars on the public registry. -func (s *Service) Search(job *engine.Job) error { - if n := len(job.Args); n != 1 { - return fmt.Errorf("Usage: %s TERM", job.Name) - } - var ( - term = job.Args[0] - metaHeaders = map[string][]string{} - authConfig = &AuthConfig{} - ) - job.GetenvJson("authConfig", authConfig) - job.GetenvJson("metaHeaders", metaHeaders) - - repoInfo, err := ResolveRepositoryInfo(job, term) +func (s *Service) Search(term string, authConfig *AuthConfig, headers map[string][]string) (*SearchResults, error) { + repoInfo, err := s.ResolveRepository(term) if err != nil { - return err + return nil, err } // *TODO: Search multiple indexes. endpoint, err := repoInfo.GetEndpoint() if err != nil { - return err + return nil, err } - r, err := NewSession(authConfig, HTTPRequestFactory(metaHeaders), endpoint, true) + r, err := NewSession(authConfig, HTTPRequestFactory(headers), endpoint, true) if err != nil { - return err + return nil, err } - results, err := r.SearchRepositories(repoInfo.GetSearchTerm()) - if err != nil { - return err - } - outs := engine.NewTable("star_count", 0) - for _, result := range results.Results { - out := &engine.Env{} - out.Import(result) - outs.Add(out) - } - outs.ReverseSort() - if _, err := outs.WriteListTo(job.Stdout); err != nil { - return err - } - return nil + return r.SearchRepositories(repoInfo.GetSearchTerm()) } // ResolveRepository splits a repository name into its components // and configuration of the associated registry. -func (s *Service) ResolveRepository(job *engine.Job) error { - var ( - reposName = job.Args[0] - ) - - repoInfo, err := s.Config.NewRepositoryInfo(reposName) - if err != nil { - return err - } - - out := engine.Env{} - err = out.SetJson("repository", repoInfo) - if err != nil { - return err - } - out.WriteTo(job.Stdout) - - return nil -} - -// Convenience wrapper for calling resolve_repository Job from a running job. -func ResolveRepositoryInfo(jobContext *engine.Job, reposName string) (*RepositoryInfo, error) { - job := jobContext.Eng.Job("resolve_repository", reposName) - env, err := job.Stdout.AddEnv() - if err != nil { - return nil, err - } - if err := job.Run(); err != nil { - return nil, err - } - info := RepositoryInfo{} - if err := env.GetJson("repository", &info); err != nil { - return nil, err - } - return &info, nil +func (s *Service) ResolveRepository(name string) (*RepositoryInfo, error) { + return s.Config.NewRepositoryInfo(name) } // ResolveIndex takes indexName and returns index info -func (s *Service) ResolveIndex(job *engine.Job) error { - var ( - indexName = job.Args[0] - ) - - index, err := s.Config.NewIndexInfo(indexName) - if err != nil { - return err - } - - out := engine.Env{} - err = out.SetJson("index", index) - if err != nil { - return err - } - out.WriteTo(job.Stdout) - - return nil -} - -// Convenience wrapper for calling resolve_index Job from a running job. -func ResolveIndexInfo(jobContext *engine.Job, indexName string) (*IndexInfo, error) { - job := jobContext.Eng.Job("resolve_index", indexName) - env, err := job.Stdout.AddEnv() - if err != nil { - return nil, err - } - if err := job.Run(); err != nil { - return nil, err - } - info := IndexInfo{} - if err := env.GetJson("index", &info); err != nil { - return nil, err - } - return &info, nil -} - -// GetRegistryConfig returns current registry configuration. -func (s *Service) GetRegistryConfig(job *engine.Job) error { - out := engine.Env{} - err := out.SetJson("config", s.Config) - if err != nil { - return err - } - out.WriteTo(job.Stdout) - - return nil +func (s *Service) ResolveIndex(name string) (*IndexInfo, error) { + return s.Config.NewIndexInfo(name) } From e5408bd911d3f1363d148eeca5ecf2ef8aea41b4 Mon Sep 17 00:00:00 2001 From: Doug Davis Date: Fri, 3 Apr 2015 10:29:30 -0700 Subject: [PATCH 258/375] Remove engine.Table from docker search and fix missing field registry/SearchResults was missing the "is_automated" field. I added it back in. Pull this 'table' removal one from the others because it fixed a bug too Signed-off-by: Doug Davis --- docs/types.go | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/types.go b/docs/types.go index bd0bf8b7..2c8369bd 100644 --- a/docs/types.go +++ b/docs/types.go @@ -5,6 +5,7 @@ type SearchResult struct { IsOfficial bool `json:"is_official"` Name string `json:"name"` IsTrusted bool `json:"is_trusted"` + IsAutomated bool `json:"is_automated"` Description string `json:"description"` } From ad3d879929c742b1e762eb2e4e9323676f3bc590 Mon Sep 17 00:00:00 2001 From: Antonio Murdaca Date: Sun, 29 Mar 2015 23:17:23 +0200 Subject: [PATCH 259/375] Refactor utils/utils, fixes #11923 Signed-off-by: Antonio Murdaca --- docs/config.go | 4 ++-- docs/session.go | 29 ++++++++++++++--------------- docs/session_v2.go | 18 +++++++++--------- 3 files changed, 25 insertions(+), 26 deletions(-) diff --git a/docs/config.go b/docs/config.go index 3515836d..a0a978cc 100644 --- a/docs/config.go +++ b/docs/config.go @@ -9,9 +9,9 @@ import ( "regexp" "strings" + "github.com/docker/docker/image" "github.com/docker/docker/opts" flag "github.com/docker/docker/pkg/mflag" - "github.com/docker/docker/utils" ) // Options holds command line options. @@ -213,7 +213,7 @@ func validateRemoteName(remoteName string) error { name = nameParts[0] // the repository name must not be a valid image ID - if err := utils.ValidateID(name); err == nil { + if err := image.ValidateID(name); err == nil { return fmt.Errorf("Invalid repository name (%s), cannot specify 64-byte hexadecimal strings", name) } } else { diff --git a/docs/session.go b/docs/session.go index 4682a507..c62745b5 100644 --- a/docs/session.go +++ b/docs/session.go @@ -21,7 +21,6 @@ import ( "github.com/docker/docker/pkg/httputils" "github.com/docker/docker/pkg/requestdecorator" "github.com/docker/docker/pkg/tarsum" - "github.com/docker/docker/utils" ) type Session struct { @@ -86,7 +85,7 @@ func (r *Session) GetRemoteHistory(imgID, registry string, token []string) ([]st if res.StatusCode == 401 { return nil, errLoginRequired } - return nil, utils.NewHTTPRequestError(fmt.Sprintf("Server error: %d trying to fetch remote history for %s", res.StatusCode, imgID), res) + return nil, httputils.NewHTTPRequestError(fmt.Sprintf("Server error: %d trying to fetch remote history for %s", res.StatusCode, imgID), res) } jsonString, err := ioutil.ReadAll(res.Body) @@ -115,7 +114,7 @@ func (r *Session) LookupRemoteImage(imgID, registry string, token []string) erro } res.Body.Close() if res.StatusCode != 200 { - return utils.NewHTTPRequestError(fmt.Sprintf("HTTP code %d", res.StatusCode), res) + return httputils.NewHTTPRequestError(fmt.Sprintf("HTTP code %d", res.StatusCode), res) } return nil } @@ -134,7 +133,7 @@ func (r *Session) GetRemoteImageJSON(imgID, registry string, token []string) ([] } defer res.Body.Close() if res.StatusCode != 200 { - return nil, -1, utils.NewHTTPRequestError(fmt.Sprintf("HTTP code %d", res.StatusCode), res) + return nil, -1, httputils.NewHTTPRequestError(fmt.Sprintf("HTTP code %d", res.StatusCode), res) } // if the size header is not present, then set it to '-1' imageSize := -1 @@ -282,13 +281,13 @@ func (r *Session) GetRepositoryData(remote string) (*RepositoryData, error) { // TODO: Right now we're ignoring checksums in the response body. // In the future, we need to use them to check image validity. if res.StatusCode == 404 { - return nil, utils.NewHTTPRequestError(fmt.Sprintf("HTTP code: %d", res.StatusCode), res) + return nil, httputils.NewHTTPRequestError(fmt.Sprintf("HTTP code: %d", res.StatusCode), res) } else if res.StatusCode != 200 { errBody, err := ioutil.ReadAll(res.Body) if err != nil { logrus.Debugf("Error reading response body: %s", err) } - return nil, utils.NewHTTPRequestError(fmt.Sprintf("Error: Status %d trying to pull repository %s: %q", res.StatusCode, remote, errBody), res) + return nil, httputils.NewHTTPRequestError(fmt.Sprintf("Error: Status %d trying to pull repository %s: %q", res.StatusCode, remote, errBody), res) } var tokens []string @@ -379,12 +378,12 @@ func (r *Session) PushImageJSONRegistry(imgData *ImgData, jsonRaw []byte, regist } defer res.Body.Close() if res.StatusCode == 401 && strings.HasPrefix(registry, "http://") { - return utils.NewHTTPRequestError("HTTP code 401, Docker will not send auth headers over HTTP.", res) + return httputils.NewHTTPRequestError("HTTP code 401, Docker will not send auth headers over HTTP.", res) } if res.StatusCode != 200 { errBody, err := ioutil.ReadAll(res.Body) if err != nil { - return utils.NewHTTPRequestError(fmt.Sprintf("HTTP code %d while uploading metadata and error when trying to parse response body: %s", res.StatusCode, err), res) + return httputils.NewHTTPRequestError(fmt.Sprintf("HTTP code %d while uploading metadata and error when trying to parse response body: %s", res.StatusCode, err), res) } var jsonBody map[string]string if err := json.Unmarshal(errBody, &jsonBody); err != nil { @@ -392,7 +391,7 @@ func (r *Session) PushImageJSONRegistry(imgData *ImgData, jsonRaw []byte, regist } else if jsonBody["error"] == "Image already exists" { return ErrAlreadyExists } - return utils.NewHTTPRequestError(fmt.Sprintf("HTTP code %d while uploading metadata: %q", res.StatusCode, errBody), res) + return httputils.NewHTTPRequestError(fmt.Sprintf("HTTP code %d while uploading metadata: %q", res.StatusCode, errBody), res) } return nil } @@ -432,9 +431,9 @@ func (r *Session) PushImageLayerRegistry(imgID string, layer io.Reader, registry if res.StatusCode != 200 { errBody, err := ioutil.ReadAll(res.Body) if err != nil { - return "", "", utils.NewHTTPRequestError(fmt.Sprintf("HTTP code %d while uploading metadata and error when trying to parse response body: %s", res.StatusCode, err), res) + return "", "", httputils.NewHTTPRequestError(fmt.Sprintf("HTTP code %d while uploading metadata and error when trying to parse response body: %s", res.StatusCode, err), res) } - return "", "", utils.NewHTTPRequestError(fmt.Sprintf("Received HTTP code %d while uploading layer: %q", res.StatusCode, errBody), res) + return "", "", httputils.NewHTTPRequestError(fmt.Sprintf("Received HTTP code %d while uploading layer: %q", res.StatusCode, errBody), res) } checksumPayload = "sha256:" + hex.EncodeToString(h.Sum(nil)) @@ -461,7 +460,7 @@ func (r *Session) PushRegistryTag(remote, revision, tag, registry string, token } res.Body.Close() if res.StatusCode != 200 && res.StatusCode != 201 { - return utils.NewHTTPRequestError(fmt.Sprintf("Internal server error: %d trying to push tag %s on %s", res.StatusCode, tag, remote), res) + return httputils.NewHTTPRequestError(fmt.Sprintf("Internal server error: %d trying to push tag %s on %s", res.StatusCode, tag, remote), res) } return nil } @@ -523,7 +522,7 @@ func (r *Session) PushImageJSONIndex(remote string, imgList []*ImgData, validate if err != nil { logrus.Debugf("Error reading response body: %s", err) } - return nil, utils.NewHTTPRequestError(fmt.Sprintf("Error: Status %d trying to push repository %s: %q", res.StatusCode, remote, errBody), res) + return nil, httputils.NewHTTPRequestError(fmt.Sprintf("Error: Status %d trying to push repository %s: %q", res.StatusCode, remote, errBody), res) } if res.Header.Get("X-Docker-Token") != "" { tokens = res.Header["X-Docker-Token"] @@ -547,7 +546,7 @@ func (r *Session) PushImageJSONIndex(remote string, imgList []*ImgData, validate if err != nil { logrus.Debugf("Error reading response body: %s", err) } - return nil, utils.NewHTTPRequestError(fmt.Sprintf("Error: Status %d trying to push checksums %s: %q", res.StatusCode, remote, errBody), res) + return nil, httputils.NewHTTPRequestError(fmt.Sprintf("Error: Status %d trying to push checksums %s: %q", res.StatusCode, remote, errBody), res) } } @@ -595,7 +594,7 @@ func (r *Session) SearchRepositories(term string) (*SearchResults, error) { } defer res.Body.Close() if res.StatusCode != 200 { - return nil, utils.NewHTTPRequestError(fmt.Sprintf("Unexpected status code %d", res.StatusCode), res) + return nil, httputils.NewHTTPRequestError(fmt.Sprintf("Unexpected status code %d", res.StatusCode), res) } result := new(SearchResults) err = json.NewDecoder(res.Body).Decode(result) diff --git a/docs/session_v2.go b/docs/session_v2.go index fb1d18e8..a14e434a 100644 --- a/docs/session_v2.go +++ b/docs/session_v2.go @@ -12,7 +12,7 @@ import ( "github.com/Sirupsen/logrus" "github.com/docker/distribution/digest" "github.com/docker/distribution/registry/api/v2" - "github.com/docker/docker/utils" + "github.com/docker/docker/pkg/httputils" ) const DockerDigestHeader = "Docker-Content-Digest" @@ -95,7 +95,7 @@ func (r *Session) GetV2ImageManifest(ep *Endpoint, imageName, tagName string, au } else if res.StatusCode == 404 { return nil, "", ErrDoesNotExist } - return nil, "", utils.NewHTTPRequestError(fmt.Sprintf("Server error: %d trying to fetch for %s:%s", res.StatusCode, imageName, tagName), res) + return nil, "", httputils.NewHTTPRequestError(fmt.Sprintf("Server error: %d trying to fetch for %s:%s", res.StatusCode, imageName, tagName), res) } manifestBytes, err := ioutil.ReadAll(res.Body) @@ -141,7 +141,7 @@ func (r *Session) HeadV2ImageBlob(ep *Endpoint, imageName string, dgst digest.Di return false, nil } - return false, utils.NewHTTPRequestError(fmt.Sprintf("Server error: %d trying head request for %s - %s", res.StatusCode, imageName, dgst), res) + return false, httputils.NewHTTPRequestError(fmt.Sprintf("Server error: %d trying head request for %s - %s", res.StatusCode, imageName, dgst), res) } func (r *Session) GetV2ImageBlob(ep *Endpoint, imageName string, dgst digest.Digest, blobWrtr io.Writer, auth *RequestAuthorization) error { @@ -168,7 +168,7 @@ func (r *Session) GetV2ImageBlob(ep *Endpoint, imageName string, dgst digest.Dig if res.StatusCode == 401 { return errLoginRequired } - return utils.NewHTTPRequestError(fmt.Sprintf("Server error: %d trying to pull %s blob", res.StatusCode, imageName), res) + return httputils.NewHTTPRequestError(fmt.Sprintf("Server error: %d trying to pull %s blob", res.StatusCode, imageName), res) } _, err = io.Copy(blobWrtr, res.Body) @@ -198,7 +198,7 @@ func (r *Session) GetV2ImageBlobReader(ep *Endpoint, imageName string, dgst dige if res.StatusCode == 401 { return nil, 0, errLoginRequired } - return nil, 0, utils.NewHTTPRequestError(fmt.Sprintf("Server error: %d trying to pull %s blob - %s", res.StatusCode, imageName, dgst), res) + return nil, 0, httputils.NewHTTPRequestError(fmt.Sprintf("Server error: %d trying to pull %s blob - %s", res.StatusCode, imageName, dgst), res) } lenStr := res.Header.Get("Content-Length") l, err := strconv.ParseInt(lenStr, 10, 64) @@ -245,7 +245,7 @@ func (r *Session) PutV2ImageBlob(ep *Endpoint, imageName string, dgst digest.Dig return err } logrus.Debugf("Unexpected response from server: %q %#v", errBody, res.Header) - return utils.NewHTTPRequestError(fmt.Sprintf("Server error: %d trying to push %s blob - %s", res.StatusCode, imageName, dgst), res) + return httputils.NewHTTPRequestError(fmt.Sprintf("Server error: %d trying to push %s blob - %s", res.StatusCode, imageName, dgst), res) } return nil @@ -286,7 +286,7 @@ func (r *Session) initiateBlobUpload(ep *Endpoint, imageName string, auth *Reque } logrus.Debugf("Unexpected response from server: %q %#v", errBody, res.Header) - return "", utils.NewHTTPRequestError(fmt.Sprintf("Server error: unexpected %d response status trying to initiate upload of %s", res.StatusCode, imageName), res) + return "", httputils.NewHTTPRequestError(fmt.Sprintf("Server error: unexpected %d response status trying to initiate upload of %s", res.StatusCode, imageName), res) } if location = res.Header.Get("Location"); location == "" { @@ -328,7 +328,7 @@ func (r *Session) PutV2ImageManifest(ep *Endpoint, imageName, tagName string, si return "", err } logrus.Debugf("Unexpected response from server: %q %#v", errBody, res.Header) - return "", utils.NewHTTPRequestError(fmt.Sprintf("Server error: %d trying to push %s:%s manifest", res.StatusCode, imageName, tagName), res) + return "", httputils.NewHTTPRequestError(fmt.Sprintf("Server error: %d trying to push %s:%s manifest", res.StatusCode, imageName, tagName), res) } hdrDigest, err := digest.ParseDigest(res.Header.Get(DockerDigestHeader)) @@ -384,7 +384,7 @@ func (r *Session) GetV2RemoteTags(ep *Endpoint, imageName string, auth *RequestA } else if res.StatusCode == 404 { return nil, ErrDoesNotExist } - return nil, utils.NewHTTPRequestError(fmt.Sprintf("Server error: %d trying to fetch for %s", res.StatusCode, imageName), res) + return nil, httputils.NewHTTPRequestError(fmt.Sprintf("Server error: %d trying to fetch for %s", res.StatusCode, imageName), res) } decoder := json.NewDecoder(res.Body) From 2bd5eb9d7c765a9d36dbab9e0640cfcaafac1f28 Mon Sep 17 00:00:00 2001 From: Steven Taylor Date: Wed, 15 Apr 2015 15:30:09 -0700 Subject: [PATCH 260/375] What if authConfig or factory is Null? Signed-off-by: Steven Taylor --- docs/session.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/session.go b/docs/session.go index c62745b5..dce4accd 100644 --- a/docs/session.go +++ b/docs/session.go @@ -53,7 +53,7 @@ func NewSession(authConfig *AuthConfig, factory *requestdecorator.RequestFactory if err != nil { return nil, err } - if info.Standalone { + if info.Standalone && authConfig != nil && factory != nil { logrus.Debugf("Endpoint %s is eligible for private registry. Enabling decorator.", r.indexEndpoint.String()) dec := requestdecorator.NewAuthDecorator(authConfig.Username, authConfig.Password) factory.AddDecorator(dec) From 742cf000d34a141866c38734f839152367446a0f Mon Sep 17 00:00:00 2001 From: Antonio Murdaca Date: Sun, 19 Apr 2015 15:23:48 +0200 Subject: [PATCH 261/375] Refactor else branches Signed-off-by: Antonio Murdaca --- docs/session.go | 24 +++++++++++------------- 1 file changed, 11 insertions(+), 13 deletions(-) diff --git a/docs/session.go b/docs/session.go index c62745b5..e9d6a33d 100644 --- a/docs/session.go +++ b/docs/session.go @@ -222,10 +222,10 @@ func (r *Session) GetRemoteTags(registries []string, repository string, token [] logrus.Debugf("Got status code %d from %s", res.StatusCode, endpoint) defer res.Body.Close() - if res.StatusCode != 200 && res.StatusCode != 404 { - continue - } else if res.StatusCode == 404 { + if res.StatusCode == 404 { return nil, fmt.Errorf("Repository not found") + } else if res.StatusCode != 200 { + continue } result := make(map[string]string) @@ -524,21 +524,19 @@ func (r *Session) PushImageJSONIndex(remote string, imgList []*ImgData, validate } return nil, httputils.NewHTTPRequestError(fmt.Sprintf("Error: Status %d trying to push repository %s: %q", res.StatusCode, remote, errBody), res) } - if res.Header.Get("X-Docker-Token") != "" { - tokens = res.Header["X-Docker-Token"] - logrus.Debugf("Auth token: %v", tokens) - } else { + if res.Header.Get("X-Docker-Token") == "" { return nil, fmt.Errorf("Index response didn't contain an access token") } + tokens = res.Header["X-Docker-Token"] + logrus.Debugf("Auth token: %v", tokens) - if res.Header.Get("X-Docker-Endpoints") != "" { - endpoints, err = buildEndpointsList(res.Header["X-Docker-Endpoints"], r.indexEndpoint.VersionString(1)) - if err != nil { - return nil, err - } - } else { + if res.Header.Get("X-Docker-Endpoints") == "" { return nil, fmt.Errorf("Index response didn't contain any endpoints") } + endpoints, err = buildEndpointsList(res.Header["X-Docker-Endpoints"], r.indexEndpoint.VersionString(1)) + if err != nil { + return nil, err + } } if validate { if res.StatusCode != 204 { From 47784682029f587384d6b850ae72b318725197cf Mon Sep 17 00:00:00 2001 From: Rick Wieman Date: Sun, 19 Apr 2015 23:36:58 +0200 Subject: [PATCH 262/375] Removes redundant else in registry/session.go Fixes #12523 Signed-off-by: Rick Wieman --- docs/session.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/docs/session.go b/docs/session.go index e9d6a33d..940e407e 100644 --- a/docs/session.go +++ b/docs/session.go @@ -224,7 +224,8 @@ func (r *Session) GetRemoteTags(registries []string, repository string, token [] if res.StatusCode == 404 { return nil, fmt.Errorf("Repository not found") - } else if res.StatusCode != 200 { + } + if res.StatusCode != 200 { continue } From 7b8b61bda1cef0211db51b44ef293b069d9a9ae8 Mon Sep 17 00:00:00 2001 From: Doug Davis Date: Wed, 1 Apr 2015 15:39:37 -0700 Subject: [PATCH 263/375] Add .docker/config.json and support for HTTP Headers This PR does the following: - migrated ~/.dockerfg to ~/.docker/config.json. The data is migrated but the old file remains in case its needed - moves the auth json in that fie into an "auth" property so we can add new top-level properties w/o messing with the auth stuff - adds support for an HttpHeaders property in ~/.docker/config.json which adds these http headers to all msgs from the cli In a follow-on PR I'll move the config file process out from under "registry" since it not specific to that any more. I didn't do it here because I wanted the diff to be smaller so people can make sure I didn't break/miss any auth code during my edits. Signed-off-by: Doug Davis --- docs/auth.go | 105 ++++++++++++++++++++++-------- docs/auth_test.go | 26 ++++---- docs/config_file_test.go | 135 +++++++++++++++++++++++++++++++++++++++ 3 files changed, 228 insertions(+), 38 deletions(-) create mode 100644 docs/config_file_test.go diff --git a/docs/auth.go b/docs/auth.go index 51b781dd..bccf58fc 100644 --- a/docs/auth.go +++ b/docs/auth.go @@ -8,24 +8,27 @@ import ( "io/ioutil" "net/http" "os" - "path" + "path/filepath" "strings" "sync" "time" "github.com/Sirupsen/logrus" + "github.com/docker/docker/pkg/homedir" "github.com/docker/docker/pkg/requestdecorator" ) const ( // Where we store the config file - CONFIGFILE = ".dockercfg" + CONFIGFILE = "config.json" + OLD_CONFIGFILE = ".dockercfg" ) var ( ErrConfigFileMissing = errors.New("The Auth config file is missing") ) +// Registry Auth Info type AuthConfig struct { Username string `json:"username,omitempty"` Password string `json:"password,omitempty"` @@ -34,9 +37,11 @@ type AuthConfig struct { ServerAddress string `json:"serveraddress,omitempty"` } +// ~/.docker/config.json file info type ConfigFile struct { - Configs map[string]AuthConfig `json:"configs,omitempty"` - rootPath string + AuthConfigs map[string]AuthConfig `json:"auths"` + HttpHeaders map[string]string `json:"HttpHeaders,omitempty"` + filename string // Note: not serialized - for internal use only } type RequestAuthorization struct { @@ -147,18 +152,58 @@ func decodeAuth(authStr string) (string, string, error) { // load up the auth config information and return values // FIXME: use the internal golang config parser -func LoadConfig(rootPath string) (*ConfigFile, error) { - configFile := ConfigFile{Configs: make(map[string]AuthConfig), rootPath: rootPath} - confFile := path.Join(rootPath, CONFIGFILE) +func LoadConfig(configDir string) (*ConfigFile, error) { + if configDir == "" { + configDir = filepath.Join(homedir.Get(), ".docker") + } + + configFile := ConfigFile{ + AuthConfigs: make(map[string]AuthConfig), + filename: filepath.Join(configDir, CONFIGFILE), + } + + // Try happy path first - latest config file + if _, err := os.Stat(configFile.filename); err == nil { + file, err := os.Open(configFile.filename) + if err != nil { + return &configFile, err + } + defer file.Close() + + if err := json.NewDecoder(file).Decode(&configFile); err != nil { + return &configFile, err + } + + for addr, ac := range configFile.AuthConfigs { + ac.Username, ac.Password, err = decodeAuth(ac.Auth) + if err != nil { + return &configFile, err + } + ac.Auth = "" + ac.ServerAddress = addr + configFile.AuthConfigs[addr] = ac + } + + return &configFile, nil + } else if !os.IsNotExist(err) { + // if file is there but we can't stat it for any reason other + // than it doesn't exist then stop + return &configFile, err + } + + // Can't find latest config file so check for the old one + confFile := filepath.Join(homedir.Get(), OLD_CONFIGFILE) + if _, err := os.Stat(confFile); err != nil { return &configFile, nil //missing file is not an error } + b, err := ioutil.ReadFile(confFile) if err != nil { return &configFile, err } - if err := json.Unmarshal(b, &configFile.Configs); err != nil { + if err := json.Unmarshal(b, &configFile.AuthConfigs); err != nil { arr := strings.Split(string(b), "\n") if len(arr) < 2 { return &configFile, fmt.Errorf("The Auth config file is empty") @@ -179,48 +224,52 @@ func LoadConfig(rootPath string) (*ConfigFile, error) { authConfig.Email = origEmail[1] authConfig.ServerAddress = IndexServerAddress() // *TODO: Switch to using IndexServerName() instead? - configFile.Configs[IndexServerAddress()] = authConfig + configFile.AuthConfigs[IndexServerAddress()] = authConfig } else { - for k, authConfig := range configFile.Configs { + for k, authConfig := range configFile.AuthConfigs { authConfig.Username, authConfig.Password, err = decodeAuth(authConfig.Auth) if err != nil { return &configFile, err } authConfig.Auth = "" authConfig.ServerAddress = k - configFile.Configs[k] = authConfig + configFile.AuthConfigs[k] = authConfig } } return &configFile, nil } -// save the auth config -func SaveConfig(configFile *ConfigFile) error { - confFile := path.Join(configFile.rootPath, CONFIGFILE) - if len(configFile.Configs) == 0 { - os.Remove(confFile) - return nil - } - - configs := make(map[string]AuthConfig, len(configFile.Configs)) - for k, authConfig := range configFile.Configs { +func (configFile *ConfigFile) Save() error { + // Encode sensitive data into a new/temp struct + tmpAuthConfigs := make(map[string]AuthConfig, len(configFile.AuthConfigs)) + for k, authConfig := range configFile.AuthConfigs { authCopy := authConfig authCopy.Auth = encodeAuth(&authCopy) authCopy.Username = "" authCopy.Password = "" authCopy.ServerAddress = "" - configs[k] = authCopy + tmpAuthConfigs[k] = authCopy } - b, err := json.MarshalIndent(configs, "", "\t") + saveAuthConfigs := configFile.AuthConfigs + configFile.AuthConfigs = tmpAuthConfigs + defer func() { configFile.AuthConfigs = saveAuthConfigs }() + + data, err := json.MarshalIndent(configFile, "", "\t") if err != nil { return err } - err = ioutil.WriteFile(confFile, b, 0600) + + if err := os.MkdirAll(filepath.Dir(configFile.filename), 0600); err != nil { + return err + } + + err = ioutil.WriteFile(configFile.filename, data, 0600) if err != nil { return err } + return nil } @@ -431,7 +480,7 @@ func tryV2TokenAuthLogin(authConfig *AuthConfig, params map[string]string, regis func (config *ConfigFile) ResolveAuthConfig(index *IndexInfo) AuthConfig { configKey := index.GetAuthConfigKey() // First try the happy case - if c, found := config.Configs[configKey]; found || index.Official { + if c, found := config.AuthConfigs[configKey]; found || index.Official { return c } @@ -450,7 +499,7 @@ func (config *ConfigFile) ResolveAuthConfig(index *IndexInfo) AuthConfig { // Maybe they have a legacy config file, we will iterate the keys converting // them to the new format and testing - for registry, config := range config.Configs { + for registry, config := range config.AuthConfigs { if configKey == convertToHostname(registry) { return config } @@ -459,3 +508,7 @@ func (config *ConfigFile) ResolveAuthConfig(index *IndexInfo) AuthConfig { // When all else fails, return an empty auth config return AuthConfig{} } + +func (config *ConfigFile) Filename() string { + return config.filename +} diff --git a/docs/auth_test.go b/docs/auth_test.go index 9cc299aa..b07aa7db 100644 --- a/docs/auth_test.go +++ b/docs/auth_test.go @@ -3,6 +3,7 @@ package registry import ( "io/ioutil" "os" + "path/filepath" "testing" ) @@ -31,13 +32,14 @@ func setupTempConfigFile() (*ConfigFile, error) { if err != nil { return nil, err } + root = filepath.Join(root, CONFIGFILE) configFile := &ConfigFile{ - rootPath: root, - Configs: make(map[string]AuthConfig), + AuthConfigs: make(map[string]AuthConfig), + filename: root, } for _, registry := range []string{"testIndex", IndexServerAddress()} { - configFile.Configs[registry] = AuthConfig{ + configFile.AuthConfigs[registry] = AuthConfig{ Username: "docker-user", Password: "docker-pass", Email: "docker@docker.io", @@ -52,14 +54,14 @@ func TestSameAuthDataPostSave(t *testing.T) { if err != nil { t.Fatal(err) } - defer os.RemoveAll(configFile.rootPath) + defer os.RemoveAll(configFile.filename) - err = SaveConfig(configFile) + err = configFile.Save() if err != nil { t.Fatal(err) } - authConfig := configFile.Configs["testIndex"] + authConfig := configFile.AuthConfigs["testIndex"] if authConfig.Username != "docker-user" { t.Fail() } @@ -79,9 +81,9 @@ func TestResolveAuthConfigIndexServer(t *testing.T) { if err != nil { t.Fatal(err) } - defer os.RemoveAll(configFile.rootPath) + defer os.RemoveAll(configFile.filename) - indexConfig := configFile.Configs[IndexServerAddress()] + indexConfig := configFile.AuthConfigs[IndexServerAddress()] officialIndex := &IndexInfo{ Official: true, @@ -102,7 +104,7 @@ func TestResolveAuthConfigFullURL(t *testing.T) { if err != nil { t.Fatal(err) } - defer os.RemoveAll(configFile.rootPath) + defer os.RemoveAll(configFile.filename) registryAuth := AuthConfig{ Username: "foo-user", @@ -119,7 +121,7 @@ func TestResolveAuthConfigFullURL(t *testing.T) { Password: "baz-pass", Email: "baz@example.com", } - configFile.Configs[IndexServerAddress()] = officialAuth + configFile.AuthConfigs[IndexServerAddress()] = officialAuth expectedAuths := map[string]AuthConfig{ "registry.example.com": registryAuth, @@ -157,12 +159,12 @@ func TestResolveAuthConfigFullURL(t *testing.T) { Name: configKey, } for _, registry := range registries { - configFile.Configs[registry] = configured + configFile.AuthConfigs[registry] = configured resolved := configFile.ResolveAuthConfig(index) if resolved.Email != configured.Email { t.Errorf("%s -> %q != %q\n", registry, resolved.Email, configured.Email) } - delete(configFile.Configs, registry) + delete(configFile.AuthConfigs, registry) resolved = configFile.ResolveAuthConfig(index) if resolved.Email == configured.Email { t.Errorf("%s -> %q == %q\n", registry, resolved.Email, configured.Email) diff --git a/docs/config_file_test.go b/docs/config_file_test.go new file mode 100644 index 00000000..9abb8ee9 --- /dev/null +++ b/docs/config_file_test.go @@ -0,0 +1,135 @@ +package registry + +import ( + "io/ioutil" + "os" + "path/filepath" + "runtime" + "strings" + "testing" + + "github.com/docker/docker/pkg/homedir" +) + +func TestMissingFile(t *testing.T) { + tmpHome, _ := ioutil.TempDir("", "config-test") + + config, err := LoadConfig(tmpHome) + if err != nil { + t.Fatalf("Failed loading on missing file: %q", err) + } + + // Now save it and make sure it shows up in new form + err = config.Save() + if err != nil { + t.Fatalf("Failed to save: %q", err) + } + + buf, err := ioutil.ReadFile(filepath.Join(tmpHome, CONFIGFILE)) + if !strings.Contains(string(buf), `"auths":`) { + t.Fatalf("Should have save in new form: %s", string(buf)) + } +} + +func TestEmptyFile(t *testing.T) { + tmpHome, _ := ioutil.TempDir("", "config-test") + fn := filepath.Join(tmpHome, CONFIGFILE) + ioutil.WriteFile(fn, []byte(""), 0600) + + _, err := LoadConfig(tmpHome) + if err == nil { + t.Fatalf("Was supposed to fail") + } +} + +func TestEmptyJson(t *testing.T) { + tmpHome, _ := ioutil.TempDir("", "config-test") + fn := filepath.Join(tmpHome, CONFIGFILE) + ioutil.WriteFile(fn, []byte("{}"), 0600) + + config, err := LoadConfig(tmpHome) + if err != nil { + t.Fatalf("Failed loading on empty json file: %q", err) + } + + // Now save it and make sure it shows up in new form + err = config.Save() + if err != nil { + t.Fatalf("Failed to save: %q", err) + } + + buf, err := ioutil.ReadFile(filepath.Join(tmpHome, CONFIGFILE)) + if !strings.Contains(string(buf), `"auths":`) { + t.Fatalf("Should have save in new form: %s", string(buf)) + } +} + +func TestOldJson(t *testing.T) { + if runtime.GOOS == "windows" { + return + } + + tmpHome, _ := ioutil.TempDir("", "config-test") + defer os.RemoveAll(tmpHome) + + homeKey := homedir.Key() + homeVal := homedir.Get() + + defer func() { os.Setenv(homeKey, homeVal) }() + os.Setenv(homeKey, tmpHome) + + fn := filepath.Join(tmpHome, OLD_CONFIGFILE) + js := `{"https://index.docker.io/v1/":{"auth":"am9lam9lOmhlbGxv","email":"user@example.com"}}` + ioutil.WriteFile(fn, []byte(js), 0600) + + config, err := LoadConfig(tmpHome) + if err != nil { + t.Fatalf("Failed loading on empty json file: %q", err) + } + + ac := config.AuthConfigs["https://index.docker.io/v1/"] + if ac.Email != "user@example.com" || ac.Username != "joejoe" || ac.Password != "hello" { + t.Fatalf("Missing data from parsing:\n%q", config) + } + + // Now save it and make sure it shows up in new form + err = config.Save() + if err != nil { + t.Fatalf("Failed to save: %q", err) + } + + buf, err := ioutil.ReadFile(filepath.Join(tmpHome, CONFIGFILE)) + if !strings.Contains(string(buf), `"auths":`) || + !strings.Contains(string(buf), "user@example.com") { + t.Fatalf("Should have save in new form: %s", string(buf)) + } +} + +func TestNewJson(t *testing.T) { + tmpHome, _ := ioutil.TempDir("", "config-test") + fn := filepath.Join(tmpHome, CONFIGFILE) + js := ` { "auths": { "https://index.docker.io/v1/": { "auth": "am9lam9lOmhlbGxv", "email": "user@example.com" } } }` + ioutil.WriteFile(fn, []byte(js), 0600) + + config, err := LoadConfig(tmpHome) + if err != nil { + t.Fatalf("Failed loading on empty json file: %q", err) + } + + ac := config.AuthConfigs["https://index.docker.io/v1/"] + if ac.Email != "user@example.com" || ac.Username != "joejoe" || ac.Password != "hello" { + t.Fatalf("Missing data from parsing:\n%q", config) + } + + // Now save it and make sure it shows up in new form + err = config.Save() + if err != nil { + t.Fatalf("Failed to save: %q", err) + } + + buf, err := ioutil.ReadFile(filepath.Join(tmpHome, CONFIGFILE)) + if !strings.Contains(string(buf), `"auths":`) || + !strings.Contains(string(buf), "user@example.com") { + t.Fatalf("Should have save in new form: %s", string(buf)) + } +} From 34d1494c7fe484a4c796bd1984da38c852fb3ad1 Mon Sep 17 00:00:00 2001 From: Doug Davis Date: Mon, 20 Apr 2015 14:09:41 -0700 Subject: [PATCH 264/375] Make .docker dir have 0700 perms not 0600 Thanks to @dmcgowan for noticing. Added a testcase to make sure Save() can create the dir and then read from it. Signed-off-by: Doug Davis --- docs/auth.go | 2 +- docs/config_file_test.go | 22 ++++++++++++++++++++++ 2 files changed, 23 insertions(+), 1 deletion(-) diff --git a/docs/auth.go b/docs/auth.go index bccf58fc..ef4985ab 100644 --- a/docs/auth.go +++ b/docs/auth.go @@ -261,7 +261,7 @@ func (configFile *ConfigFile) Save() error { return err } - if err := os.MkdirAll(filepath.Dir(configFile.filename), 0600); err != nil { + if err := os.MkdirAll(filepath.Dir(configFile.filename), 0700); err != nil { return err } diff --git a/docs/config_file_test.go b/docs/config_file_test.go index 9abb8ee9..6f8bd74f 100644 --- a/docs/config_file_test.go +++ b/docs/config_file_test.go @@ -31,6 +31,28 @@ func TestMissingFile(t *testing.T) { } } +func TestSaveFileToDirs(t *testing.T) { + tmpHome, _ := ioutil.TempDir("", "config-test") + + tmpHome += "/.docker" + + config, err := LoadConfig(tmpHome) + if err != nil { + t.Fatalf("Failed loading on missing file: %q", err) + } + + // Now save it and make sure it shows up in new form + err = config.Save() + if err != nil { + t.Fatalf("Failed to save: %q", err) + } + + buf, err := ioutil.ReadFile(filepath.Join(tmpHome, CONFIGFILE)) + if !strings.Contains(string(buf), `"auths":`) { + t.Fatalf("Should have save in new form: %s", string(buf)) + } +} + func TestEmptyFile(t *testing.T) { tmpHome, _ := ioutil.TempDir("", "config-test") fn := filepath.Join(tmpHome, CONFIGFILE) From a8b9bec1049eb9812faf032491b3b572d1df8d24 Mon Sep 17 00:00:00 2001 From: Doug Davis Date: Wed, 22 Apr 2015 05:06:58 -0700 Subject: [PATCH 265/375] Move CLI config processing out from under registry dir No logic changes should be in here, just moving things around. Signed-off-by: Doug Davis --- docs/auth.go | 210 +++------------------------------------ docs/auth_test.go | 43 ++++---- docs/config_file_test.go | 157 ----------------------------- docs/registry_test.go | 5 +- docs/service.go | 6 +- docs/session.go | 9 +- 6 files changed, 45 insertions(+), 385 deletions(-) delete mode 100644 docs/config_file_test.go diff --git a/docs/auth.go b/docs/auth.go index ef4985ab..1ac1ca98 100644 --- a/docs/auth.go +++ b/docs/auth.go @@ -1,51 +1,21 @@ package registry import ( - "encoding/base64" "encoding/json" - "errors" "fmt" "io/ioutil" "net/http" - "os" - "path/filepath" "strings" "sync" "time" "github.com/Sirupsen/logrus" - "github.com/docker/docker/pkg/homedir" + "github.com/docker/docker/cliconfig" "github.com/docker/docker/pkg/requestdecorator" ) -const ( - // Where we store the config file - CONFIGFILE = "config.json" - OLD_CONFIGFILE = ".dockercfg" -) - -var ( - ErrConfigFileMissing = errors.New("The Auth config file is missing") -) - -// Registry Auth Info -type AuthConfig struct { - Username string `json:"username,omitempty"` - Password string `json:"password,omitempty"` - Auth string `json:"auth"` - Email string `json:"email"` - ServerAddress string `json:"serveraddress,omitempty"` -} - -// ~/.docker/config.json file info -type ConfigFile struct { - AuthConfigs map[string]AuthConfig `json:"auths"` - HttpHeaders map[string]string `json:"HttpHeaders,omitempty"` - filename string // Note: not serialized - for internal use only -} - type RequestAuthorization struct { - authConfig *AuthConfig + authConfig *cliconfig.AuthConfig registryEndpoint *Endpoint resource string scope string @@ -56,7 +26,7 @@ type RequestAuthorization struct { tokenExpiration time.Time } -func NewRequestAuthorization(authConfig *AuthConfig, registryEndpoint *Endpoint, resource, scope string, actions []string) *RequestAuthorization { +func NewRequestAuthorization(authConfig *cliconfig.AuthConfig, registryEndpoint *Endpoint, resource, scope string, actions []string) *RequestAuthorization { return &RequestAuthorization{ authConfig: authConfig, registryEndpoint: registryEndpoint, @@ -121,160 +91,8 @@ func (auth *RequestAuthorization) Authorize(req *http.Request) error { return nil } -// create a base64 encoded auth string to store in config -func encodeAuth(authConfig *AuthConfig) string { - authStr := authConfig.Username + ":" + authConfig.Password - msg := []byte(authStr) - encoded := make([]byte, base64.StdEncoding.EncodedLen(len(msg))) - base64.StdEncoding.Encode(encoded, msg) - return string(encoded) -} - -// decode the auth string -func decodeAuth(authStr string) (string, string, error) { - decLen := base64.StdEncoding.DecodedLen(len(authStr)) - decoded := make([]byte, decLen) - authByte := []byte(authStr) - n, err := base64.StdEncoding.Decode(decoded, authByte) - if err != nil { - return "", "", err - } - if n > decLen { - return "", "", fmt.Errorf("Something went wrong decoding auth config") - } - arr := strings.SplitN(string(decoded), ":", 2) - if len(arr) != 2 { - return "", "", fmt.Errorf("Invalid auth configuration file") - } - password := strings.Trim(arr[1], "\x00") - return arr[0], password, nil -} - -// load up the auth config information and return values -// FIXME: use the internal golang config parser -func LoadConfig(configDir string) (*ConfigFile, error) { - if configDir == "" { - configDir = filepath.Join(homedir.Get(), ".docker") - } - - configFile := ConfigFile{ - AuthConfigs: make(map[string]AuthConfig), - filename: filepath.Join(configDir, CONFIGFILE), - } - - // Try happy path first - latest config file - if _, err := os.Stat(configFile.filename); err == nil { - file, err := os.Open(configFile.filename) - if err != nil { - return &configFile, err - } - defer file.Close() - - if err := json.NewDecoder(file).Decode(&configFile); err != nil { - return &configFile, err - } - - for addr, ac := range configFile.AuthConfigs { - ac.Username, ac.Password, err = decodeAuth(ac.Auth) - if err != nil { - return &configFile, err - } - ac.Auth = "" - ac.ServerAddress = addr - configFile.AuthConfigs[addr] = ac - } - - return &configFile, nil - } else if !os.IsNotExist(err) { - // if file is there but we can't stat it for any reason other - // than it doesn't exist then stop - return &configFile, err - } - - // Can't find latest config file so check for the old one - confFile := filepath.Join(homedir.Get(), OLD_CONFIGFILE) - - if _, err := os.Stat(confFile); err != nil { - return &configFile, nil //missing file is not an error - } - - b, err := ioutil.ReadFile(confFile) - if err != nil { - return &configFile, err - } - - if err := json.Unmarshal(b, &configFile.AuthConfigs); err != nil { - arr := strings.Split(string(b), "\n") - if len(arr) < 2 { - return &configFile, fmt.Errorf("The Auth config file is empty") - } - authConfig := AuthConfig{} - origAuth := strings.Split(arr[0], " = ") - if len(origAuth) != 2 { - return &configFile, fmt.Errorf("Invalid Auth config file") - } - authConfig.Username, authConfig.Password, err = decodeAuth(origAuth[1]) - if err != nil { - return &configFile, err - } - origEmail := strings.Split(arr[1], " = ") - if len(origEmail) != 2 { - return &configFile, fmt.Errorf("Invalid Auth config file") - } - authConfig.Email = origEmail[1] - authConfig.ServerAddress = IndexServerAddress() - // *TODO: Switch to using IndexServerName() instead? - configFile.AuthConfigs[IndexServerAddress()] = authConfig - } else { - for k, authConfig := range configFile.AuthConfigs { - authConfig.Username, authConfig.Password, err = decodeAuth(authConfig.Auth) - if err != nil { - return &configFile, err - } - authConfig.Auth = "" - authConfig.ServerAddress = k - configFile.AuthConfigs[k] = authConfig - } - } - return &configFile, nil -} - -func (configFile *ConfigFile) Save() error { - // Encode sensitive data into a new/temp struct - tmpAuthConfigs := make(map[string]AuthConfig, len(configFile.AuthConfigs)) - for k, authConfig := range configFile.AuthConfigs { - authCopy := authConfig - - authCopy.Auth = encodeAuth(&authCopy) - authCopy.Username = "" - authCopy.Password = "" - authCopy.ServerAddress = "" - tmpAuthConfigs[k] = authCopy - } - - saveAuthConfigs := configFile.AuthConfigs - configFile.AuthConfigs = tmpAuthConfigs - defer func() { configFile.AuthConfigs = saveAuthConfigs }() - - data, err := json.MarshalIndent(configFile, "", "\t") - if err != nil { - return err - } - - if err := os.MkdirAll(filepath.Dir(configFile.filename), 0700); err != nil { - return err - } - - err = ioutil.WriteFile(configFile.filename, data, 0600) - if err != nil { - return err - } - - return nil -} - // Login tries to register/login to the registry server. -func Login(authConfig *AuthConfig, registryEndpoint *Endpoint, factory *requestdecorator.RequestFactory) (string, error) { +func Login(authConfig *cliconfig.AuthConfig, registryEndpoint *Endpoint, factory *requestdecorator.RequestFactory) (string, error) { // Separates the v2 registry login logic from the v1 logic. if registryEndpoint.Version == APIVersion2 { return loginV2(authConfig, registryEndpoint, factory) @@ -283,7 +101,7 @@ func Login(authConfig *AuthConfig, registryEndpoint *Endpoint, factory *requestd } // loginV1 tries to register/login to the v1 registry server. -func loginV1(authConfig *AuthConfig, registryEndpoint *Endpoint, factory *requestdecorator.RequestFactory) (string, error) { +func loginV1(authConfig *cliconfig.AuthConfig, registryEndpoint *Endpoint, factory *requestdecorator.RequestFactory) (string, error) { var ( status string reqBody []byte @@ -396,7 +214,7 @@ func loginV1(authConfig *AuthConfig, registryEndpoint *Endpoint, factory *reques // now, users should create their account through other means like directly from a web page // served by the v2 registry service provider. Whether this will be supported in the future // is to be determined. -func loginV2(authConfig *AuthConfig, registryEndpoint *Endpoint, factory *requestdecorator.RequestFactory) (string, error) { +func loginV2(authConfig *cliconfig.AuthConfig, registryEndpoint *Endpoint, factory *requestdecorator.RequestFactory) (string, error) { logrus.Debugf("attempting v2 login to registry endpoint %s", registryEndpoint) var ( err error @@ -429,7 +247,7 @@ func loginV2(authConfig *AuthConfig, registryEndpoint *Endpoint, factory *reques return "", fmt.Errorf("no successful auth challenge for %s - errors: %s", registryEndpoint, allErrors) } -func tryV2BasicAuthLogin(authConfig *AuthConfig, params map[string]string, registryEndpoint *Endpoint, client *http.Client, factory *requestdecorator.RequestFactory) error { +func tryV2BasicAuthLogin(authConfig *cliconfig.AuthConfig, params map[string]string, registryEndpoint *Endpoint, client *http.Client, factory *requestdecorator.RequestFactory) error { req, err := factory.NewRequest("GET", registryEndpoint.Path(""), nil) if err != nil { return err @@ -450,7 +268,7 @@ func tryV2BasicAuthLogin(authConfig *AuthConfig, params map[string]string, regis return nil } -func tryV2TokenAuthLogin(authConfig *AuthConfig, params map[string]string, registryEndpoint *Endpoint, client *http.Client, factory *requestdecorator.RequestFactory) error { +func tryV2TokenAuthLogin(authConfig *cliconfig.AuthConfig, params map[string]string, registryEndpoint *Endpoint, client *http.Client, factory *requestdecorator.RequestFactory) error { token, err := getToken(authConfig.Username, authConfig.Password, params, registryEndpoint, client, factory) if err != nil { return err @@ -477,7 +295,7 @@ func tryV2TokenAuthLogin(authConfig *AuthConfig, params map[string]string, regis } // this method matches a auth configuration to a server address or a url -func (config *ConfigFile) ResolveAuthConfig(index *IndexInfo) AuthConfig { +func ResolveAuthConfig(config *cliconfig.ConfigFile, index *IndexInfo) cliconfig.AuthConfig { configKey := index.GetAuthConfigKey() // First try the happy case if c, found := config.AuthConfigs[configKey]; found || index.Official { @@ -499,16 +317,12 @@ func (config *ConfigFile) ResolveAuthConfig(index *IndexInfo) AuthConfig { // Maybe they have a legacy config file, we will iterate the keys converting // them to the new format and testing - for registry, config := range config.AuthConfigs { + for registry, ac := range config.AuthConfigs { if configKey == convertToHostname(registry) { - return config + return ac } } // When all else fails, return an empty auth config - return AuthConfig{} -} - -func (config *ConfigFile) Filename() string { - return config.filename + return cliconfig.AuthConfig{} } diff --git a/docs/auth_test.go b/docs/auth_test.go index b07aa7db..71b963a1 100644 --- a/docs/auth_test.go +++ b/docs/auth_test.go @@ -5,14 +5,16 @@ import ( "os" "path/filepath" "testing" + + "github.com/docker/docker/cliconfig" ) func TestEncodeAuth(t *testing.T) { - newAuthConfig := &AuthConfig{Username: "ken", Password: "test", Email: "test@example.com"} - authStr := encodeAuth(newAuthConfig) - decAuthConfig := &AuthConfig{} + newAuthConfig := &cliconfig.AuthConfig{Username: "ken", Password: "test", Email: "test@example.com"} + authStr := cliconfig.EncodeAuth(newAuthConfig) + decAuthConfig := &cliconfig.AuthConfig{} var err error - decAuthConfig.Username, decAuthConfig.Password, err = decodeAuth(authStr) + decAuthConfig.Username, decAuthConfig.Password, err = cliconfig.DecodeAuth(authStr) if err != nil { t.Fatal(err) } @@ -27,19 +29,16 @@ func TestEncodeAuth(t *testing.T) { } } -func setupTempConfigFile() (*ConfigFile, error) { +func setupTempConfigFile() (*cliconfig.ConfigFile, error) { root, err := ioutil.TempDir("", "docker-test-auth") if err != nil { return nil, err } - root = filepath.Join(root, CONFIGFILE) - configFile := &ConfigFile{ - AuthConfigs: make(map[string]AuthConfig), - filename: root, - } + root = filepath.Join(root, cliconfig.CONFIGFILE) + configFile := cliconfig.NewConfigFile(root) for _, registry := range []string{"testIndex", IndexServerAddress()} { - configFile.AuthConfigs[registry] = AuthConfig{ + configFile.AuthConfigs[registry] = cliconfig.AuthConfig{ Username: "docker-user", Password: "docker-pass", Email: "docker@docker.io", @@ -54,7 +53,7 @@ func TestSameAuthDataPostSave(t *testing.T) { if err != nil { t.Fatal(err) } - defer os.RemoveAll(configFile.filename) + defer os.RemoveAll(configFile.Filename()) err = configFile.Save() if err != nil { @@ -81,7 +80,7 @@ func TestResolveAuthConfigIndexServer(t *testing.T) { if err != nil { t.Fatal(err) } - defer os.RemoveAll(configFile.filename) + defer os.RemoveAll(configFile.Filename()) indexConfig := configFile.AuthConfigs[IndexServerAddress()] @@ -92,10 +91,10 @@ func TestResolveAuthConfigIndexServer(t *testing.T) { Official: false, } - resolved := configFile.ResolveAuthConfig(officialIndex) + resolved := ResolveAuthConfig(configFile, officialIndex) assertEqual(t, resolved, indexConfig, "Expected ResolveAuthConfig to return IndexServerAddress()") - resolved = configFile.ResolveAuthConfig(privateIndex) + resolved = ResolveAuthConfig(configFile, privateIndex) assertNotEqual(t, resolved, indexConfig, "Expected ResolveAuthConfig to not return IndexServerAddress()") } @@ -104,26 +103,26 @@ func TestResolveAuthConfigFullURL(t *testing.T) { if err != nil { t.Fatal(err) } - defer os.RemoveAll(configFile.filename) + defer os.RemoveAll(configFile.Filename()) - registryAuth := AuthConfig{ + registryAuth := cliconfig.AuthConfig{ Username: "foo-user", Password: "foo-pass", Email: "foo@example.com", } - localAuth := AuthConfig{ + localAuth := cliconfig.AuthConfig{ Username: "bar-user", Password: "bar-pass", Email: "bar@example.com", } - officialAuth := AuthConfig{ + officialAuth := cliconfig.AuthConfig{ Username: "baz-user", Password: "baz-pass", Email: "baz@example.com", } configFile.AuthConfigs[IndexServerAddress()] = officialAuth - expectedAuths := map[string]AuthConfig{ + expectedAuths := map[string]cliconfig.AuthConfig{ "registry.example.com": registryAuth, "localhost:8000": localAuth, "registry.com": localAuth, @@ -160,12 +159,12 @@ func TestResolveAuthConfigFullURL(t *testing.T) { } for _, registry := range registries { configFile.AuthConfigs[registry] = configured - resolved := configFile.ResolveAuthConfig(index) + resolved := ResolveAuthConfig(configFile, index) if resolved.Email != configured.Email { t.Errorf("%s -> %q != %q\n", registry, resolved.Email, configured.Email) } delete(configFile.AuthConfigs, registry) - resolved = configFile.ResolveAuthConfig(index) + resolved = ResolveAuthConfig(configFile, index) if resolved.Email == configured.Email { t.Errorf("%s -> %q == %q\n", registry, resolved.Email, configured.Email) } diff --git a/docs/config_file_test.go b/docs/config_file_test.go deleted file mode 100644 index 6f8bd74f..00000000 --- a/docs/config_file_test.go +++ /dev/null @@ -1,157 +0,0 @@ -package registry - -import ( - "io/ioutil" - "os" - "path/filepath" - "runtime" - "strings" - "testing" - - "github.com/docker/docker/pkg/homedir" -) - -func TestMissingFile(t *testing.T) { - tmpHome, _ := ioutil.TempDir("", "config-test") - - config, err := LoadConfig(tmpHome) - if err != nil { - t.Fatalf("Failed loading on missing file: %q", err) - } - - // Now save it and make sure it shows up in new form - err = config.Save() - if err != nil { - t.Fatalf("Failed to save: %q", err) - } - - buf, err := ioutil.ReadFile(filepath.Join(tmpHome, CONFIGFILE)) - if !strings.Contains(string(buf), `"auths":`) { - t.Fatalf("Should have save in new form: %s", string(buf)) - } -} - -func TestSaveFileToDirs(t *testing.T) { - tmpHome, _ := ioutil.TempDir("", "config-test") - - tmpHome += "/.docker" - - config, err := LoadConfig(tmpHome) - if err != nil { - t.Fatalf("Failed loading on missing file: %q", err) - } - - // Now save it and make sure it shows up in new form - err = config.Save() - if err != nil { - t.Fatalf("Failed to save: %q", err) - } - - buf, err := ioutil.ReadFile(filepath.Join(tmpHome, CONFIGFILE)) - if !strings.Contains(string(buf), `"auths":`) { - t.Fatalf("Should have save in new form: %s", string(buf)) - } -} - -func TestEmptyFile(t *testing.T) { - tmpHome, _ := ioutil.TempDir("", "config-test") - fn := filepath.Join(tmpHome, CONFIGFILE) - ioutil.WriteFile(fn, []byte(""), 0600) - - _, err := LoadConfig(tmpHome) - if err == nil { - t.Fatalf("Was supposed to fail") - } -} - -func TestEmptyJson(t *testing.T) { - tmpHome, _ := ioutil.TempDir("", "config-test") - fn := filepath.Join(tmpHome, CONFIGFILE) - ioutil.WriteFile(fn, []byte("{}"), 0600) - - config, err := LoadConfig(tmpHome) - if err != nil { - t.Fatalf("Failed loading on empty json file: %q", err) - } - - // Now save it and make sure it shows up in new form - err = config.Save() - if err != nil { - t.Fatalf("Failed to save: %q", err) - } - - buf, err := ioutil.ReadFile(filepath.Join(tmpHome, CONFIGFILE)) - if !strings.Contains(string(buf), `"auths":`) { - t.Fatalf("Should have save in new form: %s", string(buf)) - } -} - -func TestOldJson(t *testing.T) { - if runtime.GOOS == "windows" { - return - } - - tmpHome, _ := ioutil.TempDir("", "config-test") - defer os.RemoveAll(tmpHome) - - homeKey := homedir.Key() - homeVal := homedir.Get() - - defer func() { os.Setenv(homeKey, homeVal) }() - os.Setenv(homeKey, tmpHome) - - fn := filepath.Join(tmpHome, OLD_CONFIGFILE) - js := `{"https://index.docker.io/v1/":{"auth":"am9lam9lOmhlbGxv","email":"user@example.com"}}` - ioutil.WriteFile(fn, []byte(js), 0600) - - config, err := LoadConfig(tmpHome) - if err != nil { - t.Fatalf("Failed loading on empty json file: %q", err) - } - - ac := config.AuthConfigs["https://index.docker.io/v1/"] - if ac.Email != "user@example.com" || ac.Username != "joejoe" || ac.Password != "hello" { - t.Fatalf("Missing data from parsing:\n%q", config) - } - - // Now save it and make sure it shows up in new form - err = config.Save() - if err != nil { - t.Fatalf("Failed to save: %q", err) - } - - buf, err := ioutil.ReadFile(filepath.Join(tmpHome, CONFIGFILE)) - if !strings.Contains(string(buf), `"auths":`) || - !strings.Contains(string(buf), "user@example.com") { - t.Fatalf("Should have save in new form: %s", string(buf)) - } -} - -func TestNewJson(t *testing.T) { - tmpHome, _ := ioutil.TempDir("", "config-test") - fn := filepath.Join(tmpHome, CONFIGFILE) - js := ` { "auths": { "https://index.docker.io/v1/": { "auth": "am9lam9lOmhlbGxv", "email": "user@example.com" } } }` - ioutil.WriteFile(fn, []byte(js), 0600) - - config, err := LoadConfig(tmpHome) - if err != nil { - t.Fatalf("Failed loading on empty json file: %q", err) - } - - ac := config.AuthConfigs["https://index.docker.io/v1/"] - if ac.Email != "user@example.com" || ac.Username != "joejoe" || ac.Password != "hello" { - t.Fatalf("Missing data from parsing:\n%q", config) - } - - // Now save it and make sure it shows up in new form - err = config.Save() - if err != nil { - t.Fatalf("Failed to save: %q", err) - } - - buf, err := ioutil.ReadFile(filepath.Join(tmpHome, CONFIGFILE)) - if !strings.Contains(string(buf), `"auths":`) || - !strings.Contains(string(buf), "user@example.com") { - t.Fatalf("Should have save in new form: %s", string(buf)) - } -} diff --git a/docs/registry_test.go b/docs/registry_test.go index a066de9f..b4bd4ee7 100644 --- a/docs/registry_test.go +++ b/docs/registry_test.go @@ -7,6 +7,7 @@ import ( "strings" "testing" + "github.com/docker/docker/cliconfig" "github.com/docker/docker/pkg/requestdecorator" ) @@ -20,7 +21,7 @@ const ( ) func spawnTestRegistrySession(t *testing.T) *Session { - authConfig := &AuthConfig{} + authConfig := &cliconfig.AuthConfig{} endpoint, err := NewEndpoint(makeIndex("/v1/")) if err != nil { t.Fatal(err) @@ -33,7 +34,7 @@ func spawnTestRegistrySession(t *testing.T) *Session { } func TestPublicSession(t *testing.T) { - authConfig := &AuthConfig{} + authConfig := &cliconfig.AuthConfig{} getSessionDecorators := func(index *IndexInfo) int { endpoint, err := NewEndpoint(index) diff --git a/docs/service.go b/docs/service.go index cf29732f..87fc1d07 100644 --- a/docs/service.go +++ b/docs/service.go @@ -1,5 +1,7 @@ package registry +import "github.com/docker/docker/cliconfig" + type Service struct { Config *ServiceConfig } @@ -15,7 +17,7 @@ func NewService(options *Options) *Service { // Auth contacts the public registry with the provided credentials, // and returns OK if authentication was sucessful. // It can be used to verify the validity of a client's credentials. -func (s *Service) Auth(authConfig *AuthConfig) (string, error) { +func (s *Service) Auth(authConfig *cliconfig.AuthConfig) (string, error) { addr := authConfig.ServerAddress if addr == "" { // Use the official registry address if not specified. @@ -35,7 +37,7 @@ func (s *Service) Auth(authConfig *AuthConfig) (string, error) { // Search queries the public registry for images matching the specified // search terms, and returns the results. -func (s *Service) Search(term string, authConfig *AuthConfig, headers map[string][]string) (*SearchResults, error) { +func (s *Service) Search(term string, authConfig *cliconfig.AuthConfig, headers map[string][]string) (*SearchResults, error) { repoInfo, err := s.ResolveRepository(term) if err != nil { return nil, err diff --git a/docs/session.go b/docs/session.go index 940e407e..dd868a2b 100644 --- a/docs/session.go +++ b/docs/session.go @@ -18,20 +18,21 @@ import ( "time" "github.com/Sirupsen/logrus" + "github.com/docker/docker/cliconfig" "github.com/docker/docker/pkg/httputils" "github.com/docker/docker/pkg/requestdecorator" "github.com/docker/docker/pkg/tarsum" ) type Session struct { - authConfig *AuthConfig + authConfig *cliconfig.AuthConfig reqFactory *requestdecorator.RequestFactory indexEndpoint *Endpoint jar *cookiejar.Jar timeout TimeoutType } -func NewSession(authConfig *AuthConfig, factory *requestdecorator.RequestFactory, endpoint *Endpoint, timeout bool) (r *Session, err error) { +func NewSession(authConfig *cliconfig.AuthConfig, factory *requestdecorator.RequestFactory, endpoint *Endpoint, timeout bool) (r *Session, err error) { r = &Session{ authConfig: authConfig, indexEndpoint: endpoint, @@ -600,12 +601,12 @@ func (r *Session) SearchRepositories(term string) (*SearchResults, error) { return result, err } -func (r *Session) GetAuthConfig(withPasswd bool) *AuthConfig { +func (r *Session) GetAuthConfig(withPasswd bool) *cliconfig.AuthConfig { password := "" if withPasswd { password = r.authConfig.Password } - return &AuthConfig{ + return &cliconfig.AuthConfig{ Username: r.authConfig.Username, Password: password, Email: r.authConfig.Email, From 9a26753d4187562131380e4f714fbd381ae154ad Mon Sep 17 00:00:00 2001 From: Antonio Murdaca Date: Sun, 26 Apr 2015 18:50:25 +0200 Subject: [PATCH 266/375] Small if err cleaning Signed-off-by: Antonio Murdaca --- docs/session.go | 3 +-- docs/session_v2.go | 4 +--- 2 files changed, 2 insertions(+), 5 deletions(-) diff --git a/docs/session.go b/docs/session.go index f7358bc1..e65f82cd 100644 --- a/docs/session.go +++ b/docs/session.go @@ -597,8 +597,7 @@ func (r *Session) SearchRepositories(term string) (*SearchResults, error) { return nil, httputils.NewHTTPRequestError(fmt.Sprintf("Unexpected status code %d", res.StatusCode), res) } result := new(SearchResults) - err = json.NewDecoder(res.Body).Decode(result) - return result, err + return result, json.NewDecoder(res.Body).Decode(result) } func (r *Session) GetAuthConfig(withPasswd bool) *cliconfig.AuthConfig { diff --git a/docs/session_v2.go b/docs/session_v2.go index a14e434a..4188e505 100644 --- a/docs/session_v2.go +++ b/docs/session_v2.go @@ -387,10 +387,8 @@ func (r *Session) GetV2RemoteTags(ep *Endpoint, imageName string, auth *RequestA return nil, httputils.NewHTTPRequestError(fmt.Sprintf("Server error: %d trying to fetch for %s", res.StatusCode, imageName), res) } - decoder := json.NewDecoder(res.Body) var remote remoteTags - err = decoder.Decode(&remote) - if err != nil { + if err := json.NewDecoder(res.Body).Decode(&remote); err != nil { return nil, fmt.Errorf("Error while decoding the http response: %s", err) } return remote.Tags, nil From bb93129df4c9d0ff61859d752a4a11e5d5b400b5 Mon Sep 17 00:00:00 2001 From: David Mackey Date: Mon, 27 Apr 2015 13:33:30 -0700 Subject: [PATCH 267/375] trivial: typo cleanup Signed-off-by: David Mackey --- docs/registry_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/registry_test.go b/docs/registry_test.go index b4bd4ee7..3f63eb6e 100644 --- a/docs/registry_test.go +++ b/docs/registry_test.go @@ -736,7 +736,7 @@ func TestSearchRepositories(t *testing.T) { } assertEqual(t, results.NumResults, 1, "Expected 1 search results") assertEqual(t, results.Query, "fakequery", "Expected 'fakequery' as query") - assertEqual(t, results.Results[0].StarCount, 42, "Expected 'fakeimage' a ot hae 42 stars") + assertEqual(t, results.Results[0].StarCount, 42, "Expected 'fakeimage' to have 42 stars") } func TestValidRemoteName(t *testing.T) { From 351babbf07f56082d65bca5a18ad7b39a881c8f1 Mon Sep 17 00:00:00 2001 From: Lei Jitang Date: Wed, 13 May 2015 14:23:13 +0800 Subject: [PATCH 268/375] Fix invalid tag name Signed-off-by: Lei Jitang --- docs/config.go | 6 ++++++ docs/registry_test.go | 3 +++ 2 files changed, 9 insertions(+) diff --git a/docs/config.go b/docs/config.go index a0a978cc..568756f4 100644 --- a/docs/config.go +++ b/docs/config.go @@ -198,6 +198,9 @@ func ValidateIndexName(val string) (string, error) { if val == "index."+IndexServerName() { val = IndexServerName() } + if strings.HasPrefix(val, "-") || strings.HasSuffix(val, "-") { + return "", fmt.Errorf("Invalid index name (%s). Cannot begin or end with a hyphen.", val) + } // *TODO: Check if valid hostname[:port]/ip[:port]? return val, nil } @@ -235,6 +238,9 @@ func validateRemoteName(remoteName string) error { if !validRepo.MatchString(name) { return fmt.Errorf("Invalid repository name (%s), only [a-z0-9-_.] are allowed", name) } + if strings.HasPrefix(name, "-") || strings.HasSuffix(name, "-") { + return fmt.Errorf("Invalid repository name (%s). Cannot begin or end with a hyphen.", name) + } return nil } diff --git a/docs/registry_test.go b/docs/registry_test.go index 3f63eb6e..799d080e 100644 --- a/docs/registry_test.go +++ b/docs/registry_test.go @@ -299,6 +299,9 @@ func TestValidateRepositoryName(t *testing.T) { invalidRepoNames := []string{ "https://github.com/docker/docker", "docker/Docker", + "-docker", + "-docker/docker", + "-docker.io/docker/docker", "docker///docker", "docker.io/docker/Docker", "docker.io/docker///docker", From 3c34b3c87e3794746918294077683f0d8b1e4533 Mon Sep 17 00:00:00 2001 From: James Lal Date: Mon, 18 May 2015 13:11:36 -0700 Subject: [PATCH 269/375] Increase default connection timeout to 30s Closes #13307 Signed-off-by: James Lal --- docs/registry.go | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/docs/registry.go b/docs/registry.go index 163e2de3..aff28eaa 100644 --- a/docs/registry.go +++ b/docs/registry.go @@ -52,8 +52,9 @@ func newClient(jar http.CookieJar, roots *x509.CertPool, certs []tls.Certificate switch timeout { case ConnectTimeout: httpTransport.Dial = func(proto string, addr string) (net.Conn, error) { - // Set the connect timeout to 5 seconds - d := net.Dialer{Timeout: 5 * time.Second, DualStack: true} + // Set the connect timeout to 30 seconds to allow for slower connection + // times... + d := net.Dialer{Timeout: 30 * time.Second, DualStack: true} conn, err := d.Dial(proto, addr) if err != nil { From 89bd48481ce25ec050cd678d04e3fc3180f05b59 Mon Sep 17 00:00:00 2001 From: Tibor Vass Date: Thu, 14 May 2015 07:12:54 -0700 Subject: [PATCH 270/375] registry: Refactor requestfactory to use http.RoundTrippers This patch removes the need for requestFactories and decorators by implementing http.RoundTripper transports instead. It refactors some challenging-to-read code. NewSession now takes an *http.Client that can already have a custom Transport, it will add its own auth transport by wrapping it. The idea is that callers of http.Client should not bother setting custom headers for every handler but instead it should be transparent to the callers of a same context. This patch is needed for future refactorings of registry, namely refactoring of the v1 client code. Signed-off-by: Tibor Vass --- docs/auth.go | 32 +++-- docs/endpoint.go | 41 +++--- docs/httpfactory.go | 30 ----- docs/registry.go | 226 ++++++++++++++++++++++----------- docs/registry_test.go | 66 ++++------ docs/service.go | 5 +- docs/session.go | 288 +++++++++++++++++++++--------------------- docs/session_v2.go | 32 ++--- docs/token.go | 6 +- 9 files changed, 373 insertions(+), 353 deletions(-) delete mode 100644 docs/httpfactory.go diff --git a/docs/auth.go b/docs/auth.go index 1ac1ca98..0b6c3b0f 100644 --- a/docs/auth.go +++ b/docs/auth.go @@ -11,7 +11,6 @@ import ( "github.com/Sirupsen/logrus" "github.com/docker/docker/cliconfig" - "github.com/docker/docker/pkg/requestdecorator" ) type RequestAuthorization struct { @@ -46,7 +45,6 @@ func (auth *RequestAuthorization) getToken() (string, error) { } client := auth.registryEndpoint.HTTPClient() - factory := HTTPRequestFactory(nil) for _, challenge := range auth.registryEndpoint.AuthChallenges { switch strings.ToLower(challenge.Scheme) { @@ -59,7 +57,7 @@ func (auth *RequestAuthorization) getToken() (string, error) { params[k] = v } params["scope"] = fmt.Sprintf("%s:%s:%s", auth.resource, auth.scope, strings.Join(auth.actions, ",")) - token, err := getToken(auth.authConfig.Username, auth.authConfig.Password, params, auth.registryEndpoint, client, factory) + token, err := getToken(auth.authConfig.Username, auth.authConfig.Password, params, auth.registryEndpoint, client) if err != nil { return "", err } @@ -92,16 +90,16 @@ func (auth *RequestAuthorization) Authorize(req *http.Request) error { } // Login tries to register/login to the registry server. -func Login(authConfig *cliconfig.AuthConfig, registryEndpoint *Endpoint, factory *requestdecorator.RequestFactory) (string, error) { +func Login(authConfig *cliconfig.AuthConfig, registryEndpoint *Endpoint) (string, error) { // Separates the v2 registry login logic from the v1 logic. if registryEndpoint.Version == APIVersion2 { - return loginV2(authConfig, registryEndpoint, factory) + return loginV2(authConfig, registryEndpoint) } - return loginV1(authConfig, registryEndpoint, factory) + return loginV1(authConfig, registryEndpoint) } // loginV1 tries to register/login to the v1 registry server. -func loginV1(authConfig *cliconfig.AuthConfig, registryEndpoint *Endpoint, factory *requestdecorator.RequestFactory) (string, error) { +func loginV1(authConfig *cliconfig.AuthConfig, registryEndpoint *Endpoint) (string, error) { var ( status string reqBody []byte @@ -151,7 +149,7 @@ func loginV1(authConfig *cliconfig.AuthConfig, registryEndpoint *Endpoint, facto } } else if reqStatusCode == 400 { if string(reqBody) == "\"Username or email already exists\"" { - req, err := factory.NewRequest("GET", serverAddress+"users/", nil) + req, err := http.NewRequest("GET", serverAddress+"users/", nil) req.SetBasicAuth(authConfig.Username, authConfig.Password) resp, err := client.Do(req) if err != nil { @@ -180,7 +178,7 @@ func loginV1(authConfig *cliconfig.AuthConfig, registryEndpoint *Endpoint, facto } else if reqStatusCode == 401 { // This case would happen with private registries where /v1/users is // protected, so people can use `docker login` as an auth check. - req, err := factory.NewRequest("GET", serverAddress+"users/", nil) + req, err := http.NewRequest("GET", serverAddress+"users/", nil) req.SetBasicAuth(authConfig.Username, authConfig.Password) resp, err := client.Do(req) if err != nil { @@ -214,7 +212,7 @@ func loginV1(authConfig *cliconfig.AuthConfig, registryEndpoint *Endpoint, facto // now, users should create their account through other means like directly from a web page // served by the v2 registry service provider. Whether this will be supported in the future // is to be determined. -func loginV2(authConfig *cliconfig.AuthConfig, registryEndpoint *Endpoint, factory *requestdecorator.RequestFactory) (string, error) { +func loginV2(authConfig *cliconfig.AuthConfig, registryEndpoint *Endpoint) (string, error) { logrus.Debugf("attempting v2 login to registry endpoint %s", registryEndpoint) var ( err error @@ -227,9 +225,9 @@ func loginV2(authConfig *cliconfig.AuthConfig, registryEndpoint *Endpoint, facto switch strings.ToLower(challenge.Scheme) { case "basic": - err = tryV2BasicAuthLogin(authConfig, challenge.Parameters, registryEndpoint, client, factory) + err = tryV2BasicAuthLogin(authConfig, challenge.Parameters, registryEndpoint, client) case "bearer": - err = tryV2TokenAuthLogin(authConfig, challenge.Parameters, registryEndpoint, client, factory) + err = tryV2TokenAuthLogin(authConfig, challenge.Parameters, registryEndpoint, client) default: // Unsupported challenge types are explicitly skipped. err = fmt.Errorf("unsupported auth scheme: %q", challenge.Scheme) @@ -247,8 +245,8 @@ func loginV2(authConfig *cliconfig.AuthConfig, registryEndpoint *Endpoint, facto return "", fmt.Errorf("no successful auth challenge for %s - errors: %s", registryEndpoint, allErrors) } -func tryV2BasicAuthLogin(authConfig *cliconfig.AuthConfig, params map[string]string, registryEndpoint *Endpoint, client *http.Client, factory *requestdecorator.RequestFactory) error { - req, err := factory.NewRequest("GET", registryEndpoint.Path(""), nil) +func tryV2BasicAuthLogin(authConfig *cliconfig.AuthConfig, params map[string]string, registryEndpoint *Endpoint, client *http.Client) error { + req, err := http.NewRequest("GET", registryEndpoint.Path(""), nil) if err != nil { return err } @@ -268,13 +266,13 @@ func tryV2BasicAuthLogin(authConfig *cliconfig.AuthConfig, params map[string]str return nil } -func tryV2TokenAuthLogin(authConfig *cliconfig.AuthConfig, params map[string]string, registryEndpoint *Endpoint, client *http.Client, factory *requestdecorator.RequestFactory) error { - token, err := getToken(authConfig.Username, authConfig.Password, params, registryEndpoint, client, factory) +func tryV2TokenAuthLogin(authConfig *cliconfig.AuthConfig, params map[string]string, registryEndpoint *Endpoint, client *http.Client) error { + token, err := getToken(authConfig.Username, authConfig.Password, params, registryEndpoint, client) if err != nil { return err } - req, err := factory.NewRequest("GET", registryEndpoint.Path(""), nil) + req, err := http.NewRequest("GET", registryEndpoint.Path(""), nil) if err != nil { return err } diff --git a/docs/endpoint.go b/docs/endpoint.go index 84b11a98..25f66ad2 100644 --- a/docs/endpoint.go +++ b/docs/endpoint.go @@ -1,7 +1,6 @@ package registry import ( - "crypto/tls" "encoding/json" "fmt" "io/ioutil" @@ -12,7 +11,6 @@ import ( "github.com/Sirupsen/logrus" "github.com/docker/distribution/registry/api/v2" - "github.com/docker/docker/pkg/requestdecorator" ) // for mocking in unit tests @@ -109,6 +107,7 @@ func (repoInfo *RepositoryInfo) GetEndpoint() (*Endpoint, error) { // Endpoint stores basic information about a registry endpoint. type Endpoint struct { + client *http.Client URL *url.URL Version APIVersion IsSecure bool @@ -135,25 +134,24 @@ func (e *Endpoint) Path(path string) string { func (e *Endpoint) Ping() (RegistryInfo, error) { // The ping logic to use is determined by the registry endpoint version. - factory := HTTPRequestFactory(nil) switch e.Version { case APIVersion1: - return e.pingV1(factory) + return e.pingV1() case APIVersion2: - return e.pingV2(factory) + return e.pingV2() } // APIVersionUnknown // We should try v2 first... e.Version = APIVersion2 - regInfo, errV2 := e.pingV2(factory) + regInfo, errV2 := e.pingV2() if errV2 == nil { return regInfo, nil } // ... then fallback to v1. e.Version = APIVersion1 - regInfo, errV1 := e.pingV1(factory) + regInfo, errV1 := e.pingV1() if errV1 == nil { return regInfo, nil } @@ -162,7 +160,7 @@ func (e *Endpoint) Ping() (RegistryInfo, error) { return RegistryInfo{}, fmt.Errorf("unable to ping registry endpoint %s\nv2 ping attempt failed with error: %s\n v1 ping attempt failed with error: %s", e, errV2, errV1) } -func (e *Endpoint) pingV1(factory *requestdecorator.RequestFactory) (RegistryInfo, error) { +func (e *Endpoint) pingV1() (RegistryInfo, error) { logrus.Debugf("attempting v1 ping for registry endpoint %s", e) if e.String() == IndexServerAddress() { @@ -171,12 +169,12 @@ func (e *Endpoint) pingV1(factory *requestdecorator.RequestFactory) (RegistryInf return RegistryInfo{Standalone: false}, nil } - req, err := factory.NewRequest("GET", e.Path("_ping"), nil) + req, err := http.NewRequest("GET", e.Path("_ping"), nil) if err != nil { return RegistryInfo{Standalone: false}, err } - resp, _, err := doRequest(req, nil, ConnectTimeout, e.IsSecure) + resp, err := e.HTTPClient().Do(req) if err != nil { return RegistryInfo{Standalone: false}, err } @@ -216,15 +214,15 @@ func (e *Endpoint) pingV1(factory *requestdecorator.RequestFactory) (RegistryInf return info, nil } -func (e *Endpoint) pingV2(factory *requestdecorator.RequestFactory) (RegistryInfo, error) { +func (e *Endpoint) pingV2() (RegistryInfo, error) { logrus.Debugf("attempting v2 ping for registry endpoint %s", e) - req, err := factory.NewRequest("GET", e.Path(""), nil) + req, err := http.NewRequest("GET", e.Path(""), nil) if err != nil { return RegistryInfo{}, err } - resp, _, err := doRequest(req, nil, ConnectTimeout, e.IsSecure) + resp, err := e.HTTPClient().Do(req) if err != nil { return RegistryInfo{}, err } @@ -265,18 +263,9 @@ HeaderLoop: } func (e *Endpoint) HTTPClient() *http.Client { - tlsConfig := tls.Config{ - MinVersion: tls.VersionTLS10, - } - if !e.IsSecure { - tlsConfig.InsecureSkipVerify = true - } - return &http.Client{ - Transport: &http.Transport{ - DisableKeepAlives: true, - Proxy: http.ProxyFromEnvironment, - TLSClientConfig: &tlsConfig, - }, - CheckRedirect: AddRequiredHeadersToRedirectedRequests, + if e.client == nil { + tr := NewTransport(ConnectTimeout, e.IsSecure) + e.client = HTTPClient(tr) } + return e.client } diff --git a/docs/httpfactory.go b/docs/httpfactory.go deleted file mode 100644 index f1b89e58..00000000 --- a/docs/httpfactory.go +++ /dev/null @@ -1,30 +0,0 @@ -package registry - -import ( - "runtime" - - "github.com/docker/docker/autogen/dockerversion" - "github.com/docker/docker/pkg/parsers/kernel" - "github.com/docker/docker/pkg/requestdecorator" -) - -func HTTPRequestFactory(metaHeaders map[string][]string) *requestdecorator.RequestFactory { - // FIXME: this replicates the 'info' job. - httpVersion := make([]requestdecorator.UAVersionInfo, 0, 4) - httpVersion = append(httpVersion, requestdecorator.NewUAVersionInfo("docker", dockerversion.VERSION)) - httpVersion = append(httpVersion, requestdecorator.NewUAVersionInfo("go", runtime.Version())) - httpVersion = append(httpVersion, requestdecorator.NewUAVersionInfo("git-commit", dockerversion.GITCOMMIT)) - if kernelVersion, err := kernel.GetKernelVersion(); err == nil { - httpVersion = append(httpVersion, requestdecorator.NewUAVersionInfo("kernel", kernelVersion.String())) - } - httpVersion = append(httpVersion, requestdecorator.NewUAVersionInfo("os", runtime.GOOS)) - httpVersion = append(httpVersion, requestdecorator.NewUAVersionInfo("arch", runtime.GOARCH)) - uad := &requestdecorator.UserAgentDecorator{ - Versions: httpVersion, - } - mhd := &requestdecorator.MetaHeadersDecorator{ - Headers: metaHeaders, - } - factory := requestdecorator.NewRequestFactory(uad, mhd) - return factory -} diff --git a/docs/registry.go b/docs/registry.go index aff28eaa..db77a985 100644 --- a/docs/registry.go +++ b/docs/registry.go @@ -8,12 +8,17 @@ import ( "io/ioutil" "net" "net/http" + "net/http/httputil" "os" "path" + "runtime" "strings" "time" "github.com/Sirupsen/logrus" + "github.com/docker/docker/autogen/dockerversion" + "github.com/docker/docker/pkg/parsers/kernel" + "github.com/docker/docker/pkg/requestdecorator" "github.com/docker/docker/pkg/timeoutconn" ) @@ -31,66 +36,23 @@ const ( ConnectTimeout ) -func newClient(jar http.CookieJar, roots *x509.CertPool, certs []tls.Certificate, timeout TimeoutType, secure bool) *http.Client { - tlsConfig := tls.Config{ - RootCAs: roots, - // Avoid fallback to SSL protocols < TLS1.0 - MinVersion: tls.VersionTLS10, - Certificates: certs, - } - - if !secure { - tlsConfig.InsecureSkipVerify = true - } - - httpTransport := &http.Transport{ - DisableKeepAlives: true, - Proxy: http.ProxyFromEnvironment, - TLSClientConfig: &tlsConfig, - } - - switch timeout { - case ConnectTimeout: - httpTransport.Dial = func(proto string, addr string) (net.Conn, error) { - // Set the connect timeout to 30 seconds to allow for slower connection - // times... - d := net.Dialer{Timeout: 30 * time.Second, DualStack: true} - - conn, err := d.Dial(proto, addr) - if err != nil { - return nil, err - } - // Set the recv timeout to 10 seconds - conn.SetDeadline(time.Now().Add(10 * time.Second)) - return conn, nil - } - case ReceiveTimeout: - httpTransport.Dial = func(proto string, addr string) (net.Conn, error) { - d := net.Dialer{DualStack: true} - - conn, err := d.Dial(proto, addr) - if err != nil { - return nil, err - } - conn = timeoutconn.New(conn, 1*time.Minute) - return conn, nil - } - } - - return &http.Client{ - Transport: httpTransport, - CheckRedirect: AddRequiredHeadersToRedirectedRequests, - Jar: jar, - } +type httpsTransport struct { + *http.Transport } -func doRequest(req *http.Request, jar http.CookieJar, timeout TimeoutType, secure bool) (*http.Response, *http.Client, error) { +// DRAGONS(tiborvass): If someone wonders why do we set tlsconfig in a roundtrip, +// it's because it's so as to match the current behavior in master: we generate the +// certpool on every-goddam-request. It's not great, but it allows people to just put +// the certs in /etc/docker/certs.d/.../ and let docker "pick it up" immediately. Would +// prefer an fsnotify implementation, but that was out of scope of my refactoring. +// TODO: improve things +func (tr *httpsTransport) RoundTrip(req *http.Request) (*http.Response, error) { var ( - pool *x509.CertPool + roots *x509.CertPool certs []tls.Certificate ) - if secure && req.URL.Scheme == "https" { + if req.URL.Scheme == "https" { hasFile := func(files []os.FileInfo, name string) bool { for _, f := range files { if f.Name() == name { @@ -104,31 +66,31 @@ func doRequest(req *http.Request, jar http.CookieJar, timeout TimeoutType, secur logrus.Debugf("hostDir: %s", hostDir) fs, err := ioutil.ReadDir(hostDir) if err != nil && !os.IsNotExist(err) { - return nil, nil, err + return nil, err } for _, f := range fs { if strings.HasSuffix(f.Name(), ".crt") { - if pool == nil { - pool = x509.NewCertPool() + if roots == nil { + roots = x509.NewCertPool() } logrus.Debugf("crt: %s", hostDir+"/"+f.Name()) data, err := ioutil.ReadFile(path.Join(hostDir, f.Name())) if err != nil { - return nil, nil, err + return nil, err } - pool.AppendCertsFromPEM(data) + roots.AppendCertsFromPEM(data) } if strings.HasSuffix(f.Name(), ".cert") { certName := f.Name() keyName := certName[:len(certName)-5] + ".key" logrus.Debugf("cert: %s", hostDir+"/"+f.Name()) if !hasFile(fs, keyName) { - return nil, nil, fmt.Errorf("Missing key %s for certificate %s", keyName, certName) + return nil, fmt.Errorf("Missing key %s for certificate %s", keyName, certName) } cert, err := tls.LoadX509KeyPair(path.Join(hostDir, certName), path.Join(hostDir, keyName)) if err != nil { - return nil, nil, err + return nil, err } certs = append(certs, cert) } @@ -137,24 +99,142 @@ func doRequest(req *http.Request, jar http.CookieJar, timeout TimeoutType, secur certName := keyName[:len(keyName)-4] + ".cert" logrus.Debugf("key: %s", hostDir+"/"+f.Name()) if !hasFile(fs, certName) { - return nil, nil, fmt.Errorf("Missing certificate %s for key %s", certName, keyName) + return nil, fmt.Errorf("Missing certificate %s for key %s", certName, keyName) } } } - } - - if len(certs) == 0 { - client := newClient(jar, pool, nil, timeout, secure) - res, err := client.Do(req) - if err != nil { - return nil, nil, err + if tr.Transport.TLSClientConfig == nil { + tr.Transport.TLSClientConfig = &tls.Config{ + // Avoid fallback to SSL protocols < TLS1.0 + MinVersion: tls.VersionTLS10, + } } - return res, client, nil + tr.Transport.TLSClientConfig.RootCAs = roots + tr.Transport.TLSClientConfig.Certificates = certs + } + return tr.Transport.RoundTrip(req) +} + +func NewTransport(timeout TimeoutType, secure bool) http.RoundTripper { + tlsConfig := tls.Config{ + // Avoid fallback to SSL protocols < TLS1.0 + MinVersion: tls.VersionTLS10, + InsecureSkipVerify: !secure, } - client := newClient(jar, pool, certs, timeout, secure) - res, err := client.Do(req) - return res, client, err + transport := &http.Transport{ + DisableKeepAlives: true, + Proxy: http.ProxyFromEnvironment, + TLSClientConfig: &tlsConfig, + } + + switch timeout { + case ConnectTimeout: + transport.Dial = func(proto string, addr string) (net.Conn, error) { + // Set the connect timeout to 30 seconds to allow for slower connection + // times... + d := net.Dialer{Timeout: 30 * time.Second, DualStack: true} + + conn, err := d.Dial(proto, addr) + if err != nil { + return nil, err + } + // Set the recv timeout to 10 seconds + conn.SetDeadline(time.Now().Add(10 * time.Second)) + return conn, nil + } + case ReceiveTimeout: + transport.Dial = func(proto string, addr string) (net.Conn, error) { + d := net.Dialer{DualStack: true} + + conn, err := d.Dial(proto, addr) + if err != nil { + return nil, err + } + conn = timeoutconn.New(conn, 1*time.Minute) + return conn, nil + } + } + + if secure { + // note: httpsTransport also handles http transport + // but for HTTPS, it sets up the certs + return &httpsTransport{transport} + } + + return transport +} + +type DockerHeaders struct { + http.RoundTripper + Headers http.Header +} + +// cloneRequest returns a clone of the provided *http.Request. +// The clone is a shallow copy of the struct and its Header map +func cloneRequest(r *http.Request) *http.Request { + // shallow copy of the struct + r2 := new(http.Request) + *r2 = *r + // deep copy of the Header + r2.Header = make(http.Header, len(r.Header)) + for k, s := range r.Header { + r2.Header[k] = append([]string(nil), s...) + } + return r2 +} + +func (tr *DockerHeaders) RoundTrip(req *http.Request) (*http.Response, error) { + req = cloneRequest(req) + httpVersion := make([]requestdecorator.UAVersionInfo, 0, 4) + httpVersion = append(httpVersion, requestdecorator.NewUAVersionInfo("docker", dockerversion.VERSION)) + httpVersion = append(httpVersion, requestdecorator.NewUAVersionInfo("go", runtime.Version())) + httpVersion = append(httpVersion, requestdecorator.NewUAVersionInfo("git-commit", dockerversion.GITCOMMIT)) + if kernelVersion, err := kernel.GetKernelVersion(); err == nil { + httpVersion = append(httpVersion, requestdecorator.NewUAVersionInfo("kernel", kernelVersion.String())) + } + httpVersion = append(httpVersion, requestdecorator.NewUAVersionInfo("os", runtime.GOOS)) + httpVersion = append(httpVersion, requestdecorator.NewUAVersionInfo("arch", runtime.GOARCH)) + + userAgent := requestdecorator.AppendVersions(req.UserAgent(), httpVersion...) + + req.Header.Set("User-Agent", userAgent) + + for k, v := range tr.Headers { + req.Header[k] = v + } + return tr.RoundTripper.RoundTrip(req) +} + +type debugTransport struct{ http.RoundTripper } + +func (tr debugTransport) RoundTrip(req *http.Request) (*http.Response, error) { + dump, err := httputil.DumpRequestOut(req, false) + if err != nil { + fmt.Println("could not dump request") + } + fmt.Println(string(dump)) + resp, err := tr.RoundTripper.RoundTrip(req) + if err != nil { + return nil, err + } + dump, err = httputil.DumpResponse(resp, false) + if err != nil { + fmt.Println("could not dump response") + } + fmt.Println(string(dump)) + return resp, err +} + +func HTTPClient(transport http.RoundTripper) *http.Client { + if transport == nil { + transport = NewTransport(ConnectTimeout, true) + } + + return &http.Client{ + Transport: transport, + CheckRedirect: AddRequiredHeadersToRedirectedRequests, + } } func trustedLocation(req *http.Request) bool { diff --git a/docs/registry_test.go b/docs/registry_test.go index 799d080e..d4a5ded0 100644 --- a/docs/registry_test.go +++ b/docs/registry_test.go @@ -8,7 +8,6 @@ import ( "testing" "github.com/docker/docker/cliconfig" - "github.com/docker/docker/pkg/requestdecorator" ) var ( @@ -26,38 +25,27 @@ func spawnTestRegistrySession(t *testing.T) *Session { if err != nil { t.Fatal(err) } - r, err := NewSession(authConfig, requestdecorator.NewRequestFactory(), endpoint, true) + var tr http.RoundTripper = debugTransport{NewTransport(ReceiveTimeout, endpoint.IsSecure)} + tr = &DockerHeaders{&authTransport{RoundTripper: tr, AuthConfig: authConfig}, nil} + client := HTTPClient(tr) + r, err := NewSession(client, authConfig, endpoint) if err != nil { t.Fatal(err) } + // In a normal scenario for the v1 registry, the client should send a `X-Docker-Token: true` + // header while authenticating, in order to retrieve a token that can be later used to + // perform authenticated actions. + // + // The mock v1 registry does not support that, (TODO(tiborvass): support it), instead, + // it will consider authenticated any request with the header `X-Docker-Token: fake-token`. + // + // Because we know that the client's transport is an `*authTransport` we simply cast it, + // in order to set the internal cached token to the fake token, and thus send that fake token + // upon every subsequent requests. + r.client.Transport.(*authTransport).token = token return r } -func TestPublicSession(t *testing.T) { - authConfig := &cliconfig.AuthConfig{} - - getSessionDecorators := func(index *IndexInfo) int { - endpoint, err := NewEndpoint(index) - if err != nil { - t.Fatal(err) - } - r, err := NewSession(authConfig, requestdecorator.NewRequestFactory(), endpoint, true) - if err != nil { - t.Fatal(err) - } - return len(r.reqFactory.GetDecorators()) - } - - decorators := getSessionDecorators(makeIndex("/v1/")) - assertEqual(t, decorators, 0, "Expected no decorator on http session") - - decorators = getSessionDecorators(makeHttpsIndex("/v1/")) - assertNotEqual(t, decorators, 0, "Expected decorator on https session") - - decorators = getSessionDecorators(makePublicIndex()) - assertEqual(t, decorators, 0, "Expected no decorator on public session") -} - func TestPingRegistryEndpoint(t *testing.T) { testPing := func(index *IndexInfo, expectedStandalone bool, assertMessage string) { ep, err := NewEndpoint(index) @@ -170,7 +158,7 @@ func TestEndpoint(t *testing.T) { func TestGetRemoteHistory(t *testing.T) { r := spawnTestRegistrySession(t) - hist, err := r.GetRemoteHistory(imageID, makeURL("/v1/"), token) + hist, err := r.GetRemoteHistory(imageID, makeURL("/v1/")) if err != nil { t.Fatal(err) } @@ -182,16 +170,16 @@ func TestGetRemoteHistory(t *testing.T) { func TestLookupRemoteImage(t *testing.T) { r := spawnTestRegistrySession(t) - err := r.LookupRemoteImage(imageID, makeURL("/v1/"), token) + err := r.LookupRemoteImage(imageID, makeURL("/v1/")) assertEqual(t, err, nil, "Expected error of remote lookup to nil") - if err := r.LookupRemoteImage("abcdef", makeURL("/v1/"), token); err == nil { + if err := r.LookupRemoteImage("abcdef", makeURL("/v1/")); err == nil { t.Fatal("Expected error of remote lookup to not nil") } } func TestGetRemoteImageJSON(t *testing.T) { r := spawnTestRegistrySession(t) - json, size, err := r.GetRemoteImageJSON(imageID, makeURL("/v1/"), token) + json, size, err := r.GetRemoteImageJSON(imageID, makeURL("/v1/")) if err != nil { t.Fatal(err) } @@ -200,7 +188,7 @@ func TestGetRemoteImageJSON(t *testing.T) { t.Fatal("Expected non-empty json") } - _, _, err = r.GetRemoteImageJSON("abcdef", makeURL("/v1/"), token) + _, _, err = r.GetRemoteImageJSON("abcdef", makeURL("/v1/")) if err == nil { t.Fatal("Expected image not found error") } @@ -208,7 +196,7 @@ func TestGetRemoteImageJSON(t *testing.T) { func TestGetRemoteImageLayer(t *testing.T) { r := spawnTestRegistrySession(t) - data, err := r.GetRemoteImageLayer(imageID, makeURL("/v1/"), token, 0) + data, err := r.GetRemoteImageLayer(imageID, makeURL("/v1/"), 0) if err != nil { t.Fatal(err) } @@ -216,7 +204,7 @@ func TestGetRemoteImageLayer(t *testing.T) { t.Fatal("Expected non-nil data result") } - _, err = r.GetRemoteImageLayer("abcdef", makeURL("/v1/"), token, 0) + _, err = r.GetRemoteImageLayer("abcdef", makeURL("/v1/"), 0) if err == nil { t.Fatal("Expected image not found error") } @@ -224,14 +212,14 @@ func TestGetRemoteImageLayer(t *testing.T) { func TestGetRemoteTags(t *testing.T) { r := spawnTestRegistrySession(t) - tags, err := r.GetRemoteTags([]string{makeURL("/v1/")}, REPO, token) + tags, err := r.GetRemoteTags([]string{makeURL("/v1/")}, REPO) if err != nil { t.Fatal(err) } assertEqual(t, len(tags), 1, "Expected one tag") assertEqual(t, tags["latest"], imageID, "Expected tag latest to map to "+imageID) - _, err = r.GetRemoteTags([]string{makeURL("/v1/")}, "foo42/baz", token) + _, err = r.GetRemoteTags([]string{makeURL("/v1/")}, "foo42/baz") if err == nil { t.Fatal("Expected error when fetching tags for bogus repo") } @@ -265,7 +253,7 @@ func TestPushImageJSONRegistry(t *testing.T) { Checksum: "sha256:1ac330d56e05eef6d438586545ceff7550d3bdcb6b19961f12c5ba714ee1bb37", } - err := r.PushImageJSONRegistry(imgData, []byte{0x42, 0xdf, 0x0}, makeURL("/v1/"), token) + err := r.PushImageJSONRegistry(imgData, []byte{0x42, 0xdf, 0x0}, makeURL("/v1/")) if err != nil { t.Fatal(err) } @@ -274,7 +262,7 @@ func TestPushImageJSONRegistry(t *testing.T) { func TestPushImageLayerRegistry(t *testing.T) { r := spawnTestRegistrySession(t) layer := strings.NewReader("") - _, _, err := r.PushImageLayerRegistry(imageID, layer, makeURL("/v1/"), token, []byte{}) + _, _, err := r.PushImageLayerRegistry(imageID, layer, makeURL("/v1/"), []byte{}) if err != nil { t.Fatal(err) } @@ -694,7 +682,7 @@ func TestNewIndexInfo(t *testing.T) { func TestPushRegistryTag(t *testing.T) { r := spawnTestRegistrySession(t) - err := r.PushRegistryTag("foo42/bar", imageID, "stable", makeURL("/v1/"), token) + err := r.PushRegistryTag("foo42/bar", imageID, "stable", makeURL("/v1/")) if err != nil { t.Fatal(err) } diff --git a/docs/service.go b/docs/service.go index 87fc1d07..067df107 100644 --- a/docs/service.go +++ b/docs/service.go @@ -32,7 +32,7 @@ func (s *Service) Auth(authConfig *cliconfig.AuthConfig) (string, error) { return "", err } authConfig.ServerAddress = endpoint.String() - return Login(authConfig, endpoint, HTTPRequestFactory(nil)) + return Login(authConfig, endpoint) } // Search queries the public registry for images matching the specified @@ -42,12 +42,13 @@ func (s *Service) Search(term string, authConfig *cliconfig.AuthConfig, headers if err != nil { return nil, err } + // *TODO: Search multiple indexes. endpoint, err := repoInfo.GetEndpoint() if err != nil { return nil, err } - r, err := NewSession(authConfig, HTTPRequestFactory(headers), endpoint, true) + r, err := NewSession(endpoint.HTTPClient(), authConfig, endpoint) if err != nil { return nil, err } diff --git a/docs/session.go b/docs/session.go index e65f82cd..686e322d 100644 --- a/docs/session.go +++ b/docs/session.go @@ -3,6 +3,7 @@ package registry import ( "bytes" "crypto/sha256" + "errors" // this is required for some certificates _ "crypto/sha512" "encoding/hex" @@ -20,64 +21,105 @@ import ( "github.com/Sirupsen/logrus" "github.com/docker/docker/cliconfig" "github.com/docker/docker/pkg/httputils" - "github.com/docker/docker/pkg/requestdecorator" "github.com/docker/docker/pkg/tarsum" ) type Session struct { - authConfig *cliconfig.AuthConfig - reqFactory *requestdecorator.RequestFactory indexEndpoint *Endpoint - jar *cookiejar.Jar - timeout TimeoutType + client *http.Client + // TODO(tiborvass): remove authConfig + authConfig *cliconfig.AuthConfig } -func NewSession(authConfig *cliconfig.AuthConfig, factory *requestdecorator.RequestFactory, endpoint *Endpoint, timeout bool) (r *Session, err error) { - r = &Session{ - authConfig: authConfig, - indexEndpoint: endpoint, +// authTransport handles the auth layer when communicating with a v1 registry (private or official) +// +// For private v1 registries, set alwaysSetBasicAuth to true. +// +// For the official v1 registry, if there isn't already an Authorization header in the request, +// but there is an X-Docker-Token header set to true, then Basic Auth will be used to set the Authorization header. +// After sending the request with the provided base http.RoundTripper, if an X-Docker-Token header, representing +// a token, is present in the response, then it gets cached and sent in the Authorization header of all subsequent +// requests. +// +// If the server sends a token without the client having requested it, it is ignored. +// +// This RoundTripper also has a CancelRequest method important for correct timeout handling. +type authTransport struct { + http.RoundTripper + *cliconfig.AuthConfig + + alwaysSetBasicAuth bool + token []string +} + +func (tr *authTransport) RoundTrip(req *http.Request) (*http.Response, error) { + req = cloneRequest(req) + + if tr.alwaysSetBasicAuth { + req.SetBasicAuth(tr.Username, tr.Password) + return tr.RoundTripper.RoundTrip(req) } - if timeout { - r.timeout = ReceiveTimeout - } + var askedForToken bool - r.jar, err = cookiejar.New(nil) + // Don't override + if req.Header.Get("Authorization") == "" { + if req.Header.Get("X-Docker-Token") == "true" { + req.SetBasicAuth(tr.Username, tr.Password) + askedForToken = true + } else if len(tr.token) > 0 { + req.Header.Set("Authorization", "Token "+strings.Join(tr.token, ",")) + } + } + resp, err := tr.RoundTripper.RoundTrip(req) if err != nil { return nil, err } + if askedForToken && len(resp.Header["X-Docker-Token"]) > 0 { + tr.token = resp.Header["X-Docker-Token"] + } + return resp, nil +} + +// TODO(tiborvass): remove authConfig param once registry client v2 is vendored +func NewSession(client *http.Client, authConfig *cliconfig.AuthConfig, endpoint *Endpoint) (r *Session, err error) { + r = &Session{ + authConfig: authConfig, + client: client, + indexEndpoint: endpoint, + } + + var alwaysSetBasicAuth bool // If we're working with a standalone private registry over HTTPS, send Basic Auth headers - // alongside our requests. - if r.indexEndpoint.VersionString(1) != IndexServerAddress() && r.indexEndpoint.URL.Scheme == "https" { - info, err := r.indexEndpoint.Ping() + // alongside all our requests. + if endpoint.VersionString(1) != IndexServerAddress() && endpoint.URL.Scheme == "https" { + info, err := endpoint.Ping() if err != nil { return nil, err } - if info.Standalone && authConfig != nil && factory != nil { - logrus.Debugf("Endpoint %s is eligible for private registry. Enabling decorator.", r.indexEndpoint.String()) - dec := requestdecorator.NewAuthDecorator(authConfig.Username, authConfig.Password) - factory.AddDecorator(dec) + + if info.Standalone && authConfig != nil { + logrus.Debugf("Endpoint %s is eligible for private registry. Enabling decorator.", endpoint.String()) + alwaysSetBasicAuth = true } } - r.reqFactory = factory - return r, nil -} + client.Transport = &authTransport{RoundTripper: client.Transport, AuthConfig: authConfig, alwaysSetBasicAuth: alwaysSetBasicAuth} -func (r *Session) doRequest(req *http.Request) (*http.Response, *http.Client, error) { - return doRequest(req, r.jar, r.timeout, r.indexEndpoint.IsSecure) + jar, err := cookiejar.New(nil) + if err != nil { + return nil, errors.New("cookiejar.New is not supposed to return an error") + } + client.Jar = jar + + return r, nil } // Retrieve the history of a given image from the Registry. // Return a list of the parent's json (requested image included) -func (r *Session) GetRemoteHistory(imgID, registry string, token []string) ([]string, error) { - req, err := r.reqFactory.NewRequest("GET", registry+"images/"+imgID+"/ancestry", nil) - if err != nil { - return nil, err - } - setTokenAuth(req, token) - res, _, err := r.doRequest(req) +func (r *Session) GetRemoteHistory(imgID, registry string) ([]string, error) { + res, err := r.client.Get(registry + "images/" + imgID + "/ancestry") if err != nil { return nil, err } @@ -89,27 +131,18 @@ func (r *Session) GetRemoteHistory(imgID, registry string, token []string) ([]st return nil, httputils.NewHTTPRequestError(fmt.Sprintf("Server error: %d trying to fetch remote history for %s", res.StatusCode, imgID), res) } - jsonString, err := ioutil.ReadAll(res.Body) - if err != nil { - return nil, fmt.Errorf("Error while reading the http response: %s", err) + var history []string + if err := json.NewDecoder(res.Body).Decode(&history); err != nil { + return nil, fmt.Errorf("Error while reading the http response: %v", err) } - logrus.Debugf("Ancestry: %s", jsonString) - history := new([]string) - if err := json.Unmarshal(jsonString, history); err != nil { - return nil, err - } - return *history, nil + logrus.Debugf("Ancestry: %v", history) + return history, nil } // Check if an image exists in the Registry -func (r *Session) LookupRemoteImage(imgID, registry string, token []string) error { - req, err := r.reqFactory.NewRequest("GET", registry+"images/"+imgID+"/json", nil) - if err != nil { - return err - } - setTokenAuth(req, token) - res, _, err := r.doRequest(req) +func (r *Session) LookupRemoteImage(imgID, registry string) error { + res, err := r.client.Get(registry + "images/" + imgID + "/json") if err != nil { return err } @@ -121,14 +154,8 @@ func (r *Session) LookupRemoteImage(imgID, registry string, token []string) erro } // Retrieve an image from the Registry. -func (r *Session) GetRemoteImageJSON(imgID, registry string, token []string) ([]byte, int, error) { - // Get the JSON - req, err := r.reqFactory.NewRequest("GET", registry+"images/"+imgID+"/json", nil) - if err != nil { - return nil, -1, fmt.Errorf("Failed to download json: %s", err) - } - setTokenAuth(req, token) - res, _, err := r.doRequest(req) +func (r *Session) GetRemoteImageJSON(imgID, registry string) ([]byte, int, error) { + res, err := r.client.Get(registry + "images/" + imgID + "/json") if err != nil { return nil, -1, fmt.Errorf("Failed to download json: %s", err) } @@ -147,44 +174,44 @@ func (r *Session) GetRemoteImageJSON(imgID, registry string, token []string) ([] jsonString, err := ioutil.ReadAll(res.Body) if err != nil { - return nil, -1, fmt.Errorf("Failed to parse downloaded json: %s (%s)", err, jsonString) + return nil, -1, fmt.Errorf("Failed to parse downloaded json: %v (%s)", err, jsonString) } return jsonString, imageSize, nil } -func (r *Session) GetRemoteImageLayer(imgID, registry string, token []string, imgSize int64) (io.ReadCloser, error) { +func (r *Session) GetRemoteImageLayer(imgID, registry string, imgSize int64) (io.ReadCloser, error) { var ( retries = 5 statusCode = 0 - client *http.Client res *http.Response + err error imageURL = fmt.Sprintf("%simages/%s/layer", registry, imgID) ) - req, err := r.reqFactory.NewRequest("GET", imageURL, nil) + req, err := http.NewRequest("GET", imageURL, nil) if err != nil { - return nil, fmt.Errorf("Error while getting from the server: %s\n", err) + return nil, fmt.Errorf("Error while getting from the server: %v", err) } - setTokenAuth(req, token) + // TODO: why are we doing retries at this level? + // These retries should be generic to both v1 and v2 for i := 1; i <= retries; i++ { statusCode = 0 - res, client, err = r.doRequest(req) - if err != nil { - logrus.Debugf("Error contacting registry: %s", err) - if res != nil { - if res.Body != nil { - res.Body.Close() - } - statusCode = res.StatusCode - } - if i == retries { - return nil, fmt.Errorf("Server error: Status %d while fetching image layer (%s)", - statusCode, imgID) - } - time.Sleep(time.Duration(i) * 5 * time.Second) - continue + res, err = r.client.Do(req) + if err == nil { + break } - break + logrus.Debugf("Error contacting registry %s: %v", registry, err) + if res != nil { + if res.Body != nil { + res.Body.Close() + } + statusCode = res.StatusCode + } + if i == retries { + return nil, fmt.Errorf("Server error: Status %d while fetching image layer (%s)", + statusCode, imgID) + } + time.Sleep(time.Duration(i) * 5 * time.Second) } if res.StatusCode != 200 { @@ -195,13 +222,13 @@ func (r *Session) GetRemoteImageLayer(imgID, registry string, token []string, im if res.Header.Get("Accept-Ranges") == "bytes" && imgSize > 0 { logrus.Debugf("server supports resume") - return httputils.ResumableRequestReaderWithInitialResponse(client, req, 5, imgSize, res), nil + return httputils.ResumableRequestReaderWithInitialResponse(r.client, req, 5, imgSize, res), nil } logrus.Debugf("server doesn't support resume") return res.Body, nil } -func (r *Session) GetRemoteTags(registries []string, repository string, token []string) (map[string]string, error) { +func (r *Session) GetRemoteTags(registries []string, repository string) (map[string]string, error) { if strings.Count(repository, "/") == 0 { // This will be removed once the Registry supports auto-resolution on // the "library" namespace @@ -209,13 +236,7 @@ func (r *Session) GetRemoteTags(registries []string, repository string, token [] } for _, host := range registries { endpoint := fmt.Sprintf("%srepositories/%s/tags", host, repository) - req, err := r.reqFactory.NewRequest("GET", endpoint, nil) - - if err != nil { - return nil, err - } - setTokenAuth(req, token) - res, _, err := r.doRequest(req) + res, err := r.client.Get(endpoint) if err != nil { return nil, err } @@ -263,16 +284,13 @@ func (r *Session) GetRepositoryData(remote string) (*RepositoryData, error) { logrus.Debugf("[registry] Calling GET %s", repositoryTarget) - req, err := r.reqFactory.NewRequest("GET", repositoryTarget, nil) + req, err := http.NewRequest("GET", repositoryTarget, nil) if err != nil { return nil, err } - if r.authConfig != nil && len(r.authConfig.Username) > 0 { - req.SetBasicAuth(r.authConfig.Username, r.authConfig.Password) - } + // this will set basic auth in r.client.Transport and send cached X-Docker-Token headers for all subsequent requests req.Header.Set("X-Docker-Token", "true") - - res, _, err := r.doRequest(req) + res, err := r.client.Do(req) if err != nil { return nil, err } @@ -292,11 +310,6 @@ func (r *Session) GetRepositoryData(remote string) (*RepositoryData, error) { return nil, httputils.NewHTTPRequestError(fmt.Sprintf("Error: Status %d trying to pull repository %s: %q", res.StatusCode, remote, errBody), res) } - var tokens []string - if res.Header.Get("X-Docker-Token") != "" { - tokens = res.Header["X-Docker-Token"] - } - var endpoints []string if res.Header.Get("X-Docker-Endpoints") != "" { endpoints, err = buildEndpointsList(res.Header["X-Docker-Endpoints"], r.indexEndpoint.VersionString(1)) @@ -322,29 +335,29 @@ func (r *Session) GetRepositoryData(remote string) (*RepositoryData, error) { return &RepositoryData{ ImgList: imgsData, Endpoints: endpoints, - Tokens: tokens, }, nil } -func (r *Session) PushImageChecksumRegistry(imgData *ImgData, registry string, token []string) error { +func (r *Session) PushImageChecksumRegistry(imgData *ImgData, registry string) error { - logrus.Debugf("[registry] Calling PUT %s", registry+"images/"+imgData.ID+"/checksum") + u := registry + "images/" + imgData.ID + "/checksum" - req, err := r.reqFactory.NewRequest("PUT", registry+"images/"+imgData.ID+"/checksum", nil) + logrus.Debugf("[registry] Calling PUT %s", u) + + req, err := http.NewRequest("PUT", u, nil) if err != nil { return err } - setTokenAuth(req, token) req.Header.Set("X-Docker-Checksum", imgData.Checksum) req.Header.Set("X-Docker-Checksum-Payload", imgData.ChecksumPayload) - res, _, err := r.doRequest(req) + res, err := r.client.Do(req) if err != nil { - return fmt.Errorf("Failed to upload metadata: %s", err) + return fmt.Errorf("Failed to upload metadata: %v", err) } defer res.Body.Close() if len(res.Cookies()) > 0 { - r.jar.SetCookies(req.URL, res.Cookies()) + r.client.Jar.SetCookies(req.URL, res.Cookies()) } if res.StatusCode != 200 { errBody, err := ioutil.ReadAll(res.Body) @@ -363,18 +376,19 @@ func (r *Session) PushImageChecksumRegistry(imgData *ImgData, registry string, t } // Push a local image to the registry -func (r *Session) PushImageJSONRegistry(imgData *ImgData, jsonRaw []byte, registry string, token []string) error { +func (r *Session) PushImageJSONRegistry(imgData *ImgData, jsonRaw []byte, registry string) error { - logrus.Debugf("[registry] Calling PUT %s", registry+"images/"+imgData.ID+"/json") + u := registry + "images/" + imgData.ID + "/json" - req, err := r.reqFactory.NewRequest("PUT", registry+"images/"+imgData.ID+"/json", bytes.NewReader(jsonRaw)) + logrus.Debugf("[registry] Calling PUT %s", u) + + req, err := http.NewRequest("PUT", u, bytes.NewReader(jsonRaw)) if err != nil { return err } req.Header.Add("Content-type", "application/json") - setTokenAuth(req, token) - res, _, err := r.doRequest(req) + res, err := r.client.Do(req) if err != nil { return fmt.Errorf("Failed to upload metadata: %s", err) } @@ -398,9 +412,11 @@ func (r *Session) PushImageJSONRegistry(imgData *ImgData, jsonRaw []byte, regist return nil } -func (r *Session) PushImageLayerRegistry(imgID string, layer io.Reader, registry string, token []string, jsonRaw []byte) (checksum string, checksumPayload string, err error) { +func (r *Session) PushImageLayerRegistry(imgID string, layer io.Reader, registry string, jsonRaw []byte) (checksum string, checksumPayload string, err error) { - logrus.Debugf("[registry] Calling PUT %s", registry+"images/"+imgID+"/layer") + u := registry + "images/" + imgID + "/layer" + + logrus.Debugf("[registry] Calling PUT %s", u) tarsumLayer, err := tarsum.NewTarSum(layer, false, tarsum.Version0) if err != nil { @@ -411,17 +427,16 @@ func (r *Session) PushImageLayerRegistry(imgID string, layer io.Reader, registry h.Write([]byte{'\n'}) checksumLayer := io.TeeReader(tarsumLayer, h) - req, err := r.reqFactory.NewRequest("PUT", registry+"images/"+imgID+"/layer", checksumLayer) + req, err := http.NewRequest("PUT", u, checksumLayer) if err != nil { return "", "", err } req.Header.Add("Content-Type", "application/octet-stream") req.ContentLength = -1 req.TransferEncoding = []string{"chunked"} - setTokenAuth(req, token) - res, _, err := r.doRequest(req) + res, err := r.client.Do(req) if err != nil { - return "", "", fmt.Errorf("Failed to upload layer: %s", err) + return "", "", fmt.Errorf("Failed to upload layer: %v", err) } if rc, ok := layer.(io.Closer); ok { if err := rc.Close(); err != nil { @@ -444,19 +459,18 @@ func (r *Session) PushImageLayerRegistry(imgID string, layer io.Reader, registry // push a tag on the registry. // Remote has the format '/ -func (r *Session) PushRegistryTag(remote, revision, tag, registry string, token []string) error { +func (r *Session) PushRegistryTag(remote, revision, tag, registry string) error { // "jsonify" the string revision = "\"" + revision + "\"" path := fmt.Sprintf("repositories/%s/tags/%s", remote, tag) - req, err := r.reqFactory.NewRequest("PUT", registry+path, strings.NewReader(revision)) + req, err := http.NewRequest("PUT", registry+path, strings.NewReader(revision)) if err != nil { return err } req.Header.Add("Content-type", "application/json") - setTokenAuth(req, token) req.ContentLength = int64(len(revision)) - res, _, err := r.doRequest(req) + res, err := r.client.Do(req) if err != nil { return err } @@ -491,7 +505,8 @@ func (r *Session) PushImageJSONIndex(remote string, imgList []*ImgData, validate logrus.Debugf("[registry] PUT %s", u) logrus.Debugf("Image list pushed to index:\n%s", imgListJSON) headers := map[string][]string{ - "Content-type": {"application/json"}, + "Content-type": {"application/json"}, + // this will set basic auth in r.client.Transport and send cached X-Docker-Token headers for all subsequent requests "X-Docker-Token": {"true"}, } if validate { @@ -526,9 +541,6 @@ func (r *Session) PushImageJSONIndex(remote string, imgList []*ImgData, validate } return nil, httputils.NewHTTPRequestError(fmt.Sprintf("Error: Status %d trying to push repository %s: %q", res.StatusCode, remote, errBody), res) } - if res.Header.Get("X-Docker-Token") == "" { - return nil, fmt.Errorf("Index response didn't contain an access token") - } tokens = res.Header["X-Docker-Token"] logrus.Debugf("Auth token: %v", tokens) @@ -539,8 +551,7 @@ func (r *Session) PushImageJSONIndex(remote string, imgList []*ImgData, validate if err != nil { return nil, err } - } - if validate { + } else { if res.StatusCode != 204 { errBody, err := ioutil.ReadAll(res.Body) if err != nil { @@ -551,22 +562,20 @@ func (r *Session) PushImageJSONIndex(remote string, imgList []*ImgData, validate } return &RepositoryData{ - Tokens: tokens, Endpoints: endpoints, }, nil } func (r *Session) putImageRequest(u string, headers map[string][]string, body []byte) (*http.Response, error) { - req, err := r.reqFactory.NewRequest("PUT", u, bytes.NewReader(body)) + req, err := http.NewRequest("PUT", u, bytes.NewReader(body)) if err != nil { return nil, err } - req.SetBasicAuth(r.authConfig.Username, r.authConfig.Password) req.ContentLength = int64(len(body)) for k, v := range headers { req.Header[k] = v } - response, _, err := r.doRequest(req) + response, err := r.client.Do(req) if err != nil { return nil, err } @@ -580,15 +589,7 @@ func shouldRedirect(response *http.Response) bool { func (r *Session) SearchRepositories(term string) (*SearchResults, error) { logrus.Debugf("Index server: %s", r.indexEndpoint) u := r.indexEndpoint.VersionString(1) + "search?q=" + url.QueryEscape(term) - req, err := r.reqFactory.NewRequest("GET", u, nil) - if err != nil { - return nil, err - } - if r.authConfig != nil && len(r.authConfig.Username) > 0 { - req.SetBasicAuth(r.authConfig.Username, r.authConfig.Password) - } - req.Header.Set("X-Docker-Token", "true") - res, _, err := r.doRequest(req) + res, err := r.client.Get(u) if err != nil { return nil, err } @@ -600,6 +601,7 @@ func (r *Session) SearchRepositories(term string) (*SearchResults, error) { return result, json.NewDecoder(res.Body).Decode(result) } +// TODO(tiborvass): remove this once registry client v2 is vendored func (r *Session) GetAuthConfig(withPasswd bool) *cliconfig.AuthConfig { password := "" if withPasswd { @@ -611,9 +613,3 @@ func (r *Session) GetAuthConfig(withPasswd bool) *cliconfig.AuthConfig { Email: r.authConfig.Email, } } - -func setTokenAuth(req *http.Request, token []string) { - if req.Header.Get("Authorization") == "" { // Don't override - req.Header.Set("Authorization", "Token "+strings.Join(token, ",")) - } -} diff --git a/docs/session_v2.go b/docs/session_v2.go index 4188e505..c639f922 100644 --- a/docs/session_v2.go +++ b/docs/session_v2.go @@ -77,14 +77,14 @@ func (r *Session) GetV2ImageManifest(ep *Endpoint, imageName, tagName string, au method := "GET" logrus.Debugf("[registry] Calling %q %s", method, routeURL) - req, err := r.reqFactory.NewRequest(method, routeURL, nil) + req, err := http.NewRequest(method, routeURL, nil) if err != nil { return nil, "", err } if err := auth.Authorize(req); err != nil { return nil, "", err } - res, _, err := r.doRequest(req) + res, err := r.client.Do(req) if err != nil { return nil, "", err } @@ -118,14 +118,14 @@ func (r *Session) HeadV2ImageBlob(ep *Endpoint, imageName string, dgst digest.Di method := "HEAD" logrus.Debugf("[registry] Calling %q %s", method, routeURL) - req, err := r.reqFactory.NewRequest(method, routeURL, nil) + req, err := http.NewRequest(method, routeURL, nil) if err != nil { return false, err } if err := auth.Authorize(req); err != nil { return false, err } - res, _, err := r.doRequest(req) + res, err := r.client.Do(req) if err != nil { return false, err } @@ -152,14 +152,14 @@ func (r *Session) GetV2ImageBlob(ep *Endpoint, imageName string, dgst digest.Dig method := "GET" logrus.Debugf("[registry] Calling %q %s", method, routeURL) - req, err := r.reqFactory.NewRequest(method, routeURL, nil) + req, err := http.NewRequest(method, routeURL, nil) if err != nil { return err } if err := auth.Authorize(req); err != nil { return err } - res, _, err := r.doRequest(req) + res, err := r.client.Do(req) if err != nil { return err } @@ -183,14 +183,14 @@ func (r *Session) GetV2ImageBlobReader(ep *Endpoint, imageName string, dgst dige method := "GET" logrus.Debugf("[registry] Calling %q %s", method, routeURL) - req, err := r.reqFactory.NewRequest(method, routeURL, nil) + req, err := http.NewRequest(method, routeURL, nil) if err != nil { return nil, 0, err } if err := auth.Authorize(req); err != nil { return nil, 0, err } - res, _, err := r.doRequest(req) + res, err := r.client.Do(req) if err != nil { return nil, 0, err } @@ -220,7 +220,7 @@ func (r *Session) PutV2ImageBlob(ep *Endpoint, imageName string, dgst digest.Dig method := "PUT" logrus.Debugf("[registry] Calling %q %s", method, location) - req, err := r.reqFactory.NewRequest(method, location, ioutil.NopCloser(blobRdr)) + req, err := http.NewRequest(method, location, ioutil.NopCloser(blobRdr)) if err != nil { return err } @@ -230,7 +230,7 @@ func (r *Session) PutV2ImageBlob(ep *Endpoint, imageName string, dgst digest.Dig if err := auth.Authorize(req); err != nil { return err } - res, _, err := r.doRequest(req) + res, err := r.client.Do(req) if err != nil { return err } @@ -259,7 +259,7 @@ func (r *Session) initiateBlobUpload(ep *Endpoint, imageName string, auth *Reque } logrus.Debugf("[registry] Calling %q %s", "POST", routeURL) - req, err := r.reqFactory.NewRequest("POST", routeURL, nil) + req, err := http.NewRequest("POST", routeURL, nil) if err != nil { return "", err } @@ -267,7 +267,7 @@ func (r *Session) initiateBlobUpload(ep *Endpoint, imageName string, auth *Reque if err := auth.Authorize(req); err != nil { return "", err } - res, _, err := r.doRequest(req) + res, err := r.client.Do(req) if err != nil { return "", err } @@ -305,14 +305,14 @@ func (r *Session) PutV2ImageManifest(ep *Endpoint, imageName, tagName string, si method := "PUT" logrus.Debugf("[registry] Calling %q %s", method, routeURL) - req, err := r.reqFactory.NewRequest(method, routeURL, bytes.NewReader(signedManifest)) + req, err := http.NewRequest(method, routeURL, bytes.NewReader(signedManifest)) if err != nil { return "", err } if err := auth.Authorize(req); err != nil { return "", err } - res, _, err := r.doRequest(req) + res, err := r.client.Do(req) if err != nil { return "", err } @@ -366,14 +366,14 @@ func (r *Session) GetV2RemoteTags(ep *Endpoint, imageName string, auth *RequestA method := "GET" logrus.Debugf("[registry] Calling %q %s", method, routeURL) - req, err := r.reqFactory.NewRequest(method, routeURL, nil) + req, err := http.NewRequest(method, routeURL, nil) if err != nil { return nil, err } if err := auth.Authorize(req); err != nil { return nil, err } - res, _, err := r.doRequest(req) + res, err := r.client.Do(req) if err != nil { return nil, err } diff --git a/docs/token.go b/docs/token.go index b03bd891..af7d5f3f 100644 --- a/docs/token.go +++ b/docs/token.go @@ -7,15 +7,13 @@ import ( "net/http" "net/url" "strings" - - "github.com/docker/docker/pkg/requestdecorator" ) type tokenResponse struct { Token string `json:"token"` } -func getToken(username, password string, params map[string]string, registryEndpoint *Endpoint, client *http.Client, factory *requestdecorator.RequestFactory) (token string, err error) { +func getToken(username, password string, params map[string]string, registryEndpoint *Endpoint, client *http.Client) (token string, err error) { realm, ok := params["realm"] if !ok { return "", errors.New("no realm specified for token auth challenge") @@ -34,7 +32,7 @@ func getToken(username, password string, params map[string]string, registryEndpo } } - req, err := factory.NewRequest("GET", realmURL.String(), nil) + req, err := http.NewRequest("GET", realmURL.String(), nil) if err != nil { return "", err } From 9e6affc364d3f1e2afb06d4ea865ac9d1199aa22 Mon Sep 17 00:00:00 2001 From: Tibor Vass Date: Fri, 15 May 2015 15:03:08 -0700 Subject: [PATCH 271/375] requestdecorator: repurpose the package and rename to useragent Signed-off-by: Tibor Vass --- docs/registry.go | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/docs/registry.go b/docs/registry.go index db77a985..4f540300 100644 --- a/docs/registry.go +++ b/docs/registry.go @@ -18,8 +18,8 @@ import ( "github.com/Sirupsen/logrus" "github.com/docker/docker/autogen/dockerversion" "github.com/docker/docker/pkg/parsers/kernel" - "github.com/docker/docker/pkg/requestdecorator" "github.com/docker/docker/pkg/timeoutconn" + "github.com/docker/docker/pkg/useragent" ) var ( @@ -186,17 +186,17 @@ func cloneRequest(r *http.Request) *http.Request { func (tr *DockerHeaders) RoundTrip(req *http.Request) (*http.Response, error) { req = cloneRequest(req) - httpVersion := make([]requestdecorator.UAVersionInfo, 0, 4) - httpVersion = append(httpVersion, requestdecorator.NewUAVersionInfo("docker", dockerversion.VERSION)) - httpVersion = append(httpVersion, requestdecorator.NewUAVersionInfo("go", runtime.Version())) - httpVersion = append(httpVersion, requestdecorator.NewUAVersionInfo("git-commit", dockerversion.GITCOMMIT)) + httpVersion := make([]useragent.VersionInfo, 0, 4) + httpVersion = append(httpVersion, useragent.VersionInfo{"docker", dockerversion.VERSION}) + httpVersion = append(httpVersion, useragent.VersionInfo{"go", runtime.Version()}) + httpVersion = append(httpVersion, useragent.VersionInfo{"git-commit", dockerversion.GITCOMMIT}) if kernelVersion, err := kernel.GetKernelVersion(); err == nil { - httpVersion = append(httpVersion, requestdecorator.NewUAVersionInfo("kernel", kernelVersion.String())) + httpVersion = append(httpVersion, useragent.VersionInfo{"kernel", kernelVersion.String()}) } - httpVersion = append(httpVersion, requestdecorator.NewUAVersionInfo("os", runtime.GOOS)) - httpVersion = append(httpVersion, requestdecorator.NewUAVersionInfo("arch", runtime.GOARCH)) + httpVersion = append(httpVersion, useragent.VersionInfo{"os", runtime.GOOS}) + httpVersion = append(httpVersion, useragent.VersionInfo{"arch", runtime.GOARCH}) - userAgent := requestdecorator.AppendVersions(req.UserAgent(), httpVersion...) + userAgent := useragent.AppendVersions(req.UserAgent(), httpVersion...) req.Header.Set("User-Agent", userAgent) From 808c87ce270c78faef15703c3d208c0e5223516c Mon Sep 17 00:00:00 2001 From: Tibor Vass Date: Fri, 15 May 2015 18:35:04 -0700 Subject: [PATCH 272/375] Add transport package to support CancelRequest Signed-off-by: Tibor Vass --- docs/auth.go | 26 +++++------ docs/endpoint.go | 25 ++++------ docs/endpoint_test.go | 3 +- docs/registry.go | 106 ++++++++++++++++++------------------------ docs/registry_test.go | 15 +++--- docs/service.go | 12 +++-- docs/session.go | 59 +++++++++++++++++++---- docs/session_v2.go | 4 +- docs/token.go | 4 +- 9 files changed, 136 insertions(+), 118 deletions(-) diff --git a/docs/auth.go b/docs/auth.go index 0b6c3b0f..33f8fa06 100644 --- a/docs/auth.go +++ b/docs/auth.go @@ -44,8 +44,6 @@ func (auth *RequestAuthorization) getToken() (string, error) { return auth.tokenCache, nil } - client := auth.registryEndpoint.HTTPClient() - for _, challenge := range auth.registryEndpoint.AuthChallenges { switch strings.ToLower(challenge.Scheme) { case "basic": @@ -57,7 +55,7 @@ func (auth *RequestAuthorization) getToken() (string, error) { params[k] = v } params["scope"] = fmt.Sprintf("%s:%s:%s", auth.resource, auth.scope, strings.Join(auth.actions, ",")) - token, err := getToken(auth.authConfig.Username, auth.authConfig.Password, params, auth.registryEndpoint, client) + token, err := getToken(auth.authConfig.Username, auth.authConfig.Password, params, auth.registryEndpoint) if err != nil { return "", err } @@ -104,7 +102,6 @@ func loginV1(authConfig *cliconfig.AuthConfig, registryEndpoint *Endpoint) (stri status string reqBody []byte err error - client = registryEndpoint.HTTPClient() reqStatusCode = 0 serverAddress = authConfig.ServerAddress ) @@ -128,7 +125,7 @@ func loginV1(authConfig *cliconfig.AuthConfig, registryEndpoint *Endpoint) (stri // using `bytes.NewReader(jsonBody)` here causes the server to respond with a 411 status. b := strings.NewReader(string(jsonBody)) - req1, err := client.Post(serverAddress+"users/", "application/json; charset=utf-8", b) + req1, err := registryEndpoint.client.Post(serverAddress+"users/", "application/json; charset=utf-8", b) if err != nil { return "", fmt.Errorf("Server Error: %s", err) } @@ -151,7 +148,7 @@ func loginV1(authConfig *cliconfig.AuthConfig, registryEndpoint *Endpoint) (stri if string(reqBody) == "\"Username or email already exists\"" { req, err := http.NewRequest("GET", serverAddress+"users/", nil) req.SetBasicAuth(authConfig.Username, authConfig.Password) - resp, err := client.Do(req) + resp, err := registryEndpoint.client.Do(req) if err != nil { return "", err } @@ -180,7 +177,7 @@ func loginV1(authConfig *cliconfig.AuthConfig, registryEndpoint *Endpoint) (stri // protected, so people can use `docker login` as an auth check. req, err := http.NewRequest("GET", serverAddress+"users/", nil) req.SetBasicAuth(authConfig.Username, authConfig.Password) - resp, err := client.Do(req) + resp, err := registryEndpoint.client.Do(req) if err != nil { return "", err } @@ -217,7 +214,6 @@ func loginV2(authConfig *cliconfig.AuthConfig, registryEndpoint *Endpoint) (stri var ( err error allErrors []error - client = registryEndpoint.HTTPClient() ) for _, challenge := range registryEndpoint.AuthChallenges { @@ -225,9 +221,9 @@ func loginV2(authConfig *cliconfig.AuthConfig, registryEndpoint *Endpoint) (stri switch strings.ToLower(challenge.Scheme) { case "basic": - err = tryV2BasicAuthLogin(authConfig, challenge.Parameters, registryEndpoint, client) + err = tryV2BasicAuthLogin(authConfig, challenge.Parameters, registryEndpoint) case "bearer": - err = tryV2TokenAuthLogin(authConfig, challenge.Parameters, registryEndpoint, client) + err = tryV2TokenAuthLogin(authConfig, challenge.Parameters, registryEndpoint) default: // Unsupported challenge types are explicitly skipped. err = fmt.Errorf("unsupported auth scheme: %q", challenge.Scheme) @@ -245,7 +241,7 @@ func loginV2(authConfig *cliconfig.AuthConfig, registryEndpoint *Endpoint) (stri return "", fmt.Errorf("no successful auth challenge for %s - errors: %s", registryEndpoint, allErrors) } -func tryV2BasicAuthLogin(authConfig *cliconfig.AuthConfig, params map[string]string, registryEndpoint *Endpoint, client *http.Client) error { +func tryV2BasicAuthLogin(authConfig *cliconfig.AuthConfig, params map[string]string, registryEndpoint *Endpoint) error { req, err := http.NewRequest("GET", registryEndpoint.Path(""), nil) if err != nil { return err @@ -253,7 +249,7 @@ func tryV2BasicAuthLogin(authConfig *cliconfig.AuthConfig, params map[string]str req.SetBasicAuth(authConfig.Username, authConfig.Password) - resp, err := client.Do(req) + resp, err := registryEndpoint.client.Do(req) if err != nil { return err } @@ -266,8 +262,8 @@ func tryV2BasicAuthLogin(authConfig *cliconfig.AuthConfig, params map[string]str return nil } -func tryV2TokenAuthLogin(authConfig *cliconfig.AuthConfig, params map[string]string, registryEndpoint *Endpoint, client *http.Client) error { - token, err := getToken(authConfig.Username, authConfig.Password, params, registryEndpoint, client) +func tryV2TokenAuthLogin(authConfig *cliconfig.AuthConfig, params map[string]string, registryEndpoint *Endpoint) error { + token, err := getToken(authConfig.Username, authConfig.Password, params, registryEndpoint) if err != nil { return err } @@ -279,7 +275,7 @@ func tryV2TokenAuthLogin(authConfig *cliconfig.AuthConfig, params map[string]str req.Header.Set("Authorization", fmt.Sprintf("Bearer %s", token)) - resp, err := client.Do(req) + resp, err := registryEndpoint.client.Do(req) if err != nil { return err } diff --git a/docs/endpoint.go b/docs/endpoint.go index 25f66ad2..ce92668f 100644 --- a/docs/endpoint.go +++ b/docs/endpoint.go @@ -11,6 +11,7 @@ import ( "github.com/Sirupsen/logrus" "github.com/docker/distribution/registry/api/v2" + "github.com/docker/docker/pkg/transport" ) // for mocking in unit tests @@ -41,9 +42,9 @@ func scanForAPIVersion(address string) (string, APIVersion) { } // NewEndpoint parses the given address to return a registry endpoint. -func NewEndpoint(index *IndexInfo) (*Endpoint, error) { +func NewEndpoint(index *IndexInfo, metaHeaders http.Header) (*Endpoint, error) { // *TODO: Allow per-registry configuration of endpoints. - endpoint, err := newEndpoint(index.GetAuthConfigKey(), index.Secure) + endpoint, err := newEndpoint(index.GetAuthConfigKey(), index.Secure, metaHeaders) if err != nil { return nil, err } @@ -81,7 +82,7 @@ func validateEndpoint(endpoint *Endpoint) error { return nil } -func newEndpoint(address string, secure bool) (*Endpoint, error) { +func newEndpoint(address string, secure bool, metaHeaders http.Header) (*Endpoint, error) { var ( endpoint = new(Endpoint) trimmedAddress string @@ -98,11 +99,13 @@ func newEndpoint(address string, secure bool) (*Endpoint, error) { return nil, err } endpoint.IsSecure = secure + tr := NewTransport(ConnectTimeout, endpoint.IsSecure) + endpoint.client = HTTPClient(transport.NewTransport(tr, DockerHeaders(metaHeaders)...)) return endpoint, nil } -func (repoInfo *RepositoryInfo) GetEndpoint() (*Endpoint, error) { - return NewEndpoint(repoInfo.Index) +func (repoInfo *RepositoryInfo) GetEndpoint(metaHeaders http.Header) (*Endpoint, error) { + return NewEndpoint(repoInfo.Index, metaHeaders) } // Endpoint stores basic information about a registry endpoint. @@ -174,7 +177,7 @@ func (e *Endpoint) pingV1() (RegistryInfo, error) { return RegistryInfo{Standalone: false}, err } - resp, err := e.HTTPClient().Do(req) + resp, err := e.client.Do(req) if err != nil { return RegistryInfo{Standalone: false}, err } @@ -222,7 +225,7 @@ func (e *Endpoint) pingV2() (RegistryInfo, error) { return RegistryInfo{}, err } - resp, err := e.HTTPClient().Do(req) + resp, err := e.client.Do(req) if err != nil { return RegistryInfo{}, err } @@ -261,11 +264,3 @@ HeaderLoop: return RegistryInfo{}, fmt.Errorf("v2 registry endpoint returned status %d: %q", resp.StatusCode, http.StatusText(resp.StatusCode)) } - -func (e *Endpoint) HTTPClient() *http.Client { - if e.client == nil { - tr := NewTransport(ConnectTimeout, e.IsSecure) - e.client = HTTPClient(tr) - } - return e.client -} diff --git a/docs/endpoint_test.go b/docs/endpoint_test.go index 9567ba23..6f67867b 100644 --- a/docs/endpoint_test.go +++ b/docs/endpoint_test.go @@ -19,7 +19,7 @@ func TestEndpointParse(t *testing.T) { {"0.0.0.0:5000", "https://0.0.0.0:5000/v0/"}, } for _, td := range testData { - e, err := newEndpoint(td.str, false) + e, err := newEndpoint(td.str, false, nil) if err != nil { t.Errorf("%q: %s", td.str, err) } @@ -60,6 +60,7 @@ func TestValidateEndpointAmbiguousAPIVersion(t *testing.T) { testEndpoint := Endpoint{ URL: testServerURL, Version: APIVersionUnknown, + client: HTTPClient(NewTransport(ConnectTimeout, false)), } if err = validateEndpoint(&testEndpoint); err != nil { diff --git a/docs/registry.go b/docs/registry.go index 4f540300..b0706e34 100644 --- a/docs/registry.go +++ b/docs/registry.go @@ -19,6 +19,7 @@ import ( "github.com/docker/docker/autogen/dockerversion" "github.com/docker/docker/pkg/parsers/kernel" "github.com/docker/docker/pkg/timeoutconn" + "github.com/docker/docker/pkg/transport" "github.com/docker/docker/pkg/useragent" ) @@ -36,17 +37,32 @@ const ( ConnectTimeout ) -type httpsTransport struct { - *http.Transport +// dockerUserAgent is the User-Agent the Docker client uses to identify itself. +// It is populated on init(), comprising version information of different components. +var dockerUserAgent string + +func init() { + httpVersion := make([]useragent.VersionInfo, 0, 6) + httpVersion = append(httpVersion, useragent.VersionInfo{"docker", dockerversion.VERSION}) + httpVersion = append(httpVersion, useragent.VersionInfo{"go", runtime.Version()}) + httpVersion = append(httpVersion, useragent.VersionInfo{"git-commit", dockerversion.GITCOMMIT}) + if kernelVersion, err := kernel.GetKernelVersion(); err == nil { + httpVersion = append(httpVersion, useragent.VersionInfo{"kernel", kernelVersion.String()}) + } + httpVersion = append(httpVersion, useragent.VersionInfo{"os", runtime.GOOS}) + httpVersion = append(httpVersion, useragent.VersionInfo{"arch", runtime.GOARCH}) + + dockerUserAgent = useragent.AppendVersions("", httpVersion...) } +type httpsRequestModifier struct{ tlsConfig *tls.Config } + // DRAGONS(tiborvass): If someone wonders why do we set tlsconfig in a roundtrip, // it's because it's so as to match the current behavior in master: we generate the // certpool on every-goddam-request. It's not great, but it allows people to just put // the certs in /etc/docker/certs.d/.../ and let docker "pick it up" immediately. Would // prefer an fsnotify implementation, but that was out of scope of my refactoring. -// TODO: improve things -func (tr *httpsTransport) RoundTrip(req *http.Request) (*http.Response, error) { +func (m *httpsRequestModifier) ModifyRequest(req *http.Request) error { var ( roots *x509.CertPool certs []tls.Certificate @@ -66,7 +82,7 @@ func (tr *httpsTransport) RoundTrip(req *http.Request) (*http.Response, error) { logrus.Debugf("hostDir: %s", hostDir) fs, err := ioutil.ReadDir(hostDir) if err != nil && !os.IsNotExist(err) { - return nil, err + return nil } for _, f := range fs { @@ -77,7 +93,7 @@ func (tr *httpsTransport) RoundTrip(req *http.Request) (*http.Response, error) { logrus.Debugf("crt: %s", hostDir+"/"+f.Name()) data, err := ioutil.ReadFile(path.Join(hostDir, f.Name())) if err != nil { - return nil, err + return err } roots.AppendCertsFromPEM(data) } @@ -86,11 +102,11 @@ func (tr *httpsTransport) RoundTrip(req *http.Request) (*http.Response, error) { keyName := certName[:len(certName)-5] + ".key" logrus.Debugf("cert: %s", hostDir+"/"+f.Name()) if !hasFile(fs, keyName) { - return nil, fmt.Errorf("Missing key %s for certificate %s", keyName, certName) + return fmt.Errorf("Missing key %s for certificate %s", keyName, certName) } cert, err := tls.LoadX509KeyPair(path.Join(hostDir, certName), path.Join(hostDir, keyName)) if err != nil { - return nil, err + return err } certs = append(certs, cert) } @@ -99,38 +115,32 @@ func (tr *httpsTransport) RoundTrip(req *http.Request) (*http.Response, error) { certName := keyName[:len(keyName)-4] + ".cert" logrus.Debugf("key: %s", hostDir+"/"+f.Name()) if !hasFile(fs, certName) { - return nil, fmt.Errorf("Missing certificate %s for key %s", certName, keyName) + return fmt.Errorf("Missing certificate %s for key %s", certName, keyName) } } } - if tr.Transport.TLSClientConfig == nil { - tr.Transport.TLSClientConfig = &tls.Config{ - // Avoid fallback to SSL protocols < TLS1.0 - MinVersion: tls.VersionTLS10, - } - } - tr.Transport.TLSClientConfig.RootCAs = roots - tr.Transport.TLSClientConfig.Certificates = certs + m.tlsConfig.RootCAs = roots + m.tlsConfig.Certificates = certs } - return tr.Transport.RoundTrip(req) + return nil } func NewTransport(timeout TimeoutType, secure bool) http.RoundTripper { - tlsConfig := tls.Config{ + tlsConfig := &tls.Config{ // Avoid fallback to SSL protocols < TLS1.0 MinVersion: tls.VersionTLS10, InsecureSkipVerify: !secure, } - transport := &http.Transport{ + tr := &http.Transport{ DisableKeepAlives: true, Proxy: http.ProxyFromEnvironment, - TLSClientConfig: &tlsConfig, + TLSClientConfig: tlsConfig, } switch timeout { case ConnectTimeout: - transport.Dial = func(proto string, addr string) (net.Conn, error) { + tr.Dial = func(proto string, addr string) (net.Conn, error) { // Set the connect timeout to 30 seconds to allow for slower connection // times... d := net.Dialer{Timeout: 30 * time.Second, DualStack: true} @@ -144,7 +154,7 @@ func NewTransport(timeout TimeoutType, secure bool) http.RoundTripper { return conn, nil } case ReceiveTimeout: - transport.Dial = func(proto string, addr string) (net.Conn, error) { + tr.Dial = func(proto string, addr string) (net.Conn, error) { d := net.Dialer{DualStack: true} conn, err := d.Dial(proto, addr) @@ -159,51 +169,23 @@ func NewTransport(timeout TimeoutType, secure bool) http.RoundTripper { if secure { // note: httpsTransport also handles http transport // but for HTTPS, it sets up the certs - return &httpsTransport{transport} + return transport.NewTransport(tr, &httpsRequestModifier{tlsConfig}) } - return transport + return tr } -type DockerHeaders struct { - http.RoundTripper - Headers http.Header -} - -// cloneRequest returns a clone of the provided *http.Request. -// The clone is a shallow copy of the struct and its Header map -func cloneRequest(r *http.Request) *http.Request { - // shallow copy of the struct - r2 := new(http.Request) - *r2 = *r - // deep copy of the Header - r2.Header = make(http.Header, len(r.Header)) - for k, s := range r.Header { - r2.Header[k] = append([]string(nil), s...) +// DockerHeaders returns request modifiers that ensure requests have +// the User-Agent header set to dockerUserAgent and that metaHeaders +// are added. +func DockerHeaders(metaHeaders http.Header) []transport.RequestModifier { + modifiers := []transport.RequestModifier{ + transport.NewHeaderRequestModifier(http.Header{"User-Agent": []string{dockerUserAgent}}), } - return r2 -} - -func (tr *DockerHeaders) RoundTrip(req *http.Request) (*http.Response, error) { - req = cloneRequest(req) - httpVersion := make([]useragent.VersionInfo, 0, 4) - httpVersion = append(httpVersion, useragent.VersionInfo{"docker", dockerversion.VERSION}) - httpVersion = append(httpVersion, useragent.VersionInfo{"go", runtime.Version()}) - httpVersion = append(httpVersion, useragent.VersionInfo{"git-commit", dockerversion.GITCOMMIT}) - if kernelVersion, err := kernel.GetKernelVersion(); err == nil { - httpVersion = append(httpVersion, useragent.VersionInfo{"kernel", kernelVersion.String()}) + if metaHeaders != nil { + modifiers = append(modifiers, transport.NewHeaderRequestModifier(metaHeaders)) } - httpVersion = append(httpVersion, useragent.VersionInfo{"os", runtime.GOOS}) - httpVersion = append(httpVersion, useragent.VersionInfo{"arch", runtime.GOARCH}) - - userAgent := useragent.AppendVersions(req.UserAgent(), httpVersion...) - - req.Header.Set("User-Agent", userAgent) - - for k, v := range tr.Headers { - req.Header[k] = v - } - return tr.RoundTripper.RoundTrip(req) + return modifiers } type debugTransport struct{ http.RoundTripper } diff --git a/docs/registry_test.go b/docs/registry_test.go index d4a5ded0..33e86ff4 100644 --- a/docs/registry_test.go +++ b/docs/registry_test.go @@ -8,6 +8,7 @@ import ( "testing" "github.com/docker/docker/cliconfig" + "github.com/docker/docker/pkg/transport" ) var ( @@ -21,12 +22,12 @@ const ( func spawnTestRegistrySession(t *testing.T) *Session { authConfig := &cliconfig.AuthConfig{} - endpoint, err := NewEndpoint(makeIndex("/v1/")) + endpoint, err := NewEndpoint(makeIndex("/v1/"), nil) if err != nil { t.Fatal(err) } var tr http.RoundTripper = debugTransport{NewTransport(ReceiveTimeout, endpoint.IsSecure)} - tr = &DockerHeaders{&authTransport{RoundTripper: tr, AuthConfig: authConfig}, nil} + tr = transport.NewTransport(AuthTransport(tr, authConfig, false), DockerHeaders(nil)...) client := HTTPClient(tr) r, err := NewSession(client, authConfig, endpoint) if err != nil { @@ -48,7 +49,7 @@ func spawnTestRegistrySession(t *testing.T) *Session { func TestPingRegistryEndpoint(t *testing.T) { testPing := func(index *IndexInfo, expectedStandalone bool, assertMessage string) { - ep, err := NewEndpoint(index) + ep, err := NewEndpoint(index, nil) if err != nil { t.Fatal(err) } @@ -68,7 +69,7 @@ func TestPingRegistryEndpoint(t *testing.T) { func TestEndpoint(t *testing.T) { // Simple wrapper to fail test if err != nil expandEndpoint := func(index *IndexInfo) *Endpoint { - endpoint, err := NewEndpoint(index) + endpoint, err := NewEndpoint(index, nil) if err != nil { t.Fatal(err) } @@ -77,7 +78,7 @@ func TestEndpoint(t *testing.T) { assertInsecureIndex := func(index *IndexInfo) { index.Secure = true - _, err := NewEndpoint(index) + _, err := NewEndpoint(index, nil) assertNotEqual(t, err, nil, index.Name+": Expected error for insecure index") assertEqual(t, strings.Contains(err.Error(), "insecure-registry"), true, index.Name+": Expected insecure-registry error for insecure index") index.Secure = false @@ -85,7 +86,7 @@ func TestEndpoint(t *testing.T) { assertSecureIndex := func(index *IndexInfo) { index.Secure = true - _, err := NewEndpoint(index) + _, err := NewEndpoint(index, nil) assertNotEqual(t, err, nil, index.Name+": Expected cert error for secure index") assertEqual(t, strings.Contains(err.Error(), "certificate signed by unknown authority"), true, index.Name+": Expected cert error for secure index") index.Secure = false @@ -151,7 +152,7 @@ func TestEndpoint(t *testing.T) { } for _, address := range badEndpoints { index.Name = address - _, err := NewEndpoint(index) + _, err := NewEndpoint(index, nil) checkNotEqual(t, err, nil, "Expected error while expanding bad endpoint") } } diff --git a/docs/service.go b/docs/service.go index 067df107..68117492 100644 --- a/docs/service.go +++ b/docs/service.go @@ -1,6 +1,10 @@ package registry -import "github.com/docker/docker/cliconfig" +import ( + "net/http" + + "github.com/docker/docker/cliconfig" +) type Service struct { Config *ServiceConfig @@ -27,7 +31,7 @@ func (s *Service) Auth(authConfig *cliconfig.AuthConfig) (string, error) { if err != nil { return "", err } - endpoint, err := NewEndpoint(index) + endpoint, err := NewEndpoint(index, nil) if err != nil { return "", err } @@ -44,11 +48,11 @@ func (s *Service) Search(term string, authConfig *cliconfig.AuthConfig, headers } // *TODO: Search multiple indexes. - endpoint, err := repoInfo.GetEndpoint() + endpoint, err := repoInfo.GetEndpoint(http.Header(headers)) if err != nil { return nil, err } - r, err := NewSession(endpoint.HTTPClient(), authConfig, endpoint) + r, err := NewSession(endpoint.client, authConfig, endpoint) if err != nil { return nil, err } diff --git a/docs/session.go b/docs/session.go index 686e322d..8e54bc82 100644 --- a/docs/session.go +++ b/docs/session.go @@ -4,6 +4,7 @@ import ( "bytes" "crypto/sha256" "errors" + "sync" // this is required for some certificates _ "crypto/sha512" "encoding/hex" @@ -22,6 +23,7 @@ import ( "github.com/docker/docker/cliconfig" "github.com/docker/docker/pkg/httputils" "github.com/docker/docker/pkg/tarsum" + "github.com/docker/docker/pkg/transport" ) type Session struct { @@ -31,7 +33,18 @@ type Session struct { authConfig *cliconfig.AuthConfig } -// authTransport handles the auth layer when communicating with a v1 registry (private or official) +type authTransport struct { + http.RoundTripper + *cliconfig.AuthConfig + + alwaysSetBasicAuth bool + token []string + + mu sync.Mutex // guards modReq + modReq map[*http.Request]*http.Request // original -> modified +} + +// AuthTransport handles the auth layer when communicating with a v1 registry (private or official) // // For private v1 registries, set alwaysSetBasicAuth to true. // @@ -44,16 +57,23 @@ type Session struct { // If the server sends a token without the client having requested it, it is ignored. // // This RoundTripper also has a CancelRequest method important for correct timeout handling. -type authTransport struct { - http.RoundTripper - *cliconfig.AuthConfig - - alwaysSetBasicAuth bool - token []string +func AuthTransport(base http.RoundTripper, authConfig *cliconfig.AuthConfig, alwaysSetBasicAuth bool) http.RoundTripper { + if base == nil { + base = http.DefaultTransport + } + return &authTransport{ + RoundTripper: base, + AuthConfig: authConfig, + alwaysSetBasicAuth: alwaysSetBasicAuth, + modReq: make(map[*http.Request]*http.Request), + } } -func (tr *authTransport) RoundTrip(req *http.Request) (*http.Response, error) { - req = cloneRequest(req) +func (tr *authTransport) RoundTrip(orig *http.Request) (*http.Response, error) { + req := transport.CloneRequest(orig) + tr.mu.Lock() + tr.modReq[orig] = req + tr.mu.Unlock() if tr.alwaysSetBasicAuth { req.SetBasicAuth(tr.Username, tr.Password) @@ -73,14 +93,33 @@ func (tr *authTransport) RoundTrip(req *http.Request) (*http.Response, error) { } resp, err := tr.RoundTripper.RoundTrip(req) if err != nil { + delete(tr.modReq, orig) return nil, err } if askedForToken && len(resp.Header["X-Docker-Token"]) > 0 { tr.token = resp.Header["X-Docker-Token"] } + resp.Body = &transport.OnEOFReader{ + Rc: resp.Body, + Fn: func() { delete(tr.modReq, orig) }, + } return resp, nil } +// CancelRequest cancels an in-flight request by closing its connection. +func (tr *authTransport) CancelRequest(req *http.Request) { + type canceler interface { + CancelRequest(*http.Request) + } + if cr, ok := tr.RoundTripper.(canceler); ok { + tr.mu.Lock() + modReq := tr.modReq[req] + delete(tr.modReq, req) + tr.mu.Unlock() + cr.CancelRequest(modReq) + } +} + // TODO(tiborvass): remove authConfig param once registry client v2 is vendored func NewSession(client *http.Client, authConfig *cliconfig.AuthConfig, endpoint *Endpoint) (r *Session, err error) { r = &Session{ @@ -105,7 +144,7 @@ func NewSession(client *http.Client, authConfig *cliconfig.AuthConfig, endpoint } } - client.Transport = &authTransport{RoundTripper: client.Transport, AuthConfig: authConfig, alwaysSetBasicAuth: alwaysSetBasicAuth} + client.Transport = AuthTransport(client.Transport, authConfig, alwaysSetBasicAuth) jar, err := cookiejar.New(nil) if err != nil { diff --git a/docs/session_v2.go b/docs/session_v2.go index c639f922..b6601728 100644 --- a/docs/session_v2.go +++ b/docs/session_v2.go @@ -27,7 +27,7 @@ func getV2Builder(e *Endpoint) *v2.URLBuilder { func (r *Session) V2RegistryEndpoint(index *IndexInfo) (ep *Endpoint, err error) { // TODO check if should use Mirror if index.Official { - ep, err = newEndpoint(REGISTRYSERVER, true) + ep, err = newEndpoint(REGISTRYSERVER, true, nil) if err != nil { return } @@ -38,7 +38,7 @@ func (r *Session) V2RegistryEndpoint(index *IndexInfo) (ep *Endpoint, err error) } else if r.indexEndpoint.String() == index.GetAuthConfigKey() { ep = r.indexEndpoint } else { - ep, err = NewEndpoint(index) + ep, err = NewEndpoint(index, nil) if err != nil { return } diff --git a/docs/token.go b/docs/token.go index af7d5f3f..e27cb6f5 100644 --- a/docs/token.go +++ b/docs/token.go @@ -13,7 +13,7 @@ type tokenResponse struct { Token string `json:"token"` } -func getToken(username, password string, params map[string]string, registryEndpoint *Endpoint, client *http.Client) (token string, err error) { +func getToken(username, password string, params map[string]string, registryEndpoint *Endpoint) (token string, err error) { realm, ok := params["realm"] if !ok { return "", errors.New("no realm specified for token auth challenge") @@ -56,7 +56,7 @@ func getToken(username, password string, params map[string]string, registryEndpo req.URL.RawQuery = reqParams.Encode() - resp, err := client.Do(req) + resp, err := registryEndpoint.client.Do(req) if err != nil { return "", err } From 38f0c6fa8a6d5650046ca7ef50e860c2817ba84d Mon Sep 17 00:00:00 2001 From: Arnaud Porterie Date: Tue, 19 May 2015 10:58:45 -0700 Subject: [PATCH 273/375] Windows: fix registry filepath and location Signed-off-by: Arnaud Porterie --- docs/registry.go | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/docs/registry.go b/docs/registry.go index b0706e34..4436f135 100644 --- a/docs/registry.go +++ b/docs/registry.go @@ -11,6 +11,7 @@ import ( "net/http/httputil" "os" "path" + "path/filepath" "runtime" "strings" "time" @@ -64,8 +65,9 @@ type httpsRequestModifier struct{ tlsConfig *tls.Config } // prefer an fsnotify implementation, but that was out of scope of my refactoring. func (m *httpsRequestModifier) ModifyRequest(req *http.Request) error { var ( - roots *x509.CertPool - certs []tls.Certificate + roots *x509.CertPool + certs []tls.Certificate + hostDir string ) if req.URL.Scheme == "https" { @@ -78,7 +80,11 @@ func (m *httpsRequestModifier) ModifyRequest(req *http.Request) error { return false } - hostDir := path.Join("/etc/docker/certs.d", req.URL.Host) + if runtime.GOOS == "windows" { + hostDir = path.Join(os.TempDir(), "/docker/certs.d", req.URL.Host) + } else { + hostDir = path.Join("/etc/docker/certs.d", req.URL.Host) + } logrus.Debugf("hostDir: %s", hostDir) fs, err := ioutil.ReadDir(hostDir) if err != nil && !os.IsNotExist(err) { @@ -91,7 +97,7 @@ func (m *httpsRequestModifier) ModifyRequest(req *http.Request) error { roots = x509.NewCertPool() } logrus.Debugf("crt: %s", hostDir+"/"+f.Name()) - data, err := ioutil.ReadFile(path.Join(hostDir, f.Name())) + data, err := ioutil.ReadFile(filepath.Join(hostDir, f.Name())) if err != nil { return err } @@ -104,7 +110,7 @@ func (m *httpsRequestModifier) ModifyRequest(req *http.Request) error { if !hasFile(fs, keyName) { return fmt.Errorf("Missing key %s for certificate %s", keyName, certName) } - cert, err := tls.LoadX509KeyPair(path.Join(hostDir, certName), path.Join(hostDir, keyName)) + cert, err := tls.LoadX509KeyPair(filepath.Join(hostDir, certName), path.Join(hostDir, keyName)) if err != nil { return err } From 07e5885de1cb52c1b3a47776df0778f05f279eec Mon Sep 17 00:00:00 2001 From: Jacob Atzen Date: Wed, 20 May 2015 18:00:01 +0200 Subject: [PATCH 274/375] Fix wording in comment Signed-off-by: Jacob Atzen --- docs/session_v2.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/session_v2.go b/docs/session_v2.go index b6601728..43d638c7 100644 --- a/docs/session_v2.go +++ b/docs/session_v2.go @@ -49,7 +49,7 @@ func (r *Session) V2RegistryEndpoint(index *IndexInfo) (ep *Endpoint, err error) } // GetV2Authorization gets the authorization needed to the given image -// If readonly access is requested, then only the authorization may +// If readonly access is requested, then the authorization may // only be used for Get operations. func (r *Session) GetV2Authorization(ep *Endpoint, imageName string, readOnly bool) (auth *RequestAuthorization, err error) { scopes := []string{"pull"} From a1ade52bb6582bcb255e95d3f03f7926d40d0534 Mon Sep 17 00:00:00 2001 From: Tibor Vass Date: Thu, 21 May 2015 16:53:22 -0400 Subject: [PATCH 275/375] registry: fix auth bug Signed-off-by: Tibor Vass --- docs/session.go | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/docs/session.go b/docs/session.go index 8e54bc82..71b27bef 100644 --- a/docs/session.go +++ b/docs/session.go @@ -80,13 +80,10 @@ func (tr *authTransport) RoundTrip(orig *http.Request) (*http.Response, error) { return tr.RoundTripper.RoundTrip(req) } - var askedForToken bool - // Don't override if req.Header.Get("Authorization") == "" { - if req.Header.Get("X-Docker-Token") == "true" { + if req.Header.Get("X-Docker-Token") == "true" && len(tr.Username) > 0 { req.SetBasicAuth(tr.Username, tr.Password) - askedForToken = true } else if len(tr.token) > 0 { req.Header.Set("Authorization", "Token "+strings.Join(tr.token, ",")) } @@ -96,7 +93,7 @@ func (tr *authTransport) RoundTrip(orig *http.Request) (*http.Response, error) { delete(tr.modReq, orig) return nil, err } - if askedForToken && len(resp.Header["X-Docker-Token"]) > 0 { + if len(resp.Header["X-Docker-Token"]) > 0 { tr.token = resp.Header["X-Docker-Token"] } resp.Body = &transport.OnEOFReader{ From 8fc7d769ab3a680e2ae3a93691cbc1ecccf831ee Mon Sep 17 00:00:00 2001 From: Antonio Murdaca Date: Sat, 23 May 2015 23:50:08 +0200 Subject: [PATCH 276/375] Fix race in httpsRequestModifier.ModifyRequest when writing tlsConfig Signed-off-by: Antonio Murdaca --- docs/registry.go | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/docs/registry.go b/docs/registry.go index 4436f135..47bd2553 100644 --- a/docs/registry.go +++ b/docs/registry.go @@ -14,6 +14,7 @@ import ( "path/filepath" "runtime" "strings" + "sync" "time" "github.com/Sirupsen/logrus" @@ -56,7 +57,10 @@ func init() { dockerUserAgent = useragent.AppendVersions("", httpVersion...) } -type httpsRequestModifier struct{ tlsConfig *tls.Config } +type httpsRequestModifier struct { + mu sync.Mutex + tlsConfig *tls.Config +} // DRAGONS(tiborvass): If someone wonders why do we set tlsconfig in a roundtrip, // it's because it's so as to match the current behavior in master: we generate the @@ -125,8 +129,10 @@ func (m *httpsRequestModifier) ModifyRequest(req *http.Request) error { } } } + m.mu.Lock() m.tlsConfig.RootCAs = roots m.tlsConfig.Certificates = certs + m.mu.Unlock() } return nil } @@ -175,7 +181,7 @@ func NewTransport(timeout TimeoutType, secure bool) http.RoundTripper { if secure { // note: httpsTransport also handles http transport // but for HTTPS, it sets up the certs - return transport.NewTransport(tr, &httpsRequestModifier{tlsConfig}) + return transport.NewTransport(tr, &httpsRequestModifier{tlsConfig: tlsConfig}) } return tr From 287cf41118c281e99dbc99c3c45accaa0040fa22 Mon Sep 17 00:00:00 2001 From: Richard Date: Fri, 15 May 2015 17:48:20 -0700 Subject: [PATCH 277/375] Registry v2 mirror support. The v2 registry will act as a pull-through cache, and needs to be handled differently by the client to the v1 registry mirror. See docker/distribution#459 for details Configuration Only one v2 registry can be configured as a mirror. Acceptable configurations in this chanage are: 0...n v1 mirrors or 1 v2 mirror. A mixture of v1 and v2 mirrors is considered an error. Pull If a v2 mirror is configured, all pulls are redirected to that mirror. The mirror will serve the content locally or attempt a pull from the upstream mirror, cache it locally, and then serve to the client. Push If an image is tagged to a mirror, it will be pushed to the mirror and be stored locally there. Otherwise, images are pushed to the hub. This is unchanged behavior. Signed-off-by: Richard Scothern --- docs/config.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/docs/config.go b/docs/config.go index 568756f4..92ef4d99 100644 --- a/docs/config.go +++ b/docs/config.go @@ -189,7 +189,7 @@ func ValidateMirror(val string) (string, error) { return "", fmt.Errorf("Unsupported path/query/fragment at end of the URI") } - return fmt.Sprintf("%s://%s/v1/", uri.Scheme, uri.Host), nil + return fmt.Sprintf("%s://%s/", uri.Scheme, uri.Host), nil } // ValidateIndexName validates an index name. @@ -358,7 +358,9 @@ func (config *ServiceConfig) NewRepositoryInfo(reposName string) (*RepositoryInf // *TODO: Decouple index name from hostname (via registry configuration?) repoInfo.LocalName = repoInfo.Index.Name + "/" + repoInfo.RemoteName repoInfo.CanonicalName = repoInfo.LocalName + } + return repoInfo, nil } From 7d11fc6e5c53fa476b98440c0913aa028b120489 Mon Sep 17 00:00:00 2001 From: Antonio Murdaca Date: Sun, 24 May 2015 15:17:29 +0200 Subject: [PATCH 278/375] Remove PortSpecs from Config Signed-off-by: Antonio Murdaca --- docs/registry_mock_test.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/registry_mock_test.go b/docs/registry_mock_test.go index d58904d0..60173578 100644 --- a/docs/registry_mock_test.go +++ b/docs/registry_mock_test.go @@ -30,7 +30,7 @@ var ( "comment":"test base image","created":"2013-03-23T12:53:11.10432-07:00", "container_config":{"Hostname":"","User":"","Memory":0,"MemorySwap":0, "CpuShares":0,"AttachStdin":false,"AttachStdout":false,"AttachStderr":false, - "PortSpecs":null,"Tty":false,"OpenStdin":false,"StdinOnce":false, + "Tty":false,"OpenStdin":false,"StdinOnce":false, "Env":null,"Cmd":null,"Dns":null,"Image":"","Volumes":null, "VolumesFrom":"","Entrypoint":null},"Size":424242}`, "checksum_simple": "sha256:1ac330d56e05eef6d438586545ceff7550d3bdcb6b19961f12c5ba714ee1bb37", @@ -56,7 +56,7 @@ var ( "comment":"test base image","created":"2013-03-23T12:55:11.10432-07:00", "container_config":{"Hostname":"","User":"","Memory":0,"MemorySwap":0, "CpuShares":0,"AttachStdin":false,"AttachStdout":false,"AttachStderr":false, - "PortSpecs":null,"Tty":false,"OpenStdin":false,"StdinOnce":false, + "Tty":false,"OpenStdin":false,"StdinOnce":false, "Env":null,"Cmd":null,"Dns":null,"Image":"","Volumes":null, "VolumesFrom":"","Entrypoint":null},"Size":424242}`, "checksum_simple": "sha256:bea7bf2e4bacd479344b737328db47b18880d09096e6674165533aa994f5e9f2", From 0ecc759684ddc36cac924bc8b03d8bc4204796e4 Mon Sep 17 00:00:00 2001 From: Stephen J Day Date: Thu, 28 May 2015 22:50:56 -0700 Subject: [PATCH 279/375] Properly verify manifests and layer digests on pull To ensure manifest integrity when pulling by digest, this changeset ensures that not only the remote digest provided by the registry is verified but also that the digest provided on the command line is checked, as well. If this check fails, the pull is cancelled as with an error. Inspection also should that while layers were being verified against their digests, the error was being treated as tech preview image signing verification error. This, in fact, is not a tech preview and opens up the docker daemon to man in the middle attacks that can be avoided with the v2 registry protocol. As a matter of cleanliness, the digest package from the distribution project has been updated to latest version. There were some recent improvements in the digest package. Signed-off-by: Stephen J Day --- docs/session_v2.go | 41 ++++++++++++++++++++++++++++++----------- 1 file changed, 30 insertions(+), 11 deletions(-) diff --git a/docs/session_v2.go b/docs/session_v2.go index 43d638c7..f2b21df4 100644 --- a/docs/session_v2.go +++ b/docs/session_v2.go @@ -68,10 +68,15 @@ func (r *Session) GetV2Authorization(ep *Endpoint, imageName string, readOnly bo // 1.c) if anything else, err // 2) PUT the created/signed manifest // -func (r *Session) GetV2ImageManifest(ep *Endpoint, imageName, tagName string, auth *RequestAuthorization) ([]byte, string, error) { + +// GetV2ImageManifest simply fetches the bytes of a manifest and the remote +// digest, if available in the request. Note that the application shouldn't +// rely on the untrusted remoteDigest, and should also verify against a +// locally provided digest, if applicable. +func (r *Session) GetV2ImageManifest(ep *Endpoint, imageName, tagName string, auth *RequestAuthorization) (remoteDigest digest.Digest, p []byte, err error) { routeURL, err := getV2Builder(ep).BuildManifestURL(imageName, tagName) if err != nil { - return nil, "", err + return "", nil, err } method := "GET" @@ -79,31 +84,45 @@ func (r *Session) GetV2ImageManifest(ep *Endpoint, imageName, tagName string, au req, err := http.NewRequest(method, routeURL, nil) if err != nil { - return nil, "", err + return "", nil, err } + if err := auth.Authorize(req); err != nil { - return nil, "", err + return "", nil, err } + res, err := r.client.Do(req) if err != nil { - return nil, "", err + return "", nil, err } defer res.Body.Close() + if res.StatusCode != 200 { if res.StatusCode == 401 { - return nil, "", errLoginRequired + return "", nil, errLoginRequired } else if res.StatusCode == 404 { - return nil, "", ErrDoesNotExist + return "", nil, ErrDoesNotExist } - return nil, "", httputils.NewHTTPRequestError(fmt.Sprintf("Server error: %d trying to fetch for %s:%s", res.StatusCode, imageName, tagName), res) + return "", nil, httputils.NewHTTPRequestError(fmt.Sprintf("Server error: %d trying to fetch for %s:%s", res.StatusCode, imageName, tagName), res) } - manifestBytes, err := ioutil.ReadAll(res.Body) + p, err = ioutil.ReadAll(res.Body) if err != nil { - return nil, "", fmt.Errorf("Error while reading the http response: %s", err) + return "", nil, fmt.Errorf("Error while reading the http response: %s", err) } - return manifestBytes, res.Header.Get(DockerDigestHeader), nil + dgstHdr := res.Header.Get(DockerDigestHeader) + if dgstHdr != "" { + remoteDigest, err = digest.ParseDigest(dgstHdr) + if err != nil { + // NOTE(stevvooe): Including the remote digest is optional. We + // don't need to verify against it, but it is good practice. + remoteDigest = "" + logrus.Debugf("error parsing remote digest when fetching %v: %v", routeURL, err) + } + } + + return } // - Succeeded to head image blob (already exists) From 767c5283a28f68751c3003bc7dec4f43338020af Mon Sep 17 00:00:00 2001 From: Alexander Morozov Date: Mon, 1 Jun 2015 13:25:18 -0700 Subject: [PATCH 280/375] Fix race condition in registry/session Signed-off-by: Alexander Morozov --- docs/session.go | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/docs/session.go b/docs/session.go index 71b27bef..22702108 100644 --- a/docs/session.go +++ b/docs/session.go @@ -98,7 +98,11 @@ func (tr *authTransport) RoundTrip(orig *http.Request) (*http.Response, error) { } resp.Body = &transport.OnEOFReader{ Rc: resp.Body, - Fn: func() { delete(tr.modReq, orig) }, + Fn: func() { + tr.mu.Lock() + delete(tr.modReq, orig) + tr.mu.Unlock() + }, } return resp, nil } From 6640f60cc58587d49a4fcb37466cc5f753102cd3 Mon Sep 17 00:00:00 2001 From: Tibor Vass Date: Mon, 1 Jun 2015 17:48:30 -0400 Subject: [PATCH 281/375] registry: debugTransport should print with testing.T.Log It should not print to STDOUT so that it only prints the debugTransport output if there was an error in one of the registry tests. Signed-off-by: Tibor Vass --- docs/registry.go | 13 ++++++++----- docs/registry_test.go | 2 +- 2 files changed, 9 insertions(+), 6 deletions(-) diff --git a/docs/registry.go b/docs/registry.go index 47bd2553..80d4268e 100644 --- a/docs/registry.go +++ b/docs/registry.go @@ -200,23 +200,26 @@ func DockerHeaders(metaHeaders http.Header) []transport.RequestModifier { return modifiers } -type debugTransport struct{ http.RoundTripper } +type debugTransport struct { + http.RoundTripper + log func(...interface{}) +} func (tr debugTransport) RoundTrip(req *http.Request) (*http.Response, error) { dump, err := httputil.DumpRequestOut(req, false) if err != nil { - fmt.Println("could not dump request") + tr.log("could not dump request") } - fmt.Println(string(dump)) + tr.log(string(dump)) resp, err := tr.RoundTripper.RoundTrip(req) if err != nil { return nil, err } dump, err = httputil.DumpResponse(resp, false) if err != nil { - fmt.Println("could not dump response") + tr.log("could not dump response") } - fmt.Println(string(dump)) + tr.log(string(dump)) return resp, err } diff --git a/docs/registry_test.go b/docs/registry_test.go index 33e86ff4..eee801d4 100644 --- a/docs/registry_test.go +++ b/docs/registry_test.go @@ -26,7 +26,7 @@ func spawnTestRegistrySession(t *testing.T) *Session { if err != nil { t.Fatal(err) } - var tr http.RoundTripper = debugTransport{NewTransport(ReceiveTimeout, endpoint.IsSecure)} + var tr http.RoundTripper = debugTransport{NewTransport(ReceiveTimeout, endpoint.IsSecure), t.Log} tr = transport.NewTransport(AuthTransport(tr, authConfig, false), DockerHeaders(nil)...) client := HTTPClient(tr) r, err := NewSession(client, authConfig, endpoint) From 5418e3be0c799c868cb7440ac7837933a153b399 Mon Sep 17 00:00:00 2001 From: Jeffrey van Gogh Date: Mon, 1 Jun 2015 15:13:35 -0700 Subject: [PATCH 282/375] Upon HTTP 302 redirect do not include "Authorization" header on 'untrusted' registries. Refactoring in Docker 1.7 changed the behavior to add this header where as Docker <= 1.6 wouldn't emit this Header on a HTTP 302 redirect. This closes #13649 Signed-off-by: Jeffrey van Gogh --- docs/session.go | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/docs/session.go b/docs/session.go index 71b27bef..024685ae 100644 --- a/docs/session.go +++ b/docs/session.go @@ -84,7 +84,13 @@ func (tr *authTransport) RoundTrip(orig *http.Request) (*http.Response, error) { if req.Header.Get("Authorization") == "" { if req.Header.Get("X-Docker-Token") == "true" && len(tr.Username) > 0 { req.SetBasicAuth(tr.Username, tr.Password) - } else if len(tr.token) > 0 { + } else if len(tr.token) > 0 && + // Authorization should not be set on 302 redirect for untrusted locations. + // This logic mirrors the behavior in AddRequiredHeadersToRedirectedRequests. + // As the authorization logic is currently implemented in RoundTrip, + // a 302 redirect is detected by looking at the Referer header as go http package adds said header. + // This is safe as Docker doesn't set Referer in other scenarios. + (req.Header.Get("Referer") == "" || trustedLocation(orig)) { req.Header.Set("Authorization", "Token "+strings.Join(tr.token, ",")) } } From c01c508ea1a6500dd18df9c8034e1fff85cd30d0 Mon Sep 17 00:00:00 2001 From: Matt Moore Date: Thu, 28 May 2015 20:46:20 -0700 Subject: [PATCH 283/375] Make the v2 logic fallback on v1 when v2 requests cannot be authorized. Signed-off-by: Matt Moore --- docs/auth.go | 28 +++++++++++++++++++++++----- 1 file changed, 23 insertions(+), 5 deletions(-) diff --git a/docs/auth.go b/docs/auth.go index 33f8fa06..66b3438f 100644 --- a/docs/auth.go +++ b/docs/auth.go @@ -74,6 +74,19 @@ func (auth *RequestAuthorization) getToken() (string, error) { return "", nil } +// Checks that requests to the v2 registry can be authorized. +func (auth *RequestAuthorization) CanAuthorizeV2() bool { + if len(auth.registryEndpoint.AuthChallenges) == 0 { + return true + } + scope := fmt.Sprintf("%s:%s:%s", auth.resource, auth.scope, strings.Join(auth.actions, ",")) + if _, err := loginV2(auth.authConfig, auth.registryEndpoint, scope); err != nil { + logrus.Debugf("Cannot authorize against V2 endpoint: %s", auth.registryEndpoint) + return false + } + return true +} + func (auth *RequestAuthorization) Authorize(req *http.Request) error { token, err := auth.getToken() if err != nil { @@ -91,7 +104,7 @@ func (auth *RequestAuthorization) Authorize(req *http.Request) error { func Login(authConfig *cliconfig.AuthConfig, registryEndpoint *Endpoint) (string, error) { // Separates the v2 registry login logic from the v1 logic. if registryEndpoint.Version == APIVersion2 { - return loginV2(authConfig, registryEndpoint) + return loginV2(authConfig, registryEndpoint, "" /* scope */) } return loginV1(authConfig, registryEndpoint) } @@ -209,7 +222,7 @@ func loginV1(authConfig *cliconfig.AuthConfig, registryEndpoint *Endpoint) (stri // now, users should create their account through other means like directly from a web page // served by the v2 registry service provider. Whether this will be supported in the future // is to be determined. -func loginV2(authConfig *cliconfig.AuthConfig, registryEndpoint *Endpoint) (string, error) { +func loginV2(authConfig *cliconfig.AuthConfig, registryEndpoint *Endpoint, scope string) (string, error) { logrus.Debugf("attempting v2 login to registry endpoint %s", registryEndpoint) var ( err error @@ -217,13 +230,18 @@ func loginV2(authConfig *cliconfig.AuthConfig, registryEndpoint *Endpoint) (stri ) for _, challenge := range registryEndpoint.AuthChallenges { - logrus.Debugf("trying %q auth challenge with params %s", challenge.Scheme, challenge.Parameters) + params := make(map[string]string, len(challenge.Parameters)+1) + for k, v := range challenge.Parameters { + params[k] = v + } + params["scope"] = scope + logrus.Debugf("trying %q auth challenge with params %v", challenge.Scheme, params) switch strings.ToLower(challenge.Scheme) { case "basic": - err = tryV2BasicAuthLogin(authConfig, challenge.Parameters, registryEndpoint) + err = tryV2BasicAuthLogin(authConfig, params, registryEndpoint) case "bearer": - err = tryV2TokenAuthLogin(authConfig, challenge.Parameters, registryEndpoint) + err = tryV2TokenAuthLogin(authConfig, params, registryEndpoint) default: // Unsupported challenge types are explicitly skipped. err = fmt.Errorf("unsupported auth scheme: %q", challenge.Scheme) From 5a8f690426b9e1ea9490064b08aa9511c23075e4 Mon Sep 17 00:00:00 2001 From: Tibor Vass Date: Mon, 8 Jun 2015 19:56:37 -0400 Subject: [PATCH 284/375] Do not set auth headers if 302 This patch ensures no auth headers are set for v1 registries if there was a 302 redirect. This also ensures v2 does not use authTransport. Signed-off-by: Tibor Vass --- docs/session.go | 21 +++++++++++++-------- 1 file changed, 13 insertions(+), 8 deletions(-) diff --git a/docs/session.go b/docs/session.go index d7478f6c..ca1f8e49 100644 --- a/docs/session.go +++ b/docs/session.go @@ -70,6 +70,15 @@ func AuthTransport(base http.RoundTripper, authConfig *cliconfig.AuthConfig, alw } func (tr *authTransport) RoundTrip(orig *http.Request) (*http.Response, error) { + // Authorization should not be set on 302 redirect for untrusted locations. + // This logic mirrors the behavior in AddRequiredHeadersToRedirectedRequests. + // As the authorization logic is currently implemented in RoundTrip, + // a 302 redirect is detected by looking at the Referer header as go http package adds said header. + // This is safe as Docker doesn't set Referer in other scenarios. + if orig.Header.Get("Referer") != "" && !trustedLocation(orig) { + return tr.RoundTripper.RoundTrip(orig) + } + req := transport.CloneRequest(orig) tr.mu.Lock() tr.modReq[orig] = req @@ -84,13 +93,7 @@ func (tr *authTransport) RoundTrip(orig *http.Request) (*http.Response, error) { if req.Header.Get("Authorization") == "" { if req.Header.Get("X-Docker-Token") == "true" && len(tr.Username) > 0 { req.SetBasicAuth(tr.Username, tr.Password) - } else if len(tr.token) > 0 && - // Authorization should not be set on 302 redirect for untrusted locations. - // This logic mirrors the behavior in AddRequiredHeadersToRedirectedRequests. - // As the authorization logic is currently implemented in RoundTrip, - // a 302 redirect is detected by looking at the Referer header as go http package adds said header. - // This is safe as Docker doesn't set Referer in other scenarios. - (req.Header.Get("Referer") == "" || trustedLocation(orig)) { + } else if len(tr.token) > 0 { req.Header.Set("Authorization", "Token "+strings.Join(tr.token, ",")) } } @@ -151,7 +154,9 @@ func NewSession(client *http.Client, authConfig *cliconfig.AuthConfig, endpoint } } - client.Transport = AuthTransport(client.Transport, authConfig, alwaysSetBasicAuth) + if endpoint.Version == APIVersion1 { + client.Transport = AuthTransport(client.Transport, authConfig, alwaysSetBasicAuth) + } jar, err := cookiejar.New(nil) if err != nil { From 5b3e2c7dda7c58a137cbe4c36b809c917af96ce7 Mon Sep 17 00:00:00 2001 From: xiekeyang Date: Tue, 9 Jun 2015 10:07:48 +0800 Subject: [PATCH 285/375] Registry: remove unwanted return variable name Signed-off-by: xiekeyang --- docs/token.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/token.go b/docs/token.go index e27cb6f5..d91bd455 100644 --- a/docs/token.go +++ b/docs/token.go @@ -13,7 +13,7 @@ type tokenResponse struct { Token string `json:"token"` } -func getToken(username, password string, params map[string]string, registryEndpoint *Endpoint) (token string, err error) { +func getToken(username, password string, params map[string]string, registryEndpoint *Endpoint) (string, error) { realm, ok := params["realm"] if !ok { return "", errors.New("no realm specified for token auth challenge") From 13b279f5b6806f39f2c7bd7f86644a6550214789 Mon Sep 17 00:00:00 2001 From: Don Kjer Date: Wed, 10 Jun 2015 22:18:15 +0000 Subject: [PATCH 286/375] Only pulling single repository tag on pull for a specific tag. extending TestGetRemoteTags unit test Splitting out GetRemoteTag from GetRemoteTags. Adding registry.ErrRepoNotFound error Signed-off-by: Don Kjer --- docs/registry_mock_test.go | 1 + docs/registry_test.go | 21 ++++++++++++++++++--- docs/session.go | 38 +++++++++++++++++++++++++++++++++++++- 3 files changed, 56 insertions(+), 4 deletions(-) diff --git a/docs/registry_mock_test.go b/docs/registry_mock_test.go index 60173578..eab87d46 100644 --- a/docs/registry_mock_test.go +++ b/docs/registry_mock_test.go @@ -81,6 +81,7 @@ var ( testRepositories = map[string]map[string]string{ "foo42/bar": { "latest": "42d718c941f5c532ac049bf0b0ab53f0062f09a03afd4aa4a02c098e46032b9d", + "test": "42d718c941f5c532ac049bf0b0ab53f0062f09a03afd4aa4a02c098e46032b9d", }, } mockHosts = map[string][]net.IP{ diff --git a/docs/registry_test.go b/docs/registry_test.go index eee801d4..bb2761c5 100644 --- a/docs/registry_test.go +++ b/docs/registry_test.go @@ -211,18 +211,33 @@ func TestGetRemoteImageLayer(t *testing.T) { } } +func TestGetRemoteTag(t *testing.T) { + r := spawnTestRegistrySession(t) + tag, err := r.GetRemoteTag([]string{makeURL("/v1/")}, REPO, "test") + if err != nil { + t.Fatal(err) + } + assertEqual(t, tag, imageID, "Expected tag test to map to "+imageID) + + _, err = r.GetRemoteTag([]string{makeURL("/v1/")}, "foo42/baz", "foo") + if err != ErrRepoNotFound { + t.Fatal("Expected ErrRepoNotFound error when fetching tag for bogus repo") + } +} + func TestGetRemoteTags(t *testing.T) { r := spawnTestRegistrySession(t) tags, err := r.GetRemoteTags([]string{makeURL("/v1/")}, REPO) if err != nil { t.Fatal(err) } - assertEqual(t, len(tags), 1, "Expected one tag") + assertEqual(t, len(tags), 2, "Expected two tags") assertEqual(t, tags["latest"], imageID, "Expected tag latest to map to "+imageID) + assertEqual(t, tags["test"], imageID, "Expected tag test to map to "+imageID) _, err = r.GetRemoteTags([]string{makeURL("/v1/")}, "foo42/baz") - if err == nil { - t.Fatal("Expected error when fetching tags for bogus repo") + if err != ErrRepoNotFound { + t.Fatal("Expected ErrRepoNotFound error when fetching tags for bogus repo") } } diff --git a/docs/session.go b/docs/session.go index ca1f8e49..573a03bf 100644 --- a/docs/session.go +++ b/docs/session.go @@ -26,6 +26,10 @@ import ( "github.com/docker/docker/pkg/transport" ) +var ( + ErrRepoNotFound = errors.New("Repository not found") +) + type Session struct { indexEndpoint *Endpoint client *http.Client @@ -279,6 +283,38 @@ func (r *Session) GetRemoteImageLayer(imgID, registry string, imgSize int64) (io return res.Body, nil } +func (r *Session) GetRemoteTag(registries []string, repository string, askedTag string) (string, error) { + if strings.Count(repository, "/") == 0 { + // This will be removed once the Registry supports auto-resolution on + // the "library" namespace + repository = "library/" + repository + } + for _, host := range registries { + endpoint := fmt.Sprintf("%srepositories/%s/tags/%s", host, repository, askedTag) + res, err := r.client.Get(endpoint) + if err != nil { + return "", err + } + + logrus.Debugf("Got status code %d from %s", res.StatusCode, endpoint) + defer res.Body.Close() + + if res.StatusCode == 404 { + return "", ErrRepoNotFound + } + if res.StatusCode != 200 { + continue + } + + var tagId string + if err := json.NewDecoder(res.Body).Decode(&tagId); err != nil { + return "", err + } + return tagId, nil + } + return "", fmt.Errorf("Could not reach any registry endpoint") +} + func (r *Session) GetRemoteTags(registries []string, repository string) (map[string]string, error) { if strings.Count(repository, "/") == 0 { // This will be removed once the Registry supports auto-resolution on @@ -296,7 +332,7 @@ func (r *Session) GetRemoteTags(registries []string, repository string) (map[str defer res.Body.Close() if res.StatusCode == 404 { - return nil, fmt.Errorf("Repository not found") + return nil, ErrRepoNotFound } if res.StatusCode != 200 { continue From d4c7ea430101b29100854928e88f55d96f95445b Mon Sep 17 00:00:00 2001 From: Shishir Mahajan Date: Fri, 29 May 2015 10:22:33 -0400 Subject: [PATCH 287/375] Use distribution's ValidateRepositoryName for remote name validation. Signed-off-by: Shishir Mahajan --- docs/config.go | 42 +++++++----------------------------------- docs/registry_test.go | 8 +++++--- 2 files changed, 12 insertions(+), 38 deletions(-) diff --git a/docs/config.go b/docs/config.go index 92ef4d99..a336d743 100644 --- a/docs/config.go +++ b/docs/config.go @@ -6,9 +6,9 @@ import ( "fmt" "net" "net/url" - "regexp" "strings" + "github.com/docker/distribution/registry/api/v2" "github.com/docker/docker/image" "github.com/docker/docker/opts" flag "github.com/docker/docker/pkg/mflag" @@ -32,8 +32,6 @@ const ( var ( ErrInvalidRepositoryName = errors.New("Invalid repository name (ex: \"registry.domain.tld/myrepos\")") emptyServiceConfig = NewServiceConfig(nil) - validNamespaceChars = regexp.MustCompile(`^([a-z0-9-_]*)$`) - validRepo = regexp.MustCompile(`^([a-z0-9-_.]+)$`) ) func IndexServerAddress() string { @@ -206,42 +204,16 @@ func ValidateIndexName(val string) (string, error) { } func validateRemoteName(remoteName string) error { - var ( - namespace string - name string - ) - nameParts := strings.SplitN(remoteName, "/", 2) - if len(nameParts) < 2 { - namespace = "library" - name = nameParts[0] + + if !strings.Contains(remoteName, "/") { // the repository name must not be a valid image ID - if err := image.ValidateID(name); err == nil { - return fmt.Errorf("Invalid repository name (%s), cannot specify 64-byte hexadecimal strings", name) + if err := image.ValidateID(remoteName); err == nil { + return fmt.Errorf("Invalid repository name (%s), cannot specify 64-byte hexadecimal strings", remoteName) } - } else { - namespace = nameParts[0] - name = nameParts[1] } - if !validNamespaceChars.MatchString(namespace) { - return fmt.Errorf("Invalid namespace name (%s). Only [a-z0-9-_] are allowed.", namespace) - } - if len(namespace) < 2 || len(namespace) > 255 { - return fmt.Errorf("Invalid namespace name (%s). Cannot be fewer than 2 or more than 255 characters.", namespace) - } - if strings.HasPrefix(namespace, "-") || strings.HasSuffix(namespace, "-") { - return fmt.Errorf("Invalid namespace name (%s). Cannot begin or end with a hyphen.", namespace) - } - if strings.Contains(namespace, "--") { - return fmt.Errorf("Invalid namespace name (%s). Cannot contain consecutive hyphens.", namespace) - } - if !validRepo.MatchString(name) { - return fmt.Errorf("Invalid repository name (%s), only [a-z0-9-_.] are allowed", name) - } - if strings.HasPrefix(name, "-") || strings.HasSuffix(name, "-") { - return fmt.Errorf("Invalid repository name (%s). Cannot begin or end with a hyphen.", name) - } - return nil + + return v2.ValidateRepositoryName(remoteName) } func validateNoSchema(reposName string) error { diff --git a/docs/registry_test.go b/docs/registry_test.go index eee801d4..1ee48d00 100644 --- a/docs/registry_test.go +++ b/docs/registry_test.go @@ -742,9 +742,6 @@ func TestValidRemoteName(t *testing.T) { // Allow embedded hyphens. "docker-rules/docker", - // Allow underscores everywhere (as opposed to hyphens). - "____/____", - //Username doc and image name docker being tested. "doc/docker", } @@ -769,6 +766,11 @@ func TestValidRemoteName(t *testing.T) { "docker-/docker", "-docker-/docker", + // Don't allow underscores everywhere (as opposed to hyphens). + "____/____", + + "_docker/_docker", + // Disallow consecutive hyphens. "dock--er/docker", From 79661b8a7e7b075d9b56c2bced927e996f07d8f0 Mon Sep 17 00:00:00 2001 From: Matt Moore Date: Fri, 19 Jun 2015 10:12:52 -0700 Subject: [PATCH 288/375] Unconditionally add AuthTransport. Today, endpoints implementing v2 cannot properly fallback to v1 because the underlying transport that deals with authentication (Basic / Token) doesn't get annotated. This doesn't affect DockerHub because the DockerHub endpoint appears as 'https://index.docker.io/v1/' (in .dockercfg), and the 'v1' tricks this logic just long enough that the transport is always annotated for DockerHub accesses. Signed-off-by: Matt Moore --- docs/session.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/session.go b/docs/session.go index 573a03bf..77f6d20b 100644 --- a/docs/session.go +++ b/docs/session.go @@ -158,9 +158,9 @@ func NewSession(client *http.Client, authConfig *cliconfig.AuthConfig, endpoint } } - if endpoint.Version == APIVersion1 { - client.Transport = AuthTransport(client.Transport, authConfig, alwaysSetBasicAuth) - } + // Annotate the transport unconditionally so that v2 can + // properly fallback on v1 when an image is not found. + client.Transport = AuthTransport(client.Transport, authConfig, alwaysSetBasicAuth) jar, err := cookiejar.New(nil) if err != nil { From ebd569961dbc1f8265c56ef9db6ef3bc84b9bfd3 Mon Sep 17 00:00:00 2001 From: Antonio Murdaca Date: Sat, 20 Jun 2015 14:28:18 +0200 Subject: [PATCH 289/375] Remove dead code Signed-off-by: Antonio Murdaca --- docs/registry.go | 24 ------------------------ docs/registry_test.go | 24 ++++++++++++++++++++++++ 2 files changed, 24 insertions(+), 24 deletions(-) diff --git a/docs/registry.go b/docs/registry.go index 80d4268e..8b78af96 100644 --- a/docs/registry.go +++ b/docs/registry.go @@ -8,7 +8,6 @@ import ( "io/ioutil" "net" "net/http" - "net/http/httputil" "os" "path" "path/filepath" @@ -200,29 +199,6 @@ func DockerHeaders(metaHeaders http.Header) []transport.RequestModifier { return modifiers } -type debugTransport struct { - http.RoundTripper - log func(...interface{}) -} - -func (tr debugTransport) RoundTrip(req *http.Request) (*http.Response, error) { - dump, err := httputil.DumpRequestOut(req, false) - if err != nil { - tr.log("could not dump request") - } - tr.log(string(dump)) - resp, err := tr.RoundTripper.RoundTrip(req) - if err != nil { - return nil, err - } - dump, err = httputil.DumpResponse(resp, false) - if err != nil { - tr.log("could not dump response") - } - tr.log(string(dump)) - return resp, err -} - func HTTPClient(transport http.RoundTripper) *http.Client { if transport == nil { transport = NewTransport(ConnectTimeout, true) diff --git a/docs/registry_test.go b/docs/registry_test.go index a6bd7201..7233075b 100644 --- a/docs/registry_test.go +++ b/docs/registry_test.go @@ -3,6 +3,7 @@ package registry import ( "fmt" "net/http" + "net/http/httputil" "net/url" "strings" "testing" @@ -911,3 +912,26 @@ func TestIsSecureIndex(t *testing.T) { } } } + +type debugTransport struct { + http.RoundTripper + log func(...interface{}) +} + +func (tr debugTransport) RoundTrip(req *http.Request) (*http.Response, error) { + dump, err := httputil.DumpRequestOut(req, false) + if err != nil { + tr.log("could not dump request") + } + tr.log(string(dump)) + resp, err := tr.RoundTripper.RoundTrip(req) + if err != nil { + return nil, err + } + dump, err = httputil.DumpResponse(resp, false) + if err != nil { + tr.log("could not dump response") + } + tr.log(string(dump)) + return resp, err +} From f432bcc925a97cbf323540a3fa0776072f078d52 Mon Sep 17 00:00:00 2001 From: Eric Windisch Date: Wed, 10 Jun 2015 13:37:31 -0400 Subject: [PATCH 290/375] Remove RC4 from the list of registry cipher suites The registry client's TLS configuration used the default cipher list, including RC4. This change copies the default cipher list from Golang 1.4 and removes RC4 from that list. RC4 ciphers are considered weak and vulnerable to a number of attacks. Uses the tlsconfig package to define allowed ciphers. Signed-off-by: Eric Windisch --- docs/registry.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/registry.go b/docs/registry.go index 8b78af96..fb08e5bd 100644 --- a/docs/registry.go +++ b/docs/registry.go @@ -20,6 +20,7 @@ import ( "github.com/docker/docker/autogen/dockerversion" "github.com/docker/docker/pkg/parsers/kernel" "github.com/docker/docker/pkg/timeoutconn" + "github.com/docker/docker/pkg/tlsconfig" "github.com/docker/docker/pkg/transport" "github.com/docker/docker/pkg/useragent" ) @@ -141,6 +142,7 @@ func NewTransport(timeout TimeoutType, secure bool) http.RoundTripper { // Avoid fallback to SSL protocols < TLS1.0 MinVersion: tls.VersionTLS10, InsecureSkipVerify: !secure, + CipherSuites: tlsconfig.DefaultServerAcceptedCiphers, } tr := &http.Transport{ From 8e857d114742f633b5277e535011f53d6375e1e5 Mon Sep 17 00:00:00 2001 From: Ankush Agarwal Date: Wed, 1 Jul 2015 13:02:55 -0700 Subject: [PATCH 291/375] Add 500 check for registry api call Partially Addresses #14326 Signed-off-by: Ankush Agarwal --- docs/auth.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/docs/auth.go b/docs/auth.go index 66b3438f..65991d69 100644 --- a/docs/auth.go +++ b/docs/auth.go @@ -180,6 +180,9 @@ func loginV1(authConfig *cliconfig.AuthConfig, registryEndpoint *Endpoint) (stri } // *TODO: Use registry configuration to determine what this says, if anything? return "", fmt.Errorf("Login: Account is not Active. Please see the documentation of the registry %s for instructions how to activate it.", serverAddress) + } else if resp.StatusCode == 500 { // Issue #14326 + logrus.Errorf("%s returned status code %d. Response Body :\n%s", req.URL.String(), resp.StatusCode, body) + return "", fmt.Errorf("Internal Server Error") } return "", fmt.Errorf("Login: %s (Code: %d; Headers: %s)", body, resp.StatusCode, resp.Header) } From c82a9a817f4e565586b3e3378595e8274f860391 Mon Sep 17 00:00:00 2001 From: Matt Moore Date: Thu, 9 Jul 2015 20:56:23 -0700 Subject: [PATCH 292/375] Add the X-Docker-Token header to the /v1/search requests. By adding this header AuthTransport will add Basic authentication to the request and allow 'docker search' results to include private images. Signed-off-by: Matt Moore --- docs/session.go | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/docs/session.go b/docs/session.go index 77f6d20b..7d57f1b8 100644 --- a/docs/session.go +++ b/docs/session.go @@ -676,7 +676,14 @@ func shouldRedirect(response *http.Response) bool { func (r *Session) SearchRepositories(term string) (*SearchResults, error) { logrus.Debugf("Index server: %s", r.indexEndpoint) u := r.indexEndpoint.VersionString(1) + "search?q=" + url.QueryEscape(term) - res, err := r.client.Get(u) + + req, err := http.NewRequest("GET", u, nil) + if err != nil { + return nil, fmt.Errorf("Error while getting from the server: %v", err) + } + // Have the AuthTransport send authentication, when logged in. + req.Header.Set("X-Docker-Token", "true") + res, err := r.client.Do(req) if err != nil { return nil, err } From b425c402fb7d38088b8b461087b19b3e70231697 Mon Sep 17 00:00:00 2001 From: Stephen J Day Date: Fri, 10 Jul 2015 14:06:15 -0600 Subject: [PATCH 293/375] Allow one character repository name components The docker/distribution dependency was updated in the previous commit to allow repository name components to only consist of a single letter. The unit tests have been updated to cement this change. Signed-off-by: Stephen J Day --- docs/registry_test.go | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/docs/registry_test.go b/docs/registry_test.go index 7233075b..3a996401 100644 --- a/docs/registry_test.go +++ b/docs/registry_test.go @@ -760,6 +760,10 @@ func TestValidRemoteName(t *testing.T) { //Username doc and image name docker being tested. "doc/docker", + + // single character names are now allowed. + "d/docker", + "jess/t", } for _, repositoryName := range validRepositoryNames { if err := validateRemoteName(repositoryName); err != nil { @@ -793,9 +797,6 @@ func TestValidRemoteName(t *testing.T) { // No repository. "docker/", - //namespace too short - "d/docker", - //namespace too long "this_is_not_a_valid_namespace_because_its_lenth_is_greater_than_255_this_is_not_a_valid_namespace_because_its_lenth_is_greater_than_255_this_is_not_a_valid_namespace_because_its_lenth_is_greater_than_255_this_is_not_a_valid_namespace_because_its_lenth_is_greater_than_255/docker", } From 01c8fb36657a2147835cf45e2f4012f84e9c9e59 Mon Sep 17 00:00:00 2001 From: Derek McGowan Date: Tue, 14 Jul 2015 17:45:49 -0700 Subject: [PATCH 294/375] Set canonical name correctly Currently canonical name gets set to the local name and displayed in the errors. Canonical name should be the unique and canonical name for an image. Use docker.io as the canonical domain for images on the public registry. Signed-off-by: Derek McGowan (github: dmcgowan) --- docs/config.go | 4 +--- docs/registry_test.go | 24 ++++++++++++------------ 2 files changed, 13 insertions(+), 15 deletions(-) diff --git a/docs/config.go b/docs/config.go index a336d743..105ec61d 100644 --- a/docs/config.go +++ b/docs/config.go @@ -324,10 +324,8 @@ func (config *ServiceConfig) NewRepositoryInfo(reposName string) (*RepositoryInf repoInfo.RemoteName = "library/" + normalizedName } - // *TODO: Prefix this with 'docker.io/'. - repoInfo.CanonicalName = repoInfo.LocalName + repoInfo.CanonicalName = "docker.io/" + repoInfo.RemoteName } else { - // *TODO: Decouple index name from hostname (via registry configuration?) repoInfo.LocalName = repoInfo.Index.Name + "/" + repoInfo.RemoteName repoInfo.CanonicalName = repoInfo.LocalName diff --git a/docs/registry_test.go b/docs/registry_test.go index 3a996401..52a2dc30 100644 --- a/docs/registry_test.go +++ b/docs/registry_test.go @@ -337,7 +337,7 @@ func TestParseRepositoryInfo(t *testing.T) { }, RemoteName: "fooo/bar", LocalName: "fooo/bar", - CanonicalName: "fooo/bar", + CanonicalName: "docker.io/fooo/bar", Official: false, }, "library/ubuntu": { @@ -347,7 +347,7 @@ func TestParseRepositoryInfo(t *testing.T) { }, RemoteName: "library/ubuntu", LocalName: "ubuntu", - CanonicalName: "ubuntu", + CanonicalName: "docker.io/library/ubuntu", Official: true, }, "nonlibrary/ubuntu": { @@ -357,7 +357,7 @@ func TestParseRepositoryInfo(t *testing.T) { }, RemoteName: "nonlibrary/ubuntu", LocalName: "nonlibrary/ubuntu", - CanonicalName: "nonlibrary/ubuntu", + CanonicalName: "docker.io/nonlibrary/ubuntu", Official: false, }, "ubuntu": { @@ -367,7 +367,7 @@ func TestParseRepositoryInfo(t *testing.T) { }, RemoteName: "library/ubuntu", LocalName: "ubuntu", - CanonicalName: "ubuntu", + CanonicalName: "docker.io/library/ubuntu", Official: true, }, "other/library": { @@ -377,7 +377,7 @@ func TestParseRepositoryInfo(t *testing.T) { }, RemoteName: "other/library", LocalName: "other/library", - CanonicalName: "other/library", + CanonicalName: "docker.io/other/library", Official: false, }, "127.0.0.1:8000/private/moonbase": { @@ -487,7 +487,7 @@ func TestParseRepositoryInfo(t *testing.T) { }, RemoteName: "public/moonbase", LocalName: "public/moonbase", - CanonicalName: "public/moonbase", + CanonicalName: "docker.io/public/moonbase", Official: false, }, "index." + IndexServerName() + "/public/moonbase": { @@ -497,7 +497,7 @@ func TestParseRepositoryInfo(t *testing.T) { }, RemoteName: "public/moonbase", LocalName: "public/moonbase", - CanonicalName: "public/moonbase", + CanonicalName: "docker.io/public/moonbase", Official: false, }, IndexServerName() + "/public/moonbase": { @@ -507,7 +507,7 @@ func TestParseRepositoryInfo(t *testing.T) { }, RemoteName: "public/moonbase", LocalName: "public/moonbase", - CanonicalName: "public/moonbase", + CanonicalName: "docker.io/public/moonbase", Official: false, }, "ubuntu-12.04-base": { @@ -517,7 +517,7 @@ func TestParseRepositoryInfo(t *testing.T) { }, RemoteName: "library/ubuntu-12.04-base", LocalName: "ubuntu-12.04-base", - CanonicalName: "ubuntu-12.04-base", + CanonicalName: "docker.io/library/ubuntu-12.04-base", Official: true, }, IndexServerName() + "/ubuntu-12.04-base": { @@ -527,7 +527,7 @@ func TestParseRepositoryInfo(t *testing.T) { }, RemoteName: "library/ubuntu-12.04-base", LocalName: "ubuntu-12.04-base", - CanonicalName: "ubuntu-12.04-base", + CanonicalName: "docker.io/library/ubuntu-12.04-base", Official: true, }, IndexServerName() + "/ubuntu-12.04-base": { @@ -537,7 +537,7 @@ func TestParseRepositoryInfo(t *testing.T) { }, RemoteName: "library/ubuntu-12.04-base", LocalName: "ubuntu-12.04-base", - CanonicalName: "ubuntu-12.04-base", + CanonicalName: "docker.io/library/ubuntu-12.04-base", Official: true, }, "index." + IndexServerName() + "/ubuntu-12.04-base": { @@ -547,7 +547,7 @@ func TestParseRepositoryInfo(t *testing.T) { }, RemoteName: "library/ubuntu-12.04-base", LocalName: "ubuntu-12.04-base", - CanonicalName: "ubuntu-12.04-base", + CanonicalName: "docker.io/library/ubuntu-12.04-base", Official: true, }, } From 45bd073e54614561daf1809e0a5e601ca041eecd Mon Sep 17 00:00:00 2001 From: Tibor Vass Date: Thu, 16 Jul 2015 12:38:44 -0400 Subject: [PATCH 295/375] Fix issue where Search API endpoint would panic due to empty AuthConfig Signed-off-by: Tibor Vass --- docs/session.go | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/docs/session.go b/docs/session.go index 7d57f1b8..07195eb1 100644 --- a/docs/session.go +++ b/docs/session.go @@ -89,13 +89,16 @@ func (tr *authTransport) RoundTrip(orig *http.Request) (*http.Response, error) { tr.mu.Unlock() if tr.alwaysSetBasicAuth { + if tr.AuthConfig == nil { + return nil, errors.New("unexpected error: empty auth config") + } req.SetBasicAuth(tr.Username, tr.Password) return tr.RoundTripper.RoundTrip(req) } // Don't override if req.Header.Get("Authorization") == "" { - if req.Header.Get("X-Docker-Token") == "true" && len(tr.Username) > 0 { + if req.Header.Get("X-Docker-Token") == "true" && tr.AuthConfig != nil && len(tr.Username) > 0 { req.SetBasicAuth(tr.Username, tr.Password) } else if len(tr.token) > 0 { req.Header.Set("Authorization", "Token "+strings.Join(tr.token, ",")) From 950cf586c8336ad182793f1f3e9828f986c7a9b9 Mon Sep 17 00:00:00 2001 From: Tibor Vass Date: Sun, 17 May 2015 05:07:48 -0400 Subject: [PATCH 296/375] remove pkg/transport and use the one from distribution Signed-off-by: Tibor Vass --- docs/endpoint.go | 2 +- docs/registry.go | 4 ++-- docs/registry_test.go | 2 +- docs/session.go | 19 +++++++++++++++++-- 4 files changed, 21 insertions(+), 6 deletions(-) diff --git a/docs/endpoint.go b/docs/endpoint.go index ce92668f..c21a4265 100644 --- a/docs/endpoint.go +++ b/docs/endpoint.go @@ -11,7 +11,7 @@ import ( "github.com/Sirupsen/logrus" "github.com/docker/distribution/registry/api/v2" - "github.com/docker/docker/pkg/transport" + "github.com/docker/distribution/registry/client/transport" ) // for mocking in unit tests diff --git a/docs/registry.go b/docs/registry.go index fb08e5bd..f968daa8 100644 --- a/docs/registry.go +++ b/docs/registry.go @@ -17,11 +17,11 @@ import ( "time" "github.com/Sirupsen/logrus" + "github.com/docker/distribution/registry/client/transport" "github.com/docker/docker/autogen/dockerversion" "github.com/docker/docker/pkg/parsers/kernel" "github.com/docker/docker/pkg/timeoutconn" "github.com/docker/docker/pkg/tlsconfig" - "github.com/docker/docker/pkg/transport" "github.com/docker/docker/pkg/useragent" ) @@ -92,7 +92,7 @@ func (m *httpsRequestModifier) ModifyRequest(req *http.Request) error { logrus.Debugf("hostDir: %s", hostDir) fs, err := ioutil.ReadDir(hostDir) if err != nil && !os.IsNotExist(err) { - return nil + return err } for _, f := range fs { diff --git a/docs/registry_test.go b/docs/registry_test.go index 52a2dc30..d5e1a095 100644 --- a/docs/registry_test.go +++ b/docs/registry_test.go @@ -8,8 +8,8 @@ import ( "strings" "testing" + "github.com/docker/distribution/registry/client/transport" "github.com/docker/docker/cliconfig" - "github.com/docker/docker/pkg/transport" ) var ( diff --git a/docs/session.go b/docs/session.go index 07195eb1..3f3fd86f 100644 --- a/docs/session.go +++ b/docs/session.go @@ -22,8 +22,8 @@ import ( "github.com/Sirupsen/logrus" "github.com/docker/docker/cliconfig" "github.com/docker/docker/pkg/httputils" + "github.com/docker/docker/pkg/ioutils" "github.com/docker/docker/pkg/tarsum" - "github.com/docker/docker/pkg/transport" ) var ( @@ -73,6 +73,21 @@ func AuthTransport(base http.RoundTripper, authConfig *cliconfig.AuthConfig, alw } } +// cloneRequest returns a clone of the provided *http.Request. +// The clone is a shallow copy of the struct and its Header map. +func cloneRequest(r *http.Request) *http.Request { + // shallow copy of the struct + r2 := new(http.Request) + *r2 = *r + // deep copy of the Header + r2.Header = make(http.Header, len(r.Header)) + for k, s := range r.Header { + r2.Header[k] = append([]string(nil), s...) + } + + return r2 +} + func (tr *authTransport) RoundTrip(orig *http.Request) (*http.Response, error) { // Authorization should not be set on 302 redirect for untrusted locations. // This logic mirrors the behavior in AddRequiredHeadersToRedirectedRequests. @@ -112,7 +127,7 @@ func (tr *authTransport) RoundTrip(orig *http.Request) (*http.Response, error) { if len(resp.Header["X-Docker-Token"]) > 0 { tr.token = resp.Header["X-Docker-Token"] } - resp.Body = &transport.OnEOFReader{ + resp.Body = &ioutils.OnEOFReader{ Rc: resp.Body, Fn: func() { tr.mu.Lock() From 7fed379d95cd65796e55acdd768159191eff9109 Mon Sep 17 00:00:00 2001 From: Derek McGowan Date: Thu, 12 Feb 2015 10:23:22 -0800 Subject: [PATCH 297/375] Update graph to use vendored distribution client for the v2 codepath Signed-off-by: Derek McGowan (github: dmcgowan) Signed-off-by: Tibor Vass --- docs/auth.go | 2 +- docs/auth_test.go | 10 +- docs/config.go | 37 ++-- docs/endpoint.go | 17 +- docs/endpoint_test.go | 6 +- docs/registry.go | 194 +++++------------ docs/registry_mock_test.go | 2 +- docs/registry_test.go | 68 +++--- docs/service.go | 195 ++++++++++++++++- docs/session.go | 9 +- docs/session_v2.go | 414 ------------------------------------- 11 files changed, 320 insertions(+), 634 deletions(-) delete mode 100644 docs/session_v2.go diff --git a/docs/auth.go b/docs/auth.go index 65991d69..7111ede9 100644 --- a/docs/auth.go +++ b/docs/auth.go @@ -125,7 +125,7 @@ func loginV1(authConfig *cliconfig.AuthConfig, registryEndpoint *Endpoint) (stri return "", fmt.Errorf("Server Error: Server Address not set.") } - loginAgainstOfficialIndex := serverAddress == IndexServerAddress() + loginAgainstOfficialIndex := serverAddress == INDEXSERVER // to avoid sending the server address to the server it should be removed before being marshalled authCopy := *authConfig diff --git a/docs/auth_test.go b/docs/auth_test.go index 71b963a1..5f54add3 100644 --- a/docs/auth_test.go +++ b/docs/auth_test.go @@ -37,7 +37,7 @@ func setupTempConfigFile() (*cliconfig.ConfigFile, error) { root = filepath.Join(root, cliconfig.CONFIGFILE) configFile := cliconfig.NewConfigFile(root) - for _, registry := range []string{"testIndex", IndexServerAddress()} { + for _, registry := range []string{"testIndex", INDEXSERVER} { configFile.AuthConfigs[registry] = cliconfig.AuthConfig{ Username: "docker-user", Password: "docker-pass", @@ -82,7 +82,7 @@ func TestResolveAuthConfigIndexServer(t *testing.T) { } defer os.RemoveAll(configFile.Filename()) - indexConfig := configFile.AuthConfigs[IndexServerAddress()] + indexConfig := configFile.AuthConfigs[INDEXSERVER] officialIndex := &IndexInfo{ Official: true, @@ -92,10 +92,10 @@ func TestResolveAuthConfigIndexServer(t *testing.T) { } resolved := ResolveAuthConfig(configFile, officialIndex) - assertEqual(t, resolved, indexConfig, "Expected ResolveAuthConfig to return IndexServerAddress()") + assertEqual(t, resolved, indexConfig, "Expected ResolveAuthConfig to return INDEXSERVER") resolved = ResolveAuthConfig(configFile, privateIndex) - assertNotEqual(t, resolved, indexConfig, "Expected ResolveAuthConfig to not return IndexServerAddress()") + assertNotEqual(t, resolved, indexConfig, "Expected ResolveAuthConfig to not return INDEXSERVER") } func TestResolveAuthConfigFullURL(t *testing.T) { @@ -120,7 +120,7 @@ func TestResolveAuthConfigFullURL(t *testing.T) { Password: "baz-pass", Email: "baz@example.com", } - configFile.AuthConfigs[IndexServerAddress()] = officialAuth + configFile.AuthConfigs[INDEXSERVER] = officialAuth expectedAuths := map[string]cliconfig.AuthConfig{ "registry.example.com": registryAuth, diff --git a/docs/config.go b/docs/config.go index 105ec61d..333f1c46 100644 --- a/docs/config.go +++ b/docs/config.go @@ -21,9 +21,16 @@ type Options struct { } const ( + DEFAULT_NAMESPACE = "docker.io" + DEFAULT_V2_REGISTRY = "https://registry-1.docker.io" + DEFAULT_REGISTRY_VERSION_HEADER = "Docker-Distribution-Api-Version" + DEFAULT_V1_REGISTRY = "https://index.docker.io" + + CERTS_DIR = "/etc/docker/certs.d" + // Only used for user auth + account creation - INDEXSERVER = "https://index.docker.io/v1/" - REGISTRYSERVER = "https://registry-1.docker.io/v2/" + REGISTRYSERVER = DEFAULT_V2_REGISTRY + INDEXSERVER = DEFAULT_V1_REGISTRY + "/v1/" INDEXNAME = "docker.io" // INDEXSERVER = "https://registry-stage.hub.docker.com/v1/" @@ -34,14 +41,6 @@ var ( emptyServiceConfig = NewServiceConfig(nil) ) -func IndexServerAddress() string { - return INDEXSERVER -} - -func IndexServerName() string { - return INDEXNAME -} - // InstallFlags adds command-line options to the top-level flag parser for // the current process. func (options *Options) InstallFlags() { @@ -72,6 +71,7 @@ func (ipnet *netIPNet) UnmarshalJSON(b []byte) (err error) { type ServiceConfig struct { InsecureRegistryCIDRs []*netIPNet `json:"InsecureRegistryCIDRs"` IndexConfigs map[string]*IndexInfo `json:"IndexConfigs"` + Mirrors []string } // NewServiceConfig returns a new instance of ServiceConfig @@ -93,6 +93,9 @@ func NewServiceConfig(options *Options) *ServiceConfig { config := &ServiceConfig{ InsecureRegistryCIDRs: make([]*netIPNet, 0), IndexConfigs: make(map[string]*IndexInfo, 0), + // Hack: Bypass setting the mirrors to IndexConfigs since they are going away + // and Mirrors are only for the official registry anyways. + Mirrors: options.Mirrors.GetAll(), } // Split --insecure-registry into CIDR and registry-specific settings. for _, r := range options.InsecureRegistries.GetAll() { @@ -113,9 +116,9 @@ func NewServiceConfig(options *Options) *ServiceConfig { } // Configure public registry. - config.IndexConfigs[IndexServerName()] = &IndexInfo{ - Name: IndexServerName(), - Mirrors: options.Mirrors.GetAll(), + config.IndexConfigs[INDEXNAME] = &IndexInfo{ + Name: INDEXNAME, + Mirrors: config.Mirrors, Secure: true, Official: true, } @@ -193,8 +196,8 @@ func ValidateMirror(val string) (string, error) { // ValidateIndexName validates an index name. func ValidateIndexName(val string) (string, error) { // 'index.docker.io' => 'docker.io' - if val == "index."+IndexServerName() { - val = IndexServerName() + if val == "index."+INDEXNAME { + val = INDEXNAME } if strings.HasPrefix(val, "-") || strings.HasSuffix(val, "-") { return "", fmt.Errorf("Invalid index name (%s). Cannot begin or end with a hyphen.", val) @@ -264,7 +267,7 @@ func (config *ServiceConfig) NewIndexInfo(indexName string) (*IndexInfo, error) // index as the AuthConfig key, and uses the (host)name[:port] for private indexes. func (index *IndexInfo) GetAuthConfigKey() string { if index.Official { - return IndexServerAddress() + return INDEXSERVER } return index.Name } @@ -277,7 +280,7 @@ func splitReposName(reposName string) (string, string) { !strings.Contains(nameParts[0], ":") && nameParts[0] != "localhost") { // This is a Docker Index repos (ex: samalba/hipache or ubuntu) // 'docker.io' - indexName = IndexServerName() + indexName = INDEXNAME remoteName = reposName } else { indexName = nameParts[0] diff --git a/docs/endpoint.go b/docs/endpoint.go index c21a4265..17443543 100644 --- a/docs/endpoint.go +++ b/docs/endpoint.go @@ -1,6 +1,7 @@ package registry import ( + "crypto/tls" "encoding/json" "fmt" "io/ioutil" @@ -12,6 +13,7 @@ import ( "github.com/Sirupsen/logrus" "github.com/docker/distribution/registry/api/v2" "github.com/docker/distribution/registry/client/transport" + "github.com/docker/docker/pkg/tlsconfig" ) // for mocking in unit tests @@ -44,7 +46,9 @@ func scanForAPIVersion(address string) (string, APIVersion) { // NewEndpoint parses the given address to return a registry endpoint. func NewEndpoint(index *IndexInfo, metaHeaders http.Header) (*Endpoint, error) { // *TODO: Allow per-registry configuration of endpoints. - endpoint, err := newEndpoint(index.GetAuthConfigKey(), index.Secure, metaHeaders) + tlsConfig := tlsconfig.ServerDefault + tlsConfig.InsecureSkipVerify = !index.Secure + endpoint, err := newEndpoint(index.GetAuthConfigKey(), &tlsConfig, metaHeaders) if err != nil { return nil, err } @@ -82,7 +86,7 @@ func validateEndpoint(endpoint *Endpoint) error { return nil } -func newEndpoint(address string, secure bool, metaHeaders http.Header) (*Endpoint, error) { +func newEndpoint(address string, tlsConfig *tls.Config, metaHeaders http.Header) (*Endpoint, error) { var ( endpoint = new(Endpoint) trimmedAddress string @@ -93,13 +97,16 @@ func newEndpoint(address string, secure bool, metaHeaders http.Header) (*Endpoin address = "https://" + address } + endpoint.IsSecure = (tlsConfig == nil || !tlsConfig.InsecureSkipVerify) + trimmedAddress, endpoint.Version = scanForAPIVersion(address) if endpoint.URL, err = url.Parse(trimmedAddress); err != nil { return nil, err } - endpoint.IsSecure = secure - tr := NewTransport(ConnectTimeout, endpoint.IsSecure) + + // TODO(tiborvass): make sure a ConnectTimeout transport is used + tr := NewTransport(tlsConfig) endpoint.client = HTTPClient(transport.NewTransport(tr, DockerHeaders(metaHeaders)...)) return endpoint, nil } @@ -166,7 +173,7 @@ func (e *Endpoint) Ping() (RegistryInfo, error) { func (e *Endpoint) pingV1() (RegistryInfo, error) { logrus.Debugf("attempting v1 ping for registry endpoint %s", e) - if e.String() == IndexServerAddress() { + if e.String() == INDEXSERVER { // Skip the check, we know this one is valid // (and we never want to fallback to http in case of error) return RegistryInfo{Standalone: false}, nil diff --git a/docs/endpoint_test.go b/docs/endpoint_test.go index 6f67867b..a04f9a03 100644 --- a/docs/endpoint_test.go +++ b/docs/endpoint_test.go @@ -12,14 +12,14 @@ func TestEndpointParse(t *testing.T) { str string expected string }{ - {IndexServerAddress(), IndexServerAddress()}, + {INDEXSERVER, INDEXSERVER}, {"http://0.0.0.0:5000/v1/", "http://0.0.0.0:5000/v1/"}, {"http://0.0.0.0:5000/v2/", "http://0.0.0.0:5000/v2/"}, {"http://0.0.0.0:5000", "http://0.0.0.0:5000/v0/"}, {"0.0.0.0:5000", "https://0.0.0.0:5000/v0/"}, } for _, td := range testData { - e, err := newEndpoint(td.str, false, nil) + e, err := newEndpoint(td.str, nil, nil) if err != nil { t.Errorf("%q: %s", td.str, err) } @@ -60,7 +60,7 @@ func TestValidateEndpointAmbiguousAPIVersion(t *testing.T) { testEndpoint := Endpoint{ URL: testServerURL, Version: APIVersionUnknown, - client: HTTPClient(NewTransport(ConnectTimeout, false)), + client: HTTPClient(NewTransport(nil)), } if err = validateEndpoint(&testEndpoint); err != nil { diff --git a/docs/registry.go b/docs/registry.go index f968daa8..74a0ad5f 100644 --- a/docs/registry.go +++ b/docs/registry.go @@ -2,25 +2,20 @@ package registry import ( "crypto/tls" - "crypto/x509" "errors" - "fmt" - "io/ioutil" "net" "net/http" "os" - "path" - "path/filepath" "runtime" "strings" - "sync" "time" "github.com/Sirupsen/logrus" + "github.com/docker/distribution/registry/api/errcode" + "github.com/docker/distribution/registry/api/v2" "github.com/docker/distribution/registry/client/transport" "github.com/docker/docker/autogen/dockerversion" "github.com/docker/docker/pkg/parsers/kernel" - "github.com/docker/docker/pkg/timeoutconn" "github.com/docker/docker/pkg/tlsconfig" "github.com/docker/docker/pkg/useragent" ) @@ -57,135 +52,13 @@ func init() { dockerUserAgent = useragent.AppendVersions("", httpVersion...) } -type httpsRequestModifier struct { - mu sync.Mutex - tlsConfig *tls.Config -} - -// DRAGONS(tiborvass): If someone wonders why do we set tlsconfig in a roundtrip, -// it's because it's so as to match the current behavior in master: we generate the -// certpool on every-goddam-request. It's not great, but it allows people to just put -// the certs in /etc/docker/certs.d/.../ and let docker "pick it up" immediately. Would -// prefer an fsnotify implementation, but that was out of scope of my refactoring. -func (m *httpsRequestModifier) ModifyRequest(req *http.Request) error { - var ( - roots *x509.CertPool - certs []tls.Certificate - hostDir string - ) - - if req.URL.Scheme == "https" { - hasFile := func(files []os.FileInfo, name string) bool { - for _, f := range files { - if f.Name() == name { - return true - } - } - return false - } - - if runtime.GOOS == "windows" { - hostDir = path.Join(os.TempDir(), "/docker/certs.d", req.URL.Host) - } else { - hostDir = path.Join("/etc/docker/certs.d", req.URL.Host) - } - logrus.Debugf("hostDir: %s", hostDir) - fs, err := ioutil.ReadDir(hostDir) - if err != nil && !os.IsNotExist(err) { - return err - } - - for _, f := range fs { - if strings.HasSuffix(f.Name(), ".crt") { - if roots == nil { - roots = x509.NewCertPool() - } - logrus.Debugf("crt: %s", hostDir+"/"+f.Name()) - data, err := ioutil.ReadFile(filepath.Join(hostDir, f.Name())) - if err != nil { - return err - } - roots.AppendCertsFromPEM(data) - } - if strings.HasSuffix(f.Name(), ".cert") { - certName := f.Name() - keyName := certName[:len(certName)-5] + ".key" - logrus.Debugf("cert: %s", hostDir+"/"+f.Name()) - if !hasFile(fs, keyName) { - return fmt.Errorf("Missing key %s for certificate %s", keyName, certName) - } - cert, err := tls.LoadX509KeyPair(filepath.Join(hostDir, certName), path.Join(hostDir, keyName)) - if err != nil { - return err - } - certs = append(certs, cert) - } - if strings.HasSuffix(f.Name(), ".key") { - keyName := f.Name() - certName := keyName[:len(keyName)-4] + ".cert" - logrus.Debugf("key: %s", hostDir+"/"+f.Name()) - if !hasFile(fs, certName) { - return fmt.Errorf("Missing certificate %s for key %s", certName, keyName) - } - } - } - m.mu.Lock() - m.tlsConfig.RootCAs = roots - m.tlsConfig.Certificates = certs - m.mu.Unlock() - } - return nil -} - -func NewTransport(timeout TimeoutType, secure bool) http.RoundTripper { - tlsConfig := &tls.Config{ - // Avoid fallback to SSL protocols < TLS1.0 - MinVersion: tls.VersionTLS10, - InsecureSkipVerify: !secure, - CipherSuites: tlsconfig.DefaultServerAcceptedCiphers, - } - - tr := &http.Transport{ - DisableKeepAlives: true, - Proxy: http.ProxyFromEnvironment, - TLSClientConfig: tlsConfig, - } - - switch timeout { - case ConnectTimeout: - tr.Dial = func(proto string, addr string) (net.Conn, error) { - // Set the connect timeout to 30 seconds to allow for slower connection - // times... - d := net.Dialer{Timeout: 30 * time.Second, DualStack: true} - - conn, err := d.Dial(proto, addr) - if err != nil { - return nil, err - } - // Set the recv timeout to 10 seconds - conn.SetDeadline(time.Now().Add(10 * time.Second)) - return conn, nil - } - case ReceiveTimeout: - tr.Dial = func(proto string, addr string) (net.Conn, error) { - d := net.Dialer{DualStack: true} - - conn, err := d.Dial(proto, addr) - if err != nil { - return nil, err - } - conn = timeoutconn.New(conn, 1*time.Minute) - return conn, nil +func hasFile(files []os.FileInfo, name string) bool { + for _, f := range files { + if f.Name() == name { + return true } } - - if secure { - // note: httpsTransport also handles http transport - // but for HTTPS, it sets up the certs - return transport.NewTransport(tr, &httpsRequestModifier{tlsConfig: tlsConfig}) - } - - return tr + return false } // DockerHeaders returns request modifiers that ensure requests have @@ -202,10 +75,6 @@ func DockerHeaders(metaHeaders http.Header) []transport.RequestModifier { } func HTTPClient(transport http.RoundTripper) *http.Client { - if transport == nil { - transport = NewTransport(ConnectTimeout, true) - } - return &http.Client{ Transport: transport, CheckRedirect: AddRequiredHeadersToRedirectedRequests, @@ -245,3 +114,52 @@ func AddRequiredHeadersToRedirectedRequests(req *http.Request, via []*http.Reque } return nil } + +func shouldV2Fallback(err errcode.Error) bool { + logrus.Debugf("v2 error: %T %v", err, err) + switch err.Code { + case v2.ErrorCodeUnauthorized, v2.ErrorCodeManifestUnknown: + return true + } + return false +} + +type ErrNoSupport struct{ Err error } + +func (e ErrNoSupport) Error() string { + if e.Err == nil { + return "not supported" + } + return e.Err.Error() +} + +func ContinueOnError(err error) bool { + switch v := err.(type) { + case errcode.Errors: + return ContinueOnError(v[0]) + case ErrNoSupport: + return ContinueOnError(v.Err) + case errcode.Error: + return shouldV2Fallback(v) + } + return false +} + +func NewTransport(tlsConfig *tls.Config) *http.Transport { + if tlsConfig == nil { + var cfg = tlsconfig.ServerDefault + tlsConfig = &cfg + } + return &http.Transport{ + Proxy: http.ProxyFromEnvironment, + Dial: (&net.Dialer{ + Timeout: 30 * time.Second, + KeepAlive: 30 * time.Second, + DualStack: true, + }).Dial, + TLSHandshakeTimeout: 10 * time.Second, + TLSClientConfig: tlsConfig, + // TODO(dmcgowan): Call close idle connections when complete and use keep alive + DisableKeepAlives: true, + } +} diff --git a/docs/registry_mock_test.go b/docs/registry_mock_test.go index eab87d46..9217956c 100644 --- a/docs/registry_mock_test.go +++ b/docs/registry_mock_test.go @@ -165,7 +165,7 @@ func makeHttpsIndex(req string) *IndexInfo { func makePublicIndex() *IndexInfo { index := &IndexInfo{ - Name: IndexServerAddress(), + Name: INDEXSERVER, Secure: true, Official: true, } diff --git a/docs/registry_test.go b/docs/registry_test.go index d5e1a095..4d17a62c 100644 --- a/docs/registry_test.go +++ b/docs/registry_test.go @@ -27,7 +27,7 @@ func spawnTestRegistrySession(t *testing.T) *Session { if err != nil { t.Fatal(err) } - var tr http.RoundTripper = debugTransport{NewTransport(ReceiveTimeout, endpoint.IsSecure), t.Log} + var tr http.RoundTripper = debugTransport{NewTransport(nil), t.Log} tr = transport.NewTransport(AuthTransport(tr, authConfig, false), DockerHeaders(nil)...) client := HTTPClient(tr) r, err := NewSession(client, authConfig, endpoint) @@ -332,7 +332,7 @@ func TestParseRepositoryInfo(t *testing.T) { expectedRepoInfos := map[string]RepositoryInfo{ "fooo/bar": { Index: &IndexInfo{ - Name: IndexServerName(), + Name: INDEXNAME, Official: true, }, RemoteName: "fooo/bar", @@ -342,7 +342,7 @@ func TestParseRepositoryInfo(t *testing.T) { }, "library/ubuntu": { Index: &IndexInfo{ - Name: IndexServerName(), + Name: INDEXNAME, Official: true, }, RemoteName: "library/ubuntu", @@ -352,7 +352,7 @@ func TestParseRepositoryInfo(t *testing.T) { }, "nonlibrary/ubuntu": { Index: &IndexInfo{ - Name: IndexServerName(), + Name: INDEXNAME, Official: true, }, RemoteName: "nonlibrary/ubuntu", @@ -362,7 +362,7 @@ func TestParseRepositoryInfo(t *testing.T) { }, "ubuntu": { Index: &IndexInfo{ - Name: IndexServerName(), + Name: INDEXNAME, Official: true, }, RemoteName: "library/ubuntu", @@ -372,7 +372,7 @@ func TestParseRepositoryInfo(t *testing.T) { }, "other/library": { Index: &IndexInfo{ - Name: IndexServerName(), + Name: INDEXNAME, Official: true, }, RemoteName: "other/library", @@ -480,9 +480,9 @@ func TestParseRepositoryInfo(t *testing.T) { CanonicalName: "localhost/privatebase", Official: false, }, - IndexServerName() + "/public/moonbase": { + INDEXNAME + "/public/moonbase": { Index: &IndexInfo{ - Name: IndexServerName(), + Name: INDEXNAME, Official: true, }, RemoteName: "public/moonbase", @@ -490,19 +490,9 @@ func TestParseRepositoryInfo(t *testing.T) { CanonicalName: "docker.io/public/moonbase", Official: false, }, - "index." + IndexServerName() + "/public/moonbase": { + "index." + INDEXNAME + "/public/moonbase": { Index: &IndexInfo{ - Name: IndexServerName(), - Official: true, - }, - RemoteName: "public/moonbase", - LocalName: "public/moonbase", - CanonicalName: "docker.io/public/moonbase", - Official: false, - }, - IndexServerName() + "/public/moonbase": { - Index: &IndexInfo{ - Name: IndexServerName(), + Name: INDEXNAME, Official: true, }, RemoteName: "public/moonbase", @@ -512,7 +502,7 @@ func TestParseRepositoryInfo(t *testing.T) { }, "ubuntu-12.04-base": { Index: &IndexInfo{ - Name: IndexServerName(), + Name: INDEXNAME, Official: true, }, RemoteName: "library/ubuntu-12.04-base", @@ -520,9 +510,9 @@ func TestParseRepositoryInfo(t *testing.T) { CanonicalName: "docker.io/library/ubuntu-12.04-base", Official: true, }, - IndexServerName() + "/ubuntu-12.04-base": { + INDEXNAME + "/ubuntu-12.04-base": { Index: &IndexInfo{ - Name: IndexServerName(), + Name: INDEXNAME, Official: true, }, RemoteName: "library/ubuntu-12.04-base", @@ -530,19 +520,9 @@ func TestParseRepositoryInfo(t *testing.T) { CanonicalName: "docker.io/library/ubuntu-12.04-base", Official: true, }, - IndexServerName() + "/ubuntu-12.04-base": { + "index." + INDEXNAME + "/ubuntu-12.04-base": { Index: &IndexInfo{ - Name: IndexServerName(), - Official: true, - }, - RemoteName: "library/ubuntu-12.04-base", - LocalName: "ubuntu-12.04-base", - CanonicalName: "docker.io/library/ubuntu-12.04-base", - Official: true, - }, - "index." + IndexServerName() + "/ubuntu-12.04-base": { - Index: &IndexInfo{ - Name: IndexServerName(), + Name: INDEXNAME, Official: true, }, RemoteName: "library/ubuntu-12.04-base", @@ -585,14 +565,14 @@ func TestNewIndexInfo(t *testing.T) { config := NewServiceConfig(nil) noMirrors := make([]string, 0) expectedIndexInfos := map[string]*IndexInfo{ - IndexServerName(): { - Name: IndexServerName(), + INDEXNAME: { + Name: INDEXNAME, Official: true, Secure: true, Mirrors: noMirrors, }, - "index." + IndexServerName(): { - Name: IndexServerName(), + "index." + INDEXNAME: { + Name: INDEXNAME, Official: true, Secure: true, Mirrors: noMirrors, @@ -616,14 +596,14 @@ func TestNewIndexInfo(t *testing.T) { config = makeServiceConfig(publicMirrors, []string{"example.com"}) expectedIndexInfos = map[string]*IndexInfo{ - IndexServerName(): { - Name: IndexServerName(), + INDEXNAME: { + Name: INDEXNAME, Official: true, Secure: true, Mirrors: publicMirrors, }, - "index." + IndexServerName(): { - Name: IndexServerName(), + "index." + INDEXNAME: { + Name: INDEXNAME, Official: true, Secure: true, Mirrors: publicMirrors, @@ -880,7 +860,7 @@ func TestIsSecureIndex(t *testing.T) { insecureRegistries []string expected bool }{ - {IndexServerName(), nil, true}, + {INDEXNAME, nil, true}, {"example.com", []string{}, true}, {"example.com", []string{"example.com"}, false}, {"localhost", []string{"localhost:5000"}, false}, diff --git a/docs/service.go b/docs/service.go index 68117492..8dda537a 100644 --- a/docs/service.go +++ b/docs/service.go @@ -1,9 +1,19 @@ package registry import ( + "crypto/tls" + "crypto/x509" + "fmt" + "io/ioutil" "net/http" + "os" + "path/filepath" + "strings" + "github.com/Sirupsen/logrus" + "github.com/docker/distribution/registry/client/auth" "github.com/docker/docker/cliconfig" + "github.com/docker/docker/pkg/tlsconfig" ) type Service struct { @@ -25,7 +35,7 @@ func (s *Service) Auth(authConfig *cliconfig.AuthConfig) (string, error) { addr := authConfig.ServerAddress if addr == "" { // Use the official registry address if not specified. - addr = IndexServerAddress() + addr = INDEXSERVER } index, err := s.ResolveIndex(addr) if err != nil { @@ -69,3 +79,186 @@ func (s *Service) ResolveRepository(name string) (*RepositoryInfo, error) { func (s *Service) ResolveIndex(name string) (*IndexInfo, error) { return s.Config.NewIndexInfo(name) } + +type APIEndpoint struct { + Mirror bool + URL string + Version APIVersion + Official bool + TrimHostname bool + TLSConfig *tls.Config + VersionHeader string + Versions []auth.APIVersion +} + +func (e APIEndpoint) ToV1Endpoint(metaHeaders http.Header) (*Endpoint, error) { + return newEndpoint(e.URL, e.TLSConfig, metaHeaders) +} + +func (s *Service) TlsConfig(hostname string) (*tls.Config, error) { + // we construct a client tls config from server defaults + // PreferredServerCipherSuites should have no effect + tlsConfig := tlsconfig.ServerDefault + + isSecure := s.Config.isSecureIndex(hostname) + + tlsConfig.InsecureSkipVerify = !isSecure + + if isSecure { + hasFile := func(files []os.FileInfo, name string) bool { + for _, f := range files { + if f.Name() == name { + return true + } + } + return false + } + + hostDir := filepath.Join(CERTS_DIR, hostname) + logrus.Debugf("hostDir: %s", hostDir) + fs, err := ioutil.ReadDir(hostDir) + if err != nil && !os.IsNotExist(err) { + return nil, err + } + + for _, f := range fs { + if strings.HasSuffix(f.Name(), ".crt") { + if tlsConfig.RootCAs == nil { + // TODO(dmcgowan): Copy system pool + tlsConfig.RootCAs = x509.NewCertPool() + } + logrus.Debugf("crt: %s", filepath.Join(hostDir, f.Name())) + data, err := ioutil.ReadFile(filepath.Join(hostDir, f.Name())) + if err != nil { + return nil, err + } + tlsConfig.RootCAs.AppendCertsFromPEM(data) + } + if strings.HasSuffix(f.Name(), ".cert") { + certName := f.Name() + keyName := certName[:len(certName)-5] + ".key" + logrus.Debugf("cert: %s", filepath.Join(hostDir, f.Name())) + if !hasFile(fs, keyName) { + return nil, fmt.Errorf("Missing key %s for certificate %s", keyName, certName) + } + cert, err := tls.LoadX509KeyPair(filepath.Join(hostDir, certName), filepath.Join(hostDir, keyName)) + if err != nil { + return nil, err + } + tlsConfig.Certificates = append(tlsConfig.Certificates, cert) + } + if strings.HasSuffix(f.Name(), ".key") { + keyName := f.Name() + certName := keyName[:len(keyName)-4] + ".cert" + logrus.Debugf("key: %s", filepath.Join(hostDir, f.Name())) + if !hasFile(fs, certName) { + return nil, fmt.Errorf("Missing certificate %s for key %s", certName, keyName) + } + } + } + } + + return &tlsConfig, nil +} + +func (s *Service) LookupEndpoints(repoName string) (endpoints []APIEndpoint, err error) { + var cfg = tlsconfig.ServerDefault + tlsConfig := &cfg + if strings.HasPrefix(repoName, DEFAULT_NAMESPACE+"/") { + // v2 mirrors + for _, mirror := range s.Config.Mirrors { + endpoints = append(endpoints, APIEndpoint{ + URL: mirror, + // guess mirrors are v2 + Version: APIVersion2, + Mirror: true, + TrimHostname: true, + TLSConfig: tlsConfig, + }) + } + // v2 registry + endpoints = append(endpoints, APIEndpoint{ + URL: DEFAULT_V2_REGISTRY, + Version: APIVersion2, + Official: true, + TrimHostname: true, + TLSConfig: tlsConfig, + }) + // v1 mirrors + // TODO(tiborvass): shouldn't we remove v1 mirrors from here, since v1 mirrors are kinda special? + for _, mirror := range s.Config.Mirrors { + endpoints = append(endpoints, APIEndpoint{ + URL: mirror, + // guess mirrors are v1 + Version: APIVersion1, + Mirror: true, + TrimHostname: true, + TLSConfig: tlsConfig, + }) + } + // v1 registry + endpoints = append(endpoints, APIEndpoint{ + URL: DEFAULT_V1_REGISTRY, + Version: APIVersion1, + Official: true, + TrimHostname: true, + TLSConfig: tlsConfig, + }) + return endpoints, nil + } + + slashIndex := strings.IndexRune(repoName, '/') + if slashIndex <= 0 { + return nil, fmt.Errorf("invalid repo name: missing '/': %s", repoName) + } + hostname := repoName[:slashIndex] + + tlsConfig, err = s.TlsConfig(hostname) + if err != nil { + return nil, err + } + isSecure := !tlsConfig.InsecureSkipVerify + + v2Versions := []auth.APIVersion{ + { + Type: "registry", + Version: "2.0", + }, + } + endpoints = []APIEndpoint{ + { + URL: "https://" + hostname, + Version: APIVersion2, + TrimHostname: true, + TLSConfig: tlsConfig, + VersionHeader: DEFAULT_REGISTRY_VERSION_HEADER, + Versions: v2Versions, + }, + { + URL: "https://" + hostname, + Version: APIVersion1, + TrimHostname: true, + TLSConfig: tlsConfig, + }, + } + + if !isSecure { + endpoints = append(endpoints, APIEndpoint{ + URL: "http://" + hostname, + Version: APIVersion2, + TrimHostname: true, + // used to check if supposed to be secure via InsecureSkipVerify + TLSConfig: tlsConfig, + VersionHeader: DEFAULT_REGISTRY_VERSION_HEADER, + Versions: v2Versions, + }, APIEndpoint{ + URL: "http://" + hostname, + Version: APIVersion1, + TrimHostname: true, + // used to check if supposed to be secure via InsecureSkipVerify + TLSConfig: tlsConfig, + }) + } + + return endpoints, nil +} diff --git a/docs/session.go b/docs/session.go index 3f3fd86f..154c63e1 100644 --- a/docs/session.go +++ b/docs/session.go @@ -98,7 +98,7 @@ func (tr *authTransport) RoundTrip(orig *http.Request) (*http.Response, error) { return tr.RoundTripper.RoundTrip(orig) } - req := transport.CloneRequest(orig) + req := cloneRequest(orig) tr.mu.Lock() tr.modReq[orig] = req tr.mu.Unlock() @@ -164,12 +164,11 @@ func NewSession(client *http.Client, authConfig *cliconfig.AuthConfig, endpoint // If we're working with a standalone private registry over HTTPS, send Basic Auth headers // alongside all our requests. - if endpoint.VersionString(1) != IndexServerAddress() && endpoint.URL.Scheme == "https" { + if endpoint.VersionString(1) != INDEXSERVER && endpoint.URL.Scheme == "https" { info, err := endpoint.Ping() if err != nil { return nil, err } - if info.Standalone && authConfig != nil { logrus.Debugf("Endpoint %s is eligible for private registry. Enabling decorator.", endpoint.String()) alwaysSetBasicAuth = true @@ -265,7 +264,7 @@ func (r *Session) GetRemoteImageLayer(imgID, registry string, imgSize int64) (io if err != nil { return nil, fmt.Errorf("Error while getting from the server: %v", err) } - // TODO: why are we doing retries at this level? + // TODO(tiborvass): why are we doing retries at this level? // These retries should be generic to both v1 and v2 for i := 1; i <= retries; i++ { statusCode = 0 @@ -432,7 +431,7 @@ func (r *Session) GetRepositoryData(remote string) (*RepositoryData, error) { } // Forge a better object from the retrieved data - imgsData := make(map[string]*ImgData) + imgsData := make(map[string]*ImgData, len(remoteChecksums)) for _, elem := range remoteChecksums { imgsData[elem.ID] = elem } diff --git a/docs/session_v2.go b/docs/session_v2.go deleted file mode 100644 index f2b21df4..00000000 --- a/docs/session_v2.go +++ /dev/null @@ -1,414 +0,0 @@ -package registry - -import ( - "bytes" - "encoding/json" - "fmt" - "io" - "io/ioutil" - "net/http" - "strconv" - - "github.com/Sirupsen/logrus" - "github.com/docker/distribution/digest" - "github.com/docker/distribution/registry/api/v2" - "github.com/docker/docker/pkg/httputils" -) - -const DockerDigestHeader = "Docker-Content-Digest" - -func getV2Builder(e *Endpoint) *v2.URLBuilder { - if e.URLBuilder == nil { - e.URLBuilder = v2.NewURLBuilder(e.URL) - } - return e.URLBuilder -} - -func (r *Session) V2RegistryEndpoint(index *IndexInfo) (ep *Endpoint, err error) { - // TODO check if should use Mirror - if index.Official { - ep, err = newEndpoint(REGISTRYSERVER, true, nil) - if err != nil { - return - } - err = validateEndpoint(ep) - if err != nil { - return - } - } else if r.indexEndpoint.String() == index.GetAuthConfigKey() { - ep = r.indexEndpoint - } else { - ep, err = NewEndpoint(index, nil) - if err != nil { - return - } - } - - ep.URLBuilder = v2.NewURLBuilder(ep.URL) - return -} - -// GetV2Authorization gets the authorization needed to the given image -// If readonly access is requested, then the authorization may -// only be used for Get operations. -func (r *Session) GetV2Authorization(ep *Endpoint, imageName string, readOnly bool) (auth *RequestAuthorization, err error) { - scopes := []string{"pull"} - if !readOnly { - scopes = append(scopes, "push") - } - - logrus.Debugf("Getting authorization for %s %s", imageName, scopes) - return NewRequestAuthorization(r.GetAuthConfig(true), ep, "repository", imageName, scopes), nil -} - -// -// 1) Check if TarSum of each layer exists /v2/ -// 1.a) if 200, continue -// 1.b) if 300, then push the -// 1.c) if anything else, err -// 2) PUT the created/signed manifest -// - -// GetV2ImageManifest simply fetches the bytes of a manifest and the remote -// digest, if available in the request. Note that the application shouldn't -// rely on the untrusted remoteDigest, and should also verify against a -// locally provided digest, if applicable. -func (r *Session) GetV2ImageManifest(ep *Endpoint, imageName, tagName string, auth *RequestAuthorization) (remoteDigest digest.Digest, p []byte, err error) { - routeURL, err := getV2Builder(ep).BuildManifestURL(imageName, tagName) - if err != nil { - return "", nil, err - } - - method := "GET" - logrus.Debugf("[registry] Calling %q %s", method, routeURL) - - req, err := http.NewRequest(method, routeURL, nil) - if err != nil { - return "", nil, err - } - - if err := auth.Authorize(req); err != nil { - return "", nil, err - } - - res, err := r.client.Do(req) - if err != nil { - return "", nil, err - } - defer res.Body.Close() - - if res.StatusCode != 200 { - if res.StatusCode == 401 { - return "", nil, errLoginRequired - } else if res.StatusCode == 404 { - return "", nil, ErrDoesNotExist - } - return "", nil, httputils.NewHTTPRequestError(fmt.Sprintf("Server error: %d trying to fetch for %s:%s", res.StatusCode, imageName, tagName), res) - } - - p, err = ioutil.ReadAll(res.Body) - if err != nil { - return "", nil, fmt.Errorf("Error while reading the http response: %s", err) - } - - dgstHdr := res.Header.Get(DockerDigestHeader) - if dgstHdr != "" { - remoteDigest, err = digest.ParseDigest(dgstHdr) - if err != nil { - // NOTE(stevvooe): Including the remote digest is optional. We - // don't need to verify against it, but it is good practice. - remoteDigest = "" - logrus.Debugf("error parsing remote digest when fetching %v: %v", routeURL, err) - } - } - - return -} - -// - Succeeded to head image blob (already exists) -// - Failed with no error (continue to Push the Blob) -// - Failed with error -func (r *Session) HeadV2ImageBlob(ep *Endpoint, imageName string, dgst digest.Digest, auth *RequestAuthorization) (bool, error) { - routeURL, err := getV2Builder(ep).BuildBlobURL(imageName, dgst) - if err != nil { - return false, err - } - - method := "HEAD" - logrus.Debugf("[registry] Calling %q %s", method, routeURL) - - req, err := http.NewRequest(method, routeURL, nil) - if err != nil { - return false, err - } - if err := auth.Authorize(req); err != nil { - return false, err - } - res, err := r.client.Do(req) - if err != nil { - return false, err - } - res.Body.Close() // close early, since we're not needing a body on this call .. yet? - switch { - case res.StatusCode >= 200 && res.StatusCode < 400: - // return something indicating no push needed - return true, nil - case res.StatusCode == 401: - return false, errLoginRequired - case res.StatusCode == 404: - // return something indicating blob push needed - return false, nil - } - - return false, httputils.NewHTTPRequestError(fmt.Sprintf("Server error: %d trying head request for %s - %s", res.StatusCode, imageName, dgst), res) -} - -func (r *Session) GetV2ImageBlob(ep *Endpoint, imageName string, dgst digest.Digest, blobWrtr io.Writer, auth *RequestAuthorization) error { - routeURL, err := getV2Builder(ep).BuildBlobURL(imageName, dgst) - if err != nil { - return err - } - - method := "GET" - logrus.Debugf("[registry] Calling %q %s", method, routeURL) - req, err := http.NewRequest(method, routeURL, nil) - if err != nil { - return err - } - if err := auth.Authorize(req); err != nil { - return err - } - res, err := r.client.Do(req) - if err != nil { - return err - } - defer res.Body.Close() - if res.StatusCode != 200 { - if res.StatusCode == 401 { - return errLoginRequired - } - return httputils.NewHTTPRequestError(fmt.Sprintf("Server error: %d trying to pull %s blob", res.StatusCode, imageName), res) - } - - _, err = io.Copy(blobWrtr, res.Body) - return err -} - -func (r *Session) GetV2ImageBlobReader(ep *Endpoint, imageName string, dgst digest.Digest, auth *RequestAuthorization) (io.ReadCloser, int64, error) { - routeURL, err := getV2Builder(ep).BuildBlobURL(imageName, dgst) - if err != nil { - return nil, 0, err - } - - method := "GET" - logrus.Debugf("[registry] Calling %q %s", method, routeURL) - req, err := http.NewRequest(method, routeURL, nil) - if err != nil { - return nil, 0, err - } - if err := auth.Authorize(req); err != nil { - return nil, 0, err - } - res, err := r.client.Do(req) - if err != nil { - return nil, 0, err - } - if res.StatusCode != 200 { - if res.StatusCode == 401 { - return nil, 0, errLoginRequired - } - return nil, 0, httputils.NewHTTPRequestError(fmt.Sprintf("Server error: %d trying to pull %s blob - %s", res.StatusCode, imageName, dgst), res) - } - lenStr := res.Header.Get("Content-Length") - l, err := strconv.ParseInt(lenStr, 10, 64) - if err != nil { - return nil, 0, err - } - - return res.Body, l, err -} - -// Push the image to the server for storage. -// 'layer' is an uncompressed reader of the blob to be pushed. -// The server will generate it's own checksum calculation. -func (r *Session) PutV2ImageBlob(ep *Endpoint, imageName string, dgst digest.Digest, blobRdr io.Reader, auth *RequestAuthorization) error { - location, err := r.initiateBlobUpload(ep, imageName, auth) - if err != nil { - return err - } - - method := "PUT" - logrus.Debugf("[registry] Calling %q %s", method, location) - req, err := http.NewRequest(method, location, ioutil.NopCloser(blobRdr)) - if err != nil { - return err - } - queryParams := req.URL.Query() - queryParams.Add("digest", dgst.String()) - req.URL.RawQuery = queryParams.Encode() - if err := auth.Authorize(req); err != nil { - return err - } - res, err := r.client.Do(req) - if err != nil { - return err - } - defer res.Body.Close() - - if res.StatusCode != 201 { - if res.StatusCode == 401 { - return errLoginRequired - } - errBody, err := ioutil.ReadAll(res.Body) - if err != nil { - return err - } - logrus.Debugf("Unexpected response from server: %q %#v", errBody, res.Header) - return httputils.NewHTTPRequestError(fmt.Sprintf("Server error: %d trying to push %s blob - %s", res.StatusCode, imageName, dgst), res) - } - - return nil -} - -// initiateBlobUpload gets the blob upload location for the given image name. -func (r *Session) initiateBlobUpload(ep *Endpoint, imageName string, auth *RequestAuthorization) (location string, err error) { - routeURL, err := getV2Builder(ep).BuildBlobUploadURL(imageName) - if err != nil { - return "", err - } - - logrus.Debugf("[registry] Calling %q %s", "POST", routeURL) - req, err := http.NewRequest("POST", routeURL, nil) - if err != nil { - return "", err - } - - if err := auth.Authorize(req); err != nil { - return "", err - } - res, err := r.client.Do(req) - if err != nil { - return "", err - } - - if res.StatusCode != http.StatusAccepted { - if res.StatusCode == http.StatusUnauthorized { - return "", errLoginRequired - } - if res.StatusCode == http.StatusNotFound { - return "", ErrDoesNotExist - } - - errBody, err := ioutil.ReadAll(res.Body) - if err != nil { - return "", err - } - - logrus.Debugf("Unexpected response from server: %q %#v", errBody, res.Header) - return "", httputils.NewHTTPRequestError(fmt.Sprintf("Server error: unexpected %d response status trying to initiate upload of %s", res.StatusCode, imageName), res) - } - - if location = res.Header.Get("Location"); location == "" { - return "", fmt.Errorf("registry did not return a Location header for resumable blob upload for image %s", imageName) - } - - return -} - -// Finally Push the (signed) manifest of the blobs we've just pushed -func (r *Session) PutV2ImageManifest(ep *Endpoint, imageName, tagName string, signedManifest, rawManifest []byte, auth *RequestAuthorization) (digest.Digest, error) { - routeURL, err := getV2Builder(ep).BuildManifestURL(imageName, tagName) - if err != nil { - return "", err - } - - method := "PUT" - logrus.Debugf("[registry] Calling %q %s", method, routeURL) - req, err := http.NewRequest(method, routeURL, bytes.NewReader(signedManifest)) - if err != nil { - return "", err - } - if err := auth.Authorize(req); err != nil { - return "", err - } - res, err := r.client.Do(req) - if err != nil { - return "", err - } - defer res.Body.Close() - - // All 2xx and 3xx responses can be accepted for a put. - if res.StatusCode >= 400 { - if res.StatusCode == 401 { - return "", errLoginRequired - } - errBody, err := ioutil.ReadAll(res.Body) - if err != nil { - return "", err - } - logrus.Debugf("Unexpected response from server: %q %#v", errBody, res.Header) - return "", httputils.NewHTTPRequestError(fmt.Sprintf("Server error: %d trying to push %s:%s manifest", res.StatusCode, imageName, tagName), res) - } - - hdrDigest, err := digest.ParseDigest(res.Header.Get(DockerDigestHeader)) - if err != nil { - return "", fmt.Errorf("invalid manifest digest from registry: %s", err) - } - - dgstVerifier, err := digest.NewDigestVerifier(hdrDigest) - if err != nil { - return "", fmt.Errorf("invalid manifest digest from registry: %s", err) - } - - dgstVerifier.Write(rawManifest) - - if !dgstVerifier.Verified() { - computedDigest, _ := digest.FromBytes(rawManifest) - return "", fmt.Errorf("unable to verify manifest digest: registry has %q, computed %q", hdrDigest, computedDigest) - } - - return hdrDigest, nil -} - -type remoteTags struct { - Name string `json:"name"` - Tags []string `json:"tags"` -} - -// Given a repository name, returns a json array of string tags -func (r *Session) GetV2RemoteTags(ep *Endpoint, imageName string, auth *RequestAuthorization) ([]string, error) { - routeURL, err := getV2Builder(ep).BuildTagsURL(imageName) - if err != nil { - return nil, err - } - - method := "GET" - logrus.Debugf("[registry] Calling %q %s", method, routeURL) - - req, err := http.NewRequest(method, routeURL, nil) - if err != nil { - return nil, err - } - if err := auth.Authorize(req); err != nil { - return nil, err - } - res, err := r.client.Do(req) - if err != nil { - return nil, err - } - defer res.Body.Close() - if res.StatusCode != 200 { - if res.StatusCode == 401 { - return nil, errLoginRequired - } else if res.StatusCode == 404 { - return nil, ErrDoesNotExist - } - return nil, httputils.NewHTTPRequestError(fmt.Sprintf("Server error: %d trying to fetch for %s", res.StatusCode, imageName), res) - } - - var remote remoteTags - if err := json.NewDecoder(res.Body).Decode(&remote); err != nil { - return nil, fmt.Errorf("Error while decoding the http response: %s", err) - } - return remote.Tags, nil -} From 3552960ef83951f70f915c8ff3cc7d8fb6284ad4 Mon Sep 17 00:00:00 2001 From: Ma Shimiao Date: Wed, 8 Apr 2015 10:29:29 +0800 Subject: [PATCH 298/375] fix 8926: rmi dangling is unsafe when pulling Signed-off-by: Ma Shimiao Signed-off-by: Tibor Vass --- docs/session.go | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/docs/session.go b/docs/session.go index 154c63e1..75947e70 100644 --- a/docs/session.go +++ b/docs/session.go @@ -23,6 +23,7 @@ import ( "github.com/docker/docker/cliconfig" "github.com/docker/docker/pkg/httputils" "github.com/docker/docker/pkg/ioutils" + "github.com/docker/docker/pkg/stringid" "github.com/docker/docker/pkg/tarsum" ) @@ -35,6 +36,7 @@ type Session struct { client *http.Client // TODO(tiborvass): remove authConfig authConfig *cliconfig.AuthConfig + id string } type authTransport struct { @@ -158,6 +160,7 @@ func NewSession(client *http.Client, authConfig *cliconfig.AuthConfig, endpoint authConfig: authConfig, client: client, indexEndpoint: endpoint, + id: stringid.GenerateRandomID(), } var alwaysSetBasicAuth bool @@ -188,6 +191,11 @@ func NewSession(client *http.Client, authConfig *cliconfig.AuthConfig, endpoint return r, nil } +// ID returns this registry session's ID. +func (r *Session) ID() string { + return r.id +} + // Retrieve the history of a given image from the Registry. // Return a list of the parent's json (requested image included) func (r *Session) GetRemoteHistory(imgID, registry string) ([]string, error) { From 138ba392603a394afca5350f9639d0b6fdd1e809 Mon Sep 17 00:00:00 2001 From: Morgan Bauer Date: Mon, 20 Jul 2015 22:39:07 +0000 Subject: [PATCH 299/375] golint for cliconfig - fully capitalize HTTP in HTTPHeaders - comment for CONFIGFILE - camelcase and privatize oldConfigfile, defaultIndexserver - remove unused var errConfigFileMissing - comments for methods and functions throughout - external references to renamed variables changed Signed-off-by: Morgan Bauer --- docs/auth_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/auth_test.go b/docs/auth_test.go index 5f54add3..be520add 100644 --- a/docs/auth_test.go +++ b/docs/auth_test.go @@ -34,7 +34,7 @@ func setupTempConfigFile() (*cliconfig.ConfigFile, error) { if err != nil { return nil, err } - root = filepath.Join(root, cliconfig.CONFIGFILE) + root = filepath.Join(root, cliconfig.ConfigFileName) configFile := cliconfig.NewConfigFile(root) for _, registry := range []string{"testIndex", INDEXSERVER} { From 4c255a0d41630ccf67b8e84b65dfc0370adcea37 Mon Sep 17 00:00:00 2001 From: Aaron Lehmann Date: Tue, 21 Jul 2015 11:45:53 -0700 Subject: [PATCH 300/375] Remove dead code in registry package The only uses of RequestAuthorization and its associated functions were removed in 7fed379d95cd65796e55acdd768159191eff9109 ("Update graph to use vendored distribution client for the v2 codepath") Signed-off-by: Aaron Lehmann --- docs/auth.go | 89 ---------------------------------------------------- 1 file changed, 89 deletions(-) diff --git a/docs/auth.go b/docs/auth.go index 7111ede9..157d2140 100644 --- a/docs/auth.go +++ b/docs/auth.go @@ -6,100 +6,11 @@ import ( "io/ioutil" "net/http" "strings" - "sync" - "time" "github.com/Sirupsen/logrus" "github.com/docker/docker/cliconfig" ) -type RequestAuthorization struct { - authConfig *cliconfig.AuthConfig - registryEndpoint *Endpoint - resource string - scope string - actions []string - - tokenLock sync.Mutex - tokenCache string - tokenExpiration time.Time -} - -func NewRequestAuthorization(authConfig *cliconfig.AuthConfig, registryEndpoint *Endpoint, resource, scope string, actions []string) *RequestAuthorization { - return &RequestAuthorization{ - authConfig: authConfig, - registryEndpoint: registryEndpoint, - resource: resource, - scope: scope, - actions: actions, - } -} - -func (auth *RequestAuthorization) getToken() (string, error) { - auth.tokenLock.Lock() - defer auth.tokenLock.Unlock() - now := time.Now() - if now.Before(auth.tokenExpiration) { - logrus.Debugf("Using cached token for %s", auth.authConfig.Username) - return auth.tokenCache, nil - } - - for _, challenge := range auth.registryEndpoint.AuthChallenges { - switch strings.ToLower(challenge.Scheme) { - case "basic": - // no token necessary - case "bearer": - logrus.Debugf("Getting bearer token with %s for %s", challenge.Parameters, auth.authConfig.Username) - params := map[string]string{} - for k, v := range challenge.Parameters { - params[k] = v - } - params["scope"] = fmt.Sprintf("%s:%s:%s", auth.resource, auth.scope, strings.Join(auth.actions, ",")) - token, err := getToken(auth.authConfig.Username, auth.authConfig.Password, params, auth.registryEndpoint) - if err != nil { - return "", err - } - auth.tokenCache = token - auth.tokenExpiration = now.Add(time.Minute) - - return token, nil - default: - logrus.Infof("Unsupported auth scheme: %q", challenge.Scheme) - } - } - - // Do not expire cache since there are no challenges which use a token - auth.tokenExpiration = time.Now().Add(time.Hour * 24) - - return "", nil -} - -// Checks that requests to the v2 registry can be authorized. -func (auth *RequestAuthorization) CanAuthorizeV2() bool { - if len(auth.registryEndpoint.AuthChallenges) == 0 { - return true - } - scope := fmt.Sprintf("%s:%s:%s", auth.resource, auth.scope, strings.Join(auth.actions, ",")) - if _, err := loginV2(auth.authConfig, auth.registryEndpoint, scope); err != nil { - logrus.Debugf("Cannot authorize against V2 endpoint: %s", auth.registryEndpoint) - return false - } - return true -} - -func (auth *RequestAuthorization) Authorize(req *http.Request) error { - token, err := auth.getToken() - if err != nil { - return err - } - if token != "" { - req.Header.Set("Authorization", fmt.Sprintf("Bearer %s", token)) - } else if auth.authConfig.Username != "" && auth.authConfig.Password != "" { - req.SetBasicAuth(auth.authConfig.Username, auth.authConfig.Password) - } - return nil -} - // Login tries to register/login to the registry server. func Login(authConfig *cliconfig.AuthConfig, registryEndpoint *Endpoint) (string, error) { // Separates the v2 registry login logic from the v1 logic. From 5280103cc41ff1e6f1f5da499950d6615ed66590 Mon Sep 17 00:00:00 2001 From: Aaron Lehmann Date: Tue, 21 Jul 2015 11:53:57 -0700 Subject: [PATCH 301/375] Remove unused types in registry package Signed-off-by: Aaron Lehmann --- docs/types.go | 17 ----------------- 1 file changed, 17 deletions(-) diff --git a/docs/types.go b/docs/types.go index 2c8369bd..d02ae4fc 100644 --- a/docs/types.go +++ b/docs/types.go @@ -33,23 +33,6 @@ type RegistryInfo struct { Standalone bool `json:"standalone"` } -type FSLayer struct { - BlobSum string `json:"blobSum"` -} - -type ManifestHistory struct { - V1Compatibility string `json:"v1Compatibility"` -} - -type ManifestData struct { - Name string `json:"name"` - Tag string `json:"tag"` - Architecture string `json:"architecture"` - FSLayers []*FSLayer `json:"fsLayers"` - History []*ManifestHistory `json:"history"` - SchemaVersion int `json:"schemaVersion"` -} - type APIVersion int func (av APIVersion) String() string { From 00edb3bbcef75b26da81ae727346074b890dfa96 Mon Sep 17 00:00:00 2001 From: Richard Scothern Date: Tue, 21 Jul 2015 14:10:34 -0700 Subject: [PATCH 302/375] Configure TLS for private registry mirrors. If a registry mirror is using TLS, ensure that certs for it are picked up from /etc/docker/certs.d Signed-off-by: Richard Scothern --- docs/service.go | 21 +++++++++++++++++++-- 1 file changed, 19 insertions(+), 2 deletions(-) diff --git a/docs/service.go b/docs/service.go index 8dda537a..64ea242a 100644 --- a/docs/service.go +++ b/docs/service.go @@ -6,6 +6,7 @@ import ( "fmt" "io/ioutil" "net/http" + "net/url" "os" "path/filepath" "strings" @@ -161,19 +162,31 @@ func (s *Service) TlsConfig(hostname string) (*tls.Config, error) { return &tlsConfig, nil } +func (s *Service) tlsConfigForMirror(mirror string) (*tls.Config, error) { + mirrorUrl, err := url.Parse(mirror) + if err != nil { + return nil, err + } + return s.TlsConfig(mirrorUrl.Host) +} + func (s *Service) LookupEndpoints(repoName string) (endpoints []APIEndpoint, err error) { var cfg = tlsconfig.ServerDefault tlsConfig := &cfg if strings.HasPrefix(repoName, DEFAULT_NAMESPACE+"/") { // v2 mirrors for _, mirror := range s.Config.Mirrors { + mirrorTlsConfig, err := s.tlsConfigForMirror(mirror) + if err != nil { + return nil, err + } endpoints = append(endpoints, APIEndpoint{ URL: mirror, // guess mirrors are v2 Version: APIVersion2, Mirror: true, TrimHostname: true, - TLSConfig: tlsConfig, + TLSConfig: mirrorTlsConfig, }) } // v2 registry @@ -187,13 +200,17 @@ func (s *Service) LookupEndpoints(repoName string) (endpoints []APIEndpoint, err // v1 mirrors // TODO(tiborvass): shouldn't we remove v1 mirrors from here, since v1 mirrors are kinda special? for _, mirror := range s.Config.Mirrors { + mirrorTlsConfig, err := s.tlsConfigForMirror(mirror) + if err != nil { + return nil, err + } endpoints = append(endpoints, APIEndpoint{ URL: mirror, // guess mirrors are v1 Version: APIVersion1, Mirror: true, TrimHostname: true, - TLSConfig: tlsConfig, + TLSConfig: mirrorTlsConfig, }) } // v1 registry From 2b7788f2e8d2cf33aca06438915443c5cf9a3fb6 Mon Sep 17 00:00:00 2001 From: Richard Scothern Date: Tue, 21 Jul 2015 15:03:51 -0700 Subject: [PATCH 303/375] Remove v1 registry mirror configuration from LookupEndpoints. V1 mirrors do not mirror the index and those endpoints should only be indexes. Signed-off-by: Richard Scothern --- docs/service.go | 16 ---------------- 1 file changed, 16 deletions(-) diff --git a/docs/service.go b/docs/service.go index 64ea242a..1be448e4 100644 --- a/docs/service.go +++ b/docs/service.go @@ -197,22 +197,6 @@ func (s *Service) LookupEndpoints(repoName string) (endpoints []APIEndpoint, err TrimHostname: true, TLSConfig: tlsConfig, }) - // v1 mirrors - // TODO(tiborvass): shouldn't we remove v1 mirrors from here, since v1 mirrors are kinda special? - for _, mirror := range s.Config.Mirrors { - mirrorTlsConfig, err := s.tlsConfigForMirror(mirror) - if err != nil { - return nil, err - } - endpoints = append(endpoints, APIEndpoint{ - URL: mirror, - // guess mirrors are v1 - Version: APIVersion1, - Mirror: true, - TrimHostname: true, - TLSConfig: mirrorTlsConfig, - }) - } // v1 registry endpoints = append(endpoints, APIEndpoint{ URL: DEFAULT_V1_REGISTRY, From 726f3e07e39561bbe75b0e9696d7130adc17a97b Mon Sep 17 00:00:00 2001 From: Jessica Frazelle Date: Tue, 21 Jul 2015 18:45:17 -0700 Subject: [PATCH 304/375] better i/o timeout error on pull Signed-off-by: Jessica Frazelle --- docs/session.go | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/docs/session.go b/docs/session.go index 75947e70..cb982353 100644 --- a/docs/session.go +++ b/docs/session.go @@ -404,7 +404,14 @@ func (r *Session) GetRepositoryData(remote string) (*RepositoryData, error) { req.Header.Set("X-Docker-Token", "true") res, err := r.client.Do(req) if err != nil { - return nil, err + // check if the error is because of i/o timeout + // and return a non-obtuse error message for users + // "Get https://index.docker.io/v1/repositories/library/busybox/images: i/o timeout" + // was a top search on the docker user forum + if strings.HasSuffix(err.Error(), "i/o timeout") { + return nil, fmt.Errorf("Network timed out while trying to connect to %s. You may want to check your internet connection or if you are behind a proxy.", repositoryTarget) + } + return nil, fmt.Errorf("Error while pulling image: %v", err) } defer res.Body.Close() if res.StatusCode == 401 { From a246ab0a5e65ae553d10c24e54052ae5184305ba Mon Sep 17 00:00:00 2001 From: Tibor Vass Date: Tue, 5 May 2015 00:18:28 -0400 Subject: [PATCH 305/375] cli: new daemon command and new cli package This patch creates a new cli package that allows to combine both client and daemon commands (there is only one daemon command: docker daemon). The `-d` and `--daemon` top-level flags are deprecated and a special message is added to prompt the user to use `docker daemon`. Providing top-level daemon-specific flags for client commands result in an error message prompting the user to use `docker daemon`. This patch does not break any old but correct usages. This also makes `-d` and `--daemon` flags, as well as the `daemon` command illegal in client-only binaries. Signed-off-by: Tibor Vass --- docs/config.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/config.go b/docs/config.go index 333f1c46..a1dc3aba 100644 --- a/docs/config.go +++ b/docs/config.go @@ -43,11 +43,11 @@ var ( // InstallFlags adds command-line options to the top-level flag parser for // the current process. -func (options *Options) InstallFlags() { +func (options *Options) InstallFlags(cmd *flag.FlagSet, usageFn func(string) string) { options.Mirrors = opts.NewListOpts(ValidateMirror) - flag.Var(&options.Mirrors, []string{"-registry-mirror"}, "Preferred Docker registry mirror") + cmd.Var(&options.Mirrors, []string{"-registry-mirror"}, usageFn("Preferred Docker registry mirror")) options.InsecureRegistries = opts.NewListOpts(ValidateIndexName) - flag.Var(&options.InsecureRegistries, []string{"-insecure-registry"}, "Enable insecure registry communication") + cmd.Var(&options.InsecureRegistries, []string{"-insecure-registry"}, usageFn("Enable insecure registry communication")) } type netIPNet net.IPNet From 52136ab008e9586786fa4d81bd964e0c055d6fe5 Mon Sep 17 00:00:00 2001 From: Aaron Lehmann Date: Tue, 21 Jul 2015 12:40:36 -0700 Subject: [PATCH 306/375] Improve documentation and golint compliance of registry package * Add godoc documentation where it was missing * Change identifier names that don't match Go style, such as INDEX_NAME * Rename RegistryInfo to PingResult, which more accurately describes what this structure is for. It also has the benefit of making the name not stutter if used outside the package. Updates #14756 Signed-off-by: Aaron Lehmann --- docs/auth.go | 4 +- docs/auth_test.go | 10 ++--- docs/config.go | 43 ++++++++++-------- docs/endpoint.go | 42 ++++++++++-------- docs/endpoint_test.go | 2 +- docs/registry.go | 25 ++++++----- docs/registry_mock_test.go | 10 ++--- docs/registry_test.go | 60 ++++++++++++------------- docs/service.go | 37 +++++++++------- docs/session.go | 51 +++++++++++++++------- docs/types.go | 89 +++++++++++++++++++++++++++++--------- 11 files changed, 231 insertions(+), 142 deletions(-) diff --git a/docs/auth.go b/docs/auth.go index 157d2140..57560935 100644 --- a/docs/auth.go +++ b/docs/auth.go @@ -36,7 +36,7 @@ func loginV1(authConfig *cliconfig.AuthConfig, registryEndpoint *Endpoint) (stri return "", fmt.Errorf("Server Error: Server Address not set.") } - loginAgainstOfficialIndex := serverAddress == INDEXSERVER + loginAgainstOfficialIndex := serverAddress == IndexServer // to avoid sending the server address to the server it should be removed before being marshalled authCopy := *authConfig @@ -220,7 +220,7 @@ func tryV2TokenAuthLogin(authConfig *cliconfig.AuthConfig, params map[string]str return nil } -// this method matches a auth configuration to a server address or a url +// ResolveAuthConfig matches an auth configuration to a server address or a URL func ResolveAuthConfig(config *cliconfig.ConfigFile, index *IndexInfo) cliconfig.AuthConfig { configKey := index.GetAuthConfigKey() // First try the happy case diff --git a/docs/auth_test.go b/docs/auth_test.go index be520add..a8e3da01 100644 --- a/docs/auth_test.go +++ b/docs/auth_test.go @@ -37,7 +37,7 @@ func setupTempConfigFile() (*cliconfig.ConfigFile, error) { root = filepath.Join(root, cliconfig.ConfigFileName) configFile := cliconfig.NewConfigFile(root) - for _, registry := range []string{"testIndex", INDEXSERVER} { + for _, registry := range []string{"testIndex", IndexServer} { configFile.AuthConfigs[registry] = cliconfig.AuthConfig{ Username: "docker-user", Password: "docker-pass", @@ -82,7 +82,7 @@ func TestResolveAuthConfigIndexServer(t *testing.T) { } defer os.RemoveAll(configFile.Filename()) - indexConfig := configFile.AuthConfigs[INDEXSERVER] + indexConfig := configFile.AuthConfigs[IndexServer] officialIndex := &IndexInfo{ Official: true, @@ -92,10 +92,10 @@ func TestResolveAuthConfigIndexServer(t *testing.T) { } resolved := ResolveAuthConfig(configFile, officialIndex) - assertEqual(t, resolved, indexConfig, "Expected ResolveAuthConfig to return INDEXSERVER") + assertEqual(t, resolved, indexConfig, "Expected ResolveAuthConfig to return IndexServer") resolved = ResolveAuthConfig(configFile, privateIndex) - assertNotEqual(t, resolved, indexConfig, "Expected ResolveAuthConfig to not return INDEXSERVER") + assertNotEqual(t, resolved, indexConfig, "Expected ResolveAuthConfig to not return IndexServer") } func TestResolveAuthConfigFullURL(t *testing.T) { @@ -120,7 +120,7 @@ func TestResolveAuthConfigFullURL(t *testing.T) { Password: "baz-pass", Email: "baz@example.com", } - configFile.AuthConfigs[INDEXSERVER] = officialAuth + configFile.AuthConfigs[IndexServer] = officialAuth expectedAuths := map[string]cliconfig.AuthConfig{ "registry.example.com": registryAuth, diff --git a/docs/config.go b/docs/config.go index a1dc3aba..d2108894 100644 --- a/docs/config.go +++ b/docs/config.go @@ -21,24 +21,33 @@ type Options struct { } const ( - DEFAULT_NAMESPACE = "docker.io" - DEFAULT_V2_REGISTRY = "https://registry-1.docker.io" - DEFAULT_REGISTRY_VERSION_HEADER = "Docker-Distribution-Api-Version" - DEFAULT_V1_REGISTRY = "https://index.docker.io" + // DefaultNamespace is the default namespace + DefaultNamespace = "docker.io" + // DefaultV2Registry is the URI of the default v2 registry + DefaultV2Registry = "https://registry-1.docker.io" + // DefaultRegistryVersionHeader is the name of the default HTTP header + // that carries Registry version info + DefaultRegistryVersionHeader = "Docker-Distribution-Api-Version" + // DefaultV1Registry is the URI of the default v1 registry + DefaultV1Registry = "https://index.docker.io" - CERTS_DIR = "/etc/docker/certs.d" + // CertsDir is the directory where certificates are stored + CertsDir = "/etc/docker/certs.d" - // Only used for user auth + account creation - REGISTRYSERVER = DEFAULT_V2_REGISTRY - INDEXSERVER = DEFAULT_V1_REGISTRY + "/v1/" - INDEXNAME = "docker.io" + // IndexServer is the v1 registry server used for user auth + account creation + IndexServer = DefaultV1Registry + "/v1/" + // IndexName is the name of the index + IndexName = "docker.io" - // INDEXSERVER = "https://registry-stage.hub.docker.com/v1/" + // IndexServer = "https://registry-stage.hub.docker.com/v1/" ) var ( + // ErrInvalidRepositoryName is an error returned if the repository name did + // not have the correct form ErrInvalidRepositoryName = errors.New("Invalid repository name (ex: \"registry.domain.tld/myrepos\")") - emptyServiceConfig = NewServiceConfig(nil) + + emptyServiceConfig = NewServiceConfig(nil) ) // InstallFlags adds command-line options to the top-level flag parser for @@ -116,8 +125,8 @@ func NewServiceConfig(options *Options) *ServiceConfig { } // Configure public registry. - config.IndexConfigs[INDEXNAME] = &IndexInfo{ - Name: INDEXNAME, + config.IndexConfigs[IndexName] = &IndexInfo{ + Name: IndexName, Mirrors: config.Mirrors, Secure: true, Official: true, @@ -196,8 +205,8 @@ func ValidateMirror(val string) (string, error) { // ValidateIndexName validates an index name. func ValidateIndexName(val string) (string, error) { // 'index.docker.io' => 'docker.io' - if val == "index."+INDEXNAME { - val = INDEXNAME + if val == "index."+IndexName { + val = IndexName } if strings.HasPrefix(val, "-") || strings.HasSuffix(val, "-") { return "", fmt.Errorf("Invalid index name (%s). Cannot begin or end with a hyphen.", val) @@ -267,7 +276,7 @@ func (config *ServiceConfig) NewIndexInfo(indexName string) (*IndexInfo, error) // index as the AuthConfig key, and uses the (host)name[:port] for private indexes. func (index *IndexInfo) GetAuthConfigKey() string { if index.Official { - return INDEXSERVER + return IndexServer } return index.Name } @@ -280,7 +289,7 @@ func splitReposName(reposName string) (string, string) { !strings.Contains(nameParts[0], ":") && nameParts[0] != "localhost") { // This is a Docker Index repos (ex: samalba/hipache or ubuntu) // 'docker.io' - indexName = INDEXNAME + indexName = IndexName remoteName = reposName } else { indexName = nameParts[0] diff --git a/docs/endpoint.go b/docs/endpoint.go index 17443543..c6361346 100644 --- a/docs/endpoint.go +++ b/docs/endpoint.go @@ -111,6 +111,7 @@ func newEndpoint(address string, tlsConfig *tls.Config, metaHeaders http.Header) return endpoint, nil } +// GetEndpoint returns a new endpoint with the specified headers func (repoInfo *RepositoryInfo) GetEndpoint(metaHeaders http.Header) (*Endpoint, error) { return NewEndpoint(repoInfo.Index, metaHeaders) } @@ -142,7 +143,10 @@ func (e *Endpoint) Path(path string) string { return fmt.Sprintf("%s/v%d/%s", e.URL, e.Version, path) } -func (e *Endpoint) Ping() (RegistryInfo, error) { +// Ping pings the remote endpoint with v2 and v1 pings to determine the API +// version. It returns a PingResult containing the discovered version. The +// PingResult also indicates whether the registry is standalone or not. +func (e *Endpoint) Ping() (PingResult, error) { // The ping logic to use is determined by the registry endpoint version. switch e.Version { case APIVersion1: @@ -167,49 +171,49 @@ func (e *Endpoint) Ping() (RegistryInfo, error) { } e.Version = APIVersionUnknown - return RegistryInfo{}, fmt.Errorf("unable to ping registry endpoint %s\nv2 ping attempt failed with error: %s\n v1 ping attempt failed with error: %s", e, errV2, errV1) + return PingResult{}, fmt.Errorf("unable to ping registry endpoint %s\nv2 ping attempt failed with error: %s\n v1 ping attempt failed with error: %s", e, errV2, errV1) } -func (e *Endpoint) pingV1() (RegistryInfo, error) { +func (e *Endpoint) pingV1() (PingResult, error) { logrus.Debugf("attempting v1 ping for registry endpoint %s", e) - if e.String() == INDEXSERVER { + if e.String() == IndexServer { // Skip the check, we know this one is valid // (and we never want to fallback to http in case of error) - return RegistryInfo{Standalone: false}, nil + return PingResult{Standalone: false}, nil } req, err := http.NewRequest("GET", e.Path("_ping"), nil) if err != nil { - return RegistryInfo{Standalone: false}, err + return PingResult{Standalone: false}, err } resp, err := e.client.Do(req) if err != nil { - return RegistryInfo{Standalone: false}, err + return PingResult{Standalone: false}, err } defer resp.Body.Close() jsonString, err := ioutil.ReadAll(resp.Body) if err != nil { - return RegistryInfo{Standalone: false}, fmt.Errorf("error while reading the http response: %s", err) + return PingResult{Standalone: false}, fmt.Errorf("error while reading the http response: %s", err) } // If the header is absent, we assume true for compatibility with earlier // versions of the registry. default to true - info := RegistryInfo{ + info := PingResult{ Standalone: true, } if err := json.Unmarshal(jsonString, &info); err != nil { - logrus.Debugf("Error unmarshalling the _ping RegistryInfo: %s", err) + logrus.Debugf("Error unmarshalling the _ping PingResult: %s", err) // don't stop here. Just assume sane defaults } if hdr := resp.Header.Get("X-Docker-Registry-Version"); hdr != "" { logrus.Debugf("Registry version header: '%s'", hdr) info.Version = hdr } - logrus.Debugf("RegistryInfo.Version: %q", info.Version) + logrus.Debugf("PingResult.Version: %q", info.Version) standalone := resp.Header.Get("X-Docker-Registry-Standalone") logrus.Debugf("Registry standalone header: '%s'", standalone) @@ -220,21 +224,21 @@ func (e *Endpoint) pingV1() (RegistryInfo, error) { // there is a header set, and it is not "true" or "1", so assume fails info.Standalone = false } - logrus.Debugf("RegistryInfo.Standalone: %t", info.Standalone) + logrus.Debugf("PingResult.Standalone: %t", info.Standalone) return info, nil } -func (e *Endpoint) pingV2() (RegistryInfo, error) { +func (e *Endpoint) pingV2() (PingResult, error) { logrus.Debugf("attempting v2 ping for registry endpoint %s", e) req, err := http.NewRequest("GET", e.Path(""), nil) if err != nil { - return RegistryInfo{}, err + return PingResult{}, err } resp, err := e.client.Do(req) if err != nil { - return RegistryInfo{}, err + return PingResult{}, err } defer resp.Body.Close() @@ -253,21 +257,21 @@ HeaderLoop: } if !supportsV2 { - return RegistryInfo{}, fmt.Errorf("%s does not appear to be a v2 registry endpoint", e) + return PingResult{}, fmt.Errorf("%s does not appear to be a v2 registry endpoint", e) } if resp.StatusCode == http.StatusOK { // It would seem that no authentication/authorization is required. // So we don't need to parse/add any authorization schemes. - return RegistryInfo{Standalone: true}, nil + return PingResult{Standalone: true}, nil } if resp.StatusCode == http.StatusUnauthorized { // Parse the WWW-Authenticate Header and store the challenges // on this endpoint object. e.AuthChallenges = parseAuthHeader(resp.Header) - return RegistryInfo{}, nil + return PingResult{}, nil } - return RegistryInfo{}, fmt.Errorf("v2 registry endpoint returned status %d: %q", resp.StatusCode, http.StatusText(resp.StatusCode)) + return PingResult{}, fmt.Errorf("v2 registry endpoint returned status %d: %q", resp.StatusCode, http.StatusText(resp.StatusCode)) } diff --git a/docs/endpoint_test.go b/docs/endpoint_test.go index a04f9a03..ee301dbd 100644 --- a/docs/endpoint_test.go +++ b/docs/endpoint_test.go @@ -12,7 +12,7 @@ func TestEndpointParse(t *testing.T) { str string expected string }{ - {INDEXSERVER, INDEXSERVER}, + {IndexServer, IndexServer}, {"http://0.0.0.0:5000/v1/", "http://0.0.0.0:5000/v1/"}, {"http://0.0.0.0:5000/v2/", "http://0.0.0.0:5000/v2/"}, {"http://0.0.0.0:5000", "http://0.0.0.0:5000/v0/"}, diff --git a/docs/registry.go b/docs/registry.go index 74a0ad5f..fd85c21c 100644 --- a/docs/registry.go +++ b/docs/registry.go @@ -21,19 +21,12 @@ import ( ) var ( + // ErrAlreadyExists is an error returned if an image being pushed + // already exists on the remote side ErrAlreadyExists = errors.New("Image already exists") - ErrDoesNotExist = errors.New("Image does not exist") errLoginRequired = errors.New("Authentication is required.") ) -type TimeoutType uint32 - -const ( - NoTimeout TimeoutType = iota - ReceiveTimeout - ConnectTimeout -) - // dockerUserAgent is the User-Agent the Docker client uses to identify itself. // It is populated on init(), comprising version information of different components. var dockerUserAgent string @@ -74,10 +67,12 @@ func DockerHeaders(metaHeaders http.Header) []transport.RequestModifier { return modifiers } +// HTTPClient returns a HTTP client structure which uses the given transport +// and contains the necessary headers for redirected requests func HTTPClient(transport http.RoundTripper) *http.Client { return &http.Client{ Transport: transport, - CheckRedirect: AddRequiredHeadersToRedirectedRequests, + CheckRedirect: addRequiredHeadersToRedirectedRequests, } } @@ -98,7 +93,9 @@ func trustedLocation(req *http.Request) bool { return false } -func AddRequiredHeadersToRedirectedRequests(req *http.Request, via []*http.Request) error { +// addRequiredHeadersToRedirectedRequests adds the necessary redirection headers +// for redirected requests +func addRequiredHeadersToRedirectedRequests(req *http.Request, via []*http.Request) error { if via != nil && via[0] != nil { if trustedLocation(req) && trustedLocation(via[0]) { req.Header = via[0].Header @@ -124,6 +121,8 @@ func shouldV2Fallback(err errcode.Error) bool { return false } +// ErrNoSupport is an error type used for errors indicating that an operation +// is not supported. It encapsulates a more specific error. type ErrNoSupport struct{ Err error } func (e ErrNoSupport) Error() string { @@ -133,6 +132,8 @@ func (e ErrNoSupport) Error() string { return e.Err.Error() } +// ContinueOnError returns true if we should fallback to the next endpoint +// as a result of this error. func ContinueOnError(err error) bool { switch v := err.(type) { case errcode.Errors: @@ -145,6 +146,8 @@ func ContinueOnError(err error) bool { return false } +// NewTransport returns a new HTTP transport. If tlsConfig is nil, it uses the +// default TLS configuration. func NewTransport(tlsConfig *tls.Config) *http.Transport { if tlsConfig == nil { var cfg = tlsconfig.ServerDefault diff --git a/docs/registry_mock_test.go b/docs/registry_mock_test.go index 9217956c..fb19e577 100644 --- a/docs/registry_mock_test.go +++ b/docs/registry_mock_test.go @@ -145,7 +145,7 @@ func makeURL(req string) string { return testHTTPServer.URL + req } -func makeHttpsURL(req string) string { +func makeHTTPSURL(req string) string { return testHTTPSServer.URL + req } @@ -156,16 +156,16 @@ func makeIndex(req string) *IndexInfo { return index } -func makeHttpsIndex(req string) *IndexInfo { +func makeHTTPSIndex(req string) *IndexInfo { index := &IndexInfo{ - Name: makeHttpsURL(req), + Name: makeHTTPSURL(req), } return index } func makePublicIndex() *IndexInfo { index := &IndexInfo{ - Name: INDEXSERVER, + Name: IndexServer, Secure: true, Official: true, } @@ -468,7 +468,7 @@ func TestPing(t *testing.T) { * WARNING: Don't push on the repos uncommented, it'll block the tests * func TestWait(t *testing.T) { - logrus.Println("Test HTTP server ready and waiting:", testHttpServer.URL) + logrus.Println("Test HTTP server ready and waiting:", testHTTPServer.URL) c := make(chan int) <-c } diff --git a/docs/registry_test.go b/docs/registry_test.go index 4d17a62c..88b08dff 100644 --- a/docs/registry_test.go +++ b/docs/registry_test.go @@ -63,7 +63,7 @@ func TestPingRegistryEndpoint(t *testing.T) { } testPing(makeIndex("/v1/"), true, "Expected standalone to be true (default)") - testPing(makeHttpsIndex("/v1/"), true, "Expected standalone to be true (default)") + testPing(makeHTTPSIndex("/v1/"), true, "Expected standalone to be true (default)") testPing(makePublicIndex(), false, "Expected standalone to be false for public index") } @@ -119,7 +119,7 @@ func TestEndpoint(t *testing.T) { } assertInsecureIndex(index) - index.Name = makeHttpsURL("/v1/") + index.Name = makeHTTPSURL("/v1/") endpoint = expandEndpoint(index) assertEqual(t, endpoint.String(), index.Name, "Expected endpoint to be "+index.Name) if endpoint.Version != APIVersion1 { @@ -127,7 +127,7 @@ func TestEndpoint(t *testing.T) { } assertSecureIndex(index) - index.Name = makeHttpsURL("") + index.Name = makeHTTPSURL("") endpoint = expandEndpoint(index) assertEqual(t, endpoint.String(), index.Name+"/v1/", index.Name+": Expected endpoint to be "+index.Name+"/v1/") if endpoint.Version != APIVersion1 { @@ -135,7 +135,7 @@ func TestEndpoint(t *testing.T) { } assertSecureIndex(index) - httpsURL := makeHttpsURL("") + httpsURL := makeHTTPSURL("") index.Name = strings.SplitN(httpsURL, "://", 2)[1] endpoint = expandEndpoint(index) assertEqual(t, endpoint.String(), httpsURL+"/v1/", index.Name+": Expected endpoint to be "+httpsURL+"/v1/") @@ -332,7 +332,7 @@ func TestParseRepositoryInfo(t *testing.T) { expectedRepoInfos := map[string]RepositoryInfo{ "fooo/bar": { Index: &IndexInfo{ - Name: INDEXNAME, + Name: IndexName, Official: true, }, RemoteName: "fooo/bar", @@ -342,7 +342,7 @@ func TestParseRepositoryInfo(t *testing.T) { }, "library/ubuntu": { Index: &IndexInfo{ - Name: INDEXNAME, + Name: IndexName, Official: true, }, RemoteName: "library/ubuntu", @@ -352,7 +352,7 @@ func TestParseRepositoryInfo(t *testing.T) { }, "nonlibrary/ubuntu": { Index: &IndexInfo{ - Name: INDEXNAME, + Name: IndexName, Official: true, }, RemoteName: "nonlibrary/ubuntu", @@ -362,7 +362,7 @@ func TestParseRepositoryInfo(t *testing.T) { }, "ubuntu": { Index: &IndexInfo{ - Name: INDEXNAME, + Name: IndexName, Official: true, }, RemoteName: "library/ubuntu", @@ -372,7 +372,7 @@ func TestParseRepositoryInfo(t *testing.T) { }, "other/library": { Index: &IndexInfo{ - Name: INDEXNAME, + Name: IndexName, Official: true, }, RemoteName: "other/library", @@ -480,9 +480,9 @@ func TestParseRepositoryInfo(t *testing.T) { CanonicalName: "localhost/privatebase", Official: false, }, - INDEXNAME + "/public/moonbase": { + IndexName + "/public/moonbase": { Index: &IndexInfo{ - Name: INDEXNAME, + Name: IndexName, Official: true, }, RemoteName: "public/moonbase", @@ -490,9 +490,9 @@ func TestParseRepositoryInfo(t *testing.T) { CanonicalName: "docker.io/public/moonbase", Official: false, }, - "index." + INDEXNAME + "/public/moonbase": { + "index." + IndexName + "/public/moonbase": { Index: &IndexInfo{ - Name: INDEXNAME, + Name: IndexName, Official: true, }, RemoteName: "public/moonbase", @@ -502,7 +502,7 @@ func TestParseRepositoryInfo(t *testing.T) { }, "ubuntu-12.04-base": { Index: &IndexInfo{ - Name: INDEXNAME, + Name: IndexName, Official: true, }, RemoteName: "library/ubuntu-12.04-base", @@ -510,9 +510,9 @@ func TestParseRepositoryInfo(t *testing.T) { CanonicalName: "docker.io/library/ubuntu-12.04-base", Official: true, }, - INDEXNAME + "/ubuntu-12.04-base": { + IndexName + "/ubuntu-12.04-base": { Index: &IndexInfo{ - Name: INDEXNAME, + Name: IndexName, Official: true, }, RemoteName: "library/ubuntu-12.04-base", @@ -520,9 +520,9 @@ func TestParseRepositoryInfo(t *testing.T) { CanonicalName: "docker.io/library/ubuntu-12.04-base", Official: true, }, - "index." + INDEXNAME + "/ubuntu-12.04-base": { + "index." + IndexName + "/ubuntu-12.04-base": { Index: &IndexInfo{ - Name: INDEXNAME, + Name: IndexName, Official: true, }, RemoteName: "library/ubuntu-12.04-base", @@ -563,16 +563,16 @@ func TestNewIndexInfo(t *testing.T) { } config := NewServiceConfig(nil) - noMirrors := make([]string, 0) + noMirrors := []string{} expectedIndexInfos := map[string]*IndexInfo{ - INDEXNAME: { - Name: INDEXNAME, + IndexName: { + Name: IndexName, Official: true, Secure: true, Mirrors: noMirrors, }, - "index." + INDEXNAME: { - Name: INDEXNAME, + "index." + IndexName: { + Name: IndexName, Official: true, Secure: true, Mirrors: noMirrors, @@ -596,14 +596,14 @@ func TestNewIndexInfo(t *testing.T) { config = makeServiceConfig(publicMirrors, []string{"example.com"}) expectedIndexInfos = map[string]*IndexInfo{ - INDEXNAME: { - Name: INDEXNAME, + IndexName: { + Name: IndexName, Official: true, Secure: true, Mirrors: publicMirrors, }, - "index." + INDEXNAME: { - Name: INDEXNAME, + "index." + IndexName: { + Name: IndexName, Official: true, Secure: true, Mirrors: publicMirrors, @@ -814,7 +814,7 @@ func TestAddRequiredHeadersToRedirectedRequests(t *testing.T) { reqFrom.Header.Add("Authorization", "super_secret") reqTo, _ := http.NewRequest("GET", urls[1], nil) - AddRequiredHeadersToRedirectedRequests(reqTo, []*http.Request{reqFrom}) + addRequiredHeadersToRedirectedRequests(reqTo, []*http.Request{reqFrom}) if len(reqTo.Header) != 1 { t.Fatalf("Expected 1 headers, got %d", len(reqTo.Header)) @@ -838,7 +838,7 @@ func TestAddRequiredHeadersToRedirectedRequests(t *testing.T) { reqFrom.Header.Add("Authorization", "super_secret") reqTo, _ := http.NewRequest("GET", urls[1], nil) - AddRequiredHeadersToRedirectedRequests(reqTo, []*http.Request{reqFrom}) + addRequiredHeadersToRedirectedRequests(reqTo, []*http.Request{reqFrom}) if len(reqTo.Header) != 2 { t.Fatalf("Expected 2 headers, got %d", len(reqTo.Header)) @@ -860,7 +860,7 @@ func TestIsSecureIndex(t *testing.T) { insecureRegistries []string expected bool }{ - {INDEXNAME, nil, true}, + {IndexName, nil, true}, {"example.com", []string{}, true}, {"example.com", []string{"example.com"}, false}, {"localhost", []string{"localhost:5000"}, false}, diff --git a/docs/service.go b/docs/service.go index 1be448e4..274dfeb2 100644 --- a/docs/service.go +++ b/docs/service.go @@ -17,12 +17,14 @@ import ( "github.com/docker/docker/pkg/tlsconfig" ) +// Service is a registry service. It tracks configuration data such as a list +// of mirrors. type Service struct { Config *ServiceConfig } // NewService returns a new instance of Service ready to be -// installed no an engine. +// installed into an engine. func NewService(options *Options) *Service { return &Service{ Config: NewServiceConfig(options), @@ -36,7 +38,7 @@ func (s *Service) Auth(authConfig *cliconfig.AuthConfig) (string, error) { addr := authConfig.ServerAddress if addr == "" { // Use the official registry address if not specified. - addr = INDEXSERVER + addr = IndexServer } index, err := s.ResolveIndex(addr) if err != nil { @@ -81,6 +83,7 @@ func (s *Service) ResolveIndex(name string) (*IndexInfo, error) { return s.Config.NewIndexInfo(name) } +// APIEndpoint represents a remote API endpoint type APIEndpoint struct { Mirror bool URL string @@ -92,12 +95,13 @@ type APIEndpoint struct { Versions []auth.APIVersion } +// ToV1Endpoint returns a V1 API endpoint based on the APIEndpoint func (e APIEndpoint) ToV1Endpoint(metaHeaders http.Header) (*Endpoint, error) { return newEndpoint(e.URL, e.TLSConfig, metaHeaders) } -func (s *Service) TlsConfig(hostname string) (*tls.Config, error) { - // we construct a client tls config from server defaults +// TLSConfig constructs a client TLS configuration based on server defaults +func (s *Service) TLSConfig(hostname string) (*tls.Config, error) { // PreferredServerCipherSuites should have no effect tlsConfig := tlsconfig.ServerDefault @@ -115,7 +119,7 @@ func (s *Service) TlsConfig(hostname string) (*tls.Config, error) { return false } - hostDir := filepath.Join(CERTS_DIR, hostname) + hostDir := filepath.Join(CertsDir, hostname) logrus.Debugf("hostDir: %s", hostDir) fs, err := ioutil.ReadDir(hostDir) if err != nil && !os.IsNotExist(err) { @@ -163,20 +167,23 @@ func (s *Service) TlsConfig(hostname string) (*tls.Config, error) { } func (s *Service) tlsConfigForMirror(mirror string) (*tls.Config, error) { - mirrorUrl, err := url.Parse(mirror) + mirrorURL, err := url.Parse(mirror) if err != nil { return nil, err } - return s.TlsConfig(mirrorUrl.Host) + return s.TLSConfig(mirrorURL.Host) } +// LookupEndpoints creates an list of endpoints to try, in order of preference. +// It gives preference to v2 endpoints over v1, mirrors over the actual +// registry, and HTTPS over plain HTTP. func (s *Service) LookupEndpoints(repoName string) (endpoints []APIEndpoint, err error) { var cfg = tlsconfig.ServerDefault tlsConfig := &cfg - if strings.HasPrefix(repoName, DEFAULT_NAMESPACE+"/") { + if strings.HasPrefix(repoName, DefaultNamespace+"/") { // v2 mirrors for _, mirror := range s.Config.Mirrors { - mirrorTlsConfig, err := s.tlsConfigForMirror(mirror) + mirrorTLSConfig, err := s.tlsConfigForMirror(mirror) if err != nil { return nil, err } @@ -186,12 +193,12 @@ func (s *Service) LookupEndpoints(repoName string) (endpoints []APIEndpoint, err Version: APIVersion2, Mirror: true, TrimHostname: true, - TLSConfig: mirrorTlsConfig, + TLSConfig: mirrorTLSConfig, }) } // v2 registry endpoints = append(endpoints, APIEndpoint{ - URL: DEFAULT_V2_REGISTRY, + URL: DefaultV2Registry, Version: APIVersion2, Official: true, TrimHostname: true, @@ -199,7 +206,7 @@ func (s *Service) LookupEndpoints(repoName string) (endpoints []APIEndpoint, err }) // v1 registry endpoints = append(endpoints, APIEndpoint{ - URL: DEFAULT_V1_REGISTRY, + URL: DefaultV1Registry, Version: APIVersion1, Official: true, TrimHostname: true, @@ -214,7 +221,7 @@ func (s *Service) LookupEndpoints(repoName string) (endpoints []APIEndpoint, err } hostname := repoName[:slashIndex] - tlsConfig, err = s.TlsConfig(hostname) + tlsConfig, err = s.TLSConfig(hostname) if err != nil { return nil, err } @@ -232,7 +239,7 @@ func (s *Service) LookupEndpoints(repoName string) (endpoints []APIEndpoint, err Version: APIVersion2, TrimHostname: true, TLSConfig: tlsConfig, - VersionHeader: DEFAULT_REGISTRY_VERSION_HEADER, + VersionHeader: DefaultRegistryVersionHeader, Versions: v2Versions, }, { @@ -250,7 +257,7 @@ func (s *Service) LookupEndpoints(repoName string) (endpoints []APIEndpoint, err TrimHostname: true, // used to check if supposed to be secure via InsecureSkipVerify TLSConfig: tlsConfig, - VersionHeader: DEFAULT_REGISTRY_VERSION_HEADER, + VersionHeader: DefaultRegistryVersionHeader, Versions: v2Versions, }, APIEndpoint{ URL: "http://" + hostname, diff --git a/docs/session.go b/docs/session.go index cb982353..9bec7c1b 100644 --- a/docs/session.go +++ b/docs/session.go @@ -28,9 +28,12 @@ import ( ) var ( + // ErrRepoNotFound is returned if the repository didn't exist on the + // remote side ErrRepoNotFound = errors.New("Repository not found") ) +// A Session is used to communicate with a V1 registry type Session struct { indexEndpoint *Endpoint client *http.Client @@ -90,9 +93,11 @@ func cloneRequest(r *http.Request) *http.Request { return r2 } +// RoundTrip changes a HTTP request's headers to add the necessary +// authentication-related headers func (tr *authTransport) RoundTrip(orig *http.Request) (*http.Response, error) { // Authorization should not be set on 302 redirect for untrusted locations. - // This logic mirrors the behavior in AddRequiredHeadersToRedirectedRequests. + // This logic mirrors the behavior in addRequiredHeadersToRedirectedRequests. // As the authorization logic is currently implemented in RoundTrip, // a 302 redirect is detected by looking at the Referer header as go http package adds said header. // This is safe as Docker doesn't set Referer in other scenarios. @@ -154,6 +159,7 @@ func (tr *authTransport) CancelRequest(req *http.Request) { } } +// NewSession creates a new session // TODO(tiborvass): remove authConfig param once registry client v2 is vendored func NewSession(client *http.Client, authConfig *cliconfig.AuthConfig, endpoint *Endpoint) (r *Session, err error) { r = &Session{ @@ -167,7 +173,7 @@ func NewSession(client *http.Client, authConfig *cliconfig.AuthConfig, endpoint // If we're working with a standalone private registry over HTTPS, send Basic Auth headers // alongside all our requests. - if endpoint.VersionString(1) != INDEXSERVER && endpoint.URL.Scheme == "https" { + if endpoint.VersionString(1) != IndexServer && endpoint.URL.Scheme == "https" { info, err := endpoint.Ping() if err != nil { return nil, err @@ -196,8 +202,8 @@ func (r *Session) ID() string { return r.id } -// Retrieve the history of a given image from the Registry. -// Return a list of the parent's json (requested image included) +// GetRemoteHistory retrieves the history of a given image from the registry. +// It returns a list of the parent's JSON files (including the requested image). func (r *Session) GetRemoteHistory(imgID, registry string) ([]string, error) { res, err := r.client.Get(registry + "images/" + imgID + "/ancestry") if err != nil { @@ -220,7 +226,7 @@ func (r *Session) GetRemoteHistory(imgID, registry string) ([]string, error) { return history, nil } -// Check if an image exists in the Registry +// LookupRemoteImage checks if an image exists in the registry func (r *Session) LookupRemoteImage(imgID, registry string) error { res, err := r.client.Get(registry + "images/" + imgID + "/json") if err != nil { @@ -233,7 +239,7 @@ func (r *Session) LookupRemoteImage(imgID, registry string) error { return nil } -// Retrieve an image from the Registry. +// GetRemoteImageJSON retrieves an image's JSON metadata from the registry. func (r *Session) GetRemoteImageJSON(imgID, registry string) ([]byte, int, error) { res, err := r.client.Get(registry + "images/" + imgID + "/json") if err != nil { @@ -259,6 +265,7 @@ func (r *Session) GetRemoteImageJSON(imgID, registry string) ([]byte, int, error return jsonString, imageSize, nil } +// GetRemoteImageLayer retrieves an image layer from the registry func (r *Session) GetRemoteImageLayer(imgID, registry string, imgSize int64) (io.ReadCloser, error) { var ( retries = 5 @@ -308,9 +315,13 @@ func (r *Session) GetRemoteImageLayer(imgID, registry string, imgSize int64) (io return res.Body, nil } +// GetRemoteTag retrieves the tag named in the askedTag argument from the given +// repository. It queries each of the registries supplied in the registries +// argument, and returns data from the first one that answers the query +// successfully. func (r *Session) GetRemoteTag(registries []string, repository string, askedTag string) (string, error) { if strings.Count(repository, "/") == 0 { - // This will be removed once the Registry supports auto-resolution on + // This will be removed once the registry supports auto-resolution on // the "library" namespace repository = "library/" + repository } @@ -331,18 +342,22 @@ func (r *Session) GetRemoteTag(registries []string, repository string, askedTag continue } - var tagId string - if err := json.NewDecoder(res.Body).Decode(&tagId); err != nil { + var tagID string + if err := json.NewDecoder(res.Body).Decode(&tagID); err != nil { return "", err } - return tagId, nil + return tagID, nil } return "", fmt.Errorf("Could not reach any registry endpoint") } +// GetRemoteTags retrieves all tags from the given repository. It queries each +// of the registries supplied in the registries argument, and returns data from +// the first one that answers the query successfully. It returns a map with +// tag names as the keys and image IDs as the values. func (r *Session) GetRemoteTags(registries []string, repository string) (map[string]string, error) { if strings.Count(repository, "/") == 0 { - // This will be removed once the Registry supports auto-resolution on + // This will be removed once the registry supports auto-resolution on // the "library" namespace repository = "library/" + repository } @@ -379,7 +394,7 @@ func buildEndpointsList(headers []string, indexEp string) ([]string, error) { return nil, err } var urlScheme = parsedURL.Scheme - // The Registry's URL scheme has to match the Index' + // The registry's URL scheme has to match the Index' for _, ep := range headers { epList := strings.Split(ep, ",") for _, epListElement := range epList { @@ -391,6 +406,7 @@ func buildEndpointsList(headers []string, indexEp string) ([]string, error) { return endpoints, nil } +// GetRepositoryData returns lists of images and endpoints for the repository func (r *Session) GetRepositoryData(remote string) (*RepositoryData, error) { repositoryTarget := fmt.Sprintf("%srepositories/%s/images", r.indexEndpoint.VersionString(1), remote) @@ -457,8 +473,8 @@ func (r *Session) GetRepositoryData(remote string) (*RepositoryData, error) { }, nil } +// PushImageChecksumRegistry uploads checksums for an image func (r *Session) PushImageChecksumRegistry(imgData *ImgData, registry string) error { - u := registry + "images/" + imgData.ID + "/checksum" logrus.Debugf("[registry] Calling PUT %s", u) @@ -494,7 +510,7 @@ func (r *Session) PushImageChecksumRegistry(imgData *ImgData, registry string) e return nil } -// Push a local image to the registry +// PushImageJSONRegistry pushes JSON metadata for a local image to the registry func (r *Session) PushImageJSONRegistry(imgData *ImgData, jsonRaw []byte, registry string) error { u := registry + "images/" + imgData.ID + "/json" @@ -531,8 +547,8 @@ func (r *Session) PushImageJSONRegistry(imgData *ImgData, jsonRaw []byte, regist return nil } +// PushImageLayerRegistry sends the checksum of an image layer to the registry func (r *Session) PushImageLayerRegistry(imgID string, layer io.Reader, registry string, jsonRaw []byte) (checksum string, checksumPayload string, err error) { - u := registry + "images/" + imgID + "/layer" logrus.Debugf("[registry] Calling PUT %s", u) @@ -576,7 +592,7 @@ func (r *Session) PushImageLayerRegistry(imgID string, layer io.Reader, registry return tarsumLayer.Sum(jsonRaw), checksumPayload, nil } -// push a tag on the registry. +// PushRegistryTag pushes a tag on the registry. // Remote has the format '/ func (r *Session) PushRegistryTag(remote, revision, tag, registry string) error { // "jsonify" the string @@ -600,6 +616,7 @@ func (r *Session) PushRegistryTag(remote, revision, tag, registry string) error return nil } +// PushImageJSONIndex uploads an image list to the repository func (r *Session) PushImageJSONIndex(remote string, imgList []*ImgData, validate bool, regs []string) (*RepositoryData, error) { cleanImgList := []*ImgData{} if validate { @@ -705,6 +722,7 @@ func shouldRedirect(response *http.Response) bool { return response.StatusCode >= 300 && response.StatusCode < 400 } +// SearchRepositories performs a search against the remote repository func (r *Session) SearchRepositories(term string) (*SearchResults, error) { logrus.Debugf("Index server: %s", r.indexEndpoint) u := r.indexEndpoint.VersionString(1) + "search?q=" + url.QueryEscape(term) @@ -727,6 +745,7 @@ func (r *Session) SearchRepositories(term string) (*SearchResults, error) { return result, json.NewDecoder(res.Body).Decode(result) } +// GetAuthConfig returns the authentication settings for a session // TODO(tiborvass): remove this once registry client v2 is vendored func (r *Session) GetAuthConfig(withPasswd bool) *cliconfig.AuthConfig { password := "" diff --git a/docs/types.go b/docs/types.go index d02ae4fc..09b9d571 100644 --- a/docs/types.go +++ b/docs/types.go @@ -1,38 +1,66 @@ package registry +// SearchResult describes a search result returned from a registry type SearchResult struct { - StarCount int `json:"star_count"` - IsOfficial bool `json:"is_official"` - Name string `json:"name"` - IsTrusted bool `json:"is_trusted"` - IsAutomated bool `json:"is_automated"` + // StarCount indicates the number of stars this repository has + StarCount int `json:"star_count"` + // IsOfficial indicates whether the result is an official repository or not + IsOfficial bool `json:"is_official"` + // Name is the name of the repository + Name string `json:"name"` + // IsOfficial indicates whether the result is trusted + IsTrusted bool `json:"is_trusted"` + // IsAutomated indicates whether the result is automated + IsAutomated bool `json:"is_automated"` + // Description is a textual description of the repository Description string `json:"description"` } +// SearchResults lists a collection search results returned from a registry type SearchResults struct { - Query string `json:"query"` - NumResults int `json:"num_results"` - Results []SearchResult `json:"results"` + // Query contains the query string that generated the search results + Query string `json:"query"` + // NumResults indicates the number of results the query returned + NumResults int `json:"num_results"` + // Results is a slice containing the acutal results for the search + Results []SearchResult `json:"results"` } +// RepositoryData tracks the image list, list of endpoints, and list of tokens +// for a repository type RepositoryData struct { - ImgList map[string]*ImgData + // ImgList is a list of images in the repository + ImgList map[string]*ImgData + // Endpoints is a list of endpoints returned in X-Docker-Endpoints Endpoints []string - Tokens []string + // Tokens is currently unused (remove it?) + Tokens []string } +// ImgData is used to transfer image checksums to and from the registry type ImgData struct { + // ID is an opaque string that identifies the image ID string `json:"id"` Checksum string `json:"checksum,omitempty"` ChecksumPayload string `json:"-"` Tag string `json:",omitempty"` } -type RegistryInfo struct { - Version string `json:"version"` - Standalone bool `json:"standalone"` +// PingResult contains the information returned when pinging a registry. It +// indicates the registry's version and whether the registry claims to be a +// standalone registry. +type PingResult struct { + // Version is the registry version supplied by the registry in a HTTP + // header + Version string `json:"version"` + // Standalone is set to true if the registry indicates it is a + // standalone registry in the X-Docker-Registry-Standalone + // header + Standalone bool `json:"standalone"` } +// APIVersion is an integral representation of an API version (presently +// either 1 or 2) type APIVersion int func (av APIVersion) String() string { @@ -51,6 +79,8 @@ const ( APIVersion2 ) +// IndexInfo contains information about a registry +// // RepositoryInfo Examples: // { // "Index" : { @@ -64,7 +94,7 @@ const ( // "CanonicalName" : "docker.io/debian" // "Official" : true, // } - +// // { // "Index" : { // "Name" : "127.0.0.1:5000", @@ -78,16 +108,33 @@ const ( // "Official" : false, // } type IndexInfo struct { - Name string - Mirrors []string - Secure bool + // Name is the name of the registry, such as "docker.io" + Name string + // Mirrors is a list of mirrors, expressed as URIs + Mirrors []string + // Secure is set to false if the registry is part of the list of + // insecure registries. Insecure registries accept HTTP and/or accept + // HTTPS with certificates from unknown CAs. + Secure bool + // Official indicates whether this is an official registry Official bool } +// RepositoryInfo describes a repository type RepositoryInfo struct { - Index *IndexInfo - RemoteName string - LocalName string + // Index points to registry information + Index *IndexInfo + // RemoteName is the remote name of the repository, such as + // "library/ubuntu-12.04-base" + RemoteName string + // LocalName is the local name of the repository, such as + // "ubuntu-12.04-base" + LocalName string + // CanonicalName is the canonical name of the repository, such as + // "docker.io/library/ubuntu-12.04-base" CanonicalName string - Official bool + // Official indicates whether the repository is considered official. + // If the registry is official, and the normalized name does not + // contain a '/' (e.g. "foo"), then it is considered an official repo. + Official bool } From c219afdb4b1fed22aafe604d3861ff8ef50c9ecb Mon Sep 17 00:00:00 2001 From: Derek McGowan Date: Wed, 15 Jul 2015 13:42:45 -0700 Subject: [PATCH 307/375] Use notary library for trusted image fetch and signing Add a trusted flag to force the cli to resolve a tag into a digest via the notary trust library and pull by digest. On push the flag the trust flag will indicate the digest and size of a manifest should be signed and push to a notary server. If a tag is given, the cli will resolve the tag into a digest and pull by digest. After pulling, if a tag is given the cli makes a request to tag the image. Use certificate directory for notary requests Read certificates using same logic used by daemon for registry requests. Catch JSON syntax errors from Notary client When an uncaught error occurs in Notary it may show up in Docker as a JSON syntax error, causing a confusing error message to the user. Provide a generic error when a JSON syntax error occurs. Catch expiration errors and wrap in additional context. Signed-off-by: Derek McGowan (github: dmcgowan) --- docs/config.go | 4 +-- docs/reference.go | 68 +++++++++++++++++++++++++++++++++++++++++++++++ docs/registry.go | 52 ++++++++++++++++++++++++++++++++++++ docs/service.go | 51 +---------------------------------- 4 files changed, 123 insertions(+), 52 deletions(-) create mode 100644 docs/reference.go diff --git a/docs/config.go b/docs/config.go index d2108894..95f73129 100644 --- a/docs/config.go +++ b/docs/config.go @@ -38,8 +38,8 @@ const ( IndexServer = DefaultV1Registry + "/v1/" // IndexName is the name of the index IndexName = "docker.io" - - // IndexServer = "https://registry-stage.hub.docker.com/v1/" + // NotaryServer is the endpoint serving the Notary trust server + NotaryServer = "https://notary.docker.io" ) var ( diff --git a/docs/reference.go b/docs/reference.go new file mode 100644 index 00000000..e15f83ee --- /dev/null +++ b/docs/reference.go @@ -0,0 +1,68 @@ +package registry + +import ( + "strings" + + "github.com/docker/distribution/digest" +) + +// Reference represents a tag or digest within a repository +type Reference interface { + // HasDigest returns whether the reference has a verifiable + // content addressable reference which may be considered secure. + HasDigest() bool + + // ImageName returns an image name for the given repository + ImageName(string) string + + // Returns a string representation of the reference + String() string +} + +type tagReference struct { + tag string +} + +func (tr tagReference) HasDigest() bool { + return false +} + +func (tr tagReference) ImageName(repo string) string { + return repo + ":" + tr.tag +} + +func (tr tagReference) String() string { + return tr.tag +} + +type digestReference struct { + digest digest.Digest +} + +func (dr digestReference) HasDigest() bool { + return true +} + +func (dr digestReference) ImageName(repo string) string { + return repo + "@" + dr.String() +} + +func (dr digestReference) String() string { + return dr.digest.String() +} + +// ParseReference parses a reference into either a digest or tag reference +func ParseReference(ref string) Reference { + if strings.Contains(ref, ":") { + dgst, err := digest.ParseDigest(ref) + if err == nil { + return digestReference{digest: dgst} + } + } + return tagReference{tag: ref} +} + +// DigestReference creates a digest reference using a digest +func DigestReference(dgst digest.Digest) Reference { + return digestReference{digest: dgst} +} diff --git a/docs/registry.go b/docs/registry.go index fd85c21c..09143ba8 100644 --- a/docs/registry.go +++ b/docs/registry.go @@ -2,10 +2,14 @@ package registry import ( "crypto/tls" + "crypto/x509" "errors" + "fmt" + "io/ioutil" "net" "net/http" "os" + "path/filepath" "runtime" "strings" "time" @@ -54,6 +58,54 @@ func hasFile(files []os.FileInfo, name string) bool { return false } +// ReadCertsDirectory reads the directory for TLS certificates +// including roots and certificate pairs and updates the +// provided TLS configuration. +func ReadCertsDirectory(tlsConfig *tls.Config, directory string) error { + fs, err := ioutil.ReadDir(directory) + if err != nil && !os.IsNotExist(err) { + return err + } + + for _, f := range fs { + if strings.HasSuffix(f.Name(), ".crt") { + if tlsConfig.RootCAs == nil { + // TODO(dmcgowan): Copy system pool + tlsConfig.RootCAs = x509.NewCertPool() + } + logrus.Debugf("crt: %s", filepath.Join(directory, f.Name())) + data, err := ioutil.ReadFile(filepath.Join(directory, f.Name())) + if err != nil { + return err + } + tlsConfig.RootCAs.AppendCertsFromPEM(data) + } + if strings.HasSuffix(f.Name(), ".cert") { + certName := f.Name() + keyName := certName[:len(certName)-5] + ".key" + logrus.Debugf("cert: %s", filepath.Join(directory, f.Name())) + if !hasFile(fs, keyName) { + return fmt.Errorf("Missing key %s for certificate %s", keyName, certName) + } + cert, err := tls.LoadX509KeyPair(filepath.Join(directory, certName), filepath.Join(directory, keyName)) + if err != nil { + return err + } + tlsConfig.Certificates = append(tlsConfig.Certificates, cert) + } + if strings.HasSuffix(f.Name(), ".key") { + keyName := f.Name() + certName := keyName[:len(keyName)-4] + ".cert" + logrus.Debugf("key: %s", filepath.Join(directory, f.Name())) + if !hasFile(fs, certName) { + return fmt.Errorf("Missing certificate %s for key %s", certName, keyName) + } + } + } + + return nil +} + // DockerHeaders returns request modifiers that ensure requests have // the User-Agent header set to dockerUserAgent and that metaHeaders // are added. diff --git a/docs/service.go b/docs/service.go index 274dfeb2..fa35e313 100644 --- a/docs/service.go +++ b/docs/service.go @@ -2,12 +2,9 @@ package registry import ( "crypto/tls" - "crypto/x509" "fmt" - "io/ioutil" "net/http" "net/url" - "os" "path/filepath" "strings" @@ -110,57 +107,11 @@ func (s *Service) TLSConfig(hostname string) (*tls.Config, error) { tlsConfig.InsecureSkipVerify = !isSecure if isSecure { - hasFile := func(files []os.FileInfo, name string) bool { - for _, f := range files { - if f.Name() == name { - return true - } - } - return false - } - hostDir := filepath.Join(CertsDir, hostname) logrus.Debugf("hostDir: %s", hostDir) - fs, err := ioutil.ReadDir(hostDir) - if err != nil && !os.IsNotExist(err) { + if err := ReadCertsDirectory(&tlsConfig, hostDir); err != nil { return nil, err } - - for _, f := range fs { - if strings.HasSuffix(f.Name(), ".crt") { - if tlsConfig.RootCAs == nil { - // TODO(dmcgowan): Copy system pool - tlsConfig.RootCAs = x509.NewCertPool() - } - logrus.Debugf("crt: %s", filepath.Join(hostDir, f.Name())) - data, err := ioutil.ReadFile(filepath.Join(hostDir, f.Name())) - if err != nil { - return nil, err - } - tlsConfig.RootCAs.AppendCertsFromPEM(data) - } - if strings.HasSuffix(f.Name(), ".cert") { - certName := f.Name() - keyName := certName[:len(certName)-5] + ".key" - logrus.Debugf("cert: %s", filepath.Join(hostDir, f.Name())) - if !hasFile(fs, keyName) { - return nil, fmt.Errorf("Missing key %s for certificate %s", keyName, certName) - } - cert, err := tls.LoadX509KeyPair(filepath.Join(hostDir, certName), filepath.Join(hostDir, keyName)) - if err != nil { - return nil, err - } - tlsConfig.Certificates = append(tlsConfig.Certificates, cert) - } - if strings.HasSuffix(f.Name(), ".key") { - keyName := f.Name() - certName := keyName[:len(keyName)-4] + ".cert" - logrus.Debugf("key: %s", filepath.Join(hostDir, f.Name())) - if !hasFile(fs, certName) { - return nil, fmt.Errorf("Missing certificate %s for key %s", certName, keyName) - } - } - } } return &tlsConfig, nil From ba358690c11e3f1d868ffd8b9b9775d206a6ce0e Mon Sep 17 00:00:00 2001 From: Derek McGowan Date: Tue, 28 Jul 2015 10:36:57 -0700 Subject: [PATCH 308/375] Fix login and search TLS configuration Currently login and search do not load per registry certificates. This is a regression caused by the last refactor since this was recently fixed. Signed-off-by: Derek McGowan (github: dmcgowan) --- docs/endpoint.go | 10 +++++----- docs/registry.go | 17 +++++++++++++++++ docs/service.go | 19 +------------------ 3 files changed, 23 insertions(+), 23 deletions(-) diff --git a/docs/endpoint.go b/docs/endpoint.go index c6361346..b7aaedaa 100644 --- a/docs/endpoint.go +++ b/docs/endpoint.go @@ -13,7 +13,6 @@ import ( "github.com/Sirupsen/logrus" "github.com/docker/distribution/registry/api/v2" "github.com/docker/distribution/registry/client/transport" - "github.com/docker/docker/pkg/tlsconfig" ) // for mocking in unit tests @@ -45,10 +44,11 @@ func scanForAPIVersion(address string) (string, APIVersion) { // NewEndpoint parses the given address to return a registry endpoint. func NewEndpoint(index *IndexInfo, metaHeaders http.Header) (*Endpoint, error) { - // *TODO: Allow per-registry configuration of endpoints. - tlsConfig := tlsconfig.ServerDefault - tlsConfig.InsecureSkipVerify = !index.Secure - endpoint, err := newEndpoint(index.GetAuthConfigKey(), &tlsConfig, metaHeaders) + tlsConfig, err := newTLSConfig(index.Name, index.Secure) + if err != nil { + return nil, err + } + endpoint, err := newEndpoint(index.GetAuthConfigKey(), tlsConfig, metaHeaders) if err != nil { return nil, err } diff --git a/docs/registry.go b/docs/registry.go index 09143ba8..74f731bd 100644 --- a/docs/registry.go +++ b/docs/registry.go @@ -49,6 +49,23 @@ func init() { dockerUserAgent = useragent.AppendVersions("", httpVersion...) } +func newTLSConfig(hostname string, isSecure bool) (*tls.Config, error) { + // PreferredServerCipherSuites should have no effect + tlsConfig := tlsconfig.ServerDefault + + tlsConfig.InsecureSkipVerify = !isSecure + + if isSecure { + hostDir := filepath.Join(CertsDir, hostname) + logrus.Debugf("hostDir: %s", hostDir) + if err := ReadCertsDirectory(&tlsConfig, hostDir); err != nil { + return nil, err + } + } + + return &tlsConfig, nil +} + func hasFile(files []os.FileInfo, name string) bool { for _, f := range files { if f.Name() == name { diff --git a/docs/service.go b/docs/service.go index fa35e313..f4ea42ef 100644 --- a/docs/service.go +++ b/docs/service.go @@ -5,10 +5,8 @@ import ( "fmt" "net/http" "net/url" - "path/filepath" "strings" - "github.com/Sirupsen/logrus" "github.com/docker/distribution/registry/client/auth" "github.com/docker/docker/cliconfig" "github.com/docker/docker/pkg/tlsconfig" @@ -99,22 +97,7 @@ func (e APIEndpoint) ToV1Endpoint(metaHeaders http.Header) (*Endpoint, error) { // TLSConfig constructs a client TLS configuration based on server defaults func (s *Service) TLSConfig(hostname string) (*tls.Config, error) { - // PreferredServerCipherSuites should have no effect - tlsConfig := tlsconfig.ServerDefault - - isSecure := s.Config.isSecureIndex(hostname) - - tlsConfig.InsecureSkipVerify = !isSecure - - if isSecure { - hostDir := filepath.Join(CertsDir, hostname) - logrus.Debugf("hostDir: %s", hostDir) - if err := ReadCertsDirectory(&tlsConfig, hostDir); err != nil { - return nil, err - } - } - - return &tlsConfig, nil + return newTLSConfig(hostname, s.Config.isSecureIndex(hostname)) } func (s *Service) tlsConfigForMirror(mirror string) (*tls.Config, error) { From 6f83ba2b29f820966a60bf316f18614a3fa7c5ea Mon Sep 17 00:00:00 2001 From: Tibor Vass Date: Fri, 24 Jul 2015 14:59:36 -0400 Subject: [PATCH 309/375] registry: Change default endpoint on windows to a windows-specific one Signed-off-by: Tibor Vass --- docs/config.go | 22 ---------------------- docs/consts.go | 24 ++++++++++++++++++++++++ docs/consts_unix.go | 6 ++++++ docs/consts_windows.go | 10 ++++++++++ docs/registry.go | 1 + docs/service.go | 19 +++++++++++-------- 6 files changed, 52 insertions(+), 30 deletions(-) create mode 100644 docs/consts.go create mode 100644 docs/consts_unix.go create mode 100644 docs/consts_windows.go diff --git a/docs/config.go b/docs/config.go index 95f73129..dc1ee899 100644 --- a/docs/config.go +++ b/docs/config.go @@ -20,28 +20,6 @@ type Options struct { InsecureRegistries opts.ListOpts } -const ( - // DefaultNamespace is the default namespace - DefaultNamespace = "docker.io" - // DefaultV2Registry is the URI of the default v2 registry - DefaultV2Registry = "https://registry-1.docker.io" - // DefaultRegistryVersionHeader is the name of the default HTTP header - // that carries Registry version info - DefaultRegistryVersionHeader = "Docker-Distribution-Api-Version" - // DefaultV1Registry is the URI of the default v1 registry - DefaultV1Registry = "https://index.docker.io" - - // CertsDir is the directory where certificates are stored - CertsDir = "/etc/docker/certs.d" - - // IndexServer is the v1 registry server used for user auth + account creation - IndexServer = DefaultV1Registry + "/v1/" - // IndexName is the name of the index - IndexName = "docker.io" - // NotaryServer is the endpoint serving the Notary trust server - NotaryServer = "https://notary.docker.io" -) - var ( // ErrInvalidRepositoryName is an error returned if the repository name did // not have the correct form diff --git a/docs/consts.go b/docs/consts.go new file mode 100644 index 00000000..19471e06 --- /dev/null +++ b/docs/consts.go @@ -0,0 +1,24 @@ +package registry + +const ( + // DefaultNamespace is the default namespace + DefaultNamespace = "docker.io" + // DefaultRegistryVersionHeader is the name of the default HTTP header + // that carries Registry version info + DefaultRegistryVersionHeader = "Docker-Distribution-Api-Version" + // DefaultV1Registry is the URI of the default v1 registry + DefaultV1Registry = "https://index.docker.io" + + // CertsDir is the directory where certificates are stored + CertsDir = "/etc/docker/certs.d" + + // IndexServer is the v1 registry server used for user auth + account creation + IndexServer = DefaultV1Registry + "/v1/" + // IndexName is the name of the index + IndexName = "docker.io" + + // NotaryServer is the endpoint serving the Notary trust server + NotaryServer = "https://notary.docker.io" + + // IndexServer = "https://registry-stage.hub.docker.com/v1/" +) diff --git a/docs/consts_unix.go b/docs/consts_unix.go new file mode 100644 index 00000000..b02e579a --- /dev/null +++ b/docs/consts_unix.go @@ -0,0 +1,6 @@ +// +build !windows + +package registry + +// DefaultV2Registry is the URI of the default v2 registry +const DefaultV2Registry = "https://registry-1.docker.io" diff --git a/docs/consts_windows.go b/docs/consts_windows.go new file mode 100644 index 00000000..b62c5faf --- /dev/null +++ b/docs/consts_windows.go @@ -0,0 +1,10 @@ +// +build windows + +package registry + +// DefaultV2Registry is the URI of the default (official) v2 registry. +// This is the windows-specific endpoint. +// +// Currently it is a TEMPORARY link that allows Microsoft to continue +// development of Docker Engine for Windows. +const DefaultV2Registry = "https://ms-tp3.registry-1.docker.io" diff --git a/docs/registry.go b/docs/registry.go index 74f731bd..e353d3cc 100644 --- a/docs/registry.go +++ b/docs/registry.go @@ -1,3 +1,4 @@ +// Package registry contains client primitives to interact with a remote Docker registry. package registry import ( diff --git a/docs/service.go b/docs/service.go index f4ea42ef..0cceb23d 100644 --- a/docs/service.go +++ b/docs/service.go @@ -5,6 +5,7 @@ import ( "fmt" "net/http" "net/url" + "runtime" "strings" "github.com/docker/distribution/registry/client/auth" @@ -138,14 +139,16 @@ func (s *Service) LookupEndpoints(repoName string) (endpoints []APIEndpoint, err TrimHostname: true, TLSConfig: tlsConfig, }) - // v1 registry - endpoints = append(endpoints, APIEndpoint{ - URL: DefaultV1Registry, - Version: APIVersion1, - Official: true, - TrimHostname: true, - TLSConfig: tlsConfig, - }) + if runtime.GOOS == "linux" { // do not inherit legacy API for OSes supported in the future + // v1 registry + endpoints = append(endpoints, APIEndpoint{ + URL: DefaultV1Registry, + Version: APIVersion1, + Official: true, + TrimHostname: true, + TLSConfig: tlsConfig, + }) + } return endpoints, nil } From 048339e3f56b39ef2ae46b8311990fd68131ccf7 Mon Sep 17 00:00:00 2001 From: Tibor Vass Date: Thu, 30 Jul 2015 19:03:38 -0400 Subject: [PATCH 310/375] registry: allow fallback on unknown errors This patch fixes a bug where a user specifies a v1 mirror for --registry-mirror and pull an image from the Hub. It used to not fallback because of an unexpected error returned when trying to JSON marshal nginx output. We now ensure that any unexpected error falls back to the next endpoint in the list. Signed-off-by: Tibor Vass --- docs/registry.go | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/docs/registry.go b/docs/registry.go index 74f731bd..9fb71d17 100644 --- a/docs/registry.go +++ b/docs/registry.go @@ -17,6 +17,7 @@ import ( "github.com/Sirupsen/logrus" "github.com/docker/distribution/registry/api/errcode" "github.com/docker/distribution/registry/api/v2" + "github.com/docker/distribution/registry/client" "github.com/docker/distribution/registry/client/transport" "github.com/docker/docker/autogen/dockerversion" "github.com/docker/docker/pkg/parsers/kernel" @@ -211,8 +212,14 @@ func ContinueOnError(err error) bool { return ContinueOnError(v.Err) case errcode.Error: return shouldV2Fallback(v) + case *client.UnexpectedHTTPResponseError: + return true } - return false + // let's be nice and fallback if the error is a completely + // unexpected one. + // If new errors have to be handled in some way, please + // add them to the switch above. + return true } // NewTransport returns a new HTTP transport. If tlsConfig is nil, it uses the From cfb0b7aa77b060dbb2abf915fcc64a97690a410d Mon Sep 17 00:00:00 2001 From: Aaron Lehmann Date: Thu, 23 Jul 2015 14:19:58 -0700 Subject: [PATCH 311/375] Fix uses of "int" where "int64" should be used instead Some structures use int for sizes and UNIX timestamps. On some platforms, int is 32 bits, so this can lead to the year 2038 issues and overflows when dealing with large containers or layers. Consistently use int64 to store sizes and UNIX timestamps in api/types/types.go. Update related to code accordingly (i.e. strconv.FormatInt instead of strconv.Itoa). Use int64 in progressreader package to avoid integer overflow when dealing with large quantities. Update related code accordingly. Signed-off-by: Aaron Lehmann --- docs/registry_test.go | 2 +- docs/session.go | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/docs/registry_test.go b/docs/registry_test.go index 88b08dff..d9ac5c6f 100644 --- a/docs/registry_test.go +++ b/docs/registry_test.go @@ -185,7 +185,7 @@ func TestGetRemoteImageJSON(t *testing.T) { if err != nil { t.Fatal(err) } - assertEqual(t, size, 154, "Expected size 154") + assertEqual(t, size, int64(154), "Expected size 154") if len(json) <= 0 { t.Fatal("Expected non-empty json") } diff --git a/docs/session.go b/docs/session.go index 9bec7c1b..a9c4daf3 100644 --- a/docs/session.go +++ b/docs/session.go @@ -240,7 +240,7 @@ func (r *Session) LookupRemoteImage(imgID, registry string) error { } // GetRemoteImageJSON retrieves an image's JSON metadata from the registry. -func (r *Session) GetRemoteImageJSON(imgID, registry string) ([]byte, int, error) { +func (r *Session) GetRemoteImageJSON(imgID, registry string) ([]byte, int64, error) { res, err := r.client.Get(registry + "images/" + imgID + "/json") if err != nil { return nil, -1, fmt.Errorf("Failed to download json: %s", err) @@ -250,9 +250,9 @@ func (r *Session) GetRemoteImageJSON(imgID, registry string) ([]byte, int, error return nil, -1, httputils.NewHTTPRequestError(fmt.Sprintf("HTTP code %d", res.StatusCode), res) } // if the size header is not present, then set it to '-1' - imageSize := -1 + imageSize := int64(-1) if hdr := res.Header.Get("X-Docker-Size"); hdr != "" { - imageSize, err = strconv.Atoi(hdr) + imageSize, err = strconv.ParseInt(hdr, 10, 64) if err != nil { return nil, -1, err } From 86a3ea91b817ef42d22748ee01ddfc9f2b881fe9 Mon Sep 17 00:00:00 2001 From: John Howard Date: Tue, 4 Aug 2015 16:30:00 -0700 Subject: [PATCH 312/375] Windows: Fix certificate directory for registry Signed-off-by: John Howard --- docs/config.go | 20 ++++++++++++++++++++ docs/config_unix.go | 19 +++++++++++++++++++ docs/config_windows.go | 25 +++++++++++++++++++++++++ docs/consts.go | 24 ------------------------ docs/consts_unix.go | 6 ------ docs/consts_windows.go | 10 ---------- docs/registry.go | 2 +- 7 files changed, 65 insertions(+), 41 deletions(-) create mode 100644 docs/config_unix.go create mode 100644 docs/config_windows.go delete mode 100644 docs/consts.go delete mode 100644 docs/consts_unix.go delete mode 100644 docs/consts_windows.go diff --git a/docs/config.go b/docs/config.go index dc1ee899..678a330e 100644 --- a/docs/config.go +++ b/docs/config.go @@ -20,6 +20,26 @@ type Options struct { InsecureRegistries opts.ListOpts } +const ( + // DefaultNamespace is the default namespace + DefaultNamespace = "docker.io" + // DefaultRegistryVersionHeader is the name of the default HTTP header + // that carries Registry version info + DefaultRegistryVersionHeader = "Docker-Distribution-Api-Version" + // DefaultV1Registry is the URI of the default v1 registry + DefaultV1Registry = "https://index.docker.io" + + // IndexServer is the v1 registry server used for user auth + account creation + IndexServer = DefaultV1Registry + "/v1/" + // IndexName is the name of the index + IndexName = "docker.io" + + // NotaryServer is the endpoint serving the Notary trust server + NotaryServer = "https://notary.docker.io" + + // IndexServer = "https://registry-stage.hub.docker.com/v1/" +) + var ( // ErrInvalidRepositoryName is an error returned if the repository name did // not have the correct form diff --git a/docs/config_unix.go b/docs/config_unix.go new file mode 100644 index 00000000..908ca2f6 --- /dev/null +++ b/docs/config_unix.go @@ -0,0 +1,19 @@ +// +build !windows + +package registry + +const ( + // DefaultV2Registry is the URI of the default v2 registry + DefaultV2Registry = "https://registry-1.docker.io" + + // CertsDir is the directory where certificates are stored + CertsDir = "/etc/docker/certs.d" +) + +// cleanPath is used to ensure that a directory name is valid on the target +// platform. It will be passed in something *similar* to a URL such as +// https:/index.docker.io/v1. Not all platforms support directory names +// which contain those characters (such as : on Windows) +func cleanPath(s string) string { + return s +} diff --git a/docs/config_windows.go b/docs/config_windows.go new file mode 100644 index 00000000..3ebc0448 --- /dev/null +++ b/docs/config_windows.go @@ -0,0 +1,25 @@ +package registry + +import ( + "os" + "path/filepath" + "strings" +) + +// DefaultV2Registry is the URI of the default (official) v2 registry. +// This is the windows-specific endpoint. +// +// Currently it is a TEMPORARY link that allows Microsoft to continue +// development of Docker Engine for Windows. +const DefaultV2Registry = "https://ms-tp3.registry-1.docker.io" + +// CertsDir is the directory where certificates are stored +var CertsDir = os.Getenv("programdata") + `\docker\certs.d` + +// cleanPath is used to ensure that a directory name is valid on the target +// platform. It will be passed in something *similar* to a URL such as +// https:\index.docker.io\v1. Not all platforms support directory names +// which contain those characters (such as : on Windows) +func cleanPath(s string) string { + return filepath.FromSlash(strings.Replace(s, ":", "", -1)) +} diff --git a/docs/consts.go b/docs/consts.go deleted file mode 100644 index 19471e06..00000000 --- a/docs/consts.go +++ /dev/null @@ -1,24 +0,0 @@ -package registry - -const ( - // DefaultNamespace is the default namespace - DefaultNamespace = "docker.io" - // DefaultRegistryVersionHeader is the name of the default HTTP header - // that carries Registry version info - DefaultRegistryVersionHeader = "Docker-Distribution-Api-Version" - // DefaultV1Registry is the URI of the default v1 registry - DefaultV1Registry = "https://index.docker.io" - - // CertsDir is the directory where certificates are stored - CertsDir = "/etc/docker/certs.d" - - // IndexServer is the v1 registry server used for user auth + account creation - IndexServer = DefaultV1Registry + "/v1/" - // IndexName is the name of the index - IndexName = "docker.io" - - // NotaryServer is the endpoint serving the Notary trust server - NotaryServer = "https://notary.docker.io" - - // IndexServer = "https://registry-stage.hub.docker.com/v1/" -) diff --git a/docs/consts_unix.go b/docs/consts_unix.go deleted file mode 100644 index b02e579a..00000000 --- a/docs/consts_unix.go +++ /dev/null @@ -1,6 +0,0 @@ -// +build !windows - -package registry - -// DefaultV2Registry is the URI of the default v2 registry -const DefaultV2Registry = "https://registry-1.docker.io" diff --git a/docs/consts_windows.go b/docs/consts_windows.go deleted file mode 100644 index b62c5faf..00000000 --- a/docs/consts_windows.go +++ /dev/null @@ -1,10 +0,0 @@ -// +build windows - -package registry - -// DefaultV2Registry is the URI of the default (official) v2 registry. -// This is the windows-specific endpoint. -// -// Currently it is a TEMPORARY link that allows Microsoft to continue -// development of Docker Engine for Windows. -const DefaultV2Registry = "https://ms-tp3.registry-1.docker.io" diff --git a/docs/registry.go b/docs/registry.go index a3123b96..408bc8e1 100644 --- a/docs/registry.go +++ b/docs/registry.go @@ -58,7 +58,7 @@ func newTLSConfig(hostname string, isSecure bool) (*tls.Config, error) { tlsConfig.InsecureSkipVerify = !isSecure if isSecure { - hostDir := filepath.Join(CertsDir, hostname) + hostDir := filepath.Join(CertsDir, cleanPath(hostname)) logrus.Debugf("hostDir: %s", hostDir) if err := ReadCertsDirectory(&tlsConfig, hostDir); err != nil { return nil, err From 66761c0284e101de1e380dcdfa210a9f94c086a7 Mon Sep 17 00:00:00 2001 From: Brian Goff Date: Thu, 6 Aug 2015 12:35:43 -0400 Subject: [PATCH 313/375] Better/more specific error messages on connect Closes #15309 Signed-off-by: Brian Goff --- docs/session.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/docs/session.go b/docs/session.go index a9c4daf3..d497cb95 100644 --- a/docs/session.go +++ b/docs/session.go @@ -20,6 +20,7 @@ import ( "time" "github.com/Sirupsen/logrus" + "github.com/docker/docker/api/types" "github.com/docker/docker/cliconfig" "github.com/docker/docker/pkg/httputils" "github.com/docker/docker/pkg/ioutils" @@ -424,7 +425,7 @@ func (r *Session) GetRepositoryData(remote string) (*RepositoryData, error) { // and return a non-obtuse error message for users // "Get https://index.docker.io/v1/repositories/library/busybox/images: i/o timeout" // was a top search on the docker user forum - if strings.HasSuffix(err.Error(), "i/o timeout") { + if types.IsTimeout(err) { return nil, fmt.Errorf("Network timed out while trying to connect to %s. You may want to check your internet connection or if you are behind a proxy.", repositoryTarget) } return nil, fmt.Errorf("Error while pulling image: %v", err) From a7eb16ad1c64cc9a3ea4c93e41353fe15126259a Mon Sep 17 00:00:00 2001 From: Tibor Vass Date: Thu, 6 Aug 2015 17:41:59 -0400 Subject: [PATCH 314/375] registry: Do not push to mirrors This patch splits LookupEndpoints into LookupPullEndpoints and LookupPushEndpoints so that mirrors added with --registry-mirror are skipped in the list returned by LookupPushEndpoints. Fixes https://github.com/docker/distribution/issues/823 Signed-off-by: Tibor Vass --- docs/service.go | 43 ++++++++++++++++++++++++++++--------------- 1 file changed, 28 insertions(+), 15 deletions(-) diff --git a/docs/service.go b/docs/service.go index 0cceb23d..0f665629 100644 --- a/docs/service.go +++ b/docs/service.go @@ -109,27 +109,40 @@ func (s *Service) tlsConfigForMirror(mirror string) (*tls.Config, error) { return s.TLSConfig(mirrorURL.Host) } -// LookupEndpoints creates an list of endpoints to try, in order of preference. +// LookupPullEndpoints creates an list of endpoints to try to pull from, in order of preference. // It gives preference to v2 endpoints over v1, mirrors over the actual // registry, and HTTPS over plain HTTP. -func (s *Service) LookupEndpoints(repoName string) (endpoints []APIEndpoint, err error) { +func (s *Service) LookupPullEndpoints(repoName string) (endpoints []APIEndpoint, err error) { + return s.lookupEndpoints(repoName, false) +} + +// LookupPushEndpoints creates an list of endpoints to try to push to, in order of preference. +// It gives preference to v2 endpoints over v1, and HTTPS over plain HTTP. +// Mirrors are not included. +func (s *Service) LookupPushEndpoints(repoName string) (endpoints []APIEndpoint, err error) { + return s.lookupEndpoints(repoName, true) +} + +func (s *Service) lookupEndpoints(repoName string, isPush bool) (endpoints []APIEndpoint, err error) { var cfg = tlsconfig.ServerDefault tlsConfig := &cfg if strings.HasPrefix(repoName, DefaultNamespace+"/") { - // v2 mirrors - for _, mirror := range s.Config.Mirrors { - mirrorTLSConfig, err := s.tlsConfigForMirror(mirror) - if err != nil { - return nil, err + if !isPush { + // v2 mirrors for pull only + for _, mirror := range s.Config.Mirrors { + mirrorTLSConfig, err := s.tlsConfigForMirror(mirror) + if err != nil { + return nil, err + } + endpoints = append(endpoints, APIEndpoint{ + URL: mirror, + // guess mirrors are v2 + Version: APIVersion2, + Mirror: true, + TrimHostname: true, + TLSConfig: mirrorTLSConfig, + }) } - endpoints = append(endpoints, APIEndpoint{ - URL: mirror, - // guess mirrors are v2 - Version: APIVersion2, - Mirror: true, - TrimHostname: true, - TLSConfig: mirrorTLSConfig, - }) } // v2 registry endpoints = append(endpoints, APIEndpoint{ From b82b069475859faa1d7059ea95ab47b4c42ca314 Mon Sep 17 00:00:00 2001 From: Aidan Hobson Sayers Date: Fri, 7 Aug 2015 02:21:02 +0100 Subject: [PATCH 315/375] Remove unnecessary func parameter, add mirror endpoint test Signed-off-by: Aidan Hobson Sayers --- docs/registry_test.go | 29 +++++++++++++++++++++++++++++ docs/service.go | 42 ++++++++++++++++++++++++------------------ 2 files changed, 53 insertions(+), 18 deletions(-) diff --git a/docs/registry_test.go b/docs/registry_test.go index d9ac5c6f..160d3440 100644 --- a/docs/registry_test.go +++ b/docs/registry_test.go @@ -677,6 +677,35 @@ func TestNewIndexInfo(t *testing.T) { testIndexInfo(config, expectedIndexInfos) } +func TestMirrorEndpointLookup(t *testing.T) { + containsMirror := func(endpoints []APIEndpoint) bool { + for _, pe := range endpoints { + if pe.URL == "my.mirror" { + return true + } + } + return false + } + s := Service{Config: makeServiceConfig([]string{"my.mirror"}, nil)} + imageName := IndexName + "/test/image" + + pushAPIEndpoints, err := s.LookupPushEndpoints(imageName) + if err != nil { + t.Fatal(err) + } + if containsMirror(pushAPIEndpoints) { + t.Fatal("Push endpoint should not contain mirror") + } + + pullAPIEndpoints, err := s.LookupPullEndpoints(imageName) + if err != nil { + t.Fatal(err) + } + if !containsMirror(pullAPIEndpoints) { + t.Fatal("Pull endpoint should contain mirror") + } +} + func TestPushRegistryTag(t *testing.T) { r := spawnTestRegistrySession(t) err := r.PushRegistryTag("foo42/bar", imageID, "stable", makeURL("/v1/")) diff --git a/docs/service.go b/docs/service.go index 0f665629..11912c57 100644 --- a/docs/service.go +++ b/docs/service.go @@ -113,36 +113,42 @@ func (s *Service) tlsConfigForMirror(mirror string) (*tls.Config, error) { // It gives preference to v2 endpoints over v1, mirrors over the actual // registry, and HTTPS over plain HTTP. func (s *Service) LookupPullEndpoints(repoName string) (endpoints []APIEndpoint, err error) { - return s.lookupEndpoints(repoName, false) + return s.lookupEndpoints(repoName) } // LookupPushEndpoints creates an list of endpoints to try to push to, in order of preference. // It gives preference to v2 endpoints over v1, and HTTPS over plain HTTP. // Mirrors are not included. func (s *Service) LookupPushEndpoints(repoName string) (endpoints []APIEndpoint, err error) { - return s.lookupEndpoints(repoName, true) + allEndpoints, err := s.lookupEndpoints(repoName) + if err == nil { + for _, endpoint := range allEndpoints { + if !endpoint.Mirror { + endpoints = append(endpoints, endpoint) + } + } + } + return endpoints, err } -func (s *Service) lookupEndpoints(repoName string, isPush bool) (endpoints []APIEndpoint, err error) { +func (s *Service) lookupEndpoints(repoName string) (endpoints []APIEndpoint, err error) { var cfg = tlsconfig.ServerDefault tlsConfig := &cfg if strings.HasPrefix(repoName, DefaultNamespace+"/") { - if !isPush { - // v2 mirrors for pull only - for _, mirror := range s.Config.Mirrors { - mirrorTLSConfig, err := s.tlsConfigForMirror(mirror) - if err != nil { - return nil, err - } - endpoints = append(endpoints, APIEndpoint{ - URL: mirror, - // guess mirrors are v2 - Version: APIVersion2, - Mirror: true, - TrimHostname: true, - TLSConfig: mirrorTLSConfig, - }) + // v2 mirrors + for _, mirror := range s.Config.Mirrors { + mirrorTLSConfig, err := s.tlsConfigForMirror(mirror) + if err != nil { + return nil, err } + endpoints = append(endpoints, APIEndpoint{ + URL: mirror, + // guess mirrors are v2 + Version: APIVersion2, + Mirror: true, + TrimHostname: true, + TLSConfig: mirrorTLSConfig, + }) } // v2 registry endpoints = append(endpoints, APIEndpoint{ From 777fd4c7aabea981a2c27c5fc24d7848d2e23b20 Mon Sep 17 00:00:00 2001 From: Arnaud Porterie Date: Fri, 7 Aug 2015 14:01:34 -0700 Subject: [PATCH 316/375] Update Windows TP3 registry endpoints Signed-off-by: Arnaud Porterie --- docs/config.go | 2 -- docs/config_unix.go | 3 +++ docs/config_windows.go | 17 +++++++++++------ 3 files changed, 14 insertions(+), 8 deletions(-) diff --git a/docs/config.go b/docs/config.go index 678a330e..5fca9df0 100644 --- a/docs/config.go +++ b/docs/config.go @@ -26,8 +26,6 @@ const ( // DefaultRegistryVersionHeader is the name of the default HTTP header // that carries Registry version info DefaultRegistryVersionHeader = "Docker-Distribution-Api-Version" - // DefaultV1Registry is the URI of the default v1 registry - DefaultV1Registry = "https://index.docker.io" // IndexServer is the v1 registry server used for user auth + account creation IndexServer = DefaultV1Registry + "/v1/" diff --git a/docs/config_unix.go b/docs/config_unix.go index 908ca2f6..32f167d0 100644 --- a/docs/config_unix.go +++ b/docs/config_unix.go @@ -3,6 +3,9 @@ package registry const ( + // DefaultV1Registry is the URI of the default v1 registry + DefaultV1Registry = "https://index.docker.io" + // DefaultV2Registry is the URI of the default v2 registry DefaultV2Registry = "https://registry-1.docker.io" diff --git a/docs/config_windows.go b/docs/config_windows.go index 3ebc0448..d01b2618 100644 --- a/docs/config_windows.go +++ b/docs/config_windows.go @@ -6,12 +6,17 @@ import ( "strings" ) -// DefaultV2Registry is the URI of the default (official) v2 registry. -// This is the windows-specific endpoint. -// -// Currently it is a TEMPORARY link that allows Microsoft to continue -// development of Docker Engine for Windows. -const DefaultV2Registry = "https://ms-tp3.registry-1.docker.io" +const ( + // DefaultV1Registry is the URI of the default v1 registry + DefaultV1Registry = "https://registry-win-tp3.docker.io" + + // DefaultV2Registry is the URI of the default (official) v2 registry. + // This is the windows-specific endpoint. + // + // Currently it is a TEMPORARY link that allows Microsoft to continue + // development of Docker Engine for Windows. + DefaultV2Registry = "https://registry-win-tp3.docker.io" +) // CertsDir is the directory where certificates are stored var CertsDir = os.Getenv("programdata") + `\docker\certs.d` From cf9016592ef7a0d382ebc794b84a9fe334a4792c Mon Sep 17 00:00:00 2001 From: Veres Lajos Date: Fri, 7 Aug 2015 23:24:18 +0100 Subject: [PATCH 317/375] typofix - https://github.com/vlajos/misspell_fixer Signed-off-by: Veres Lajos --- docs/service.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/service.go b/docs/service.go index 0f665629..3637b4b5 100644 --- a/docs/service.go +++ b/docs/service.go @@ -28,7 +28,7 @@ func NewService(options *Options) *Service { } // Auth contacts the public registry with the provided credentials, -// and returns OK if authentication was sucessful. +// and returns OK if authentication was successful. // It can be used to verify the validity of a client's credentials. func (s *Service) Auth(authConfig *cliconfig.AuthConfig) (string, error) { addr := authConfig.ServerAddress From 2b658054bbccd0f9f78a89ee6deb3fe7826645bd Mon Sep 17 00:00:00 2001 From: David Calavera Date: Mon, 7 Sep 2015 19:29:33 -0400 Subject: [PATCH 318/375] Make RegistryConfig a typed value in the api. Remove possible circular dependency that prevented us from using a real type. Signed-off-by: David Calavera --- docs/session.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/session.go b/docs/session.go index d497cb95..2a20d321 100644 --- a/docs/session.go +++ b/docs/session.go @@ -20,12 +20,12 @@ import ( "time" "github.com/Sirupsen/logrus" - "github.com/docker/docker/api/types" "github.com/docker/docker/cliconfig" "github.com/docker/docker/pkg/httputils" "github.com/docker/docker/pkg/ioutils" "github.com/docker/docker/pkg/stringid" "github.com/docker/docker/pkg/tarsum" + "github.com/docker/docker/utils" ) var ( @@ -425,7 +425,7 @@ func (r *Session) GetRepositoryData(remote string) (*RepositoryData, error) { // and return a non-obtuse error message for users // "Get https://index.docker.io/v1/repositories/library/busybox/images: i/o timeout" // was a top search on the docker user forum - if types.IsTimeout(err) { + if utils.IsTimeout(err) { return nil, fmt.Errorf("Network timed out while trying to connect to %s. You may want to check your internet connection or if you are behind a proxy.", repositoryTarget) } return nil, fmt.Errorf("Error while pulling image: %v", err) From ebaa771c3b18683fe56abd85261329176e33cae7 Mon Sep 17 00:00:00 2001 From: Richard Scothern Date: Wed, 16 Sep 2015 10:42:17 -0700 Subject: [PATCH 319/375] Prevent push and pull to v1 registries by filtering the available endpoints. Add a daemon flag to control this behaviour. Add a warning message when pulling an image from a v1 registry. The default order of pull is slightly altered with this changset. Previously it was: https v2, https v1, http v2, http v1 now it is: https v2, http v2, https v1, http v1 Prevent login to v1 registries by explicitly setting the version before ping to prevent fallback to v1. Add unit tests for v2 only mode. Create a mock server that can register handlers for various endpoints. Assert no v1 endpoints are hit with legacy registries disabled for the following commands: pull, push, build, run and login. Assert the opposite when legacy registries are not disabled. Signed-off-by: Richard Scothern --- docs/config.go | 5 ++ docs/endpoint.go | 13 +++-- docs/registry.go | 4 ++ docs/registry_test.go | 12 ++--- docs/service.go | 107 +++++++----------------------------------- docs/service_v1.go | 54 +++++++++++++++++++++ docs/service_v2.go | 83 ++++++++++++++++++++++++++++++++ 7 files changed, 175 insertions(+), 103 deletions(-) create mode 100644 docs/service_v1.go create mode 100644 docs/service_v2.go diff --git a/docs/config.go b/docs/config.go index 5fca9df0..5ab3e08c 100644 --- a/docs/config.go +++ b/docs/config.go @@ -44,6 +44,10 @@ var ( ErrInvalidRepositoryName = errors.New("Invalid repository name (ex: \"registry.domain.tld/myrepos\")") emptyServiceConfig = NewServiceConfig(nil) + + // V2Only controls access to legacy registries. If it is set to true via the + // command line flag the daemon will not attempt to contact v1 legacy registries + V2Only = false ) // InstallFlags adds command-line options to the top-level flag parser for @@ -53,6 +57,7 @@ func (options *Options) InstallFlags(cmd *flag.FlagSet, usageFn func(string) str cmd.Var(&options.Mirrors, []string{"-registry-mirror"}, usageFn("Preferred Docker registry mirror")) options.InsecureRegistries = opts.NewListOpts(ValidateIndexName) cmd.Var(&options.InsecureRegistries, []string{"-insecure-registry"}, usageFn("Enable insecure registry communication")) + cmd.BoolVar(&V2Only, []string{"-no-legacy-registry"}, false, "Do not contact legacy registries") } type netIPNet net.IPNet diff --git a/docs/endpoint.go b/docs/endpoint.go index b7aaedaa..20805767 100644 --- a/docs/endpoint.go +++ b/docs/endpoint.go @@ -42,8 +42,9 @@ func scanForAPIVersion(address string) (string, APIVersion) { return address, APIVersionUnknown } -// NewEndpoint parses the given address to return a registry endpoint. -func NewEndpoint(index *IndexInfo, metaHeaders http.Header) (*Endpoint, error) { +// NewEndpoint parses the given address to return a registry endpoint. v can be used to +// specify a specific endpoint version +func NewEndpoint(index *IndexInfo, metaHeaders http.Header, v APIVersion) (*Endpoint, error) { tlsConfig, err := newTLSConfig(index.Name, index.Secure) if err != nil { return nil, err @@ -52,6 +53,9 @@ func NewEndpoint(index *IndexInfo, metaHeaders http.Header) (*Endpoint, error) { if err != nil { return nil, err } + if v != APIVersionUnknown { + endpoint.Version = v + } if err := validateEndpoint(endpoint); err != nil { return nil, err } @@ -111,11 +115,6 @@ func newEndpoint(address string, tlsConfig *tls.Config, metaHeaders http.Header) return endpoint, nil } -// GetEndpoint returns a new endpoint with the specified headers -func (repoInfo *RepositoryInfo) GetEndpoint(metaHeaders http.Header) (*Endpoint, error) { - return NewEndpoint(repoInfo.Index, metaHeaders) -} - // Endpoint stores basic information about a registry endpoint. type Endpoint struct { client *http.Client diff --git a/docs/registry.go b/docs/registry.go index 408bc8e1..389bd959 100644 --- a/docs/registry.go +++ b/docs/registry.go @@ -49,6 +49,10 @@ func init() { httpVersion = append(httpVersion, useragent.VersionInfo{"arch", runtime.GOARCH}) dockerUserAgent = useragent.AppendVersions("", httpVersion...) + + if runtime.GOOS != "linux" { + V2Only = true + } } func newTLSConfig(hostname string, isSecure bool) (*tls.Config, error) { diff --git a/docs/registry_test.go b/docs/registry_test.go index 160d3440..f75d7d66 100644 --- a/docs/registry_test.go +++ b/docs/registry_test.go @@ -23,7 +23,7 @@ const ( func spawnTestRegistrySession(t *testing.T) *Session { authConfig := &cliconfig.AuthConfig{} - endpoint, err := NewEndpoint(makeIndex("/v1/"), nil) + endpoint, err := NewEndpoint(makeIndex("/v1/"), nil, APIVersionUnknown) if err != nil { t.Fatal(err) } @@ -50,7 +50,7 @@ func spawnTestRegistrySession(t *testing.T) *Session { func TestPingRegistryEndpoint(t *testing.T) { testPing := func(index *IndexInfo, expectedStandalone bool, assertMessage string) { - ep, err := NewEndpoint(index, nil) + ep, err := NewEndpoint(index, nil, APIVersionUnknown) if err != nil { t.Fatal(err) } @@ -70,7 +70,7 @@ func TestPingRegistryEndpoint(t *testing.T) { func TestEndpoint(t *testing.T) { // Simple wrapper to fail test if err != nil expandEndpoint := func(index *IndexInfo) *Endpoint { - endpoint, err := NewEndpoint(index, nil) + endpoint, err := NewEndpoint(index, nil, APIVersionUnknown) if err != nil { t.Fatal(err) } @@ -79,7 +79,7 @@ func TestEndpoint(t *testing.T) { assertInsecureIndex := func(index *IndexInfo) { index.Secure = true - _, err := NewEndpoint(index, nil) + _, err := NewEndpoint(index, nil, APIVersionUnknown) assertNotEqual(t, err, nil, index.Name+": Expected error for insecure index") assertEqual(t, strings.Contains(err.Error(), "insecure-registry"), true, index.Name+": Expected insecure-registry error for insecure index") index.Secure = false @@ -87,7 +87,7 @@ func TestEndpoint(t *testing.T) { assertSecureIndex := func(index *IndexInfo) { index.Secure = true - _, err := NewEndpoint(index, nil) + _, err := NewEndpoint(index, nil, APIVersionUnknown) assertNotEqual(t, err, nil, index.Name+": Expected cert error for secure index") assertEqual(t, strings.Contains(err.Error(), "certificate signed by unknown authority"), true, index.Name+": Expected cert error for secure index") index.Secure = false @@ -153,7 +153,7 @@ func TestEndpoint(t *testing.T) { } for _, address := range badEndpoints { index.Name = address - _, err := NewEndpoint(index, nil) + _, err := NewEndpoint(index, nil, APIVersionUnknown) checkNotEqual(t, err, nil, "Expected error while expanding bad endpoint") } } diff --git a/docs/service.go b/docs/service.go index 36d63091..1335fe3a 100644 --- a/docs/service.go +++ b/docs/service.go @@ -2,15 +2,11 @@ package registry import ( "crypto/tls" - "fmt" "net/http" "net/url" - "runtime" - "strings" "github.com/docker/distribution/registry/client/auth" "github.com/docker/docker/cliconfig" - "github.com/docker/docker/pkg/tlsconfig" ) // Service is a registry service. It tracks configuration data such as a list @@ -40,7 +36,14 @@ func (s *Service) Auth(authConfig *cliconfig.AuthConfig) (string, error) { if err != nil { return "", err } - endpoint, err := NewEndpoint(index, nil) + + endpointVersion := APIVersion(APIVersionUnknown) + if V2Only { + // Override the endpoint to only attempt a v2 ping + endpointVersion = APIVersion2 + } + + endpoint, err := NewEndpoint(index, nil, endpointVersion) if err != nil { return "", err } @@ -57,10 +60,11 @@ func (s *Service) Search(term string, authConfig *cliconfig.AuthConfig, headers } // *TODO: Search multiple indexes. - endpoint, err := repoInfo.GetEndpoint(http.Header(headers)) + endpoint, err := NewEndpoint(repoInfo.Index, http.Header(headers), APIVersionUnknown) if err != nil { return nil, err } + r, err := NewSession(endpoint.client, authConfig, endpoint) if err != nil { return nil, err @@ -132,97 +136,20 @@ func (s *Service) LookupPushEndpoints(repoName string) (endpoints []APIEndpoint, } func (s *Service) lookupEndpoints(repoName string) (endpoints []APIEndpoint, err error) { - var cfg = tlsconfig.ServerDefault - tlsConfig := &cfg - if strings.HasPrefix(repoName, DefaultNamespace+"/") { - // v2 mirrors - for _, mirror := range s.Config.Mirrors { - mirrorTLSConfig, err := s.tlsConfigForMirror(mirror) - if err != nil { - return nil, err - } - endpoints = append(endpoints, APIEndpoint{ - URL: mirror, - // guess mirrors are v2 - Version: APIVersion2, - Mirror: true, - TrimHostname: true, - TLSConfig: mirrorTLSConfig, - }) - } - // v2 registry - endpoints = append(endpoints, APIEndpoint{ - URL: DefaultV2Registry, - Version: APIVersion2, - Official: true, - TrimHostname: true, - TLSConfig: tlsConfig, - }) - if runtime.GOOS == "linux" { // do not inherit legacy API for OSes supported in the future - // v1 registry - endpoints = append(endpoints, APIEndpoint{ - URL: DefaultV1Registry, - Version: APIVersion1, - Official: true, - TrimHostname: true, - TLSConfig: tlsConfig, - }) - } - return endpoints, nil - } - - slashIndex := strings.IndexRune(repoName, '/') - if slashIndex <= 0 { - return nil, fmt.Errorf("invalid repo name: missing '/': %s", repoName) - } - hostname := repoName[:slashIndex] - - tlsConfig, err = s.TLSConfig(hostname) + endpoints, err = s.lookupV2Endpoints(repoName) if err != nil { return nil, err } - isSecure := !tlsConfig.InsecureSkipVerify - v2Versions := []auth.APIVersion{ - { - Type: "registry", - Version: "2.0", - }, - } - endpoints = []APIEndpoint{ - { - URL: "https://" + hostname, - Version: APIVersion2, - TrimHostname: true, - TLSConfig: tlsConfig, - VersionHeader: DefaultRegistryVersionHeader, - Versions: v2Versions, - }, - { - URL: "https://" + hostname, - Version: APIVersion1, - TrimHostname: true, - TLSConfig: tlsConfig, - }, + if V2Only { + return endpoints, nil } - if !isSecure { - endpoints = append(endpoints, APIEndpoint{ - URL: "http://" + hostname, - Version: APIVersion2, - TrimHostname: true, - // used to check if supposed to be secure via InsecureSkipVerify - TLSConfig: tlsConfig, - VersionHeader: DefaultRegistryVersionHeader, - Versions: v2Versions, - }, APIEndpoint{ - URL: "http://" + hostname, - Version: APIVersion1, - TrimHostname: true, - // used to check if supposed to be secure via InsecureSkipVerify - TLSConfig: tlsConfig, - }) + legacyEndpoints, err := s.lookupV1Endpoints(repoName) + if err != nil { + return nil, err } + endpoints = append(endpoints, legacyEndpoints...) return endpoints, nil } diff --git a/docs/service_v1.go b/docs/service_v1.go new file mode 100644 index 00000000..ddb78ee6 --- /dev/null +++ b/docs/service_v1.go @@ -0,0 +1,54 @@ +package registry + +import ( + "fmt" + "strings" + + "github.com/docker/docker/pkg/tlsconfig" +) + +func (s *Service) lookupV1Endpoints(repoName string) (endpoints []APIEndpoint, err error) { + var cfg = tlsconfig.ServerDefault + tlsConfig := &cfg + if strings.HasPrefix(repoName, DefaultNamespace+"/") { + endpoints = append(endpoints, APIEndpoint{ + URL: DefaultV1Registry, + Version: APIVersion1, + Official: true, + TrimHostname: true, + TLSConfig: tlsConfig, + }) + return endpoints, nil + } + + slashIndex := strings.IndexRune(repoName, '/') + if slashIndex <= 0 { + return nil, fmt.Errorf("invalid repo name: missing '/': %s", repoName) + } + hostname := repoName[:slashIndex] + + tlsConfig, err = s.TLSConfig(hostname) + if err != nil { + return nil, err + } + + endpoints = []APIEndpoint{ + { + URL: "https://" + hostname, + Version: APIVersion1, + TrimHostname: true, + TLSConfig: tlsConfig, + }, + } + + if tlsConfig.InsecureSkipVerify { + endpoints = append(endpoints, APIEndpoint{ // or this + URL: "http://" + hostname, + Version: APIVersion1, + TrimHostname: true, + // used to check if supposed to be secure via InsecureSkipVerify + TLSConfig: tlsConfig, + }) + } + return endpoints, nil +} diff --git a/docs/service_v2.go b/docs/service_v2.go new file mode 100644 index 00000000..70d5fd71 --- /dev/null +++ b/docs/service_v2.go @@ -0,0 +1,83 @@ +package registry + +import ( + "fmt" + "strings" + + "github.com/docker/distribution/registry/client/auth" + "github.com/docker/docker/pkg/tlsconfig" +) + +func (s *Service) lookupV2Endpoints(repoName string) (endpoints []APIEndpoint, err error) { + var cfg = tlsconfig.ServerDefault + tlsConfig := &cfg + if strings.HasPrefix(repoName, DefaultNamespace+"/") { + // v2 mirrors + for _, mirror := range s.Config.Mirrors { + mirrorTLSConfig, err := s.tlsConfigForMirror(mirror) + if err != nil { + return nil, err + } + endpoints = append(endpoints, APIEndpoint{ + URL: mirror, + // guess mirrors are v2 + Version: APIVersion2, + Mirror: true, + TrimHostname: true, + TLSConfig: mirrorTLSConfig, + }) + } + // v2 registry + endpoints = append(endpoints, APIEndpoint{ + URL: DefaultV2Registry, + Version: APIVersion2, + Official: true, + TrimHostname: true, + TLSConfig: tlsConfig, + }) + + return endpoints, nil + } + + slashIndex := strings.IndexRune(repoName, '/') + if slashIndex <= 0 { + return nil, fmt.Errorf("invalid repo name: missing '/': %s", repoName) + } + hostname := repoName[:slashIndex] + + tlsConfig, err = s.TLSConfig(hostname) + if err != nil { + return nil, err + } + + v2Versions := []auth.APIVersion{ + { + Type: "registry", + Version: "2.0", + }, + } + endpoints = []APIEndpoint{ + { + URL: "https://" + hostname, + Version: APIVersion2, + TrimHostname: true, + TLSConfig: tlsConfig, + VersionHeader: DefaultRegistryVersionHeader, + Versions: v2Versions, + }, + } + + if tlsConfig.InsecureSkipVerify { + endpoints = append(endpoints, APIEndpoint{ + URL: "http://" + hostname, + Version: APIVersion2, + TrimHostname: true, + // used to check if supposed to be secure via InsecureSkipVerify + TLSConfig: tlsConfig, + VersionHeader: DefaultRegistryVersionHeader, + Versions: v2Versions, + }) + } + + return endpoints, nil +} From 82965f6c84f207c53321c5720072785111ab3fb5 Mon Sep 17 00:00:00 2001 From: Hu Keping Date: Tue, 22 Sep 2015 19:44:40 +0800 Subject: [PATCH 320/375] Fix docker search problem Search terms shouldn't be restricted to only full valid repository names. It should be perfectly valid to search using a part of a name, even if it ends with a period, dash or underscore. Signed-off-by: Hu Keping --- docs/config.go | 22 ++++++++++++++++++---- docs/service.go | 11 +++++++++-- 2 files changed, 27 insertions(+), 6 deletions(-) diff --git a/docs/config.go b/docs/config.go index 5fca9df0..c73b6c5d 100644 --- a/docs/config.go +++ b/docs/config.go @@ -295,14 +295,17 @@ func splitReposName(reposName string) (string, string) { } // NewRepositoryInfo validates and breaks down a repository name into a RepositoryInfo -func (config *ServiceConfig) NewRepositoryInfo(reposName string) (*RepositoryInfo, error) { +func (config *ServiceConfig) NewRepositoryInfo(reposName string, bySearch bool) (*RepositoryInfo, error) { if err := validateNoSchema(reposName); err != nil { return nil, err } indexName, remoteName := splitReposName(reposName) - if err := validateRemoteName(remoteName); err != nil { - return nil, err + + if !bySearch { + if err := validateRemoteName(remoteName); err != nil { + return nil, err + } } repoInfo := &RepositoryInfo{ @@ -354,7 +357,18 @@ func (repoInfo *RepositoryInfo) GetSearchTerm() string { // ParseRepositoryInfo performs the breakdown of a repository name into a RepositoryInfo, but // lacks registry configuration. func ParseRepositoryInfo(reposName string) (*RepositoryInfo, error) { - return emptyServiceConfig.NewRepositoryInfo(reposName) + return emptyServiceConfig.NewRepositoryInfo(reposName, false) +} + +// ParseIndexInfo will use repository name to get back an indexInfo. +func ParseIndexInfo(reposName string) (*IndexInfo, error) { + indexName, _ := splitReposName(reposName) + + indexInfo, err := emptyServiceConfig.NewIndexInfo(indexName) + if err != nil { + return nil, err + } + return indexInfo, nil } // NormalizeLocalName transforms a repository name into a normalize LocalName diff --git a/docs/service.go b/docs/service.go index 36d63091..bb38b2c0 100644 --- a/docs/service.go +++ b/docs/service.go @@ -51,7 +51,8 @@ func (s *Service) Auth(authConfig *cliconfig.AuthConfig) (string, error) { // Search queries the public registry for images matching the specified // search terms, and returns the results. func (s *Service) Search(term string, authConfig *cliconfig.AuthConfig, headers map[string][]string) (*SearchResults, error) { - repoInfo, err := s.ResolveRepository(term) + + repoInfo, err := s.ResolveRepositoryBySearch(term) if err != nil { return nil, err } @@ -71,7 +72,13 @@ func (s *Service) Search(term string, authConfig *cliconfig.AuthConfig, headers // ResolveRepository splits a repository name into its components // and configuration of the associated registry. func (s *Service) ResolveRepository(name string) (*RepositoryInfo, error) { - return s.Config.NewRepositoryInfo(name) + return s.Config.NewRepositoryInfo(name, false) +} + +// ResolveRepositoryBySearch splits a repository name into its components +// and configuration of the associated registry. +func (s *Service) ResolveRepositoryBySearch(name string) (*RepositoryInfo, error) { + return s.Config.NewRepositoryInfo(name, true) } // ResolveIndex takes indexName and returns index info From 0b543b4767b242eecd0ba708eacada42b47f4588 Mon Sep 17 00:00:00 2001 From: Jessica Frazelle Date: Tue, 6 Oct 2015 15:45:32 -0700 Subject: [PATCH 321/375] change flag name to better follow the other flags that start with disable; Signed-off-by: Jessica Frazelle --- docs/config.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/config.go b/docs/config.go index edae558e..b49bd910 100644 --- a/docs/config.go +++ b/docs/config.go @@ -57,7 +57,7 @@ func (options *Options) InstallFlags(cmd *flag.FlagSet, usageFn func(string) str cmd.Var(&options.Mirrors, []string{"-registry-mirror"}, usageFn("Preferred Docker registry mirror")) options.InsecureRegistries = opts.NewListOpts(ValidateIndexName) cmd.Var(&options.InsecureRegistries, []string{"-insecure-registry"}, usageFn("Enable insecure registry communication")) - cmd.BoolVar(&V2Only, []string{"-no-legacy-registry"}, false, "Do not contact legacy registries") + cmd.BoolVar(&V2Only, []string{"-disable-legacy-registry"}, false, "Do not contact legacy registries") } type netIPNet net.IPNet From ed69ef01ee1d5d218fdcc44839942cbd77d0d0db Mon Sep 17 00:00:00 2001 From: Derek McGowan Date: Thu, 8 Oct 2015 17:16:43 -0700 Subject: [PATCH 322/375] Update distribution package Pick up name regexp change in distribution to allow matching of hostnames as a valid component of a repository. Signed-off-by: Derek McGowan (github: dmcgowan) --- docs/registry_test.go | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/docs/registry_test.go b/docs/registry_test.go index f75d7d66..5b36210a 100644 --- a/docs/registry_test.go +++ b/docs/registry_test.go @@ -767,6 +767,9 @@ func TestValidRemoteName(t *testing.T) { // Allow embedded hyphens. "docker-rules/docker", + // Allow multiple hyphens as well. + "docker---rules/docker", + //Username doc and image name docker being tested. "doc/docker", @@ -800,8 +803,11 @@ func TestValidRemoteName(t *testing.T) { "_docker/_docker", - // Disallow consecutive hyphens. - "dock--er/docker", + // Disallow consecutive underscores and periods. + "dock__er/docker", + "dock..er/docker", + "dock_.er/docker", + "dock-.er/docker", // No repository. "docker/", From c2d5c29437cd11e1a01cc2c574e02ed9865eb647 Mon Sep 17 00:00:00 2001 From: Vincent Demeester Date: Sun, 25 Oct 2015 19:18:23 +0100 Subject: [PATCH 323/375] dockerversion placeholder for library-import - Move autogen/dockerversion to version - Update autogen and "builds" to use this package and a build flag Signed-off-by: Vincent Demeester --- docs/registry.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/registry.go b/docs/registry.go index 389bd959..16310235 100644 --- a/docs/registry.go +++ b/docs/registry.go @@ -20,10 +20,10 @@ import ( "github.com/docker/distribution/registry/api/v2" "github.com/docker/distribution/registry/client" "github.com/docker/distribution/registry/client/transport" - "github.com/docker/docker/autogen/dockerversion" "github.com/docker/docker/pkg/parsers/kernel" "github.com/docker/docker/pkg/tlsconfig" "github.com/docker/docker/pkg/useragent" + "github.com/docker/docker/version" ) var ( @@ -39,9 +39,9 @@ var dockerUserAgent string func init() { httpVersion := make([]useragent.VersionInfo, 0, 6) - httpVersion = append(httpVersion, useragent.VersionInfo{"docker", dockerversion.VERSION}) + httpVersion = append(httpVersion, useragent.VersionInfo{"docker", version.VERSION}) httpVersion = append(httpVersion, useragent.VersionInfo{"go", runtime.Version()}) - httpVersion = append(httpVersion, useragent.VersionInfo{"git-commit", dockerversion.GITCOMMIT}) + httpVersion = append(httpVersion, useragent.VersionInfo{"git-commit", version.GITCOMMIT}) if kernelVersion, err := kernel.GetKernelVersion(); err == nil { httpVersion = append(httpVersion, useragent.VersionInfo{"kernel", kernelVersion.String()}) } From 24de26a805ba4fa1c583d3742cfe19f93751fc5c Mon Sep 17 00:00:00 2001 From: Brian Goff Date: Tue, 27 Oct 2015 21:19:14 -0400 Subject: [PATCH 324/375] Revert "dockerversion placeholder for library-import" This reverts commit c2d5c29437cd11e1a01cc2c574e02ed9865eb647. Commit caused issues on systems with case-insensitive filesystems. Revert for now Signed-off-by: Brian Goff --- docs/registry.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/registry.go b/docs/registry.go index 16310235..389bd959 100644 --- a/docs/registry.go +++ b/docs/registry.go @@ -20,10 +20,10 @@ import ( "github.com/docker/distribution/registry/api/v2" "github.com/docker/distribution/registry/client" "github.com/docker/distribution/registry/client/transport" + "github.com/docker/docker/autogen/dockerversion" "github.com/docker/docker/pkg/parsers/kernel" "github.com/docker/docker/pkg/tlsconfig" "github.com/docker/docker/pkg/useragent" - "github.com/docker/docker/version" ) var ( @@ -39,9 +39,9 @@ var dockerUserAgent string func init() { httpVersion := make([]useragent.VersionInfo, 0, 6) - httpVersion = append(httpVersion, useragent.VersionInfo{"docker", version.VERSION}) + httpVersion = append(httpVersion, useragent.VersionInfo{"docker", dockerversion.VERSION}) httpVersion = append(httpVersion, useragent.VersionInfo{"go", runtime.Version()}) - httpVersion = append(httpVersion, useragent.VersionInfo{"git-commit", version.GITCOMMIT}) + httpVersion = append(httpVersion, useragent.VersionInfo{"git-commit", dockerversion.GITCOMMIT}) if kernelVersion, err := kernel.GetKernelVersion(); err == nil { httpVersion = append(httpVersion, useragent.VersionInfo{"kernel", kernelVersion.String()}) } From 94913b8f7ff03570eef62a66afa66fbacccb85bb Mon Sep 17 00:00:00 2001 From: Alexander Morozov Date: Mon, 2 Nov 2015 08:28:34 -0800 Subject: [PATCH 325/375] Fix go vet warnings Signed-off-by: Alexander Morozov --- docs/registry.go | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/docs/registry.go b/docs/registry.go index 389bd959..bd5251e1 100644 --- a/docs/registry.go +++ b/docs/registry.go @@ -39,14 +39,14 @@ var dockerUserAgent string func init() { httpVersion := make([]useragent.VersionInfo, 0, 6) - httpVersion = append(httpVersion, useragent.VersionInfo{"docker", dockerversion.VERSION}) - httpVersion = append(httpVersion, useragent.VersionInfo{"go", runtime.Version()}) - httpVersion = append(httpVersion, useragent.VersionInfo{"git-commit", dockerversion.GITCOMMIT}) + httpVersion = append(httpVersion, useragent.VersionInfo{Name: "docker", Version: dockerversion.VERSION}) + httpVersion = append(httpVersion, useragent.VersionInfo{Name: "go", Version: runtime.Version()}) + httpVersion = append(httpVersion, useragent.VersionInfo{Name: "git-commit", Version: dockerversion.GITCOMMIT}) if kernelVersion, err := kernel.GetKernelVersion(); err == nil { - httpVersion = append(httpVersion, useragent.VersionInfo{"kernel", kernelVersion.String()}) + httpVersion = append(httpVersion, useragent.VersionInfo{Name: "kernel", Version: kernelVersion.String()}) } - httpVersion = append(httpVersion, useragent.VersionInfo{"os", runtime.GOOS}) - httpVersion = append(httpVersion, useragent.VersionInfo{"arch", runtime.GOARCH}) + httpVersion = append(httpVersion, useragent.VersionInfo{Name: "os", Version: runtime.GOOS}) + httpVersion = append(httpVersion, useragent.VersionInfo{Name: "arch", Version: runtime.GOARCH}) dockerUserAgent = useragent.AppendVersions("", httpVersion...) From afd61ce8f2341785448155a9607ca3b58b9761c5 Mon Sep 17 00:00:00 2001 From: Aaron Lehmann Date: Fri, 30 Oct 2015 17:46:25 -0700 Subject: [PATCH 326/375] Vendor updated version of docker/distribution This updates the vendored docker/distribution to the current master branch. Note the following changes: - The manifest package was split into manifest/schema1. Most references to the manifest package in the engine needed to be updated to use schema1 instead. - Validation functions in api/v2 were replaced by the distribution/reference package. The engine code has been updated to use the reference package for validation where necessary. A future PR will change the engine to use the types defined in distribution/reference more comprehensively. - The reference package explicitly allows double _ characters in repository names. registry_test.go was updated for this. - TestPullFailsWithAlteredManifest was corrupting the manifest JSON, now that the schema1 package unmarshals the correct payload. The test is being changed to modify the JSON without affecting its length, which allows the pull to succeed to the point where digest validation happens. Signed-off-by: Aaron Lehmann --- docs/config.go | 5 +++-- docs/registry.go | 2 +- docs/registry_test.go | 6 ++++-- 3 files changed, 8 insertions(+), 5 deletions(-) diff --git a/docs/config.go b/docs/config.go index b49bd910..e8f2287e 100644 --- a/docs/config.go +++ b/docs/config.go @@ -8,7 +8,7 @@ import ( "net/url" "strings" - "github.com/docker/distribution/registry/api/v2" + "github.com/docker/distribution/reference" "github.com/docker/docker/image" "github.com/docker/docker/opts" flag "github.com/docker/docker/pkg/mflag" @@ -226,7 +226,8 @@ func validateRemoteName(remoteName string) error { } } - return v2.ValidateRepositoryName(remoteName) + _, err := reference.WithName(remoteName) + return err } func validateNoSchema(reposName string) error { diff --git a/docs/registry.go b/docs/registry.go index 389bd959..02f18921 100644 --- a/docs/registry.go +++ b/docs/registry.go @@ -190,7 +190,7 @@ func addRequiredHeadersToRedirectedRequests(req *http.Request, via []*http.Reque func shouldV2Fallback(err errcode.Error) bool { logrus.Debugf("v2 error: %T %v", err, err) switch err.Code { - case v2.ErrorCodeUnauthorized, v2.ErrorCodeManifestUnknown: + case errcode.ErrorCodeUnauthorized, v2.ErrorCodeManifestUnknown: return true } return false diff --git a/docs/registry_test.go b/docs/registry_test.go index 5b36210a..7714310d 100644 --- a/docs/registry_test.go +++ b/docs/registry_test.go @@ -776,6 +776,9 @@ func TestValidRemoteName(t *testing.T) { // single character names are now allowed. "d/docker", "jess/t", + + // Consecutive underscores. + "dock__er/docker", } for _, repositoryName := range validRepositoryNames { if err := validateRemoteName(repositoryName); err != nil { @@ -803,8 +806,7 @@ func TestValidRemoteName(t *testing.T) { "_docker/_docker", - // Disallow consecutive underscores and periods. - "dock__er/docker", + // Disallow consecutive periods. "dock..er/docker", "dock_.er/docker", "dock-.er/docker", From 9516a01c56885b4b763951c15921303ecaac28f6 Mon Sep 17 00:00:00 2001 From: Vincent Demeester Date: Mon, 9 Nov 2015 19:32:46 +0100 Subject: [PATCH 327/375] dockerversion placeholder for library import - Add a *version* file placeholder. - Update autogen and builds to use it and an autogen build flag Signed-off-by: Vincent Demeester --- docs/registry.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/registry.go b/docs/registry.go index f93811e6..e8eb4785 100644 --- a/docs/registry.go +++ b/docs/registry.go @@ -20,7 +20,7 @@ import ( "github.com/docker/distribution/registry/api/v2" "github.com/docker/distribution/registry/client" "github.com/docker/distribution/registry/client/transport" - "github.com/docker/docker/autogen/dockerversion" + "github.com/docker/docker/dockerversion" "github.com/docker/docker/pkg/parsers/kernel" "github.com/docker/docker/pkg/tlsconfig" "github.com/docker/docker/pkg/useragent" @@ -39,9 +39,9 @@ var dockerUserAgent string func init() { httpVersion := make([]useragent.VersionInfo, 0, 6) - httpVersion = append(httpVersion, useragent.VersionInfo{Name: "docker", Version: dockerversion.VERSION}) + httpVersion = append(httpVersion, useragent.VersionInfo{Name: "docker", Version: dockerversion.Version}) httpVersion = append(httpVersion, useragent.VersionInfo{Name: "go", Version: runtime.Version()}) - httpVersion = append(httpVersion, useragent.VersionInfo{Name: "git-commit", Version: dockerversion.GITCOMMIT}) + httpVersion = append(httpVersion, useragent.VersionInfo{Name: "git-commit", Version: dockerversion.GitCommit}) if kernelVersion, err := kernel.GetKernelVersion(); err == nil { httpVersion = append(httpVersion, useragent.VersionInfo{Name: "kernel", Version: kernelVersion.String()}) } From 1820704288cadc715ca98518dd9f43cdee3e05ff Mon Sep 17 00:00:00 2001 From: Anil Belur Date: Mon, 2 Nov 2015 13:38:02 +0530 Subject: [PATCH 328/375] Fix for #17168 misleading pull error This fix avoids overwritting the previous error messages, ensures the client gets the correct error messages and not just the most recent message during the pull request. For this `var lastErr` replaced with a slice which acts as a temp place holder for the list of returned error messages for every attempt. The slice is later joined and returned to the caller function after searching for the image with diffirent versions(v2,v1,v0). Updated the code with check for no space left on device error occurance and prevent the daemon on falling back to v1,v0. Incorporated the comments from @calavera, @RichardScothern, @cpuguy83 Signed-off-by: Anil Belur --- docs/registry.go | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/docs/registry.go b/docs/registry.go index e8eb4785..d018f922 100644 --- a/docs/registry.go +++ b/docs/registry.go @@ -13,6 +13,7 @@ import ( "path/filepath" "runtime" "strings" + "syscall" "time" "github.com/Sirupsen/logrus" @@ -219,6 +220,10 @@ func ContinueOnError(err error) bool { return shouldV2Fallback(v) case *client.UnexpectedHTTPResponseError: return true + case error: + if val := strings.Contains(err.Error(), strings.ToLower(syscall.ENOSPC.Error())); val { + return false + } } // let's be nice and fallback if the error is a completely // unexpected one. From 3eeebe7be30a8b9e9e71d05839cd17c37bf2013f Mon Sep 17 00:00:00 2001 From: David Calavera Date: Mon, 16 Nov 2015 10:27:22 -0500 Subject: [PATCH 329/375] Make NormalizeLocalName to not reach the network to normalize names. Signed-off-by: David Calavera --- docs/config.go | 77 ++++++++++++++++++++++++++++++++++---------------- 1 file changed, 53 insertions(+), 24 deletions(-) diff --git a/docs/config.go b/docs/config.go index e8f2287e..7cac7158 100644 --- a/docs/config.go +++ b/docs/config.go @@ -240,15 +240,28 @@ func validateNoSchema(reposName string) error { // ValidateRepositoryName validates a repository name func ValidateRepositoryName(reposName string) error { - var err error - if err = validateNoSchema(reposName); err != nil { - return err + _, _, err := loadRepositoryName(reposName, true) + return err +} + +// loadRepositoryName returns the repo name splitted into index name +// and remote repo name. It returns an error if the name is not valid. +func loadRepositoryName(reposName string, checkRemoteName bool) (string, string, error) { + if err := validateNoSchema(reposName); err != nil { + return "", "", err } indexName, remoteName := splitReposName(reposName) - if _, err = ValidateIndexName(indexName); err != nil { - return err + + var err error + if indexName, err = ValidateIndexName(indexName); err != nil { + return "", "", err } - return validateRemoteName(remoteName) + if checkRemoteName { + if err = validateRemoteName(remoteName); err != nil { + return "", "", err + } + } + return indexName, remoteName, nil } // NewIndexInfo returns IndexInfo configuration from indexName @@ -302,34 +315,22 @@ func splitReposName(reposName string) (string, string) { // NewRepositoryInfo validates and breaks down a repository name into a RepositoryInfo func (config *ServiceConfig) NewRepositoryInfo(reposName string, bySearch bool) (*RepositoryInfo, error) { - if err := validateNoSchema(reposName); err != nil { + indexName, remoteName, err := loadRepositoryName(reposName, !bySearch) + if err != nil { return nil, err } - indexName, remoteName := splitReposName(reposName) - - if !bySearch { - if err := validateRemoteName(remoteName); err != nil { - return nil, err - } - } - repoInfo := &RepositoryInfo{ RemoteName: remoteName, } - var err error repoInfo.Index, err = config.NewIndexInfo(indexName) if err != nil { return nil, err } if repoInfo.Index.Official { - normalizedName := repoInfo.RemoteName - if strings.HasPrefix(normalizedName, "library/") { - // If pull "library/foo", it's stored locally under "foo" - normalizedName = strings.SplitN(normalizedName, "/", 2)[1] - } + normalizedName := normalizeLibraryRepoName(repoInfo.RemoteName) repoInfo.LocalName = normalizedName repoInfo.RemoteName = normalizedName @@ -343,7 +344,7 @@ func (config *ServiceConfig) NewRepositoryInfo(reposName string, bySearch bool) repoInfo.CanonicalName = "docker.io/" + repoInfo.RemoteName } else { - repoInfo.LocalName = repoInfo.Index.Name + "/" + repoInfo.RemoteName + repoInfo.LocalName = localNameFromRemote(repoInfo.Index.Name, repoInfo.RemoteName) repoInfo.CanonicalName = repoInfo.LocalName } @@ -379,10 +380,38 @@ func ParseIndexInfo(reposName string) (*IndexInfo, error) { // NormalizeLocalName transforms a repository name into a normalize LocalName // Passes through the name without transformation on error (image id, etc) +// It does not use the repository info because we don't want to load +// the repository index and do request over the network. func NormalizeLocalName(name string) string { - repoInfo, err := ParseRepositoryInfo(name) + indexName, remoteName, err := loadRepositoryName(name, true) if err != nil { return name } - return repoInfo.LocalName + + var officialIndex bool + // Return any configured index info, first. + if index, ok := emptyServiceConfig.IndexConfigs[indexName]; ok { + officialIndex = index.Official + } + + if officialIndex { + return normalizeLibraryRepoName(remoteName) + } + return localNameFromRemote(indexName, remoteName) +} + +// normalizeLibraryRepoName removes the library prefix from +// the repository name for official repos. +func normalizeLibraryRepoName(name string) string { + if strings.HasPrefix(name, "library/") { + // If pull "library/foo", it's stored locally under "foo" + name = strings.SplitN(name, "/", 2)[1] + } + return name +} + +// localNameFromRemote combines the index name and the repo remote name +// to generate a repo local name. +func localNameFromRemote(indexName, remoteName string) string { + return indexName + "/" + remoteName } From dde006ba6b929e5a57644334ac962a544cce1e78 Mon Sep 17 00:00:00 2001 From: Alexander Morozov Date: Tue, 17 Nov 2015 16:12:11 -0800 Subject: [PATCH 330/375] registry/registry.go: simplify logical expression Signed-off-by: Alexander Morozov --- docs/registry.go | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/docs/registry.go b/docs/registry.go index d018f922..9c8666ea 100644 --- a/docs/registry.go +++ b/docs/registry.go @@ -221,9 +221,7 @@ func ContinueOnError(err error) bool { case *client.UnexpectedHTTPResponseError: return true case error: - if val := strings.Contains(err.Error(), strings.ToLower(syscall.ENOSPC.Error())); val { - return false - } + return !strings.Contains(err.Error(), strings.ToLower(syscall.ENOSPC.Error())) } // let's be nice and fallback if the error is a completely // unexpected one. From 7efcb7496c64226050319c91b4c554c88dadd6d3 Mon Sep 17 00:00:00 2001 From: Tonis Tiigi Date: Wed, 18 Nov 2015 14:20:54 -0800 Subject: [PATCH 331/375] Update daemon and docker core to use new content addressable storage Add distribution package for managing pulls and pushes. This is based on the old code in the graph package, with major changes to work with the new image/layer model. Add v1 migration code. Update registry, api/*, and daemon packages to use the reference package's types where applicable. Update daemon package to use image/layer/tag stores instead of the graph package Signed-off-by: Aaron Lehmann Signed-off-by: Tonis Tiigi --- docs/config.go | 163 +++++++++++++++------------ docs/registry_mock_test.go | 31 ++++-- docs/registry_test.go | 219 +++++++++++++++++++++++-------------- docs/service.go | 56 +++++++--- docs/service_v1.go | 12 +- docs/service_v2.go | 12 +- docs/session.go | 29 +++-- docs/types.go | 10 +- 8 files changed, 336 insertions(+), 196 deletions(-) diff --git a/docs/config.go b/docs/config.go index 7cac7158..8d7962f8 100644 --- a/docs/config.go +++ b/docs/config.go @@ -9,7 +9,7 @@ import ( "strings" "github.com/docker/distribution/reference" - "github.com/docker/docker/image" + "github.com/docker/docker/image/v1" "github.com/docker/docker/opts" flag "github.com/docker/docker/pkg/mflag" ) @@ -216,18 +216,15 @@ func ValidateIndexName(val string) (string, error) { return val, nil } -func validateRemoteName(remoteName string) error { - - if !strings.Contains(remoteName, "/") { - +func validateRemoteName(remoteName reference.Named) error { + remoteNameStr := remoteName.Name() + if !strings.Contains(remoteNameStr, "/") { // the repository name must not be a valid image ID - if err := image.ValidateID(remoteName); err == nil { + if err := v1.ValidateID(remoteNameStr); err == nil { return fmt.Errorf("Invalid repository name (%s), cannot specify 64-byte hexadecimal strings", remoteName) } } - - _, err := reference.WithName(remoteName) - return err + return nil } func validateNoSchema(reposName string) error { @@ -239,27 +236,24 @@ func validateNoSchema(reposName string) error { } // ValidateRepositoryName validates a repository name -func ValidateRepositoryName(reposName string) error { - _, _, err := loadRepositoryName(reposName, true) +func ValidateRepositoryName(reposName reference.Named) error { + _, _, err := loadRepositoryName(reposName) return err } // loadRepositoryName returns the repo name splitted into index name // and remote repo name. It returns an error if the name is not valid. -func loadRepositoryName(reposName string, checkRemoteName bool) (string, string, error) { - if err := validateNoSchema(reposName); err != nil { - return "", "", err +func loadRepositoryName(reposName reference.Named) (string, reference.Named, error) { + if err := validateNoSchema(reposName.Name()); err != nil { + return "", nil, err } - indexName, remoteName := splitReposName(reposName) + indexName, remoteName, err := splitReposName(reposName) - var err error if indexName, err = ValidateIndexName(indexName); err != nil { - return "", "", err + return "", nil, err } - if checkRemoteName { - if err = validateRemoteName(remoteName); err != nil { - return "", "", err - } + if err = validateRemoteName(remoteName); err != nil { + return "", nil, err } return indexName, remoteName, nil } @@ -297,31 +291,36 @@ func (index *IndexInfo) GetAuthConfigKey() string { } // splitReposName breaks a reposName into an index name and remote name -func splitReposName(reposName string) (string, string) { - nameParts := strings.SplitN(reposName, "/", 2) - var indexName, remoteName string - if len(nameParts) == 1 || (!strings.Contains(nameParts[0], ".") && - !strings.Contains(nameParts[0], ":") && nameParts[0] != "localhost") { +func splitReposName(reposName reference.Named) (indexName string, remoteName reference.Named, err error) { + var remoteNameStr string + indexName, remoteNameStr = reference.SplitHostname(reposName) + if indexName == "" || (!strings.Contains(indexName, ".") && + !strings.Contains(indexName, ":") && indexName != "localhost") { // This is a Docker Index repos (ex: samalba/hipache or ubuntu) // 'docker.io' indexName = IndexName remoteName = reposName } else { - indexName = nameParts[0] - remoteName = nameParts[1] + remoteName, err = reference.WithName(remoteNameStr) } - return indexName, remoteName + return } // NewRepositoryInfo validates and breaks down a repository name into a RepositoryInfo -func (config *ServiceConfig) NewRepositoryInfo(reposName string, bySearch bool) (*RepositoryInfo, error) { - indexName, remoteName, err := loadRepositoryName(reposName, !bySearch) - if err != nil { +func (config *ServiceConfig) NewRepositoryInfo(reposName reference.Named) (*RepositoryInfo, error) { + if err := validateNoSchema(reposName.Name()); err != nil { return nil, err } - repoInfo := &RepositoryInfo{ - RemoteName: remoteName, + repoInfo := &RepositoryInfo{} + var ( + indexName string + err error + ) + + indexName, repoInfo.RemoteName, err = loadRepositoryName(reposName) + if err != nil { + return nil, err } repoInfo.Index, err = config.NewIndexInfo(indexName) @@ -330,46 +329,47 @@ func (config *ServiceConfig) NewRepositoryInfo(reposName string, bySearch bool) } if repoInfo.Index.Official { - normalizedName := normalizeLibraryRepoName(repoInfo.RemoteName) + repoInfo.LocalName, err = normalizeLibraryRepoName(repoInfo.RemoteName) + if err != nil { + return nil, err + } + repoInfo.RemoteName = repoInfo.LocalName - repoInfo.LocalName = normalizedName - repoInfo.RemoteName = normalizedName // If the normalized name does not contain a '/' (e.g. "foo") // then it is an official repo. - if strings.IndexRune(normalizedName, '/') == -1 { + if strings.IndexRune(repoInfo.RemoteName.Name(), '/') == -1 { repoInfo.Official = true // Fix up remote name for official repos. - repoInfo.RemoteName = "library/" + normalizedName + repoInfo.RemoteName, err = reference.WithName("library/" + repoInfo.RemoteName.Name()) + if err != nil { + return nil, err + } } - repoInfo.CanonicalName = "docker.io/" + repoInfo.RemoteName + repoInfo.CanonicalName, err = reference.WithName("docker.io/" + repoInfo.RemoteName.Name()) + if err != nil { + return nil, err + } } else { - repoInfo.LocalName = localNameFromRemote(repoInfo.Index.Name, repoInfo.RemoteName) + repoInfo.LocalName, err = localNameFromRemote(repoInfo.Index.Name, repoInfo.RemoteName) + if err != nil { + return nil, err + } repoInfo.CanonicalName = repoInfo.LocalName - } return repoInfo, nil } -// GetSearchTerm special-cases using local name for official index, and -// remote name for private indexes. -func (repoInfo *RepositoryInfo) GetSearchTerm() string { - if repoInfo.Index.Official { - return repoInfo.LocalName - } - return repoInfo.RemoteName -} - // ParseRepositoryInfo performs the breakdown of a repository name into a RepositoryInfo, but // lacks registry configuration. -func ParseRepositoryInfo(reposName string) (*RepositoryInfo, error) { - return emptyServiceConfig.NewRepositoryInfo(reposName, false) +func ParseRepositoryInfo(reposName reference.Named) (*RepositoryInfo, error) { + return emptyServiceConfig.NewRepositoryInfo(reposName) } -// ParseIndexInfo will use repository name to get back an indexInfo. -func ParseIndexInfo(reposName string) (*IndexInfo, error) { - indexName, _ := splitReposName(reposName) +// ParseSearchIndexInfo will use repository name to get back an indexInfo. +func ParseSearchIndexInfo(reposName string) (*IndexInfo, error) { + indexName, _ := splitReposSearchTerm(reposName) indexInfo, err := emptyServiceConfig.NewIndexInfo(indexName) if err != nil { @@ -378,12 +378,12 @@ func ParseIndexInfo(reposName string) (*IndexInfo, error) { return indexInfo, nil } -// NormalizeLocalName transforms a repository name into a normalize LocalName +// NormalizeLocalName transforms a repository name into a normalized LocalName // Passes through the name without transformation on error (image id, etc) // It does not use the repository info because we don't want to load // the repository index and do request over the network. -func NormalizeLocalName(name string) string { - indexName, remoteName, err := loadRepositoryName(name, true) +func NormalizeLocalName(name reference.Named) reference.Named { + indexName, remoteName, err := loadRepositoryName(name) if err != nil { return name } @@ -395,23 +395,52 @@ func NormalizeLocalName(name string) string { } if officialIndex { - return normalizeLibraryRepoName(remoteName) + localName, err := normalizeLibraryRepoName(remoteName) + if err != nil { + return name + } + return localName } - return localNameFromRemote(indexName, remoteName) + localName, err := localNameFromRemote(indexName, remoteName) + if err != nil { + return name + } + return localName } // normalizeLibraryRepoName removes the library prefix from // the repository name for official repos. -func normalizeLibraryRepoName(name string) string { - if strings.HasPrefix(name, "library/") { +func normalizeLibraryRepoName(name reference.Named) (reference.Named, error) { + if strings.HasPrefix(name.Name(), "library/") { // If pull "library/foo", it's stored locally under "foo" - name = strings.SplitN(name, "/", 2)[1] + return reference.WithName(strings.SplitN(name.Name(), "/", 2)[1]) } - return name + return name, nil } // localNameFromRemote combines the index name and the repo remote name // to generate a repo local name. -func localNameFromRemote(indexName, remoteName string) string { - return indexName + "/" + remoteName +func localNameFromRemote(indexName string, remoteName reference.Named) (reference.Named, error) { + return reference.WithName(indexName + "/" + remoteName.Name()) +} + +// NormalizeLocalReference transforms a reference to use a normalized LocalName +// for the name poriton. Passes through the reference without transformation on +// error. +func NormalizeLocalReference(ref reference.Named) reference.Named { + localName := NormalizeLocalName(ref) + if tagged, isTagged := ref.(reference.Tagged); isTagged { + newRef, err := reference.WithTag(localName, tagged.Tag()) + if err != nil { + return ref + } + return newRef + } else if digested, isDigested := ref.(reference.Digested); isDigested { + newRef, err := reference.WithDigest(localName, digested.Digest()) + if err != nil { + return ref + } + return newRef + } + return localName } diff --git a/docs/registry_mock_test.go b/docs/registry_mock_test.go index fb19e577..3c75dea6 100644 --- a/docs/registry_mock_test.go +++ b/docs/registry_mock_test.go @@ -15,6 +15,7 @@ import ( "testing" "time" + "github.com/docker/distribution/reference" "github.com/docker/docker/opts" "github.com/gorilla/mux" @@ -349,15 +350,19 @@ func handlerGetDeleteTags(w http.ResponseWriter, r *http.Request) { if !requiresAuth(w, r) { return } - repositoryName := mux.Vars(r)["repository"] + repositoryName, err := reference.WithName(mux.Vars(r)["repository"]) + if err != nil { + apiError(w, "Could not parse repository", 400) + return + } repositoryName = NormalizeLocalName(repositoryName) - tags, exists := testRepositories[repositoryName] + tags, exists := testRepositories[repositoryName.String()] if !exists { apiError(w, "Repository not found", 404) return } if r.Method == "DELETE" { - delete(testRepositories, repositoryName) + delete(testRepositories, repositoryName.String()) writeResponse(w, true, 200) return } @@ -369,10 +374,14 @@ func handlerGetTag(w http.ResponseWriter, r *http.Request) { return } vars := mux.Vars(r) - repositoryName := vars["repository"] + repositoryName, err := reference.WithName(vars["repository"]) + if err != nil { + apiError(w, "Could not parse repository", 400) + return + } repositoryName = NormalizeLocalName(repositoryName) tagName := vars["tag"] - tags, exists := testRepositories[repositoryName] + tags, exists := testRepositories[repositoryName.String()] if !exists { apiError(w, "Repository not found", 404) return @@ -390,13 +399,17 @@ func handlerPutTag(w http.ResponseWriter, r *http.Request) { return } vars := mux.Vars(r) - repositoryName := vars["repository"] + repositoryName, err := reference.WithName(vars["repository"]) + if err != nil { + apiError(w, "Could not parse repository", 400) + return + } repositoryName = NormalizeLocalName(repositoryName) tagName := vars["tag"] - tags, exists := testRepositories[repositoryName] + tags, exists := testRepositories[repositoryName.String()] if !exists { - tags := make(map[string]string) - testRepositories[repositoryName] = tags + tags = make(map[string]string) + testRepositories[repositoryName.String()] = tags } tagValue := "" readJSON(r, tagValue) diff --git a/docs/registry_test.go b/docs/registry_test.go index 7714310d..2bc1edff 100644 --- a/docs/registry_test.go +++ b/docs/registry_test.go @@ -8,6 +8,7 @@ import ( "strings" "testing" + "github.com/docker/distribution/reference" "github.com/docker/distribution/registry/client/transport" "github.com/docker/docker/cliconfig" ) @@ -214,13 +215,21 @@ func TestGetRemoteImageLayer(t *testing.T) { func TestGetRemoteTag(t *testing.T) { r := spawnTestRegistrySession(t) - tag, err := r.GetRemoteTag([]string{makeURL("/v1/")}, REPO, "test") + repoRef, err := reference.ParseNamed(REPO) + if err != nil { + t.Fatal(err) + } + tag, err := r.GetRemoteTag([]string{makeURL("/v1/")}, repoRef, "test") if err != nil { t.Fatal(err) } assertEqual(t, tag, imageID, "Expected tag test to map to "+imageID) - _, err = r.GetRemoteTag([]string{makeURL("/v1/")}, "foo42/baz", "foo") + bazRef, err := reference.ParseNamed("foo42/baz") + if err != nil { + t.Fatal(err) + } + _, err = r.GetRemoteTag([]string{makeURL("/v1/")}, bazRef, "foo") if err != ErrRepoNotFound { t.Fatal("Expected ErrRepoNotFound error when fetching tag for bogus repo") } @@ -228,7 +237,11 @@ func TestGetRemoteTag(t *testing.T) { func TestGetRemoteTags(t *testing.T) { r := spawnTestRegistrySession(t) - tags, err := r.GetRemoteTags([]string{makeURL("/v1/")}, REPO) + repoRef, err := reference.ParseNamed(REPO) + if err != nil { + t.Fatal(err) + } + tags, err := r.GetRemoteTags([]string{makeURL("/v1/")}, repoRef) if err != nil { t.Fatal(err) } @@ -236,7 +249,11 @@ func TestGetRemoteTags(t *testing.T) { assertEqual(t, tags["latest"], imageID, "Expected tag latest to map to "+imageID) assertEqual(t, tags["test"], imageID, "Expected tag test to map to "+imageID) - _, err = r.GetRemoteTags([]string{makeURL("/v1/")}, "foo42/baz") + bazRef, err := reference.ParseNamed("foo42/baz") + if err != nil { + t.Fatal(err) + } + _, err = r.GetRemoteTags([]string{makeURL("/v1/")}, bazRef) if err != ErrRepoNotFound { t.Fatal("Expected ErrRepoNotFound error when fetching tags for bogus repo") } @@ -249,7 +266,11 @@ func TestGetRepositoryData(t *testing.T) { t.Fatal(err) } host := "http://" + parsedURL.Host + "/v1/" - data, err := r.GetRepositoryData("foo42/bar") + repoRef, err := reference.ParseNamed(REPO) + if err != nil { + t.Fatal(err) + } + data, err := r.GetRepositoryData(repoRef) if err != nil { t.Fatal(err) } @@ -315,29 +336,41 @@ func TestValidateRepositoryName(t *testing.T) { } for _, name := range invalidRepoNames { - err := ValidateRepositoryName(name) - assertNotEqual(t, err, nil, "Expected invalid repo name: "+name) + named, err := reference.WithName(name) + if err == nil { + err := ValidateRepositoryName(named) + assertNotEqual(t, err, nil, "Expected invalid repo name: "+name) + } } for _, name := range validRepoNames { - err := ValidateRepositoryName(name) + named, err := reference.WithName(name) + if err != nil { + t.Fatalf("could not parse valid name: %s", name) + } + err = ValidateRepositoryName(named) assertEqual(t, err, nil, "Expected valid repo name: "+name) } - - err := ValidateRepositoryName(invalidRepoNames[0]) - assertEqual(t, err, ErrInvalidRepositoryName, "Expected ErrInvalidRepositoryName: "+invalidRepoNames[0]) } func TestParseRepositoryInfo(t *testing.T) { + withName := func(name string) reference.Named { + named, err := reference.WithName(name) + if err != nil { + t.Fatalf("could not parse reference %s", name) + } + return named + } + expectedRepoInfos := map[string]RepositoryInfo{ "fooo/bar": { Index: &IndexInfo{ Name: IndexName, Official: true, }, - RemoteName: "fooo/bar", - LocalName: "fooo/bar", - CanonicalName: "docker.io/fooo/bar", + RemoteName: withName("fooo/bar"), + LocalName: withName("fooo/bar"), + CanonicalName: withName("docker.io/fooo/bar"), Official: false, }, "library/ubuntu": { @@ -345,9 +378,9 @@ func TestParseRepositoryInfo(t *testing.T) { Name: IndexName, Official: true, }, - RemoteName: "library/ubuntu", - LocalName: "ubuntu", - CanonicalName: "docker.io/library/ubuntu", + RemoteName: withName("library/ubuntu"), + LocalName: withName("ubuntu"), + CanonicalName: withName("docker.io/library/ubuntu"), Official: true, }, "nonlibrary/ubuntu": { @@ -355,9 +388,9 @@ func TestParseRepositoryInfo(t *testing.T) { Name: IndexName, Official: true, }, - RemoteName: "nonlibrary/ubuntu", - LocalName: "nonlibrary/ubuntu", - CanonicalName: "docker.io/nonlibrary/ubuntu", + RemoteName: withName("nonlibrary/ubuntu"), + LocalName: withName("nonlibrary/ubuntu"), + CanonicalName: withName("docker.io/nonlibrary/ubuntu"), Official: false, }, "ubuntu": { @@ -365,9 +398,9 @@ func TestParseRepositoryInfo(t *testing.T) { Name: IndexName, Official: true, }, - RemoteName: "library/ubuntu", - LocalName: "ubuntu", - CanonicalName: "docker.io/library/ubuntu", + RemoteName: withName("library/ubuntu"), + LocalName: withName("ubuntu"), + CanonicalName: withName("docker.io/library/ubuntu"), Official: true, }, "other/library": { @@ -375,9 +408,9 @@ func TestParseRepositoryInfo(t *testing.T) { Name: IndexName, Official: true, }, - RemoteName: "other/library", - LocalName: "other/library", - CanonicalName: "docker.io/other/library", + RemoteName: withName("other/library"), + LocalName: withName("other/library"), + CanonicalName: withName("docker.io/other/library"), Official: false, }, "127.0.0.1:8000/private/moonbase": { @@ -385,9 +418,9 @@ func TestParseRepositoryInfo(t *testing.T) { Name: "127.0.0.1:8000", Official: false, }, - RemoteName: "private/moonbase", - LocalName: "127.0.0.1:8000/private/moonbase", - CanonicalName: "127.0.0.1:8000/private/moonbase", + RemoteName: withName("private/moonbase"), + LocalName: withName("127.0.0.1:8000/private/moonbase"), + CanonicalName: withName("127.0.0.1:8000/private/moonbase"), Official: false, }, "127.0.0.1:8000/privatebase": { @@ -395,9 +428,9 @@ func TestParseRepositoryInfo(t *testing.T) { Name: "127.0.0.1:8000", Official: false, }, - RemoteName: "privatebase", - LocalName: "127.0.0.1:8000/privatebase", - CanonicalName: "127.0.0.1:8000/privatebase", + RemoteName: withName("privatebase"), + LocalName: withName("127.0.0.1:8000/privatebase"), + CanonicalName: withName("127.0.0.1:8000/privatebase"), Official: false, }, "localhost:8000/private/moonbase": { @@ -405,9 +438,9 @@ func TestParseRepositoryInfo(t *testing.T) { Name: "localhost:8000", Official: false, }, - RemoteName: "private/moonbase", - LocalName: "localhost:8000/private/moonbase", - CanonicalName: "localhost:8000/private/moonbase", + RemoteName: withName("private/moonbase"), + LocalName: withName("localhost:8000/private/moonbase"), + CanonicalName: withName("localhost:8000/private/moonbase"), Official: false, }, "localhost:8000/privatebase": { @@ -415,9 +448,9 @@ func TestParseRepositoryInfo(t *testing.T) { Name: "localhost:8000", Official: false, }, - RemoteName: "privatebase", - LocalName: "localhost:8000/privatebase", - CanonicalName: "localhost:8000/privatebase", + RemoteName: withName("privatebase"), + LocalName: withName("localhost:8000/privatebase"), + CanonicalName: withName("localhost:8000/privatebase"), Official: false, }, "example.com/private/moonbase": { @@ -425,9 +458,9 @@ func TestParseRepositoryInfo(t *testing.T) { Name: "example.com", Official: false, }, - RemoteName: "private/moonbase", - LocalName: "example.com/private/moonbase", - CanonicalName: "example.com/private/moonbase", + RemoteName: withName("private/moonbase"), + LocalName: withName("example.com/private/moonbase"), + CanonicalName: withName("example.com/private/moonbase"), Official: false, }, "example.com/privatebase": { @@ -435,9 +468,9 @@ func TestParseRepositoryInfo(t *testing.T) { Name: "example.com", Official: false, }, - RemoteName: "privatebase", - LocalName: "example.com/privatebase", - CanonicalName: "example.com/privatebase", + RemoteName: withName("privatebase"), + LocalName: withName("example.com/privatebase"), + CanonicalName: withName("example.com/privatebase"), Official: false, }, "example.com:8000/private/moonbase": { @@ -445,9 +478,9 @@ func TestParseRepositoryInfo(t *testing.T) { Name: "example.com:8000", Official: false, }, - RemoteName: "private/moonbase", - LocalName: "example.com:8000/private/moonbase", - CanonicalName: "example.com:8000/private/moonbase", + RemoteName: withName("private/moonbase"), + LocalName: withName("example.com:8000/private/moonbase"), + CanonicalName: withName("example.com:8000/private/moonbase"), Official: false, }, "example.com:8000/privatebase": { @@ -455,9 +488,9 @@ func TestParseRepositoryInfo(t *testing.T) { Name: "example.com:8000", Official: false, }, - RemoteName: "privatebase", - LocalName: "example.com:8000/privatebase", - CanonicalName: "example.com:8000/privatebase", + RemoteName: withName("privatebase"), + LocalName: withName("example.com:8000/privatebase"), + CanonicalName: withName("example.com:8000/privatebase"), Official: false, }, "localhost/private/moonbase": { @@ -465,9 +498,9 @@ func TestParseRepositoryInfo(t *testing.T) { Name: "localhost", Official: false, }, - RemoteName: "private/moonbase", - LocalName: "localhost/private/moonbase", - CanonicalName: "localhost/private/moonbase", + RemoteName: withName("private/moonbase"), + LocalName: withName("localhost/private/moonbase"), + CanonicalName: withName("localhost/private/moonbase"), Official: false, }, "localhost/privatebase": { @@ -475,9 +508,9 @@ func TestParseRepositoryInfo(t *testing.T) { Name: "localhost", Official: false, }, - RemoteName: "privatebase", - LocalName: "localhost/privatebase", - CanonicalName: "localhost/privatebase", + RemoteName: withName("privatebase"), + LocalName: withName("localhost/privatebase"), + CanonicalName: withName("localhost/privatebase"), Official: false, }, IndexName + "/public/moonbase": { @@ -485,9 +518,9 @@ func TestParseRepositoryInfo(t *testing.T) { Name: IndexName, Official: true, }, - RemoteName: "public/moonbase", - LocalName: "public/moonbase", - CanonicalName: "docker.io/public/moonbase", + RemoteName: withName("public/moonbase"), + LocalName: withName("public/moonbase"), + CanonicalName: withName("docker.io/public/moonbase"), Official: false, }, "index." + IndexName + "/public/moonbase": { @@ -495,9 +528,9 @@ func TestParseRepositoryInfo(t *testing.T) { Name: IndexName, Official: true, }, - RemoteName: "public/moonbase", - LocalName: "public/moonbase", - CanonicalName: "docker.io/public/moonbase", + RemoteName: withName("public/moonbase"), + LocalName: withName("public/moonbase"), + CanonicalName: withName("docker.io/public/moonbase"), Official: false, }, "ubuntu-12.04-base": { @@ -505,9 +538,9 @@ func TestParseRepositoryInfo(t *testing.T) { Name: IndexName, Official: true, }, - RemoteName: "library/ubuntu-12.04-base", - LocalName: "ubuntu-12.04-base", - CanonicalName: "docker.io/library/ubuntu-12.04-base", + RemoteName: withName("library/ubuntu-12.04-base"), + LocalName: withName("ubuntu-12.04-base"), + CanonicalName: withName("docker.io/library/ubuntu-12.04-base"), Official: true, }, IndexName + "/ubuntu-12.04-base": { @@ -515,9 +548,9 @@ func TestParseRepositoryInfo(t *testing.T) { Name: IndexName, Official: true, }, - RemoteName: "library/ubuntu-12.04-base", - LocalName: "ubuntu-12.04-base", - CanonicalName: "docker.io/library/ubuntu-12.04-base", + RemoteName: withName("library/ubuntu-12.04-base"), + LocalName: withName("ubuntu-12.04-base"), + CanonicalName: withName("docker.io/library/ubuntu-12.04-base"), Official: true, }, "index." + IndexName + "/ubuntu-12.04-base": { @@ -525,22 +558,27 @@ func TestParseRepositoryInfo(t *testing.T) { Name: IndexName, Official: true, }, - RemoteName: "library/ubuntu-12.04-base", - LocalName: "ubuntu-12.04-base", - CanonicalName: "docker.io/library/ubuntu-12.04-base", + RemoteName: withName("library/ubuntu-12.04-base"), + LocalName: withName("ubuntu-12.04-base"), + CanonicalName: withName("docker.io/library/ubuntu-12.04-base"), Official: true, }, } for reposName, expectedRepoInfo := range expectedRepoInfos { - repoInfo, err := ParseRepositoryInfo(reposName) + named, err := reference.WithName(reposName) + if err != nil { + t.Error(err) + } + + repoInfo, err := ParseRepositoryInfo(named) if err != nil { t.Error(err) } else { checkEqual(t, repoInfo.Index.Name, expectedRepoInfo.Index.Name, reposName) - checkEqual(t, repoInfo.RemoteName, expectedRepoInfo.RemoteName, reposName) - checkEqual(t, repoInfo.LocalName, expectedRepoInfo.LocalName, reposName) - checkEqual(t, repoInfo.CanonicalName, expectedRepoInfo.CanonicalName, reposName) + checkEqual(t, repoInfo.RemoteName.String(), expectedRepoInfo.RemoteName.String(), reposName) + checkEqual(t, repoInfo.LocalName.String(), expectedRepoInfo.LocalName.String(), reposName) + checkEqual(t, repoInfo.CanonicalName.String(), expectedRepoInfo.CanonicalName.String(), reposName) checkEqual(t, repoInfo.Index.Official, expectedRepoInfo.Index.Official, reposName) checkEqual(t, repoInfo.Official, expectedRepoInfo.Official, reposName) } @@ -687,8 +725,11 @@ func TestMirrorEndpointLookup(t *testing.T) { return false } s := Service{Config: makeServiceConfig([]string{"my.mirror"}, nil)} - imageName := IndexName + "/test/image" + imageName, err := reference.WithName(IndexName + "/test/image") + if err != nil { + t.Error(err) + } pushAPIEndpoints, err := s.LookupPushEndpoints(imageName) if err != nil { t.Fatal(err) @@ -708,7 +749,11 @@ func TestMirrorEndpointLookup(t *testing.T) { func TestPushRegistryTag(t *testing.T) { r := spawnTestRegistrySession(t) - err := r.PushRegistryTag("foo42/bar", imageID, "stable", makeURL("/v1/")) + repoRef, err := reference.ParseNamed(REPO) + if err != nil { + t.Fatal(err) + } + err = r.PushRegistryTag(repoRef, imageID, "stable", makeURL("/v1/")) if err != nil { t.Fatal(err) } @@ -726,14 +771,18 @@ func TestPushImageJSONIndex(t *testing.T) { Checksum: "sha256:bea7bf2e4bacd479344b737328db47b18880d09096e6674165533aa994f5e9f2", }, } - repoData, err := r.PushImageJSONIndex("foo42/bar", imgData, false, nil) + repoRef, err := reference.ParseNamed(REPO) + if err != nil { + t.Fatal(err) + } + repoData, err := r.PushImageJSONIndex(repoRef, imgData, false, nil) if err != nil { t.Fatal(err) } if repoData == nil { t.Fatal("Expected RepositoryData object") } - repoData, err = r.PushImageJSONIndex("foo42/bar", imgData, true, []string{r.indexEndpoint.String()}) + repoData, err = r.PushImageJSONIndex(repoRef, imgData, true, []string{r.indexEndpoint.String()}) if err != nil { t.Fatal(err) } @@ -781,7 +830,11 @@ func TestValidRemoteName(t *testing.T) { "dock__er/docker", } for _, repositoryName := range validRepositoryNames { - if err := validateRemoteName(repositoryName); err != nil { + repositoryRef, err := reference.WithName(repositoryName) + if err != nil { + t.Errorf("Repository name should be valid: %v. Error: %v", repositoryName, err) + } + if err := validateRemoteName(repositoryRef); err != nil { t.Errorf("Repository name should be valid: %v. Error: %v", repositoryName, err) } } @@ -818,7 +871,11 @@ func TestValidRemoteName(t *testing.T) { "this_is_not_a_valid_namespace_because_its_lenth_is_greater_than_255_this_is_not_a_valid_namespace_because_its_lenth_is_greater_than_255_this_is_not_a_valid_namespace_because_its_lenth_is_greater_than_255_this_is_not_a_valid_namespace_because_its_lenth_is_greater_than_255/docker", } for _, repositoryName := range invalidRepositoryNames { - if err := validateRemoteName(repositoryName); err == nil { + repositoryRef, err := reference.ParseNamed(repositoryName) + if err != nil { + continue + } + if err := validateRemoteName(repositoryRef); err == nil { t.Errorf("Repository name should be invalid: %v", repositoryName) } } diff --git a/docs/service.go b/docs/service.go index 6ac930d6..1ef96827 100644 --- a/docs/service.go +++ b/docs/service.go @@ -4,7 +4,9 @@ import ( "crypto/tls" "net/http" "net/url" + "strings" + "github.com/docker/distribution/reference" "github.com/docker/distribution/registry/client/auth" "github.com/docker/docker/cliconfig" ) @@ -51,17 +53,39 @@ func (s *Service) Auth(authConfig *cliconfig.AuthConfig) (string, error) { return Login(authConfig, endpoint) } +// splitReposSearchTerm breaks a search term into an index name and remote name +func splitReposSearchTerm(reposName string) (string, string) { + nameParts := strings.SplitN(reposName, "/", 2) + var indexName, remoteName string + if len(nameParts) == 1 || (!strings.Contains(nameParts[0], ".") && + !strings.Contains(nameParts[0], ":") && nameParts[0] != "localhost") { + // This is a Docker Index repos (ex: samalba/hipache or ubuntu) + // 'docker.io' + indexName = IndexName + remoteName = reposName + } else { + indexName = nameParts[0] + remoteName = nameParts[1] + } + return indexName, remoteName +} + // Search queries the public registry for images matching the specified // search terms, and returns the results. func (s *Service) Search(term string, authConfig *cliconfig.AuthConfig, headers map[string][]string) (*SearchResults, error) { + if err := validateNoSchema(term); err != nil { + return nil, err + } - repoInfo, err := s.ResolveRepositoryBySearch(term) + indexName, remoteName := splitReposSearchTerm(term) + + index, err := s.Config.NewIndexInfo(indexName) if err != nil { return nil, err } // *TODO: Search multiple indexes. - endpoint, err := NewEndpoint(repoInfo.Index, http.Header(headers), APIVersionUnknown) + endpoint, err := NewEndpoint(index, http.Header(headers), APIVersionUnknown) if err != nil { return nil, err } @@ -70,19 +94,23 @@ func (s *Service) Search(term string, authConfig *cliconfig.AuthConfig, headers if err != nil { return nil, err } - return r.SearchRepositories(repoInfo.GetSearchTerm()) + + if index.Official { + localName := remoteName + if strings.HasPrefix(localName, "library/") { + // If pull "library/foo", it's stored locally under "foo" + localName = strings.SplitN(localName, "/", 2)[1] + } + + return r.SearchRepositories(localName) + } + return r.SearchRepositories(remoteName) } // ResolveRepository splits a repository name into its components // and configuration of the associated registry. -func (s *Service) ResolveRepository(name string) (*RepositoryInfo, error) { - return s.Config.NewRepositoryInfo(name, false) -} - -// ResolveRepositoryBySearch splits a repository name into its components -// and configuration of the associated registry. -func (s *Service) ResolveRepositoryBySearch(name string) (*RepositoryInfo, error) { - return s.Config.NewRepositoryInfo(name, true) +func (s *Service) ResolveRepository(name reference.Named) (*RepositoryInfo, error) { + return s.Config.NewRepositoryInfo(name) } // ResolveIndex takes indexName and returns index info @@ -123,14 +151,14 @@ func (s *Service) tlsConfigForMirror(mirror string) (*tls.Config, error) { // LookupPullEndpoints creates an list of endpoints to try to pull from, in order of preference. // It gives preference to v2 endpoints over v1, mirrors over the actual // registry, and HTTPS over plain HTTP. -func (s *Service) LookupPullEndpoints(repoName string) (endpoints []APIEndpoint, err error) { +func (s *Service) LookupPullEndpoints(repoName reference.Named) (endpoints []APIEndpoint, err error) { return s.lookupEndpoints(repoName) } // LookupPushEndpoints creates an list of endpoints to try to push to, in order of preference. // It gives preference to v2 endpoints over v1, and HTTPS over plain HTTP. // Mirrors are not included. -func (s *Service) LookupPushEndpoints(repoName string) (endpoints []APIEndpoint, err error) { +func (s *Service) LookupPushEndpoints(repoName reference.Named) (endpoints []APIEndpoint, err error) { allEndpoints, err := s.lookupEndpoints(repoName) if err == nil { for _, endpoint := range allEndpoints { @@ -142,7 +170,7 @@ func (s *Service) LookupPushEndpoints(repoName string) (endpoints []APIEndpoint, return endpoints, err } -func (s *Service) lookupEndpoints(repoName string) (endpoints []APIEndpoint, err error) { +func (s *Service) lookupEndpoints(repoName reference.Named) (endpoints []APIEndpoint, err error) { endpoints, err = s.lookupV2Endpoints(repoName) if err != nil { return nil, err diff --git a/docs/service_v1.go b/docs/service_v1.go index ddb78ee6..5fdc1ece 100644 --- a/docs/service_v1.go +++ b/docs/service_v1.go @@ -4,13 +4,15 @@ import ( "fmt" "strings" + "github.com/docker/distribution/reference" "github.com/docker/docker/pkg/tlsconfig" ) -func (s *Service) lookupV1Endpoints(repoName string) (endpoints []APIEndpoint, err error) { +func (s *Service) lookupV1Endpoints(repoName reference.Named) (endpoints []APIEndpoint, err error) { var cfg = tlsconfig.ServerDefault tlsConfig := &cfg - if strings.HasPrefix(repoName, DefaultNamespace+"/") { + nameString := repoName.Name() + if strings.HasPrefix(nameString, DefaultNamespace+"/") { endpoints = append(endpoints, APIEndpoint{ URL: DefaultV1Registry, Version: APIVersion1, @@ -21,11 +23,11 @@ func (s *Service) lookupV1Endpoints(repoName string) (endpoints []APIEndpoint, e return endpoints, nil } - slashIndex := strings.IndexRune(repoName, '/') + slashIndex := strings.IndexRune(nameString, '/') if slashIndex <= 0 { - return nil, fmt.Errorf("invalid repo name: missing '/': %s", repoName) + return nil, fmt.Errorf("invalid repo name: missing '/': %s", nameString) } - hostname := repoName[:slashIndex] + hostname := nameString[:slashIndex] tlsConfig, err = s.TLSConfig(hostname) if err != nil { diff --git a/docs/service_v2.go b/docs/service_v2.go index 70d5fd71..56a3d2ee 100644 --- a/docs/service_v2.go +++ b/docs/service_v2.go @@ -4,14 +4,16 @@ import ( "fmt" "strings" + "github.com/docker/distribution/reference" "github.com/docker/distribution/registry/client/auth" "github.com/docker/docker/pkg/tlsconfig" ) -func (s *Service) lookupV2Endpoints(repoName string) (endpoints []APIEndpoint, err error) { +func (s *Service) lookupV2Endpoints(repoName reference.Named) (endpoints []APIEndpoint, err error) { var cfg = tlsconfig.ServerDefault tlsConfig := &cfg - if strings.HasPrefix(repoName, DefaultNamespace+"/") { + nameString := repoName.Name() + if strings.HasPrefix(nameString, DefaultNamespace+"/") { // v2 mirrors for _, mirror := range s.Config.Mirrors { mirrorTLSConfig, err := s.tlsConfigForMirror(mirror) @@ -39,11 +41,11 @@ func (s *Service) lookupV2Endpoints(repoName string) (endpoints []APIEndpoint, e return endpoints, nil } - slashIndex := strings.IndexRune(repoName, '/') + slashIndex := strings.IndexRune(nameString, '/') if slashIndex <= 0 { - return nil, fmt.Errorf("invalid repo name: missing '/': %s", repoName) + return nil, fmt.Errorf("invalid repo name: missing '/': %s", nameString) } - hostname := repoName[:slashIndex] + hostname := nameString[:slashIndex] tlsConfig, err = s.TLSConfig(hostname) if err != nil { diff --git a/docs/session.go b/docs/session.go index 2a20d321..645e5d44 100644 --- a/docs/session.go +++ b/docs/session.go @@ -20,6 +20,7 @@ import ( "time" "github.com/Sirupsen/logrus" + "github.com/docker/distribution/reference" "github.com/docker/docker/cliconfig" "github.com/docker/docker/pkg/httputils" "github.com/docker/docker/pkg/ioutils" @@ -320,7 +321,9 @@ func (r *Session) GetRemoteImageLayer(imgID, registry string, imgSize int64) (io // repository. It queries each of the registries supplied in the registries // argument, and returns data from the first one that answers the query // successfully. -func (r *Session) GetRemoteTag(registries []string, repository string, askedTag string) (string, error) { +func (r *Session) GetRemoteTag(registries []string, repositoryRef reference.Named, askedTag string) (string, error) { + repository := repositoryRef.Name() + if strings.Count(repository, "/") == 0 { // This will be removed once the registry supports auto-resolution on // the "library" namespace @@ -356,7 +359,9 @@ func (r *Session) GetRemoteTag(registries []string, repository string, askedTag // of the registries supplied in the registries argument, and returns data from // the first one that answers the query successfully. It returns a map with // tag names as the keys and image IDs as the values. -func (r *Session) GetRemoteTags(registries []string, repository string) (map[string]string, error) { +func (r *Session) GetRemoteTags(registries []string, repositoryRef reference.Named) (map[string]string, error) { + repository := repositoryRef.Name() + if strings.Count(repository, "/") == 0 { // This will be removed once the registry supports auto-resolution on // the "library" namespace @@ -408,8 +413,8 @@ func buildEndpointsList(headers []string, indexEp string) ([]string, error) { } // GetRepositoryData returns lists of images and endpoints for the repository -func (r *Session) GetRepositoryData(remote string) (*RepositoryData, error) { - repositoryTarget := fmt.Sprintf("%srepositories/%s/images", r.indexEndpoint.VersionString(1), remote) +func (r *Session) GetRepositoryData(remote reference.Named) (*RepositoryData, error) { + repositoryTarget := fmt.Sprintf("%srepositories/%s/images", r.indexEndpoint.VersionString(1), remote.Name()) logrus.Debugf("[registry] Calling GET %s", repositoryTarget) @@ -443,7 +448,7 @@ func (r *Session) GetRepositoryData(remote string) (*RepositoryData, error) { if err != nil { logrus.Debugf("Error reading response body: %s", err) } - return nil, httputils.NewHTTPRequestError(fmt.Sprintf("Error: Status %d trying to pull repository %s: %q", res.StatusCode, remote, errBody), res) + return nil, httputils.NewHTTPRequestError(fmt.Sprintf("Error: Status %d trying to pull repository %s: %q", res.StatusCode, remote.Name(), errBody), res) } var endpoints []string @@ -595,10 +600,10 @@ func (r *Session) PushImageLayerRegistry(imgID string, layer io.Reader, registry // PushRegistryTag pushes a tag on the registry. // Remote has the format '/ -func (r *Session) PushRegistryTag(remote, revision, tag, registry string) error { +func (r *Session) PushRegistryTag(remote reference.Named, revision, tag, registry string) error { // "jsonify" the string revision = "\"" + revision + "\"" - path := fmt.Sprintf("repositories/%s/tags/%s", remote, tag) + path := fmt.Sprintf("repositories/%s/tags/%s", remote.Name(), tag) req, err := http.NewRequest("PUT", registry+path, strings.NewReader(revision)) if err != nil { @@ -612,13 +617,13 @@ func (r *Session) PushRegistryTag(remote, revision, tag, registry string) error } res.Body.Close() if res.StatusCode != 200 && res.StatusCode != 201 { - return httputils.NewHTTPRequestError(fmt.Sprintf("Internal server error: %d trying to push tag %s on %s", res.StatusCode, tag, remote), res) + return httputils.NewHTTPRequestError(fmt.Sprintf("Internal server error: %d trying to push tag %s on %s", res.StatusCode, tag, remote.Name()), res) } return nil } // PushImageJSONIndex uploads an image list to the repository -func (r *Session) PushImageJSONIndex(remote string, imgList []*ImgData, validate bool, regs []string) (*RepositoryData, error) { +func (r *Session) PushImageJSONIndex(remote reference.Named, imgList []*ImgData, validate bool, regs []string) (*RepositoryData, error) { cleanImgList := []*ImgData{} if validate { for _, elem := range imgList { @@ -638,7 +643,7 @@ func (r *Session) PushImageJSONIndex(remote string, imgList []*ImgData, validate if validate { suffix = "images" } - u := fmt.Sprintf("%srepositories/%s/%s", r.indexEndpoint.VersionString(1), remote, suffix) + u := fmt.Sprintf("%srepositories/%s/%s", r.indexEndpoint.VersionString(1), remote.Name(), suffix) logrus.Debugf("[registry] PUT %s", u) logrus.Debugf("Image list pushed to index:\n%s", imgListJSON) headers := map[string][]string{ @@ -676,7 +681,7 @@ func (r *Session) PushImageJSONIndex(remote string, imgList []*ImgData, validate if err != nil { logrus.Debugf("Error reading response body: %s", err) } - return nil, httputils.NewHTTPRequestError(fmt.Sprintf("Error: Status %d trying to push repository %s: %q", res.StatusCode, remote, errBody), res) + return nil, httputils.NewHTTPRequestError(fmt.Sprintf("Error: Status %d trying to push repository %s: %q", res.StatusCode, remote.Name(), errBody), res) } tokens = res.Header["X-Docker-Token"] logrus.Debugf("Auth token: %v", tokens) @@ -694,7 +699,7 @@ func (r *Session) PushImageJSONIndex(remote string, imgList []*ImgData, validate if err != nil { logrus.Debugf("Error reading response body: %s", err) } - return nil, httputils.NewHTTPRequestError(fmt.Sprintf("Error: Status %d trying to push checksums %s: %q", res.StatusCode, remote, errBody), res) + return nil, httputils.NewHTTPRequestError(fmt.Sprintf("Error: Status %d trying to push checksums %s: %q", res.StatusCode, remote.Name(), errBody), res) } } diff --git a/docs/types.go b/docs/types.go index 09b9d571..8a201a91 100644 --- a/docs/types.go +++ b/docs/types.go @@ -1,5 +1,9 @@ package registry +import ( + "github.com/docker/distribution/reference" +) + // SearchResult describes a search result returned from a registry type SearchResult struct { // StarCount indicates the number of stars this repository has @@ -126,13 +130,13 @@ type RepositoryInfo struct { Index *IndexInfo // RemoteName is the remote name of the repository, such as // "library/ubuntu-12.04-base" - RemoteName string + RemoteName reference.Named // LocalName is the local name of the repository, such as // "ubuntu-12.04-base" - LocalName string + LocalName reference.Named // CanonicalName is the canonical name of the repository, such as // "docker.io/library/ubuntu-12.04-base" - CanonicalName string + CanonicalName reference.Named // Official indicates whether the repository is considered official. // If the registry is official, and the normalized name does not // contain a '/' (e.g. "foo"), then it is considered an official repo. From 6bb27bcfda34c21c54bb999824469125ee534e01 Mon Sep 17 00:00:00 2001 From: mqliang Date: Wed, 25 Nov 2015 22:33:15 +0800 Subject: [PATCH 332/375] move defer statement for readability Signed-off-by: mqliang --- docs/auth.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/auth.go b/docs/auth.go index 57560935..92045c05 100644 --- a/docs/auth.go +++ b/docs/auth.go @@ -53,8 +53,8 @@ func loginV1(authConfig *cliconfig.AuthConfig, registryEndpoint *Endpoint) (stri if err != nil { return "", fmt.Errorf("Server Error: %s", err) } - reqStatusCode = req1.StatusCode defer req1.Body.Close() + reqStatusCode = req1.StatusCode reqBody, err = ioutil.ReadAll(req1.Body) if err != nil { return "", fmt.Errorf("Server Error: [%#v] %s", reqStatusCode, err) From 73a209107e3e17f5700b8fa494243027d683745a Mon Sep 17 00:00:00 2001 From: Michal Gebauer Date: Thu, 19 Nov 2015 23:30:29 +0100 Subject: [PATCH 333/375] Check if CertsDir is not empty Signed-off-by: Michal Gebauer --- docs/config_unix.go | 2 ++ docs/registry.go | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/docs/config_unix.go b/docs/config_unix.go index 32f167d0..df970181 100644 --- a/docs/config_unix.go +++ b/docs/config_unix.go @@ -8,7 +8,9 @@ const ( // DefaultV2Registry is the URI of the default v2 registry DefaultV2Registry = "https://registry-1.docker.io" +) +var ( // CertsDir is the directory where certificates are stored CertsDir = "/etc/docker/certs.d" ) diff --git a/docs/registry.go b/docs/registry.go index 9c8666ea..6a0587a2 100644 --- a/docs/registry.go +++ b/docs/registry.go @@ -62,7 +62,7 @@ func newTLSConfig(hostname string, isSecure bool) (*tls.Config, error) { tlsConfig.InsecureSkipVerify = !isSecure - if isSecure { + if isSecure && CertsDir != "" { hostDir := filepath.Join(CertsDir, cleanPath(hostname)) logrus.Debugf("hostDir: %s", hostDir) if err := ReadCertsDirectory(&tlsConfig, hostDir); err != nil { From b7d246effb016793ee50984ce1fac0e9bc5ca4ae Mon Sep 17 00:00:00 2001 From: mqliang Date: Wed, 25 Nov 2015 22:29:23 +0800 Subject: [PATCH 334/375] rename req to resp Signed-off-by: mqliang --- docs/auth.go | 32 ++++++++++++++++---------------- 1 file changed, 16 insertions(+), 16 deletions(-) diff --git a/docs/auth.go b/docs/auth.go index 92045c05..c3f09a42 100644 --- a/docs/auth.go +++ b/docs/auth.go @@ -23,11 +23,11 @@ func Login(authConfig *cliconfig.AuthConfig, registryEndpoint *Endpoint) (string // loginV1 tries to register/login to the v1 registry server. func loginV1(authConfig *cliconfig.AuthConfig, registryEndpoint *Endpoint) (string, error) { var ( - status string - reqBody []byte - err error - reqStatusCode = 0 - serverAddress = authConfig.ServerAddress + status string + respBody []byte + err error + respStatusCode = 0 + serverAddress = authConfig.ServerAddress ) logrus.Debugf("attempting v1 login to registry endpoint %s", registryEndpoint) @@ -49,18 +49,18 @@ func loginV1(authConfig *cliconfig.AuthConfig, registryEndpoint *Endpoint) (stri // using `bytes.NewReader(jsonBody)` here causes the server to respond with a 411 status. b := strings.NewReader(string(jsonBody)) - req1, err := registryEndpoint.client.Post(serverAddress+"users/", "application/json; charset=utf-8", b) + resp1, err := registryEndpoint.client.Post(serverAddress+"users/", "application/json; charset=utf-8", b) if err != nil { return "", fmt.Errorf("Server Error: %s", err) } - defer req1.Body.Close() - reqStatusCode = req1.StatusCode - reqBody, err = ioutil.ReadAll(req1.Body) + defer resp1.Body.Close() + respStatusCode = resp1.StatusCode + respBody, err = ioutil.ReadAll(resp1.Body) if err != nil { - return "", fmt.Errorf("Server Error: [%#v] %s", reqStatusCode, err) + return "", fmt.Errorf("Server Error: [%#v] %s", respStatusCode, err) } - if reqStatusCode == 201 { + if respStatusCode == 201 { if loginAgainstOfficialIndex { status = "Account created. Please use the confirmation link we sent" + " to your e-mail to activate it." @@ -68,8 +68,8 @@ func loginV1(authConfig *cliconfig.AuthConfig, registryEndpoint *Endpoint) (stri // *TODO: Use registry configuration to determine what this says, if anything? status = "Account created. Please see the documentation of the registry " + serverAddress + " for instructions how to activate it." } - } else if reqStatusCode == 400 { - if string(reqBody) == "\"Username or email already exists\"" { + } else if respStatusCode == 400 { + if string(respBody) == "\"Username or email already exists\"" { req, err := http.NewRequest("GET", serverAddress+"users/", nil) req.SetBasicAuth(authConfig.Username, authConfig.Password) resp, err := registryEndpoint.client.Do(req) @@ -97,9 +97,9 @@ func loginV1(authConfig *cliconfig.AuthConfig, registryEndpoint *Endpoint) (stri } return "", fmt.Errorf("Login: %s (Code: %d; Headers: %s)", body, resp.StatusCode, resp.Header) } - return "", fmt.Errorf("Registration: %s", reqBody) + return "", fmt.Errorf("Registration: %s", respBody) - } else if reqStatusCode == 401 { + } else if respStatusCode == 401 { // This case would happen with private registries where /v1/users is // protected, so people can use `docker login` as an auth check. req, err := http.NewRequest("GET", serverAddress+"users/", nil) @@ -122,7 +122,7 @@ func loginV1(authConfig *cliconfig.AuthConfig, registryEndpoint *Endpoint) (stri resp.StatusCode, resp.Header) } } else { - return "", fmt.Errorf("Unexpected status code [%d] : %s", reqStatusCode, reqBody) + return "", fmt.Errorf("Unexpected status code [%d] : %s", respStatusCode, respBody) } return status, nil } From 03778bd1d2c6ecea3b040fedad081f87e4f74317 Mon Sep 17 00:00:00 2001 From: Aaron Lehmann Date: Mon, 7 Dec 2015 17:28:10 -0800 Subject: [PATCH 335/375] Add missing bounds in ContinueOnError ContinueOnError assumes that something of type errcode.Errors contains at least one error. This is generally true, but might not be true if the remote registry returns an empty error body or invalid JSON. Add the bounds check, and in the case where it fails, allow fallbacks to v1. Fixes #18481 Signed-off-by: Aaron Lehmann --- docs/registry.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/docs/registry.go b/docs/registry.go index 6a0587a2..fc2959a5 100644 --- a/docs/registry.go +++ b/docs/registry.go @@ -213,6 +213,9 @@ func (e ErrNoSupport) Error() string { func ContinueOnError(err error) bool { switch v := err.(type) { case errcode.Errors: + if len(v) == 0 { + return true + } return ContinueOnError(v[0]) case ErrNoSupport: return ContinueOnError(v.Err) From 00cca12e77708639d5609e06733da9b9a2c13119 Mon Sep 17 00:00:00 2001 From: Aaron Lehmann Date: Fri, 13 Nov 2015 16:59:01 -0800 Subject: [PATCH 336/375] Improved push and pull with upload manager and download manager This commit adds a transfer manager which deduplicates and schedules transfers, and also an upload manager and download manager that build on top of the transfer manager to provide high-level interfaces for uploads and downloads. The push and pull code is modified to use these building blocks. Some benefits of the changes: - Simplification of push/pull code - Pushes can upload layers concurrently - Failed downloads and uploads are retried after backoff delays - Cancellation is supported, but individual transfers will only be cancelled if all pushes or pulls using them are cancelled. - The distribution code is decoupled from Docker Engine packages and API conventions (i.e. streamformatter), which will make it easier to split out. This commit also includes unit tests for the new distribution/xfer package. The tests cover 87.8% of the statements in the package. Signed-off-by: Aaron Lehmann --- docs/session.go | 20 +++++--------------- 1 file changed, 5 insertions(+), 15 deletions(-) diff --git a/docs/session.go b/docs/session.go index 645e5d44..5017aeac 100644 --- a/docs/session.go +++ b/docs/session.go @@ -17,7 +17,6 @@ import ( "net/url" "strconv" "strings" - "time" "github.com/Sirupsen/logrus" "github.com/docker/distribution/reference" @@ -270,7 +269,6 @@ func (r *Session) GetRemoteImageJSON(imgID, registry string) ([]byte, int64, err // GetRemoteImageLayer retrieves an image layer from the registry func (r *Session) GetRemoteImageLayer(imgID, registry string, imgSize int64) (io.ReadCloser, error) { var ( - retries = 5 statusCode = 0 res *http.Response err error @@ -281,14 +279,9 @@ func (r *Session) GetRemoteImageLayer(imgID, registry string, imgSize int64) (io if err != nil { return nil, fmt.Errorf("Error while getting from the server: %v", err) } - // TODO(tiborvass): why are we doing retries at this level? - // These retries should be generic to both v1 and v2 - for i := 1; i <= retries; i++ { - statusCode = 0 - res, err = r.client.Do(req) - if err == nil { - break - } + statusCode = 0 + res, err = r.client.Do(req) + if err != nil { logrus.Debugf("Error contacting registry %s: %v", registry, err) if res != nil { if res.Body != nil { @@ -296,11 +289,8 @@ func (r *Session) GetRemoteImageLayer(imgID, registry string, imgSize int64) (io } statusCode = res.StatusCode } - if i == retries { - return nil, fmt.Errorf("Server error: Status %d while fetching image layer (%s)", - statusCode, imgID) - } - time.Sleep(time.Duration(i) * 5 * time.Second) + return nil, fmt.Errorf("Server error: Status %d while fetching image layer (%s)", + statusCode, imgID) } if res.StatusCode != 200 { From f7bb65ca8b8d931f5da77f308091572acd38e7af Mon Sep 17 00:00:00 2001 From: Daniel Nephin Date: Fri, 11 Dec 2015 19:11:20 -0800 Subject: [PATCH 337/375] Refactor ResolveAuthConfig to remove the builder dependency on cli code. registry.ResolveAuthConfig() only needs the AuthConfigs from the ConfigFile, so this change passed just the AuthConfigs. Signed-off-by: Daniel Nephin --- docs/auth.go | 6 ++--- docs/auth_test.go | 59 +++++++++++++---------------------------------- 2 files changed, 19 insertions(+), 46 deletions(-) diff --git a/docs/auth.go b/docs/auth.go index c3f09a42..6bdf3701 100644 --- a/docs/auth.go +++ b/docs/auth.go @@ -221,10 +221,10 @@ func tryV2TokenAuthLogin(authConfig *cliconfig.AuthConfig, params map[string]str } // ResolveAuthConfig matches an auth configuration to a server address or a URL -func ResolveAuthConfig(config *cliconfig.ConfigFile, index *IndexInfo) cliconfig.AuthConfig { +func ResolveAuthConfig(authConfigs map[string]cliconfig.AuthConfig, index *IndexInfo) cliconfig.AuthConfig { configKey := index.GetAuthConfigKey() // First try the happy case - if c, found := config.AuthConfigs[configKey]; found || index.Official { + if c, found := authConfigs[configKey]; found || index.Official { return c } @@ -243,7 +243,7 @@ func ResolveAuthConfig(config *cliconfig.ConfigFile, index *IndexInfo) cliconfig // Maybe they have a legacy config file, we will iterate the keys converting // them to the new format and testing - for registry, ac := range config.AuthConfigs { + for registry, ac := range authConfigs { if configKey == convertToHostname(registry) { return ac } diff --git a/docs/auth_test.go b/docs/auth_test.go index a8e3da01..a4085bb9 100644 --- a/docs/auth_test.go +++ b/docs/auth_test.go @@ -1,9 +1,6 @@ package registry import ( - "io/ioutil" - "os" - "path/filepath" "testing" "github.com/docker/docker/cliconfig" @@ -29,38 +26,23 @@ func TestEncodeAuth(t *testing.T) { } } -func setupTempConfigFile() (*cliconfig.ConfigFile, error) { - root, err := ioutil.TempDir("", "docker-test-auth") - if err != nil { - return nil, err - } - root = filepath.Join(root, cliconfig.ConfigFileName) - configFile := cliconfig.NewConfigFile(root) +func buildAuthConfigs() map[string]cliconfig.AuthConfig { + authConfigs := map[string]cliconfig.AuthConfig{} for _, registry := range []string{"testIndex", IndexServer} { - configFile.AuthConfigs[registry] = cliconfig.AuthConfig{ + authConfigs[registry] = cliconfig.AuthConfig{ Username: "docker-user", Password: "docker-pass", Email: "docker@docker.io", } } - return configFile, nil + return authConfigs } func TestSameAuthDataPostSave(t *testing.T) { - configFile, err := setupTempConfigFile() - if err != nil { - t.Fatal(err) - } - defer os.RemoveAll(configFile.Filename()) - - err = configFile.Save() - if err != nil { - t.Fatal(err) - } - - authConfig := configFile.AuthConfigs["testIndex"] + authConfigs := buildAuthConfigs() + authConfig := authConfigs["testIndex"] if authConfig.Username != "docker-user" { t.Fail() } @@ -76,13 +58,8 @@ func TestSameAuthDataPostSave(t *testing.T) { } func TestResolveAuthConfigIndexServer(t *testing.T) { - configFile, err := setupTempConfigFile() - if err != nil { - t.Fatal(err) - } - defer os.RemoveAll(configFile.Filename()) - - indexConfig := configFile.AuthConfigs[IndexServer] + authConfigs := buildAuthConfigs() + indexConfig := authConfigs[IndexServer] officialIndex := &IndexInfo{ Official: true, @@ -91,19 +68,15 @@ func TestResolveAuthConfigIndexServer(t *testing.T) { Official: false, } - resolved := ResolveAuthConfig(configFile, officialIndex) + resolved := ResolveAuthConfig(authConfigs, officialIndex) assertEqual(t, resolved, indexConfig, "Expected ResolveAuthConfig to return IndexServer") - resolved = ResolveAuthConfig(configFile, privateIndex) + resolved = ResolveAuthConfig(authConfigs, privateIndex) assertNotEqual(t, resolved, indexConfig, "Expected ResolveAuthConfig to not return IndexServer") } func TestResolveAuthConfigFullURL(t *testing.T) { - configFile, err := setupTempConfigFile() - if err != nil { - t.Fatal(err) - } - defer os.RemoveAll(configFile.Filename()) + authConfigs := buildAuthConfigs() registryAuth := cliconfig.AuthConfig{ Username: "foo-user", @@ -120,7 +93,7 @@ func TestResolveAuthConfigFullURL(t *testing.T) { Password: "baz-pass", Email: "baz@example.com", } - configFile.AuthConfigs[IndexServer] = officialAuth + authConfigs[IndexServer] = officialAuth expectedAuths := map[string]cliconfig.AuthConfig{ "registry.example.com": registryAuth, @@ -158,13 +131,13 @@ func TestResolveAuthConfigFullURL(t *testing.T) { Name: configKey, } for _, registry := range registries { - configFile.AuthConfigs[registry] = configured - resolved := ResolveAuthConfig(configFile, index) + authConfigs[registry] = configured + resolved := ResolveAuthConfig(authConfigs, index) if resolved.Email != configured.Email { t.Errorf("%s -> %q != %q\n", registry, resolved.Email, configured.Email) } - delete(configFile.AuthConfigs, registry) - resolved = ResolveAuthConfig(configFile, index) + delete(authConfigs, registry) + resolved = ResolveAuthConfig(authConfigs, index) if resolved.Email == configured.Email { t.Errorf("%s -> %q == %q\n", registry, resolved.Email, configured.Email) } From 11e8c03c18ed46ed56e25fec01662e150de2961c Mon Sep 17 00:00:00 2001 From: Justas Brazauskas Date: Sun, 13 Dec 2015 18:00:39 +0200 Subject: [PATCH 338/375] Fix typos found across repository Signed-off-by: Justas Brazauskas --- docs/auth.go | 2 +- docs/endpoint.go | 2 +- docs/session.go | 4 ++-- docs/types.go | 2 +- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/docs/auth.go b/docs/auth.go index c3f09a42..e21ee4bc 100644 --- a/docs/auth.go +++ b/docs/auth.go @@ -38,7 +38,7 @@ func loginV1(authConfig *cliconfig.AuthConfig, registryEndpoint *Endpoint) (stri loginAgainstOfficialIndex := serverAddress == IndexServer - // to avoid sending the server address to the server it should be removed before being marshalled + // to avoid sending the server address to the server it should be removed before being marshaled authCopy := *authConfig authCopy.ServerAddress = "" diff --git a/docs/endpoint.go b/docs/endpoint.go index 20805767..72892a99 100644 --- a/docs/endpoint.go +++ b/docs/endpoint.go @@ -125,7 +125,7 @@ type Endpoint struct { URLBuilder *v2.URLBuilder } -// Get the formated URL for the root of this registry Endpoint +// Get the formatted URL for the root of this registry Endpoint func (e *Endpoint) String() string { return fmt.Sprintf("%s/v%d/", e.URL, e.Version) } diff --git a/docs/session.go b/docs/session.go index 5017aeac..cecf936b 100644 --- a/docs/session.go +++ b/docs/session.go @@ -100,8 +100,8 @@ func (tr *authTransport) RoundTrip(orig *http.Request) (*http.Response, error) { // Authorization should not be set on 302 redirect for untrusted locations. // This logic mirrors the behavior in addRequiredHeadersToRedirectedRequests. // As the authorization logic is currently implemented in RoundTrip, - // a 302 redirect is detected by looking at the Referer header as go http package adds said header. - // This is safe as Docker doesn't set Referer in other scenarios. + // a 302 redirect is detected by looking at the Referrer header as go http package adds said header. + // This is safe as Docker doesn't set Referrer in other scenarios. if orig.Header.Get("Referer") != "" && !trustedLocation(orig) { return tr.RoundTripper.RoundTrip(orig) } diff --git a/docs/types.go b/docs/types.go index 8a201a91..9b2562f9 100644 --- a/docs/types.go +++ b/docs/types.go @@ -26,7 +26,7 @@ type SearchResults struct { Query string `json:"query"` // NumResults indicates the number of results the query returned NumResults int `json:"num_results"` - // Results is a slice containing the acutal results for the search + // Results is a slice containing the actual results for the search Results []SearchResult `json:"results"` } From 6fc54d049befbea1afcf578617a6f0dfa0fb48a7 Mon Sep 17 00:00:00 2001 From: Daniel Nephin Date: Fri, 11 Dec 2015 20:11:42 -0800 Subject: [PATCH 339/375] Move AuthConfig to api/types Signed-off-by: Daniel Nephin --- docs/auth.go | 14 +++++++------- docs/auth_test.go | 14 +++++++------- docs/registry_test.go | 3 +-- docs/service.go | 5 ++--- docs/session.go | 13 ++++++------- 5 files changed, 23 insertions(+), 26 deletions(-) diff --git a/docs/auth.go b/docs/auth.go index 6307768b..9964b953 100644 --- a/docs/auth.go +++ b/docs/auth.go @@ -12,7 +12,7 @@ import ( ) // Login tries to register/login to the registry server. -func Login(authConfig *cliconfig.AuthConfig, registryEndpoint *Endpoint) (string, error) { +func Login(authConfig *types.AuthConfig, registryEndpoint *Endpoint) (string, error) { // Separates the v2 registry login logic from the v1 logic. if registryEndpoint.Version == APIVersion2 { return loginV2(authConfig, registryEndpoint, "" /* scope */) @@ -21,7 +21,7 @@ func Login(authConfig *cliconfig.AuthConfig, registryEndpoint *Endpoint) (string } // loginV1 tries to register/login to the v1 registry server. -func loginV1(authConfig *cliconfig.AuthConfig, registryEndpoint *Endpoint) (string, error) { +func loginV1(authConfig *types.AuthConfig, registryEndpoint *Endpoint) (string, error) { var ( status string respBody []byte @@ -136,7 +136,7 @@ func loginV1(authConfig *cliconfig.AuthConfig, registryEndpoint *Endpoint) (stri // now, users should create their account through other means like directly from a web page // served by the v2 registry service provider. Whether this will be supported in the future // is to be determined. -func loginV2(authConfig *cliconfig.AuthConfig, registryEndpoint *Endpoint, scope string) (string, error) { +func loginV2(authConfig *types.AuthConfig, registryEndpoint *Endpoint, scope string) (string, error) { logrus.Debugf("attempting v2 login to registry endpoint %s", registryEndpoint) var ( err error @@ -173,7 +173,7 @@ func loginV2(authConfig *cliconfig.AuthConfig, registryEndpoint *Endpoint, scope return "", fmt.Errorf("no successful auth challenge for %s - errors: %s", registryEndpoint, allErrors) } -func tryV2BasicAuthLogin(authConfig *cliconfig.AuthConfig, params map[string]string, registryEndpoint *Endpoint) error { +func tryV2BasicAuthLogin(authConfig *types.AuthConfig, params map[string]string, registryEndpoint *Endpoint) error { req, err := http.NewRequest("GET", registryEndpoint.Path(""), nil) if err != nil { return err @@ -194,7 +194,7 @@ func tryV2BasicAuthLogin(authConfig *cliconfig.AuthConfig, params map[string]str return nil } -func tryV2TokenAuthLogin(authConfig *cliconfig.AuthConfig, params map[string]string, registryEndpoint *Endpoint) error { +func tryV2TokenAuthLogin(authConfig *types.AuthConfig, params map[string]string, registryEndpoint *Endpoint) error { token, err := getToken(authConfig.Username, authConfig.Password, params, registryEndpoint) if err != nil { return err @@ -221,7 +221,7 @@ func tryV2TokenAuthLogin(authConfig *cliconfig.AuthConfig, params map[string]str } // ResolveAuthConfig matches an auth configuration to a server address or a URL -func ResolveAuthConfig(authConfigs map[string]cliconfig.AuthConfig, index *IndexInfo) cliconfig.AuthConfig { +func ResolveAuthConfig(authConfigs map[string]types.AuthConfig, index *IndexInfo) types.AuthConfig { configKey := index.GetAuthConfigKey() // First try the happy case if c, found := authConfigs[configKey]; found || index.Official { @@ -250,5 +250,5 @@ func ResolveAuthConfig(authConfigs map[string]cliconfig.AuthConfig, index *Index } // When all else fails, return an empty auth config - return cliconfig.AuthConfig{} + return types.AuthConfig{} } diff --git a/docs/auth_test.go b/docs/auth_test.go index a4085bb9..fe59658e 100644 --- a/docs/auth_test.go +++ b/docs/auth_test.go @@ -7,9 +7,9 @@ import ( ) func TestEncodeAuth(t *testing.T) { - newAuthConfig := &cliconfig.AuthConfig{Username: "ken", Password: "test", Email: "test@example.com"} + newAuthConfig := &types.AuthConfig{Username: "ken", Password: "test", Email: "test@example.com"} authStr := cliconfig.EncodeAuth(newAuthConfig) - decAuthConfig := &cliconfig.AuthConfig{} + decAuthConfig := &types.AuthConfig{} var err error decAuthConfig.Username, decAuthConfig.Password, err = cliconfig.DecodeAuth(authStr) if err != nil { @@ -30,7 +30,7 @@ func buildAuthConfigs() map[string]cliconfig.AuthConfig { authConfigs := map[string]cliconfig.AuthConfig{} for _, registry := range []string{"testIndex", IndexServer} { - authConfigs[registry] = cliconfig.AuthConfig{ + authConfigs[registry] = types.AuthConfig{ Username: "docker-user", Password: "docker-pass", Email: "docker@docker.io", @@ -78,24 +78,24 @@ func TestResolveAuthConfigIndexServer(t *testing.T) { func TestResolveAuthConfigFullURL(t *testing.T) { authConfigs := buildAuthConfigs() - registryAuth := cliconfig.AuthConfig{ + registryAuth := types.AuthConfig{ Username: "foo-user", Password: "foo-pass", Email: "foo@example.com", } - localAuth := cliconfig.AuthConfig{ + localAuth := types.AuthConfig{ Username: "bar-user", Password: "bar-pass", Email: "bar@example.com", } - officialAuth := cliconfig.AuthConfig{ + officialAuth := types.AuthConfig{ Username: "baz-user", Password: "baz-pass", Email: "baz@example.com", } authConfigs[IndexServer] = officialAuth - expectedAuths := map[string]cliconfig.AuthConfig{ + expectedAuths := map[string]types.AuthConfig{ "registry.example.com": registryAuth, "localhost:8000": localAuth, "registry.com": localAuth, diff --git a/docs/registry_test.go b/docs/registry_test.go index 2bc1edff..95f57593 100644 --- a/docs/registry_test.go +++ b/docs/registry_test.go @@ -10,7 +10,6 @@ import ( "github.com/docker/distribution/reference" "github.com/docker/distribution/registry/client/transport" - "github.com/docker/docker/cliconfig" ) var ( @@ -23,7 +22,7 @@ const ( ) func spawnTestRegistrySession(t *testing.T) *Session { - authConfig := &cliconfig.AuthConfig{} + authConfig := &types.AuthConfig{} endpoint, err := NewEndpoint(makeIndex("/v1/"), nil, APIVersionUnknown) if err != nil { t.Fatal(err) diff --git a/docs/service.go b/docs/service.go index 1ef96827..e5f79af1 100644 --- a/docs/service.go +++ b/docs/service.go @@ -8,7 +8,6 @@ import ( "github.com/docker/distribution/reference" "github.com/docker/distribution/registry/client/auth" - "github.com/docker/docker/cliconfig" ) // Service is a registry service. It tracks configuration data such as a list @@ -28,7 +27,7 @@ func NewService(options *Options) *Service { // Auth contacts the public registry with the provided credentials, // and returns OK if authentication was successful. // It can be used to verify the validity of a client's credentials. -func (s *Service) Auth(authConfig *cliconfig.AuthConfig) (string, error) { +func (s *Service) Auth(authConfig *types.AuthConfig) (string, error) { addr := authConfig.ServerAddress if addr == "" { // Use the official registry address if not specified. @@ -72,7 +71,7 @@ func splitReposSearchTerm(reposName string) (string, string) { // Search queries the public registry for images matching the specified // search terms, and returns the results. -func (s *Service) Search(term string, authConfig *cliconfig.AuthConfig, headers map[string][]string) (*SearchResults, error) { +func (s *Service) Search(term string, authConfig *types.AuthConfig, headers map[string][]string) (*SearchResults, error) { if err := validateNoSchema(term); err != nil { return nil, err } diff --git a/docs/session.go b/docs/session.go index cecf936b..774b1f5b 100644 --- a/docs/session.go +++ b/docs/session.go @@ -20,7 +20,6 @@ import ( "github.com/Sirupsen/logrus" "github.com/docker/distribution/reference" - "github.com/docker/docker/cliconfig" "github.com/docker/docker/pkg/httputils" "github.com/docker/docker/pkg/ioutils" "github.com/docker/docker/pkg/stringid" @@ -39,13 +38,13 @@ type Session struct { indexEndpoint *Endpoint client *http.Client // TODO(tiborvass): remove authConfig - authConfig *cliconfig.AuthConfig + authConfig *types.AuthConfig id string } type authTransport struct { http.RoundTripper - *cliconfig.AuthConfig + *types.AuthConfig alwaysSetBasicAuth bool token []string @@ -67,7 +66,7 @@ type authTransport struct { // If the server sends a token without the client having requested it, it is ignored. // // This RoundTripper also has a CancelRequest method important for correct timeout handling. -func AuthTransport(base http.RoundTripper, authConfig *cliconfig.AuthConfig, alwaysSetBasicAuth bool) http.RoundTripper { +func AuthTransport(base http.RoundTripper, authConfig *types.AuthConfig, alwaysSetBasicAuth bool) http.RoundTripper { if base == nil { base = http.DefaultTransport } @@ -162,7 +161,7 @@ func (tr *authTransport) CancelRequest(req *http.Request) { // NewSession creates a new session // TODO(tiborvass): remove authConfig param once registry client v2 is vendored -func NewSession(client *http.Client, authConfig *cliconfig.AuthConfig, endpoint *Endpoint) (r *Session, err error) { +func NewSession(client *http.Client, authConfig *types.AuthConfig, endpoint *Endpoint) (r *Session, err error) { r = &Session{ authConfig: authConfig, client: client, @@ -743,12 +742,12 @@ func (r *Session) SearchRepositories(term string) (*SearchResults, error) { // GetAuthConfig returns the authentication settings for a session // TODO(tiborvass): remove this once registry client v2 is vendored -func (r *Session) GetAuthConfig(withPasswd bool) *cliconfig.AuthConfig { +func (r *Session) GetAuthConfig(withPasswd bool) *types.AuthConfig { password := "" if withPasswd { password = r.authConfig.Password } - return &cliconfig.AuthConfig{ + return &types.AuthConfig{ Username: r.authConfig.Username, Password: password, Email: r.authConfig.Email, From aead731d54f4d777fc7a2a41c65abdf901efdcd3 Mon Sep 17 00:00:00 2001 From: Daniel Nephin Date: Fri, 11 Dec 2015 18:14:52 -0800 Subject: [PATCH 340/375] Move IndexInfo and ServiceConfig types to api/types/registry/registry.go Signed-off-by: Daniel Nephin --- docs/auth.go | 7 ++-- docs/auth_test.go | 12 ++++--- docs/config.go | 66 ++++++++++++-------------------------- docs/endpoint.go | 5 +-- docs/registry_mock_test.go | 15 +++++---- docs/registry_test.go | 64 ++++++++++++++++++------------------ docs/service.go | 14 ++++---- docs/session.go | 1 + docs/types.go | 44 ++----------------------- 9 files changed, 87 insertions(+), 141 deletions(-) diff --git a/docs/auth.go b/docs/auth.go index 9964b953..34d5d670 100644 --- a/docs/auth.go +++ b/docs/auth.go @@ -8,7 +8,8 @@ import ( "strings" "github.com/Sirupsen/logrus" - "github.com/docker/docker/cliconfig" + "github.com/docker/docker/api/types" + registrytypes "github.com/docker/docker/api/types/registry" ) // Login tries to register/login to the registry server. @@ -221,8 +222,8 @@ func tryV2TokenAuthLogin(authConfig *types.AuthConfig, params map[string]string, } // ResolveAuthConfig matches an auth configuration to a server address or a URL -func ResolveAuthConfig(authConfigs map[string]types.AuthConfig, index *IndexInfo) types.AuthConfig { - configKey := index.GetAuthConfigKey() +func ResolveAuthConfig(authConfigs map[string]types.AuthConfig, index *registrytypes.IndexInfo) types.AuthConfig { + configKey := GetAuthConfigKey(index) // First try the happy case if c, found := authConfigs[configKey]; found || index.Official { return c diff --git a/docs/auth_test.go b/docs/auth_test.go index fe59658e..a2c5c804 100644 --- a/docs/auth_test.go +++ b/docs/auth_test.go @@ -3,6 +3,8 @@ package registry import ( "testing" + "github.com/docker/docker/api/types" + registrytypes "github.com/docker/docker/api/types/registry" "github.com/docker/docker/cliconfig" ) @@ -26,8 +28,8 @@ func TestEncodeAuth(t *testing.T) { } } -func buildAuthConfigs() map[string]cliconfig.AuthConfig { - authConfigs := map[string]cliconfig.AuthConfig{} +func buildAuthConfigs() map[string]types.AuthConfig { + authConfigs := map[string]types.AuthConfig{} for _, registry := range []string{"testIndex", IndexServer} { authConfigs[registry] = types.AuthConfig{ @@ -61,10 +63,10 @@ func TestResolveAuthConfigIndexServer(t *testing.T) { authConfigs := buildAuthConfigs() indexConfig := authConfigs[IndexServer] - officialIndex := &IndexInfo{ + officialIndex := ®istrytypes.IndexInfo{ Official: true, } - privateIndex := &IndexInfo{ + privateIndex := ®istrytypes.IndexInfo{ Official: false, } @@ -127,7 +129,7 @@ func TestResolveAuthConfigFullURL(t *testing.T) { if !ok || configured.Email == "" { t.Fail() } - index := &IndexInfo{ + index := ®istrytypes.IndexInfo{ Name: configKey, } for _, registry := range registries { diff --git a/docs/config.go b/docs/config.go index 8d7962f8..2eeba140 100644 --- a/docs/config.go +++ b/docs/config.go @@ -1,7 +1,6 @@ package registry import ( - "encoding/json" "errors" "fmt" "net" @@ -9,6 +8,7 @@ import ( "strings" "github.com/docker/distribution/reference" + registrytypes "github.com/docker/docker/api/types/registry" "github.com/docker/docker/image/v1" "github.com/docker/docker/opts" flag "github.com/docker/docker/pkg/mflag" @@ -60,32 +60,8 @@ func (options *Options) InstallFlags(cmd *flag.FlagSet, usageFn func(string) str cmd.BoolVar(&V2Only, []string{"-disable-legacy-registry"}, false, "Do not contact legacy registries") } -type netIPNet net.IPNet - -func (ipnet *netIPNet) MarshalJSON() ([]byte, error) { - return json.Marshal((*net.IPNet)(ipnet).String()) -} - -func (ipnet *netIPNet) UnmarshalJSON(b []byte) (err error) { - var ipnetStr string - if err = json.Unmarshal(b, &ipnetStr); err == nil { - var cidr *net.IPNet - if _, cidr, err = net.ParseCIDR(ipnetStr); err == nil { - *ipnet = netIPNet(*cidr) - } - } - return -} - -// ServiceConfig stores daemon registry services configuration. -type ServiceConfig struct { - InsecureRegistryCIDRs []*netIPNet `json:"InsecureRegistryCIDRs"` - IndexConfigs map[string]*IndexInfo `json:"IndexConfigs"` - Mirrors []string -} - // NewServiceConfig returns a new instance of ServiceConfig -func NewServiceConfig(options *Options) *ServiceConfig { +func NewServiceConfig(options *Options) *registrytypes.ServiceConfig { if options == nil { options = &Options{ Mirrors: opts.NewListOpts(nil), @@ -100,9 +76,9 @@ func NewServiceConfig(options *Options) *ServiceConfig { // daemon flags on boot2docker? options.InsecureRegistries.Set("127.0.0.0/8") - config := &ServiceConfig{ - InsecureRegistryCIDRs: make([]*netIPNet, 0), - IndexConfigs: make(map[string]*IndexInfo, 0), + config := ®istrytypes.ServiceConfig{ + InsecureRegistryCIDRs: make([]*registrytypes.NetIPNet, 0), + IndexConfigs: make(map[string]*registrytypes.IndexInfo, 0), // Hack: Bypass setting the mirrors to IndexConfigs since they are going away // and Mirrors are only for the official registry anyways. Mirrors: options.Mirrors.GetAll(), @@ -113,10 +89,10 @@ func NewServiceConfig(options *Options) *ServiceConfig { _, ipnet, err := net.ParseCIDR(r) if err == nil { // Valid CIDR. - config.InsecureRegistryCIDRs = append(config.InsecureRegistryCIDRs, (*netIPNet)(ipnet)) + config.InsecureRegistryCIDRs = append(config.InsecureRegistryCIDRs, (*registrytypes.NetIPNet)(ipnet)) } else { // Assume `host:port` if not CIDR. - config.IndexConfigs[r] = &IndexInfo{ + config.IndexConfigs[r] = ®istrytypes.IndexInfo{ Name: r, Mirrors: make([]string, 0), Secure: false, @@ -126,7 +102,7 @@ func NewServiceConfig(options *Options) *ServiceConfig { } // Configure public registry. - config.IndexConfigs[IndexName] = &IndexInfo{ + config.IndexConfigs[IndexName] = ®istrytypes.IndexInfo{ Name: IndexName, Mirrors: config.Mirrors, Secure: true, @@ -147,9 +123,9 @@ func NewServiceConfig(options *Options) *ServiceConfig { // or an IP address. If it is a domain name, then it will be resolved in order to check if the IP is contained // in a subnet. If the resolving is not successful, isSecureIndex will only try to match hostname to any element // of insecureRegistries. -func (config *ServiceConfig) isSecureIndex(indexName string) bool { +func isSecureIndex(config *registrytypes.ServiceConfig, indexName string) bool { // Check for configured index, first. This is needed in case isSecureIndex - // is called from anything besides NewIndexInfo, in order to honor per-index configurations. + // is called from anything besides newIndexInfo, in order to honor per-index configurations. if index, ok := config.IndexConfigs[indexName]; ok { return index.Secure } @@ -258,8 +234,8 @@ func loadRepositoryName(reposName reference.Named) (string, reference.Named, err return indexName, remoteName, nil } -// NewIndexInfo returns IndexInfo configuration from indexName -func (config *ServiceConfig) NewIndexInfo(indexName string) (*IndexInfo, error) { +// newIndexInfo returns IndexInfo configuration from indexName +func newIndexInfo(config *registrytypes.ServiceConfig, indexName string) (*registrytypes.IndexInfo, error) { var err error indexName, err = ValidateIndexName(indexName) if err != nil { @@ -272,18 +248,18 @@ func (config *ServiceConfig) NewIndexInfo(indexName string) (*IndexInfo, error) } // Construct a non-configured index info. - index := &IndexInfo{ + index := ®istrytypes.IndexInfo{ Name: indexName, Mirrors: make([]string, 0), Official: false, } - index.Secure = config.isSecureIndex(indexName) + index.Secure = isSecureIndex(config, indexName) return index, nil } // GetAuthConfigKey special-cases using the full index address of the official // index as the AuthConfig key, and uses the (host)name[:port] for private indexes. -func (index *IndexInfo) GetAuthConfigKey() string { +func GetAuthConfigKey(index *registrytypes.IndexInfo) string { if index.Official { return IndexServer } @@ -306,8 +282,8 @@ func splitReposName(reposName reference.Named) (indexName string, remoteName ref return } -// NewRepositoryInfo validates and breaks down a repository name into a RepositoryInfo -func (config *ServiceConfig) NewRepositoryInfo(reposName reference.Named) (*RepositoryInfo, error) { +// newRepositoryInfo validates and breaks down a repository name into a RepositoryInfo +func newRepositoryInfo(config *registrytypes.ServiceConfig, reposName reference.Named) (*RepositoryInfo, error) { if err := validateNoSchema(reposName.Name()); err != nil { return nil, err } @@ -323,7 +299,7 @@ func (config *ServiceConfig) NewRepositoryInfo(reposName reference.Named) (*Repo return nil, err } - repoInfo.Index, err = config.NewIndexInfo(indexName) + repoInfo.Index, err = newIndexInfo(config, indexName) if err != nil { return nil, err } @@ -364,14 +340,14 @@ func (config *ServiceConfig) NewRepositoryInfo(reposName reference.Named) (*Repo // ParseRepositoryInfo performs the breakdown of a repository name into a RepositoryInfo, but // lacks registry configuration. func ParseRepositoryInfo(reposName reference.Named) (*RepositoryInfo, error) { - return emptyServiceConfig.NewRepositoryInfo(reposName) + return newRepositoryInfo(emptyServiceConfig, reposName) } // ParseSearchIndexInfo will use repository name to get back an indexInfo. -func ParseSearchIndexInfo(reposName string) (*IndexInfo, error) { +func ParseSearchIndexInfo(reposName string) (*registrytypes.IndexInfo, error) { indexName, _ := splitReposSearchTerm(reposName) - indexInfo, err := emptyServiceConfig.NewIndexInfo(indexName) + indexInfo, err := newIndexInfo(emptyServiceConfig, indexName) if err != nil { return nil, err } diff --git a/docs/endpoint.go b/docs/endpoint.go index 72892a99..43ac9053 100644 --- a/docs/endpoint.go +++ b/docs/endpoint.go @@ -13,6 +13,7 @@ import ( "github.com/Sirupsen/logrus" "github.com/docker/distribution/registry/api/v2" "github.com/docker/distribution/registry/client/transport" + registrytypes "github.com/docker/docker/api/types/registry" ) // for mocking in unit tests @@ -44,12 +45,12 @@ func scanForAPIVersion(address string) (string, APIVersion) { // NewEndpoint parses the given address to return a registry endpoint. v can be used to // specify a specific endpoint version -func NewEndpoint(index *IndexInfo, metaHeaders http.Header, v APIVersion) (*Endpoint, error) { +func NewEndpoint(index *registrytypes.IndexInfo, metaHeaders http.Header, v APIVersion) (*Endpoint, error) { tlsConfig, err := newTLSConfig(index.Name, index.Secure) if err != nil { return nil, err } - endpoint, err := newEndpoint(index.GetAuthConfigKey(), tlsConfig, metaHeaders) + endpoint, err := newEndpoint(GetAuthConfigKey(index), tlsConfig, metaHeaders) if err != nil { return nil, err } diff --git a/docs/registry_mock_test.go b/docs/registry_mock_test.go index 3c75dea6..89059e8e 100644 --- a/docs/registry_mock_test.go +++ b/docs/registry_mock_test.go @@ -16,6 +16,7 @@ import ( "time" "github.com/docker/distribution/reference" + registrytypes "github.com/docker/docker/api/types/registry" "github.com/docker/docker/opts" "github.com/gorilla/mux" @@ -150,22 +151,22 @@ func makeHTTPSURL(req string) string { return testHTTPSServer.URL + req } -func makeIndex(req string) *IndexInfo { - index := &IndexInfo{ +func makeIndex(req string) *registrytypes.IndexInfo { + index := ®istrytypes.IndexInfo{ Name: makeURL(req), } return index } -func makeHTTPSIndex(req string) *IndexInfo { - index := &IndexInfo{ +func makeHTTPSIndex(req string) *registrytypes.IndexInfo { + index := ®istrytypes.IndexInfo{ Name: makeHTTPSURL(req), } return index } -func makePublicIndex() *IndexInfo { - index := &IndexInfo{ +func makePublicIndex() *registrytypes.IndexInfo { + index := ®istrytypes.IndexInfo{ Name: IndexServer, Secure: true, Official: true, @@ -173,7 +174,7 @@ func makePublicIndex() *IndexInfo { return index } -func makeServiceConfig(mirrors []string, insecureRegistries []string) *ServiceConfig { +func makeServiceConfig(mirrors []string, insecureRegistries []string) *registrytypes.ServiceConfig { options := &Options{ Mirrors: opts.NewListOpts(nil), InsecureRegistries: opts.NewListOpts(nil), diff --git a/docs/registry_test.go b/docs/registry_test.go index 95f57593..7e352441 100644 --- a/docs/registry_test.go +++ b/docs/registry_test.go @@ -10,6 +10,8 @@ import ( "github.com/docker/distribution/reference" "github.com/docker/distribution/registry/client/transport" + "github.com/docker/docker/api/types" + registrytypes "github.com/docker/docker/api/types/registry" ) var ( @@ -49,7 +51,7 @@ func spawnTestRegistrySession(t *testing.T) *Session { } func TestPingRegistryEndpoint(t *testing.T) { - testPing := func(index *IndexInfo, expectedStandalone bool, assertMessage string) { + testPing := func(index *registrytypes.IndexInfo, expectedStandalone bool, assertMessage string) { ep, err := NewEndpoint(index, nil, APIVersionUnknown) if err != nil { t.Fatal(err) @@ -69,7 +71,7 @@ func TestPingRegistryEndpoint(t *testing.T) { func TestEndpoint(t *testing.T) { // Simple wrapper to fail test if err != nil - expandEndpoint := func(index *IndexInfo) *Endpoint { + expandEndpoint := func(index *registrytypes.IndexInfo) *Endpoint { endpoint, err := NewEndpoint(index, nil, APIVersionUnknown) if err != nil { t.Fatal(err) @@ -77,7 +79,7 @@ func TestEndpoint(t *testing.T) { return endpoint } - assertInsecureIndex := func(index *IndexInfo) { + assertInsecureIndex := func(index *registrytypes.IndexInfo) { index.Secure = true _, err := NewEndpoint(index, nil, APIVersionUnknown) assertNotEqual(t, err, nil, index.Name+": Expected error for insecure index") @@ -85,7 +87,7 @@ func TestEndpoint(t *testing.T) { index.Secure = false } - assertSecureIndex := func(index *IndexInfo) { + assertSecureIndex := func(index *registrytypes.IndexInfo) { index.Secure = true _, err := NewEndpoint(index, nil, APIVersionUnknown) assertNotEqual(t, err, nil, index.Name+": Expected cert error for secure index") @@ -93,7 +95,7 @@ func TestEndpoint(t *testing.T) { index.Secure = false } - index := &IndexInfo{} + index := ®istrytypes.IndexInfo{} index.Name = makeURL("/v1/") endpoint := expandEndpoint(index) assertEqual(t, endpoint.String(), index.Name, "Expected endpoint to be "+index.Name) @@ -363,7 +365,7 @@ func TestParseRepositoryInfo(t *testing.T) { expectedRepoInfos := map[string]RepositoryInfo{ "fooo/bar": { - Index: &IndexInfo{ + Index: ®istrytypes.IndexInfo{ Name: IndexName, Official: true, }, @@ -373,7 +375,7 @@ func TestParseRepositoryInfo(t *testing.T) { Official: false, }, "library/ubuntu": { - Index: &IndexInfo{ + Index: ®istrytypes.IndexInfo{ Name: IndexName, Official: true, }, @@ -383,7 +385,7 @@ func TestParseRepositoryInfo(t *testing.T) { Official: true, }, "nonlibrary/ubuntu": { - Index: &IndexInfo{ + Index: ®istrytypes.IndexInfo{ Name: IndexName, Official: true, }, @@ -393,7 +395,7 @@ func TestParseRepositoryInfo(t *testing.T) { Official: false, }, "ubuntu": { - Index: &IndexInfo{ + Index: ®istrytypes.IndexInfo{ Name: IndexName, Official: true, }, @@ -403,7 +405,7 @@ func TestParseRepositoryInfo(t *testing.T) { Official: true, }, "other/library": { - Index: &IndexInfo{ + Index: ®istrytypes.IndexInfo{ Name: IndexName, Official: true, }, @@ -413,7 +415,7 @@ func TestParseRepositoryInfo(t *testing.T) { Official: false, }, "127.0.0.1:8000/private/moonbase": { - Index: &IndexInfo{ + Index: ®istrytypes.IndexInfo{ Name: "127.0.0.1:8000", Official: false, }, @@ -423,7 +425,7 @@ func TestParseRepositoryInfo(t *testing.T) { Official: false, }, "127.0.0.1:8000/privatebase": { - Index: &IndexInfo{ + Index: ®istrytypes.IndexInfo{ Name: "127.0.0.1:8000", Official: false, }, @@ -433,7 +435,7 @@ func TestParseRepositoryInfo(t *testing.T) { Official: false, }, "localhost:8000/private/moonbase": { - Index: &IndexInfo{ + Index: ®istrytypes.IndexInfo{ Name: "localhost:8000", Official: false, }, @@ -443,7 +445,7 @@ func TestParseRepositoryInfo(t *testing.T) { Official: false, }, "localhost:8000/privatebase": { - Index: &IndexInfo{ + Index: ®istrytypes.IndexInfo{ Name: "localhost:8000", Official: false, }, @@ -453,7 +455,7 @@ func TestParseRepositoryInfo(t *testing.T) { Official: false, }, "example.com/private/moonbase": { - Index: &IndexInfo{ + Index: ®istrytypes.IndexInfo{ Name: "example.com", Official: false, }, @@ -463,7 +465,7 @@ func TestParseRepositoryInfo(t *testing.T) { Official: false, }, "example.com/privatebase": { - Index: &IndexInfo{ + Index: ®istrytypes.IndexInfo{ Name: "example.com", Official: false, }, @@ -473,7 +475,7 @@ func TestParseRepositoryInfo(t *testing.T) { Official: false, }, "example.com:8000/private/moonbase": { - Index: &IndexInfo{ + Index: ®istrytypes.IndexInfo{ Name: "example.com:8000", Official: false, }, @@ -483,7 +485,7 @@ func TestParseRepositoryInfo(t *testing.T) { Official: false, }, "example.com:8000/privatebase": { - Index: &IndexInfo{ + Index: ®istrytypes.IndexInfo{ Name: "example.com:8000", Official: false, }, @@ -493,7 +495,7 @@ func TestParseRepositoryInfo(t *testing.T) { Official: false, }, "localhost/private/moonbase": { - Index: &IndexInfo{ + Index: ®istrytypes.IndexInfo{ Name: "localhost", Official: false, }, @@ -503,7 +505,7 @@ func TestParseRepositoryInfo(t *testing.T) { Official: false, }, "localhost/privatebase": { - Index: &IndexInfo{ + Index: ®istrytypes.IndexInfo{ Name: "localhost", Official: false, }, @@ -513,7 +515,7 @@ func TestParseRepositoryInfo(t *testing.T) { Official: false, }, IndexName + "/public/moonbase": { - Index: &IndexInfo{ + Index: ®istrytypes.IndexInfo{ Name: IndexName, Official: true, }, @@ -523,7 +525,7 @@ func TestParseRepositoryInfo(t *testing.T) { Official: false, }, "index." + IndexName + "/public/moonbase": { - Index: &IndexInfo{ + Index: ®istrytypes.IndexInfo{ Name: IndexName, Official: true, }, @@ -533,7 +535,7 @@ func TestParseRepositoryInfo(t *testing.T) { Official: false, }, "ubuntu-12.04-base": { - Index: &IndexInfo{ + Index: ®istrytypes.IndexInfo{ Name: IndexName, Official: true, }, @@ -543,7 +545,7 @@ func TestParseRepositoryInfo(t *testing.T) { Official: true, }, IndexName + "/ubuntu-12.04-base": { - Index: &IndexInfo{ + Index: ®istrytypes.IndexInfo{ Name: IndexName, Official: true, }, @@ -553,7 +555,7 @@ func TestParseRepositoryInfo(t *testing.T) { Official: true, }, "index." + IndexName + "/ubuntu-12.04-base": { - Index: &IndexInfo{ + Index: ®istrytypes.IndexInfo{ Name: IndexName, Official: true, }, @@ -585,9 +587,9 @@ func TestParseRepositoryInfo(t *testing.T) { } func TestNewIndexInfo(t *testing.T) { - testIndexInfo := func(config *ServiceConfig, expectedIndexInfos map[string]*IndexInfo) { + testIndexInfo := func(config *registrytypes.ServiceConfig, expectedIndexInfos map[string]*registrytypes.IndexInfo) { for indexName, expectedIndexInfo := range expectedIndexInfos { - index, err := config.NewIndexInfo(indexName) + index, err := newIndexInfo(config, indexName) if err != nil { t.Fatal(err) } else { @@ -601,7 +603,7 @@ func TestNewIndexInfo(t *testing.T) { config := NewServiceConfig(nil) noMirrors := []string{} - expectedIndexInfos := map[string]*IndexInfo{ + expectedIndexInfos := map[string]*registrytypes.IndexInfo{ IndexName: { Name: IndexName, Official: true, @@ -632,7 +634,7 @@ func TestNewIndexInfo(t *testing.T) { publicMirrors := []string{"http://mirror1.local", "http://mirror2.local"} config = makeServiceConfig(publicMirrors, []string{"example.com"}) - expectedIndexInfos = map[string]*IndexInfo{ + expectedIndexInfos = map[string]*registrytypes.IndexInfo{ IndexName: { Name: IndexName, Official: true, @@ -679,7 +681,7 @@ func TestNewIndexInfo(t *testing.T) { testIndexInfo(config, expectedIndexInfos) config = makeServiceConfig(nil, []string{"42.42.0.0/16"}) - expectedIndexInfos = map[string]*IndexInfo{ + expectedIndexInfos = map[string]*registrytypes.IndexInfo{ "example.com": { Name: "example.com", Official: false, @@ -981,7 +983,7 @@ func TestIsSecureIndex(t *testing.T) { } for _, tt := range tests { config := makeServiceConfig(nil, tt.insecureRegistries) - if sec := config.isSecureIndex(tt.addr); sec != tt.expected { + if sec := isSecureIndex(config, tt.addr); sec != tt.expected { t.Errorf("isSecureIndex failed for %q %v, expected %v got %v", tt.addr, tt.insecureRegistries, tt.expected, sec) } } diff --git a/docs/service.go b/docs/service.go index e5f79af1..b04fd00c 100644 --- a/docs/service.go +++ b/docs/service.go @@ -8,12 +8,14 @@ import ( "github.com/docker/distribution/reference" "github.com/docker/distribution/registry/client/auth" + "github.com/docker/docker/api/types" + registrytypes "github.com/docker/docker/api/types/registry" ) // Service is a registry service. It tracks configuration data such as a list // of mirrors. type Service struct { - Config *ServiceConfig + Config *registrytypes.ServiceConfig } // NewService returns a new instance of Service ready to be @@ -78,7 +80,7 @@ func (s *Service) Search(term string, authConfig *types.AuthConfig, headers map[ indexName, remoteName := splitReposSearchTerm(term) - index, err := s.Config.NewIndexInfo(indexName) + index, err := newIndexInfo(s.Config, indexName) if err != nil { return nil, err } @@ -109,12 +111,12 @@ func (s *Service) Search(term string, authConfig *types.AuthConfig, headers map[ // ResolveRepository splits a repository name into its components // and configuration of the associated registry. func (s *Service) ResolveRepository(name reference.Named) (*RepositoryInfo, error) { - return s.Config.NewRepositoryInfo(name) + return newRepositoryInfo(s.Config, name) } // ResolveIndex takes indexName and returns index info -func (s *Service) ResolveIndex(name string) (*IndexInfo, error) { - return s.Config.NewIndexInfo(name) +func (s *Service) ResolveIndex(name string) (*registrytypes.IndexInfo, error) { + return newIndexInfo(s.Config, name) } // APIEndpoint represents a remote API endpoint @@ -136,7 +138,7 @@ func (e APIEndpoint) ToV1Endpoint(metaHeaders http.Header) (*Endpoint, error) { // TLSConfig constructs a client TLS configuration based on server defaults func (s *Service) TLSConfig(hostname string) (*tls.Config, error) { - return newTLSConfig(hostname, s.Config.isSecureIndex(hostname)) + return newTLSConfig(hostname, isSecureIndex(s.Config, hostname)) } func (s *Service) tlsConfigForMirror(mirror string) (*tls.Config, error) { diff --git a/docs/session.go b/docs/session.go index 774b1f5b..25bffc7f 100644 --- a/docs/session.go +++ b/docs/session.go @@ -20,6 +20,7 @@ import ( "github.com/Sirupsen/logrus" "github.com/docker/distribution/reference" + "github.com/docker/docker/api/types" "github.com/docker/docker/pkg/httputils" "github.com/docker/docker/pkg/ioutils" "github.com/docker/docker/pkg/stringid" diff --git a/docs/types.go b/docs/types.go index 9b2562f9..5068e00b 100644 --- a/docs/types.go +++ b/docs/types.go @@ -2,6 +2,7 @@ package registry import ( "github.com/docker/distribution/reference" + registrytypes "github.com/docker/docker/api/types/registry" ) // SearchResult describes a search result returned from a registry @@ -83,51 +84,10 @@ const ( APIVersion2 ) -// IndexInfo contains information about a registry -// -// RepositoryInfo Examples: -// { -// "Index" : { -// "Name" : "docker.io", -// "Mirrors" : ["https://registry-2.docker.io/v1/", "https://registry-3.docker.io/v1/"], -// "Secure" : true, -// "Official" : true, -// }, -// "RemoteName" : "library/debian", -// "LocalName" : "debian", -// "CanonicalName" : "docker.io/debian" -// "Official" : true, -// } -// -// { -// "Index" : { -// "Name" : "127.0.0.1:5000", -// "Mirrors" : [], -// "Secure" : false, -// "Official" : false, -// }, -// "RemoteName" : "user/repo", -// "LocalName" : "127.0.0.1:5000/user/repo", -// "CanonicalName" : "127.0.0.1:5000/user/repo", -// "Official" : false, -// } -type IndexInfo struct { - // Name is the name of the registry, such as "docker.io" - Name string - // Mirrors is a list of mirrors, expressed as URIs - Mirrors []string - // Secure is set to false if the registry is part of the list of - // insecure registries. Insecure registries accept HTTP and/or accept - // HTTPS with certificates from unknown CAs. - Secure bool - // Official indicates whether this is an official registry - Official bool -} - // RepositoryInfo describes a repository type RepositoryInfo struct { // Index points to registry information - Index *IndexInfo + Index *registrytypes.IndexInfo // RemoteName is the remote name of the repository, such as // "library/ubuntu-12.04-base" RemoteName reference.Named From 55fad57ac8e8f16e893466504e9b221ffb942c25 Mon Sep 17 00:00:00 2001 From: David Calavera Date: Mon, 14 Dec 2015 14:23:21 -0500 Subject: [PATCH 341/375] Remove timeout shared function. Handle timeouts when it's necessary based on a Timeout interface. Signed-off-by: David Calavera --- docs/session.go | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/docs/session.go b/docs/session.go index cecf936b..04c66f86 100644 --- a/docs/session.go +++ b/docs/session.go @@ -25,7 +25,6 @@ import ( "github.com/docker/docker/pkg/ioutils" "github.com/docker/docker/pkg/stringid" "github.com/docker/docker/pkg/tarsum" - "github.com/docker/docker/utils" ) var ( @@ -420,7 +419,7 @@ func (r *Session) GetRepositoryData(remote reference.Named) (*RepositoryData, er // and return a non-obtuse error message for users // "Get https://index.docker.io/v1/repositories/library/busybox/images: i/o timeout" // was a top search on the docker user forum - if utils.IsTimeout(err) { + if isTimeout(err) { return nil, fmt.Errorf("Network timed out while trying to connect to %s. You may want to check your internet connection or if you are behind a proxy.", repositoryTarget) } return nil, fmt.Errorf("Error while pulling image: %v", err) @@ -754,3 +753,16 @@ func (r *Session) GetAuthConfig(withPasswd bool) *cliconfig.AuthConfig { Email: r.authConfig.Email, } } + +func isTimeout(err error) bool { + type timeout interface { + Timeout() bool + } + e := err + switch urlErr := err.(type) { + case *url.Error: + e = urlErr.Err + } + t, ok := e.(timeout) + return ok && t.Timeout() +} From 0a56a1cbd2821f0be8790b5d4a08ccc1b58fec99 Mon Sep 17 00:00:00 2001 From: Daniel Nephin Date: Tue, 15 Dec 2015 11:44:20 -0500 Subject: [PATCH 342/375] Move registry.SearchResult types to api/types/registry. Signed-off-by: Daniel Nephin --- docs/registry_mock_test.go | 4 ++-- docs/service.go | 2 +- docs/session.go | 5 +++-- docs/types.go | 26 -------------------------- 4 files changed, 6 insertions(+), 31 deletions(-) diff --git a/docs/registry_mock_test.go b/docs/registry_mock_test.go index 89059e8e..f45de5c8 100644 --- a/docs/registry_mock_test.go +++ b/docs/registry_mock_test.go @@ -460,10 +460,10 @@ func handlerAuth(w http.ResponseWriter, r *http.Request) { } func handlerSearch(w http.ResponseWriter, r *http.Request) { - result := &SearchResults{ + result := ®istrytypes.SearchResults{ Query: "fakequery", NumResults: 1, - Results: []SearchResult{{Name: "fakeimage", StarCount: 42}}, + Results: []registrytypes.SearchResult{{Name: "fakeimage", StarCount: 42}}, } writeResponse(w, result, 200) } diff --git a/docs/service.go b/docs/service.go index b04fd00c..b826f117 100644 --- a/docs/service.go +++ b/docs/service.go @@ -73,7 +73,7 @@ func splitReposSearchTerm(reposName string) (string, string) { // Search queries the public registry for images matching the specified // search terms, and returns the results. -func (s *Service) Search(term string, authConfig *types.AuthConfig, headers map[string][]string) (*SearchResults, error) { +func (s *Service) Search(term string, authConfig *types.AuthConfig, headers map[string][]string) (*registrytypes.SearchResults, error) { if err := validateNoSchema(term); err != nil { return nil, err } diff --git a/docs/session.go b/docs/session.go index 4be9b7af..d09babd4 100644 --- a/docs/session.go +++ b/docs/session.go @@ -21,6 +21,7 @@ import ( "github.com/Sirupsen/logrus" "github.com/docker/distribution/reference" "github.com/docker/docker/api/types" + registrytypes "github.com/docker/docker/api/types/registry" "github.com/docker/docker/pkg/httputils" "github.com/docker/docker/pkg/ioutils" "github.com/docker/docker/pkg/stringid" @@ -718,7 +719,7 @@ func shouldRedirect(response *http.Response) bool { } // SearchRepositories performs a search against the remote repository -func (r *Session) SearchRepositories(term string) (*SearchResults, error) { +func (r *Session) SearchRepositories(term string) (*registrytypes.SearchResults, error) { logrus.Debugf("Index server: %s", r.indexEndpoint) u := r.indexEndpoint.VersionString(1) + "search?q=" + url.QueryEscape(term) @@ -736,7 +737,7 @@ func (r *Session) SearchRepositories(term string) (*SearchResults, error) { if res.StatusCode != 200 { return nil, httputils.NewHTTPRequestError(fmt.Sprintf("Unexpected status code %d", res.StatusCode), res) } - result := new(SearchResults) + result := new(registrytypes.SearchResults) return result, json.NewDecoder(res.Body).Decode(result) } diff --git a/docs/types.go b/docs/types.go index 5068e00b..03657820 100644 --- a/docs/types.go +++ b/docs/types.go @@ -5,32 +5,6 @@ import ( registrytypes "github.com/docker/docker/api/types/registry" ) -// SearchResult describes a search result returned from a registry -type SearchResult struct { - // StarCount indicates the number of stars this repository has - StarCount int `json:"star_count"` - // IsOfficial indicates whether the result is an official repository or not - IsOfficial bool `json:"is_official"` - // Name is the name of the repository - Name string `json:"name"` - // IsOfficial indicates whether the result is trusted - IsTrusted bool `json:"is_trusted"` - // IsAutomated indicates whether the result is automated - IsAutomated bool `json:"is_automated"` - // Description is a textual description of the repository - Description string `json:"description"` -} - -// SearchResults lists a collection search results returned from a registry -type SearchResults struct { - // Query contains the query string that generated the search results - Query string `json:"query"` - // NumResults indicates the number of results the query returned - NumResults int `json:"num_results"` - // Results is a slice containing the actual results for the search - Results []SearchResult `json:"results"` -} - // RepositoryData tracks the image list, list of endpoints, and list of tokens // for a repository type RepositoryData struct { From 14d27ab761797b45e0fc3242cd6067773a1214c7 Mon Sep 17 00:00:00 2001 From: Daniel Nephin Date: Tue, 15 Dec 2015 13:36:47 -0500 Subject: [PATCH 343/375] Move the TestEncodeAuth test to the correct package. Also make EncodeAuth and DecodeAuth private because they're only used by cliconfig. Signed-off-by: Daniel Nephin --- docs/auth_test.go | 21 --------------------- 1 file changed, 21 deletions(-) diff --git a/docs/auth_test.go b/docs/auth_test.go index a2c5c804..ff1bd547 100644 --- a/docs/auth_test.go +++ b/docs/auth_test.go @@ -5,29 +5,8 @@ import ( "github.com/docker/docker/api/types" registrytypes "github.com/docker/docker/api/types/registry" - "github.com/docker/docker/cliconfig" ) -func TestEncodeAuth(t *testing.T) { - newAuthConfig := &types.AuthConfig{Username: "ken", Password: "test", Email: "test@example.com"} - authStr := cliconfig.EncodeAuth(newAuthConfig) - decAuthConfig := &types.AuthConfig{} - var err error - decAuthConfig.Username, decAuthConfig.Password, err = cliconfig.DecodeAuth(authStr) - if err != nil { - t.Fatal(err) - } - if newAuthConfig.Username != decAuthConfig.Username { - t.Fatal("Encode Username doesn't match decoded Username") - } - if newAuthConfig.Password != decAuthConfig.Password { - t.Fatal("Encode Password doesn't match decoded Password") - } - if authStr != "a2VuOnRlc3Q=" { - t.Fatal("AuthString encoding isn't correct.") - } -} - func buildAuthConfigs() map[string]types.AuthConfig { authConfigs := map[string]types.AuthConfig{} From 9b8f1a08957674e3e0c908f6cac01a4c3e56b39d Mon Sep 17 00:00:00 2001 From: Tonis Tiigi Date: Fri, 4 Dec 2015 13:55:15 -0800 Subject: [PATCH 344/375] Add own reference package wrapper Signed-off-by: Tonis Tiigi --- docs/config.go | 9 +++++---- docs/registry_mock_test.go | 2 +- docs/registry_test.go | 2 +- docs/service.go | 2 +- docs/service_v1.go | 2 +- docs/service_v2.go | 2 +- docs/session.go | 2 +- docs/types.go | 2 +- 8 files changed, 12 insertions(+), 11 deletions(-) diff --git a/docs/config.go b/docs/config.go index 2eeba140..45caf578 100644 --- a/docs/config.go +++ b/docs/config.go @@ -7,11 +7,12 @@ import ( "net/url" "strings" - "github.com/docker/distribution/reference" + distreference "github.com/docker/distribution/reference" registrytypes "github.com/docker/docker/api/types/registry" "github.com/docker/docker/image/v1" "github.com/docker/docker/opts" flag "github.com/docker/docker/pkg/mflag" + "github.com/docker/docker/reference" ) // Options holds command line options. @@ -269,7 +270,7 @@ func GetAuthConfigKey(index *registrytypes.IndexInfo) string { // splitReposName breaks a reposName into an index name and remote name func splitReposName(reposName reference.Named) (indexName string, remoteName reference.Named, err error) { var remoteNameStr string - indexName, remoteNameStr = reference.SplitHostname(reposName) + indexName, remoteNameStr = distreference.SplitHostname(reposName) if indexName == "" || (!strings.Contains(indexName, ".") && !strings.Contains(indexName, ":") && indexName != "localhost") { // This is a Docker Index repos (ex: samalba/hipache or ubuntu) @@ -405,13 +406,13 @@ func localNameFromRemote(indexName string, remoteName reference.Named) (referenc // error. func NormalizeLocalReference(ref reference.Named) reference.Named { localName := NormalizeLocalName(ref) - if tagged, isTagged := ref.(reference.Tagged); isTagged { + if tagged, isTagged := ref.(reference.NamedTagged); isTagged { newRef, err := reference.WithTag(localName, tagged.Tag()) if err != nil { return ref } return newRef - } else if digested, isDigested := ref.(reference.Digested); isDigested { + } else if digested, isCanonical := ref.(reference.Canonical); isCanonical { newRef, err := reference.WithDigest(localName, digested.Digest()) if err != nil { return ref diff --git a/docs/registry_mock_test.go b/docs/registry_mock_test.go index f45de5c8..017d08bb 100644 --- a/docs/registry_mock_test.go +++ b/docs/registry_mock_test.go @@ -15,9 +15,9 @@ import ( "testing" "time" - "github.com/docker/distribution/reference" registrytypes "github.com/docker/docker/api/types/registry" "github.com/docker/docker/opts" + "github.com/docker/docker/reference" "github.com/gorilla/mux" "github.com/Sirupsen/logrus" diff --git a/docs/registry_test.go b/docs/registry_test.go index 7e352441..31ae2d5f 100644 --- a/docs/registry_test.go +++ b/docs/registry_test.go @@ -8,10 +8,10 @@ import ( "strings" "testing" - "github.com/docker/distribution/reference" "github.com/docker/distribution/registry/client/transport" "github.com/docker/docker/api/types" registrytypes "github.com/docker/docker/api/types/registry" + "github.com/docker/docker/reference" ) var ( diff --git a/docs/service.go b/docs/service.go index b826f117..eb5cd6bf 100644 --- a/docs/service.go +++ b/docs/service.go @@ -6,10 +6,10 @@ import ( "net/url" "strings" - "github.com/docker/distribution/reference" "github.com/docker/distribution/registry/client/auth" "github.com/docker/docker/api/types" registrytypes "github.com/docker/docker/api/types/registry" + "github.com/docker/docker/reference" ) // Service is a registry service. It tracks configuration data such as a list diff --git a/docs/service_v1.go b/docs/service_v1.go index 5fdc1ece..3b3cc780 100644 --- a/docs/service_v1.go +++ b/docs/service_v1.go @@ -4,8 +4,8 @@ import ( "fmt" "strings" - "github.com/docker/distribution/reference" "github.com/docker/docker/pkg/tlsconfig" + "github.com/docker/docker/reference" ) func (s *Service) lookupV1Endpoints(repoName reference.Named) (endpoints []APIEndpoint, err error) { diff --git a/docs/service_v2.go b/docs/service_v2.go index 56a3d2ee..3a2c32a5 100644 --- a/docs/service_v2.go +++ b/docs/service_v2.go @@ -4,9 +4,9 @@ import ( "fmt" "strings" - "github.com/docker/distribution/reference" "github.com/docker/distribution/registry/client/auth" "github.com/docker/docker/pkg/tlsconfig" + "github.com/docker/docker/reference" ) func (s *Service) lookupV2Endpoints(repoName reference.Named) (endpoints []APIEndpoint, err error) { diff --git a/docs/session.go b/docs/session.go index d09babd4..a1206206 100644 --- a/docs/session.go +++ b/docs/session.go @@ -19,13 +19,13 @@ import ( "strings" "github.com/Sirupsen/logrus" - "github.com/docker/distribution/reference" "github.com/docker/docker/api/types" registrytypes "github.com/docker/docker/api/types/registry" "github.com/docker/docker/pkg/httputils" "github.com/docker/docker/pkg/ioutils" "github.com/docker/docker/pkg/stringid" "github.com/docker/docker/pkg/tarsum" + "github.com/docker/docker/reference" ) var ( diff --git a/docs/types.go b/docs/types.go index 03657820..939f44b1 100644 --- a/docs/types.go +++ b/docs/types.go @@ -1,8 +1,8 @@ package registry import ( - "github.com/docker/distribution/reference" registrytypes "github.com/docker/docker/api/types/registry" + "github.com/docker/docker/reference" ) // RepositoryData tracks the image list, list of endpoints, and list of tokens From 46683f619203bc4d85f6b3087f78b2445e8b4b0a Mon Sep 17 00:00:00 2001 From: Tonis Tiigi Date: Fri, 11 Dec 2015 11:00:13 -0800 Subject: [PATCH 345/375] Update Named reference with validation of conversions Signed-off-by: Tonis Tiigi --- docs/config.go | 178 +------------------------ docs/registry_mock_test.go | 3 - docs/registry_test.go | 263 ++++++++++--------------------------- docs/service_v1.go | 2 +- docs/service_v2.go | 2 +- docs/session.go | 20 +-- docs/types.go | 10 +- 7 files changed, 89 insertions(+), 389 deletions(-) diff --git a/docs/config.go b/docs/config.go index 45caf578..ca7beec4 100644 --- a/docs/config.go +++ b/docs/config.go @@ -7,9 +7,7 @@ import ( "net/url" "strings" - distreference "github.com/docker/distribution/reference" registrytypes "github.com/docker/docker/api/types/registry" - "github.com/docker/docker/image/v1" "github.com/docker/docker/opts" flag "github.com/docker/docker/pkg/mflag" "github.com/docker/docker/reference" @@ -182,28 +180,15 @@ func ValidateMirror(val string) (string, error) { // ValidateIndexName validates an index name. func ValidateIndexName(val string) (string, error) { - // 'index.docker.io' => 'docker.io' - if val == "index."+IndexName { - val = IndexName + if val == reference.LegacyDefaultHostname { + val = reference.DefaultHostname } if strings.HasPrefix(val, "-") || strings.HasSuffix(val, "-") { return "", fmt.Errorf("Invalid index name (%s). Cannot begin or end with a hyphen.", val) } - // *TODO: Check if valid hostname[:port]/ip[:port]? return val, nil } -func validateRemoteName(remoteName reference.Named) error { - remoteNameStr := remoteName.Name() - if !strings.Contains(remoteNameStr, "/") { - // the repository name must not be a valid image ID - if err := v1.ValidateID(remoteNameStr); err == nil { - return fmt.Errorf("Invalid repository name (%s), cannot specify 64-byte hexadecimal strings", remoteName) - } - } - return nil -} - func validateNoSchema(reposName string) error { if strings.Contains(reposName, "://") { // It cannot contain a scheme! @@ -212,29 +197,6 @@ func validateNoSchema(reposName string) error { return nil } -// ValidateRepositoryName validates a repository name -func ValidateRepositoryName(reposName reference.Named) error { - _, _, err := loadRepositoryName(reposName) - return err -} - -// loadRepositoryName returns the repo name splitted into index name -// and remote repo name. It returns an error if the name is not valid. -func loadRepositoryName(reposName reference.Named) (string, reference.Named, error) { - if err := validateNoSchema(reposName.Name()); err != nil { - return "", nil, err - } - indexName, remoteName, err := splitReposName(reposName) - - if indexName, err = ValidateIndexName(indexName); err != nil { - return "", nil, err - } - if err = validateRemoteName(remoteName); err != nil { - return "", nil, err - } - return indexName, remoteName, nil -} - // newIndexInfo returns IndexInfo configuration from indexName func newIndexInfo(config *registrytypes.ServiceConfig, indexName string) (*registrytypes.IndexInfo, error) { var err error @@ -267,75 +229,14 @@ func GetAuthConfigKey(index *registrytypes.IndexInfo) string { return index.Name } -// splitReposName breaks a reposName into an index name and remote name -func splitReposName(reposName reference.Named) (indexName string, remoteName reference.Named, err error) { - var remoteNameStr string - indexName, remoteNameStr = distreference.SplitHostname(reposName) - if indexName == "" || (!strings.Contains(indexName, ".") && - !strings.Contains(indexName, ":") && indexName != "localhost") { - // This is a Docker Index repos (ex: samalba/hipache or ubuntu) - // 'docker.io' - indexName = IndexName - remoteName = reposName - } else { - remoteName, err = reference.WithName(remoteNameStr) - } - return -} - // newRepositoryInfo validates and breaks down a repository name into a RepositoryInfo -func newRepositoryInfo(config *registrytypes.ServiceConfig, reposName reference.Named) (*RepositoryInfo, error) { - if err := validateNoSchema(reposName.Name()); err != nil { - return nil, err - } - - repoInfo := &RepositoryInfo{} - var ( - indexName string - err error - ) - - indexName, repoInfo.RemoteName, err = loadRepositoryName(reposName) +func newRepositoryInfo(config *registrytypes.ServiceConfig, name reference.Named) (*RepositoryInfo, error) { + index, err := newIndexInfo(config, name.Hostname()) if err != nil { return nil, err } - - repoInfo.Index, err = newIndexInfo(config, indexName) - if err != nil { - return nil, err - } - - if repoInfo.Index.Official { - repoInfo.LocalName, err = normalizeLibraryRepoName(repoInfo.RemoteName) - if err != nil { - return nil, err - } - repoInfo.RemoteName = repoInfo.LocalName - - // If the normalized name does not contain a '/' (e.g. "foo") - // then it is an official repo. - if strings.IndexRune(repoInfo.RemoteName.Name(), '/') == -1 { - repoInfo.Official = true - // Fix up remote name for official repos. - repoInfo.RemoteName, err = reference.WithName("library/" + repoInfo.RemoteName.Name()) - if err != nil { - return nil, err - } - } - - repoInfo.CanonicalName, err = reference.WithName("docker.io/" + repoInfo.RemoteName.Name()) - if err != nil { - return nil, err - } - } else { - repoInfo.LocalName, err = localNameFromRemote(repoInfo.Index.Name, repoInfo.RemoteName) - if err != nil { - return nil, err - } - repoInfo.CanonicalName = repoInfo.LocalName - } - - return repoInfo, nil + official := !strings.ContainsRune(name.Name(), '/') + return &RepositoryInfo{name, index, official}, nil } // ParseRepositoryInfo performs the breakdown of a repository name into a RepositoryInfo, but @@ -354,70 +255,3 @@ func ParseSearchIndexInfo(reposName string) (*registrytypes.IndexInfo, error) { } return indexInfo, nil } - -// NormalizeLocalName transforms a repository name into a normalized LocalName -// Passes through the name without transformation on error (image id, etc) -// It does not use the repository info because we don't want to load -// the repository index and do request over the network. -func NormalizeLocalName(name reference.Named) reference.Named { - indexName, remoteName, err := loadRepositoryName(name) - if err != nil { - return name - } - - var officialIndex bool - // Return any configured index info, first. - if index, ok := emptyServiceConfig.IndexConfigs[indexName]; ok { - officialIndex = index.Official - } - - if officialIndex { - localName, err := normalizeLibraryRepoName(remoteName) - if err != nil { - return name - } - return localName - } - localName, err := localNameFromRemote(indexName, remoteName) - if err != nil { - return name - } - return localName -} - -// normalizeLibraryRepoName removes the library prefix from -// the repository name for official repos. -func normalizeLibraryRepoName(name reference.Named) (reference.Named, error) { - if strings.HasPrefix(name.Name(), "library/") { - // If pull "library/foo", it's stored locally under "foo" - return reference.WithName(strings.SplitN(name.Name(), "/", 2)[1]) - } - return name, nil -} - -// localNameFromRemote combines the index name and the repo remote name -// to generate a repo local name. -func localNameFromRemote(indexName string, remoteName reference.Named) (reference.Named, error) { - return reference.WithName(indexName + "/" + remoteName.Name()) -} - -// NormalizeLocalReference transforms a reference to use a normalized LocalName -// for the name poriton. Passes through the reference without transformation on -// error. -func NormalizeLocalReference(ref reference.Named) reference.Named { - localName := NormalizeLocalName(ref) - if tagged, isTagged := ref.(reference.NamedTagged); isTagged { - newRef, err := reference.WithTag(localName, tagged.Tag()) - if err != nil { - return ref - } - return newRef - } else if digested, isCanonical := ref.(reference.Canonical); isCanonical { - newRef, err := reference.WithDigest(localName, digested.Digest()) - if err != nil { - return ref - } - return newRef - } - return localName -} diff --git a/docs/registry_mock_test.go b/docs/registry_mock_test.go index 017d08bb..be04e346 100644 --- a/docs/registry_mock_test.go +++ b/docs/registry_mock_test.go @@ -356,7 +356,6 @@ func handlerGetDeleteTags(w http.ResponseWriter, r *http.Request) { apiError(w, "Could not parse repository", 400) return } - repositoryName = NormalizeLocalName(repositoryName) tags, exists := testRepositories[repositoryName.String()] if !exists { apiError(w, "Repository not found", 404) @@ -380,7 +379,6 @@ func handlerGetTag(w http.ResponseWriter, r *http.Request) { apiError(w, "Could not parse repository", 400) return } - repositoryName = NormalizeLocalName(repositoryName) tagName := vars["tag"] tags, exists := testRepositories[repositoryName.String()] if !exists { @@ -405,7 +403,6 @@ func handlerPutTag(w http.ResponseWriter, r *http.Request) { apiError(w, "Could not parse repository", 400) return } - repositoryName = NormalizeLocalName(repositoryName) tagName := vars["tag"] tags, exists := testRepositories[repositoryName.String()] if !exists { diff --git a/docs/registry_test.go b/docs/registry_test.go index 31ae2d5f..46d2818f 100644 --- a/docs/registry_test.go +++ b/docs/registry_test.go @@ -307,71 +307,24 @@ func TestPushImageLayerRegistry(t *testing.T) { } } -func TestValidateRepositoryName(t *testing.T) { - validRepoNames := []string{ - "docker/docker", - "library/debian", - "debian", - "docker.io/docker/docker", - "docker.io/library/debian", - "docker.io/debian", - "index.docker.io/docker/docker", - "index.docker.io/library/debian", - "index.docker.io/debian", - "127.0.0.1:5000/docker/docker", - "127.0.0.1:5000/library/debian", - "127.0.0.1:5000/debian", - "thisisthesongthatneverendsitgoesonandonandonthisisthesongthatnev", - } - invalidRepoNames := []string{ - "https://github.com/docker/docker", - "docker/Docker", - "-docker", - "-docker/docker", - "-docker.io/docker/docker", - "docker///docker", - "docker.io/docker/Docker", - "docker.io/docker///docker", - "1a3f5e7d9c1b3a5f7e9d1c3b5a7f9e1d3c5b7a9f1e3d5d7c9b1a3f5e7d9c1b3a", - "docker.io/1a3f5e7d9c1b3a5f7e9d1c3b5a7f9e1d3c5b7a9f1e3d5d7c9b1a3f5e7d9c1b3a", - } - - for _, name := range invalidRepoNames { - named, err := reference.WithName(name) - if err == nil { - err := ValidateRepositoryName(named) - assertNotEqual(t, err, nil, "Expected invalid repo name: "+name) - } - } - - for _, name := range validRepoNames { - named, err := reference.WithName(name) - if err != nil { - t.Fatalf("could not parse valid name: %s", name) - } - err = ValidateRepositoryName(named) - assertEqual(t, err, nil, "Expected valid repo name: "+name) - } -} - func TestParseRepositoryInfo(t *testing.T) { - withName := func(name string) reference.Named { - named, err := reference.WithName(name) - if err != nil { - t.Fatalf("could not parse reference %s", name) - } - return named + type staticRepositoryInfo struct { + Index *registrytypes.IndexInfo + RemoteName string + CanonicalName string + LocalName string + Official bool } - expectedRepoInfos := map[string]RepositoryInfo{ + expectedRepoInfos := map[string]staticRepositoryInfo{ "fooo/bar": { Index: ®istrytypes.IndexInfo{ Name: IndexName, Official: true, }, - RemoteName: withName("fooo/bar"), - LocalName: withName("fooo/bar"), - CanonicalName: withName("docker.io/fooo/bar"), + RemoteName: "fooo/bar", + LocalName: "fooo/bar", + CanonicalName: "docker.io/fooo/bar", Official: false, }, "library/ubuntu": { @@ -379,9 +332,9 @@ func TestParseRepositoryInfo(t *testing.T) { Name: IndexName, Official: true, }, - RemoteName: withName("library/ubuntu"), - LocalName: withName("ubuntu"), - CanonicalName: withName("docker.io/library/ubuntu"), + RemoteName: "library/ubuntu", + LocalName: "ubuntu", + CanonicalName: "docker.io/library/ubuntu", Official: true, }, "nonlibrary/ubuntu": { @@ -389,9 +342,9 @@ func TestParseRepositoryInfo(t *testing.T) { Name: IndexName, Official: true, }, - RemoteName: withName("nonlibrary/ubuntu"), - LocalName: withName("nonlibrary/ubuntu"), - CanonicalName: withName("docker.io/nonlibrary/ubuntu"), + RemoteName: "nonlibrary/ubuntu", + LocalName: "nonlibrary/ubuntu", + CanonicalName: "docker.io/nonlibrary/ubuntu", Official: false, }, "ubuntu": { @@ -399,9 +352,9 @@ func TestParseRepositoryInfo(t *testing.T) { Name: IndexName, Official: true, }, - RemoteName: withName("library/ubuntu"), - LocalName: withName("ubuntu"), - CanonicalName: withName("docker.io/library/ubuntu"), + RemoteName: "library/ubuntu", + LocalName: "ubuntu", + CanonicalName: "docker.io/library/ubuntu", Official: true, }, "other/library": { @@ -409,9 +362,9 @@ func TestParseRepositoryInfo(t *testing.T) { Name: IndexName, Official: true, }, - RemoteName: withName("other/library"), - LocalName: withName("other/library"), - CanonicalName: withName("docker.io/other/library"), + RemoteName: "other/library", + LocalName: "other/library", + CanonicalName: "docker.io/other/library", Official: false, }, "127.0.0.1:8000/private/moonbase": { @@ -419,9 +372,9 @@ func TestParseRepositoryInfo(t *testing.T) { Name: "127.0.0.1:8000", Official: false, }, - RemoteName: withName("private/moonbase"), - LocalName: withName("127.0.0.1:8000/private/moonbase"), - CanonicalName: withName("127.0.0.1:8000/private/moonbase"), + RemoteName: "private/moonbase", + LocalName: "127.0.0.1:8000/private/moonbase", + CanonicalName: "127.0.0.1:8000/private/moonbase", Official: false, }, "127.0.0.1:8000/privatebase": { @@ -429,9 +382,9 @@ func TestParseRepositoryInfo(t *testing.T) { Name: "127.0.0.1:8000", Official: false, }, - RemoteName: withName("privatebase"), - LocalName: withName("127.0.0.1:8000/privatebase"), - CanonicalName: withName("127.0.0.1:8000/privatebase"), + RemoteName: "privatebase", + LocalName: "127.0.0.1:8000/privatebase", + CanonicalName: "127.0.0.1:8000/privatebase", Official: false, }, "localhost:8000/private/moonbase": { @@ -439,9 +392,9 @@ func TestParseRepositoryInfo(t *testing.T) { Name: "localhost:8000", Official: false, }, - RemoteName: withName("private/moonbase"), - LocalName: withName("localhost:8000/private/moonbase"), - CanonicalName: withName("localhost:8000/private/moonbase"), + RemoteName: "private/moonbase", + LocalName: "localhost:8000/private/moonbase", + CanonicalName: "localhost:8000/private/moonbase", Official: false, }, "localhost:8000/privatebase": { @@ -449,9 +402,9 @@ func TestParseRepositoryInfo(t *testing.T) { Name: "localhost:8000", Official: false, }, - RemoteName: withName("privatebase"), - LocalName: withName("localhost:8000/privatebase"), - CanonicalName: withName("localhost:8000/privatebase"), + RemoteName: "privatebase", + LocalName: "localhost:8000/privatebase", + CanonicalName: "localhost:8000/privatebase", Official: false, }, "example.com/private/moonbase": { @@ -459,9 +412,9 @@ func TestParseRepositoryInfo(t *testing.T) { Name: "example.com", Official: false, }, - RemoteName: withName("private/moonbase"), - LocalName: withName("example.com/private/moonbase"), - CanonicalName: withName("example.com/private/moonbase"), + RemoteName: "private/moonbase", + LocalName: "example.com/private/moonbase", + CanonicalName: "example.com/private/moonbase", Official: false, }, "example.com/privatebase": { @@ -469,9 +422,9 @@ func TestParseRepositoryInfo(t *testing.T) { Name: "example.com", Official: false, }, - RemoteName: withName("privatebase"), - LocalName: withName("example.com/privatebase"), - CanonicalName: withName("example.com/privatebase"), + RemoteName: "privatebase", + LocalName: "example.com/privatebase", + CanonicalName: "example.com/privatebase", Official: false, }, "example.com:8000/private/moonbase": { @@ -479,9 +432,9 @@ func TestParseRepositoryInfo(t *testing.T) { Name: "example.com:8000", Official: false, }, - RemoteName: withName("private/moonbase"), - LocalName: withName("example.com:8000/private/moonbase"), - CanonicalName: withName("example.com:8000/private/moonbase"), + RemoteName: "private/moonbase", + LocalName: "example.com:8000/private/moonbase", + CanonicalName: "example.com:8000/private/moonbase", Official: false, }, "example.com:8000/privatebase": { @@ -489,9 +442,9 @@ func TestParseRepositoryInfo(t *testing.T) { Name: "example.com:8000", Official: false, }, - RemoteName: withName("privatebase"), - LocalName: withName("example.com:8000/privatebase"), - CanonicalName: withName("example.com:8000/privatebase"), + RemoteName: "privatebase", + LocalName: "example.com:8000/privatebase", + CanonicalName: "example.com:8000/privatebase", Official: false, }, "localhost/private/moonbase": { @@ -499,9 +452,9 @@ func TestParseRepositoryInfo(t *testing.T) { Name: "localhost", Official: false, }, - RemoteName: withName("private/moonbase"), - LocalName: withName("localhost/private/moonbase"), - CanonicalName: withName("localhost/private/moonbase"), + RemoteName: "private/moonbase", + LocalName: "localhost/private/moonbase", + CanonicalName: "localhost/private/moonbase", Official: false, }, "localhost/privatebase": { @@ -509,9 +462,9 @@ func TestParseRepositoryInfo(t *testing.T) { Name: "localhost", Official: false, }, - RemoteName: withName("privatebase"), - LocalName: withName("localhost/privatebase"), - CanonicalName: withName("localhost/privatebase"), + RemoteName: "privatebase", + LocalName: "localhost/privatebase", + CanonicalName: "localhost/privatebase", Official: false, }, IndexName + "/public/moonbase": { @@ -519,9 +472,9 @@ func TestParseRepositoryInfo(t *testing.T) { Name: IndexName, Official: true, }, - RemoteName: withName("public/moonbase"), - LocalName: withName("public/moonbase"), - CanonicalName: withName("docker.io/public/moonbase"), + RemoteName: "public/moonbase", + LocalName: "public/moonbase", + CanonicalName: "docker.io/public/moonbase", Official: false, }, "index." + IndexName + "/public/moonbase": { @@ -529,9 +482,9 @@ func TestParseRepositoryInfo(t *testing.T) { Name: IndexName, Official: true, }, - RemoteName: withName("public/moonbase"), - LocalName: withName("public/moonbase"), - CanonicalName: withName("docker.io/public/moonbase"), + RemoteName: "public/moonbase", + LocalName: "public/moonbase", + CanonicalName: "docker.io/public/moonbase", Official: false, }, "ubuntu-12.04-base": { @@ -539,9 +492,9 @@ func TestParseRepositoryInfo(t *testing.T) { Name: IndexName, Official: true, }, - RemoteName: withName("library/ubuntu-12.04-base"), - LocalName: withName("ubuntu-12.04-base"), - CanonicalName: withName("docker.io/library/ubuntu-12.04-base"), + RemoteName: "library/ubuntu-12.04-base", + LocalName: "ubuntu-12.04-base", + CanonicalName: "docker.io/library/ubuntu-12.04-base", Official: true, }, IndexName + "/ubuntu-12.04-base": { @@ -549,9 +502,9 @@ func TestParseRepositoryInfo(t *testing.T) { Name: IndexName, Official: true, }, - RemoteName: withName("library/ubuntu-12.04-base"), - LocalName: withName("ubuntu-12.04-base"), - CanonicalName: withName("docker.io/library/ubuntu-12.04-base"), + RemoteName: "library/ubuntu-12.04-base", + LocalName: "ubuntu-12.04-base", + CanonicalName: "docker.io/library/ubuntu-12.04-base", Official: true, }, "index." + IndexName + "/ubuntu-12.04-base": { @@ -559,9 +512,9 @@ func TestParseRepositoryInfo(t *testing.T) { Name: IndexName, Official: true, }, - RemoteName: withName("library/ubuntu-12.04-base"), - LocalName: withName("ubuntu-12.04-base"), - CanonicalName: withName("docker.io/library/ubuntu-12.04-base"), + RemoteName: "library/ubuntu-12.04-base", + LocalName: "ubuntu-12.04-base", + CanonicalName: "docker.io/library/ubuntu-12.04-base", Official: true, }, } @@ -577,9 +530,9 @@ func TestParseRepositoryInfo(t *testing.T) { t.Error(err) } else { checkEqual(t, repoInfo.Index.Name, expectedRepoInfo.Index.Name, reposName) - checkEqual(t, repoInfo.RemoteName.String(), expectedRepoInfo.RemoteName.String(), reposName) - checkEqual(t, repoInfo.LocalName.String(), expectedRepoInfo.LocalName.String(), reposName) - checkEqual(t, repoInfo.CanonicalName.String(), expectedRepoInfo.CanonicalName.String(), reposName) + checkEqual(t, repoInfo.RemoteName(), expectedRepoInfo.RemoteName, reposName) + checkEqual(t, repoInfo.Name(), expectedRepoInfo.LocalName, reposName) + checkEqual(t, repoInfo.FullName(), expectedRepoInfo.CanonicalName, reposName) checkEqual(t, repoInfo.Index.Official, expectedRepoInfo.Index.Official, reposName) checkEqual(t, repoInfo.Official, expectedRepoInfo.Official, reposName) } @@ -806,82 +759,6 @@ func TestSearchRepositories(t *testing.T) { assertEqual(t, results.Results[0].StarCount, 42, "Expected 'fakeimage' to have 42 stars") } -func TestValidRemoteName(t *testing.T) { - validRepositoryNames := []string{ - // Sanity check. - "docker/docker", - - // Allow 64-character non-hexadecimal names (hexadecimal names are forbidden). - "thisisthesongthatneverendsitgoesonandonandonthisisthesongthatnev", - - // Allow embedded hyphens. - "docker-rules/docker", - - // Allow multiple hyphens as well. - "docker---rules/docker", - - //Username doc and image name docker being tested. - "doc/docker", - - // single character names are now allowed. - "d/docker", - "jess/t", - - // Consecutive underscores. - "dock__er/docker", - } - for _, repositoryName := range validRepositoryNames { - repositoryRef, err := reference.WithName(repositoryName) - if err != nil { - t.Errorf("Repository name should be valid: %v. Error: %v", repositoryName, err) - } - if err := validateRemoteName(repositoryRef); err != nil { - t.Errorf("Repository name should be valid: %v. Error: %v", repositoryName, err) - } - } - - invalidRepositoryNames := []string{ - // Disallow capital letters. - "docker/Docker", - - // Only allow one slash. - "docker///docker", - - // Disallow 64-character hexadecimal. - "1a3f5e7d9c1b3a5f7e9d1c3b5a7f9e1d3c5b7a9f1e3d5d7c9b1a3f5e7d9c1b3a", - - // Disallow leading and trailing hyphens in namespace. - "-docker/docker", - "docker-/docker", - "-docker-/docker", - - // Don't allow underscores everywhere (as opposed to hyphens). - "____/____", - - "_docker/_docker", - - // Disallow consecutive periods. - "dock..er/docker", - "dock_.er/docker", - "dock-.er/docker", - - // No repository. - "docker/", - - //namespace too long - "this_is_not_a_valid_namespace_because_its_lenth_is_greater_than_255_this_is_not_a_valid_namespace_because_its_lenth_is_greater_than_255_this_is_not_a_valid_namespace_because_its_lenth_is_greater_than_255_this_is_not_a_valid_namespace_because_its_lenth_is_greater_than_255/docker", - } - for _, repositoryName := range invalidRepositoryNames { - repositoryRef, err := reference.ParseNamed(repositoryName) - if err != nil { - continue - } - if err := validateRemoteName(repositoryRef); err == nil { - t.Errorf("Repository name should be invalid: %v", repositoryName) - } - } -} - func TestTrustedLocation(t *testing.T) { for _, url := range []string{"http://example.com", "https://example.com:7777", "http://docker.io", "http://test.docker.com", "https://fakedocker.com"} { req, _ := http.NewRequest("GET", url, nil) diff --git a/docs/service_v1.go b/docs/service_v1.go index 3b3cc780..cd565bc4 100644 --- a/docs/service_v1.go +++ b/docs/service_v1.go @@ -11,7 +11,7 @@ import ( func (s *Service) lookupV1Endpoints(repoName reference.Named) (endpoints []APIEndpoint, err error) { var cfg = tlsconfig.ServerDefault tlsConfig := &cfg - nameString := repoName.Name() + nameString := repoName.FullName() if strings.HasPrefix(nameString, DefaultNamespace+"/") { endpoints = append(endpoints, APIEndpoint{ URL: DefaultV1Registry, diff --git a/docs/service_v2.go b/docs/service_v2.go index 3a2c32a5..8a8cd260 100644 --- a/docs/service_v2.go +++ b/docs/service_v2.go @@ -12,7 +12,7 @@ import ( func (s *Service) lookupV2Endpoints(repoName reference.Named) (endpoints []APIEndpoint, err error) { var cfg = tlsconfig.ServerDefault tlsConfig := &cfg - nameString := repoName.Name() + nameString := repoName.FullName() if strings.HasPrefix(nameString, DefaultNamespace+"/") { // v2 mirrors for _, mirror := range s.Config.Mirrors { diff --git a/docs/session.go b/docs/session.go index a1206206..494b84bf 100644 --- a/docs/session.go +++ b/docs/session.go @@ -312,7 +312,7 @@ func (r *Session) GetRemoteImageLayer(imgID, registry string, imgSize int64) (io // argument, and returns data from the first one that answers the query // successfully. func (r *Session) GetRemoteTag(registries []string, repositoryRef reference.Named, askedTag string) (string, error) { - repository := repositoryRef.Name() + repository := repositoryRef.RemoteName() if strings.Count(repository, "/") == 0 { // This will be removed once the registry supports auto-resolution on @@ -350,7 +350,7 @@ func (r *Session) GetRemoteTag(registries []string, repositoryRef reference.Name // the first one that answers the query successfully. It returns a map with // tag names as the keys and image IDs as the values. func (r *Session) GetRemoteTags(registries []string, repositoryRef reference.Named) (map[string]string, error) { - repository := repositoryRef.Name() + repository := repositoryRef.RemoteName() if strings.Count(repository, "/") == 0 { // This will be removed once the registry supports auto-resolution on @@ -403,8 +403,8 @@ func buildEndpointsList(headers []string, indexEp string) ([]string, error) { } // GetRepositoryData returns lists of images and endpoints for the repository -func (r *Session) GetRepositoryData(remote reference.Named) (*RepositoryData, error) { - repositoryTarget := fmt.Sprintf("%srepositories/%s/images", r.indexEndpoint.VersionString(1), remote.Name()) +func (r *Session) GetRepositoryData(name reference.Named) (*RepositoryData, error) { + repositoryTarget := fmt.Sprintf("%srepositories/%s/images", r.indexEndpoint.VersionString(1), name.RemoteName()) logrus.Debugf("[registry] Calling GET %s", repositoryTarget) @@ -438,7 +438,7 @@ func (r *Session) GetRepositoryData(remote reference.Named) (*RepositoryData, er if err != nil { logrus.Debugf("Error reading response body: %s", err) } - return nil, httputils.NewHTTPRequestError(fmt.Sprintf("Error: Status %d trying to pull repository %s: %q", res.StatusCode, remote.Name(), errBody), res) + return nil, httputils.NewHTTPRequestError(fmt.Sprintf("Error: Status %d trying to pull repository %s: %q", res.StatusCode, name.RemoteName(), errBody), res) } var endpoints []string @@ -593,7 +593,7 @@ func (r *Session) PushImageLayerRegistry(imgID string, layer io.Reader, registry func (r *Session) PushRegistryTag(remote reference.Named, revision, tag, registry string) error { // "jsonify" the string revision = "\"" + revision + "\"" - path := fmt.Sprintf("repositories/%s/tags/%s", remote.Name(), tag) + path := fmt.Sprintf("repositories/%s/tags/%s", remote.RemoteName(), tag) req, err := http.NewRequest("PUT", registry+path, strings.NewReader(revision)) if err != nil { @@ -607,7 +607,7 @@ func (r *Session) PushRegistryTag(remote reference.Named, revision, tag, registr } res.Body.Close() if res.StatusCode != 200 && res.StatusCode != 201 { - return httputils.NewHTTPRequestError(fmt.Sprintf("Internal server error: %d trying to push tag %s on %s", res.StatusCode, tag, remote.Name()), res) + return httputils.NewHTTPRequestError(fmt.Sprintf("Internal server error: %d trying to push tag %s on %s", res.StatusCode, tag, remote.RemoteName()), res) } return nil } @@ -633,7 +633,7 @@ func (r *Session) PushImageJSONIndex(remote reference.Named, imgList []*ImgData, if validate { suffix = "images" } - u := fmt.Sprintf("%srepositories/%s/%s", r.indexEndpoint.VersionString(1), remote.Name(), suffix) + u := fmt.Sprintf("%srepositories/%s/%s", r.indexEndpoint.VersionString(1), remote.RemoteName(), suffix) logrus.Debugf("[registry] PUT %s", u) logrus.Debugf("Image list pushed to index:\n%s", imgListJSON) headers := map[string][]string{ @@ -671,7 +671,7 @@ func (r *Session) PushImageJSONIndex(remote reference.Named, imgList []*ImgData, if err != nil { logrus.Debugf("Error reading response body: %s", err) } - return nil, httputils.NewHTTPRequestError(fmt.Sprintf("Error: Status %d trying to push repository %s: %q", res.StatusCode, remote.Name(), errBody), res) + return nil, httputils.NewHTTPRequestError(fmt.Sprintf("Error: Status %d trying to push repository %s: %q", res.StatusCode, remote.RemoteName(), errBody), res) } tokens = res.Header["X-Docker-Token"] logrus.Debugf("Auth token: %v", tokens) @@ -689,7 +689,7 @@ func (r *Session) PushImageJSONIndex(remote reference.Named, imgList []*ImgData, if err != nil { logrus.Debugf("Error reading response body: %s", err) } - return nil, httputils.NewHTTPRequestError(fmt.Sprintf("Error: Status %d trying to push checksums %s: %q", res.StatusCode, remote.Name(), errBody), res) + return nil, httputils.NewHTTPRequestError(fmt.Sprintf("Error: Status %d trying to push checksums %s: %q", res.StatusCode, remote.RemoteName(), errBody), res) } } diff --git a/docs/types.go b/docs/types.go index 939f44b1..da3eaacb 100644 --- a/docs/types.go +++ b/docs/types.go @@ -60,17 +60,9 @@ const ( // RepositoryInfo describes a repository type RepositoryInfo struct { + reference.Named // Index points to registry information Index *registrytypes.IndexInfo - // RemoteName is the remote name of the repository, such as - // "library/ubuntu-12.04-base" - RemoteName reference.Named - // LocalName is the local name of the repository, such as - // "ubuntu-12.04-base" - LocalName reference.Named - // CanonicalName is the canonical name of the repository, such as - // "docker.io/library/ubuntu-12.04-base" - CanonicalName reference.Named // Official indicates whether the repository is considered official. // If the registry is official, and the normalized name does not // contain a '/' (e.g. "foo"), then it is considered an official repo. From 5717c8243d1a92562bf9226b17f11e3ae492f21c Mon Sep 17 00:00:00 2001 From: Aaron Lehmann Date: Fri, 4 Dec 2015 13:42:33 -0800 Subject: [PATCH 346/375] Do not fall back to the V1 protocol when we know we are talking to a V2 registry If we detect a Docker-Distribution-Api-Version header indicating that the registry speaks the V2 protocol, no fallback to V1 should take place. The same applies if a V2 registry operation succeeds while attempting a push or pull. Signed-off-by: Aaron Lehmann --- docs/service.go | 15 ++++++--------- docs/service_v2.go | 21 +++++---------------- 2 files changed, 11 insertions(+), 25 deletions(-) diff --git a/docs/service.go b/docs/service.go index eb5cd6bf..7223cbd8 100644 --- a/docs/service.go +++ b/docs/service.go @@ -6,7 +6,6 @@ import ( "net/url" "strings" - "github.com/docker/distribution/registry/client/auth" "github.com/docker/docker/api/types" registrytypes "github.com/docker/docker/api/types/registry" "github.com/docker/docker/reference" @@ -121,14 +120,12 @@ func (s *Service) ResolveIndex(name string) (*registrytypes.IndexInfo, error) { // APIEndpoint represents a remote API endpoint type APIEndpoint struct { - Mirror bool - URL string - Version APIVersion - Official bool - TrimHostname bool - TLSConfig *tls.Config - VersionHeader string - Versions []auth.APIVersion + Mirror bool + URL string + Version APIVersion + Official bool + TrimHostname bool + TLSConfig *tls.Config } // ToV1Endpoint returns a V1 API endpoint based on the APIEndpoint diff --git a/docs/service_v2.go b/docs/service_v2.go index 8a8cd260..dfdc1569 100644 --- a/docs/service_v2.go +++ b/docs/service_v2.go @@ -4,7 +4,6 @@ import ( "fmt" "strings" - "github.com/docker/distribution/registry/client/auth" "github.com/docker/docker/pkg/tlsconfig" "github.com/docker/docker/reference" ) @@ -52,20 +51,12 @@ func (s *Service) lookupV2Endpoints(repoName reference.Named) (endpoints []APIEn return nil, err } - v2Versions := []auth.APIVersion{ - { - Type: "registry", - Version: "2.0", - }, - } endpoints = []APIEndpoint{ { - URL: "https://" + hostname, - Version: APIVersion2, - TrimHostname: true, - TLSConfig: tlsConfig, - VersionHeader: DefaultRegistryVersionHeader, - Versions: v2Versions, + URL: "https://" + hostname, + Version: APIVersion2, + TrimHostname: true, + TLSConfig: tlsConfig, }, } @@ -75,9 +66,7 @@ func (s *Service) lookupV2Endpoints(repoName reference.Named) (endpoints []APIEn Version: APIVersion2, TrimHostname: true, // used to check if supposed to be secure via InsecureSkipVerify - TLSConfig: tlsConfig, - VersionHeader: DefaultRegistryVersionHeader, - Versions: v2Versions, + TLSConfig: tlsConfig, }) } From 71ddfd40efd02560fb7fe1ed1ae7facd6b82fd5b Mon Sep 17 00:00:00 2001 From: Aaron Lehmann Date: Mon, 21 Dec 2015 15:42:04 -0800 Subject: [PATCH 347/375] When a manifest is not found, allow fallback to v1 PR #18590 caused compatibility issues with registries such as gcr.io which support both the v1 and v2 protocols, but do not provide the same set of images over both protocols. After #18590, pulls from these registries would never use the v1 protocol, because of the Docker-Distribution-Api-Version header indicating that v2 was supported. Fix the problem by making an exception for the case where a manifest is not found. This should allow fallback to v1 in case that image is exposed over the v1 protocol but not the v2 protocol. This avoids the overly aggressive fallback behavior before #18590 which would allow protocol fallback after almost any error, but restores interoperability with mixed v1/v2 registry setups. Fixes #18832 Signed-off-by: Aaron Lehmann --- docs/registry.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/registry.go b/docs/registry.go index fc2959a5..ba1626c1 100644 --- a/docs/registry.go +++ b/docs/registry.go @@ -188,8 +188,8 @@ func addRequiredHeadersToRedirectedRequests(req *http.Request, via []*http.Reque return nil } -func shouldV2Fallback(err errcode.Error) bool { - logrus.Debugf("v2 error: %T %v", err, err) +// ShouldV2Fallback returns true if this error is a reason to fall back to v1. +func ShouldV2Fallback(err errcode.Error) bool { switch err.Code { case errcode.ErrorCodeUnauthorized, v2.ErrorCodeManifestUnknown: return true @@ -220,7 +220,7 @@ func ContinueOnError(err error) bool { case ErrNoSupport: return ContinueOnError(v.Err) case errcode.Error: - return shouldV2Fallback(v) + return ShouldV2Fallback(v) case *client.UnexpectedHTTPResponseError: return true case error: From 693eb14e730b6675455fdc3b01d48b023331663e Mon Sep 17 00:00:00 2001 From: Aaron Lehmann Date: Wed, 23 Dec 2015 15:21:43 -0800 Subject: [PATCH 348/375] Allow v1 protocol fallback when pulling all tags from a repository unknown to v2 registry This is a followup to #18839. That PR relaxed the fallback logic so that if a manifest doesn't exist on v2, or the user is unauthorized to access it, we try again with the v1 protocol. A similar special case is needed for "pull all tags" (docker pull -a). If the v2 registry doesn't recognize the repository, or doesn't allow the user to access it, we should fall back to v1 and try to pull all tags from the v1 registry. Conversely, if the v2 registry does allow us to list the tags, there should be no fallback, even if there are errors pulling those tags. Signed-off-by: Aaron Lehmann --- docs/registry.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/registry.go b/docs/registry.go index ba1626c1..53832f47 100644 --- a/docs/registry.go +++ b/docs/registry.go @@ -191,7 +191,7 @@ func addRequiredHeadersToRedirectedRequests(req *http.Request, via []*http.Reque // ShouldV2Fallback returns true if this error is a reason to fall back to v1. func ShouldV2Fallback(err errcode.Error) bool { switch err.Code { - case errcode.ErrorCodeUnauthorized, v2.ErrorCodeManifestUnknown: + case errcode.ErrorCodeUnauthorized, v2.ErrorCodeManifestUnknown, v2.ErrorCodeNameUnknown: return true } return false From 4f4b3d525784b8b7e473ec9aaae59dfc55c12cce Mon Sep 17 00:00:00 2001 From: David Calavera Date: Tue, 29 Dec 2015 19:27:12 -0500 Subject: [PATCH 349/375] Remove usage of pkg sockets and tlsconfig. - Use the ones provided by docker/go-connections, they are a drop in replacement. - Remove pkg/sockets from docker. - Keep pkg/tlsconfig because libnetwork still needs it and there is a circular dependency issue. Signed-off-by: David Calavera --- docs/registry.go | 2 +- docs/service_v1.go | 2 +- docs/service_v2.go | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/registry.go b/docs/registry.go index ba1626c1..883879cd 100644 --- a/docs/registry.go +++ b/docs/registry.go @@ -23,8 +23,8 @@ import ( "github.com/docker/distribution/registry/client/transport" "github.com/docker/docker/dockerversion" "github.com/docker/docker/pkg/parsers/kernel" - "github.com/docker/docker/pkg/tlsconfig" "github.com/docker/docker/pkg/useragent" + "github.com/docker/go-connections/tlsconfig" ) var ( diff --git a/docs/service_v1.go b/docs/service_v1.go index cd565bc4..340ce957 100644 --- a/docs/service_v1.go +++ b/docs/service_v1.go @@ -4,8 +4,8 @@ import ( "fmt" "strings" - "github.com/docker/docker/pkg/tlsconfig" "github.com/docker/docker/reference" + "github.com/docker/go-connections/tlsconfig" ) func (s *Service) lookupV1Endpoints(repoName reference.Named) (endpoints []APIEndpoint, err error) { diff --git a/docs/service_v2.go b/docs/service_v2.go index dfdc1569..f89326d5 100644 --- a/docs/service_v2.go +++ b/docs/service_v2.go @@ -4,8 +4,8 @@ import ( "fmt" "strings" - "github.com/docker/docker/pkg/tlsconfig" "github.com/docker/docker/reference" + "github.com/docker/go-connections/tlsconfig" ) func (s *Service) lookupV2Endpoints(repoName reference.Named) (endpoints []APIEndpoint, err error) { From 72432a701a7c39cb394c6bc974256137e705ea11 Mon Sep 17 00:00:00 2001 From: Richard Scothern Date: Mon, 4 Jan 2016 14:45:17 -0800 Subject: [PATCH 350/375] Show the legacy registry flag only in the daemon arguments Signed-off-by: Richard Scothern --- docs/config.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/config.go b/docs/config.go index ca7beec4..9708d036 100644 --- a/docs/config.go +++ b/docs/config.go @@ -56,7 +56,7 @@ func (options *Options) InstallFlags(cmd *flag.FlagSet, usageFn func(string) str cmd.Var(&options.Mirrors, []string{"-registry-mirror"}, usageFn("Preferred Docker registry mirror")) options.InsecureRegistries = opts.NewListOpts(ValidateIndexName) cmd.Var(&options.InsecureRegistries, []string{"-insecure-registry"}, usageFn("Enable insecure registry communication")) - cmd.BoolVar(&V2Only, []string{"-disable-legacy-registry"}, false, "Do not contact legacy registries") + cmd.BoolVar(&V2Only, []string{"-disable-legacy-registry"}, false, usageFn("Do not contact legacy registries")) } // NewServiceConfig returns a new instance of ServiceConfig From 981a573eaf22434ebec754ac03e41cbac3f50395 Mon Sep 17 00:00:00 2001 From: David Calavera Date: Mon, 4 Jan 2016 19:05:26 -0500 Subject: [PATCH 351/375] Modify import paths to point to the new engine-api package. Signed-off-by: David Calavera --- docs/auth.go | 4 ++-- docs/auth_test.go | 4 ++-- docs/config.go | 2 +- docs/endpoint.go | 2 +- docs/registry_mock_test.go | 2 +- docs/registry_test.go | 4 ++-- docs/service.go | 4 ++-- docs/session.go | 4 ++-- docs/types.go | 2 +- 9 files changed, 14 insertions(+), 14 deletions(-) diff --git a/docs/auth.go b/docs/auth.go index 34d5d670..7175598c 100644 --- a/docs/auth.go +++ b/docs/auth.go @@ -8,8 +8,8 @@ import ( "strings" "github.com/Sirupsen/logrus" - "github.com/docker/docker/api/types" - registrytypes "github.com/docker/docker/api/types/registry" + "github.com/docker/engine-api/types" + registrytypes "github.com/docker/engine-api/types/registry" ) // Login tries to register/login to the registry server. diff --git a/docs/auth_test.go b/docs/auth_test.go index ff1bd547..caff8667 100644 --- a/docs/auth_test.go +++ b/docs/auth_test.go @@ -3,8 +3,8 @@ package registry import ( "testing" - "github.com/docker/docker/api/types" - registrytypes "github.com/docker/docker/api/types/registry" + "github.com/docker/engine-api/types" + registrytypes "github.com/docker/engine-api/types/registry" ) func buildAuthConfigs() map[string]types.AuthConfig { diff --git a/docs/config.go b/docs/config.go index 9708d036..ec8ec271 100644 --- a/docs/config.go +++ b/docs/config.go @@ -7,10 +7,10 @@ import ( "net/url" "strings" - registrytypes "github.com/docker/docker/api/types/registry" "github.com/docker/docker/opts" flag "github.com/docker/docker/pkg/mflag" "github.com/docker/docker/reference" + registrytypes "github.com/docker/engine-api/types/registry" ) // Options holds command line options. diff --git a/docs/endpoint.go b/docs/endpoint.go index 43ac9053..258a9c28 100644 --- a/docs/endpoint.go +++ b/docs/endpoint.go @@ -13,7 +13,7 @@ import ( "github.com/Sirupsen/logrus" "github.com/docker/distribution/registry/api/v2" "github.com/docker/distribution/registry/client/transport" - registrytypes "github.com/docker/docker/api/types/registry" + registrytypes "github.com/docker/engine-api/types/registry" ) // for mocking in unit tests diff --git a/docs/registry_mock_test.go b/docs/registry_mock_test.go index be04e346..057afac1 100644 --- a/docs/registry_mock_test.go +++ b/docs/registry_mock_test.go @@ -15,9 +15,9 @@ import ( "testing" "time" - registrytypes "github.com/docker/docker/api/types/registry" "github.com/docker/docker/opts" "github.com/docker/docker/reference" + registrytypes "github.com/docker/engine-api/types/registry" "github.com/gorilla/mux" "github.com/Sirupsen/logrus" diff --git a/docs/registry_test.go b/docs/registry_test.go index 46d2818f..7630d9a5 100644 --- a/docs/registry_test.go +++ b/docs/registry_test.go @@ -9,9 +9,9 @@ import ( "testing" "github.com/docker/distribution/registry/client/transport" - "github.com/docker/docker/api/types" - registrytypes "github.com/docker/docker/api/types/registry" "github.com/docker/docker/reference" + "github.com/docker/engine-api/types" + registrytypes "github.com/docker/engine-api/types/registry" ) var ( diff --git a/docs/service.go b/docs/service.go index 7223cbd8..dbdf1731 100644 --- a/docs/service.go +++ b/docs/service.go @@ -6,9 +6,9 @@ import ( "net/url" "strings" - "github.com/docker/docker/api/types" - registrytypes "github.com/docker/docker/api/types/registry" "github.com/docker/docker/reference" + "github.com/docker/engine-api/types" + registrytypes "github.com/docker/engine-api/types/registry" ) // Service is a registry service. It tracks configuration data such as a list diff --git a/docs/session.go b/docs/session.go index 494b84bf..57acbc0c 100644 --- a/docs/session.go +++ b/docs/session.go @@ -19,13 +19,13 @@ import ( "strings" "github.com/Sirupsen/logrus" - "github.com/docker/docker/api/types" - registrytypes "github.com/docker/docker/api/types/registry" "github.com/docker/docker/pkg/httputils" "github.com/docker/docker/pkg/ioutils" "github.com/docker/docker/pkg/stringid" "github.com/docker/docker/pkg/tarsum" "github.com/docker/docker/reference" + "github.com/docker/engine-api/types" + registrytypes "github.com/docker/engine-api/types/registry" ) var ( diff --git a/docs/types.go b/docs/types.go index da3eaacb..ee88276e 100644 --- a/docs/types.go +++ b/docs/types.go @@ -1,8 +1,8 @@ package registry import ( - registrytypes "github.com/docker/docker/api/types/registry" "github.com/docker/docker/reference" + registrytypes "github.com/docker/engine-api/types/registry" ) // RepositoryData tracks the image list, list of endpoints, and list of tokens From 6e85a8d94aa1ac0320e9c88ddd69eba39ea0b388 Mon Sep 17 00:00:00 2001 From: Daniel Nephin Date: Mon, 4 Jan 2016 13:36:01 -0500 Subject: [PATCH 352/375] Remove the use of dockerversion from the registry package Signed-off-by: Daniel Nephin --- docs/endpoint.go | 8 ++++---- docs/endpoint_test.go | 2 +- docs/registry.go | 32 +++++++------------------------- docs/registry_test.go | 15 ++++++++------- docs/service.go | 12 ++++++------ 5 files changed, 26 insertions(+), 43 deletions(-) diff --git a/docs/endpoint.go b/docs/endpoint.go index 258a9c28..ef00431f 100644 --- a/docs/endpoint.go +++ b/docs/endpoint.go @@ -45,12 +45,12 @@ func scanForAPIVersion(address string) (string, APIVersion) { // NewEndpoint parses the given address to return a registry endpoint. v can be used to // specify a specific endpoint version -func NewEndpoint(index *registrytypes.IndexInfo, metaHeaders http.Header, v APIVersion) (*Endpoint, error) { +func NewEndpoint(index *registrytypes.IndexInfo, userAgent string, metaHeaders http.Header, v APIVersion) (*Endpoint, error) { tlsConfig, err := newTLSConfig(index.Name, index.Secure) if err != nil { return nil, err } - endpoint, err := newEndpoint(GetAuthConfigKey(index), tlsConfig, metaHeaders) + endpoint, err := newEndpoint(GetAuthConfigKey(index), tlsConfig, userAgent, metaHeaders) if err != nil { return nil, err } @@ -91,7 +91,7 @@ func validateEndpoint(endpoint *Endpoint) error { return nil } -func newEndpoint(address string, tlsConfig *tls.Config, metaHeaders http.Header) (*Endpoint, error) { +func newEndpoint(address string, tlsConfig *tls.Config, userAgent string, metaHeaders http.Header) (*Endpoint, error) { var ( endpoint = new(Endpoint) trimmedAddress string @@ -112,7 +112,7 @@ func newEndpoint(address string, tlsConfig *tls.Config, metaHeaders http.Header) // TODO(tiborvass): make sure a ConnectTimeout transport is used tr := NewTransport(tlsConfig) - endpoint.client = HTTPClient(transport.NewTransport(tr, DockerHeaders(metaHeaders)...)) + endpoint.client = HTTPClient(transport.NewTransport(tr, DockerHeaders(userAgent, metaHeaders)...)) return endpoint, nil } diff --git a/docs/endpoint_test.go b/docs/endpoint_test.go index ee301dbd..4677e0c9 100644 --- a/docs/endpoint_test.go +++ b/docs/endpoint_test.go @@ -19,7 +19,7 @@ func TestEndpointParse(t *testing.T) { {"0.0.0.0:5000", "https://0.0.0.0:5000/v0/"}, } for _, td := range testData { - e, err := newEndpoint(td.str, nil, nil) + e, err := newEndpoint(td.str, nil, "", nil) if err != nil { t.Errorf("%q: %s", td.str, err) } diff --git a/docs/registry.go b/docs/registry.go index 643fa56e..f4ddc15a 100644 --- a/docs/registry.go +++ b/docs/registry.go @@ -21,9 +21,6 @@ import ( "github.com/docker/distribution/registry/api/v2" "github.com/docker/distribution/registry/client" "github.com/docker/distribution/registry/client/transport" - "github.com/docker/docker/dockerversion" - "github.com/docker/docker/pkg/parsers/kernel" - "github.com/docker/docker/pkg/useragent" "github.com/docker/go-connections/tlsconfig" ) @@ -34,23 +31,7 @@ var ( errLoginRequired = errors.New("Authentication is required.") ) -// dockerUserAgent is the User-Agent the Docker client uses to identify itself. -// It is populated on init(), comprising version information of different components. -var dockerUserAgent string - func init() { - httpVersion := make([]useragent.VersionInfo, 0, 6) - httpVersion = append(httpVersion, useragent.VersionInfo{Name: "docker", Version: dockerversion.Version}) - httpVersion = append(httpVersion, useragent.VersionInfo{Name: "go", Version: runtime.Version()}) - httpVersion = append(httpVersion, useragent.VersionInfo{Name: "git-commit", Version: dockerversion.GitCommit}) - if kernelVersion, err := kernel.GetKernelVersion(); err == nil { - httpVersion = append(httpVersion, useragent.VersionInfo{Name: "kernel", Version: kernelVersion.String()}) - } - httpVersion = append(httpVersion, useragent.VersionInfo{Name: "os", Version: runtime.GOOS}) - httpVersion = append(httpVersion, useragent.VersionInfo{Name: "arch", Version: runtime.GOARCH}) - - dockerUserAgent = useragent.AppendVersions("", httpVersion...) - if runtime.GOOS != "linux" { V2Only = true } @@ -130,12 +111,13 @@ func ReadCertsDirectory(tlsConfig *tls.Config, directory string) error { return nil } -// DockerHeaders returns request modifiers that ensure requests have -// the User-Agent header set to dockerUserAgent and that metaHeaders -// are added. -func DockerHeaders(metaHeaders http.Header) []transport.RequestModifier { - modifiers := []transport.RequestModifier{ - transport.NewHeaderRequestModifier(http.Header{"User-Agent": []string{dockerUserAgent}}), +// DockerHeaders returns request modifiers with a User-Agent and metaHeaders +func DockerHeaders(userAgent string, metaHeaders http.Header) []transport.RequestModifier { + modifiers := []transport.RequestModifier{} + if userAgent != "" { + modifiers = append(modifiers, transport.NewHeaderRequestModifier(http.Header{ + "User-Agent": []string{userAgent}, + })) } if metaHeaders != nil { modifiers = append(modifiers, transport.NewHeaderRequestModifier(metaHeaders)) diff --git a/docs/registry_test.go b/docs/registry_test.go index 7630d9a5..98a3aa1c 100644 --- a/docs/registry_test.go +++ b/docs/registry_test.go @@ -25,12 +25,13 @@ const ( func spawnTestRegistrySession(t *testing.T) *Session { authConfig := &types.AuthConfig{} - endpoint, err := NewEndpoint(makeIndex("/v1/"), nil, APIVersionUnknown) + endpoint, err := NewEndpoint(makeIndex("/v1/"), "", nil, APIVersionUnknown) if err != nil { t.Fatal(err) } + userAgent := "docker test client" var tr http.RoundTripper = debugTransport{NewTransport(nil), t.Log} - tr = transport.NewTransport(AuthTransport(tr, authConfig, false), DockerHeaders(nil)...) + tr = transport.NewTransport(AuthTransport(tr, authConfig, false), DockerHeaders(userAgent, nil)...) client := HTTPClient(tr) r, err := NewSession(client, authConfig, endpoint) if err != nil { @@ -52,7 +53,7 @@ func spawnTestRegistrySession(t *testing.T) *Session { func TestPingRegistryEndpoint(t *testing.T) { testPing := func(index *registrytypes.IndexInfo, expectedStandalone bool, assertMessage string) { - ep, err := NewEndpoint(index, nil, APIVersionUnknown) + ep, err := NewEndpoint(index, "", nil, APIVersionUnknown) if err != nil { t.Fatal(err) } @@ -72,7 +73,7 @@ func TestPingRegistryEndpoint(t *testing.T) { func TestEndpoint(t *testing.T) { // Simple wrapper to fail test if err != nil expandEndpoint := func(index *registrytypes.IndexInfo) *Endpoint { - endpoint, err := NewEndpoint(index, nil, APIVersionUnknown) + endpoint, err := NewEndpoint(index, "", nil, APIVersionUnknown) if err != nil { t.Fatal(err) } @@ -81,7 +82,7 @@ func TestEndpoint(t *testing.T) { assertInsecureIndex := func(index *registrytypes.IndexInfo) { index.Secure = true - _, err := NewEndpoint(index, nil, APIVersionUnknown) + _, err := NewEndpoint(index, "", nil, APIVersionUnknown) assertNotEqual(t, err, nil, index.Name+": Expected error for insecure index") assertEqual(t, strings.Contains(err.Error(), "insecure-registry"), true, index.Name+": Expected insecure-registry error for insecure index") index.Secure = false @@ -89,7 +90,7 @@ func TestEndpoint(t *testing.T) { assertSecureIndex := func(index *registrytypes.IndexInfo) { index.Secure = true - _, err := NewEndpoint(index, nil, APIVersionUnknown) + _, err := NewEndpoint(index, "", nil, APIVersionUnknown) assertNotEqual(t, err, nil, index.Name+": Expected cert error for secure index") assertEqual(t, strings.Contains(err.Error(), "certificate signed by unknown authority"), true, index.Name+": Expected cert error for secure index") index.Secure = false @@ -155,7 +156,7 @@ func TestEndpoint(t *testing.T) { } for _, address := range badEndpoints { index.Name = address - _, err := NewEndpoint(index, nil, APIVersionUnknown) + _, err := NewEndpoint(index, "", nil, APIVersionUnknown) checkNotEqual(t, err, nil, "Expected error while expanding bad endpoint") } } diff --git a/docs/service.go b/docs/service.go index dbdf1731..861cdb46 100644 --- a/docs/service.go +++ b/docs/service.go @@ -28,7 +28,7 @@ func NewService(options *Options) *Service { // Auth contacts the public registry with the provided credentials, // and returns OK if authentication was successful. // It can be used to verify the validity of a client's credentials. -func (s *Service) Auth(authConfig *types.AuthConfig) (string, error) { +func (s *Service) Auth(authConfig *types.AuthConfig, userAgent string) (string, error) { addr := authConfig.ServerAddress if addr == "" { // Use the official registry address if not specified. @@ -45,7 +45,7 @@ func (s *Service) Auth(authConfig *types.AuthConfig) (string, error) { endpointVersion = APIVersion2 } - endpoint, err := NewEndpoint(index, nil, endpointVersion) + endpoint, err := NewEndpoint(index, userAgent, nil, endpointVersion) if err != nil { return "", err } @@ -72,7 +72,7 @@ func splitReposSearchTerm(reposName string) (string, string) { // Search queries the public registry for images matching the specified // search terms, and returns the results. -func (s *Service) Search(term string, authConfig *types.AuthConfig, headers map[string][]string) (*registrytypes.SearchResults, error) { +func (s *Service) Search(term string, authConfig *types.AuthConfig, userAgent string, headers map[string][]string) (*registrytypes.SearchResults, error) { if err := validateNoSchema(term); err != nil { return nil, err } @@ -85,7 +85,7 @@ func (s *Service) Search(term string, authConfig *types.AuthConfig, headers map[ } // *TODO: Search multiple indexes. - endpoint, err := NewEndpoint(index, http.Header(headers), APIVersionUnknown) + endpoint, err := NewEndpoint(index, userAgent, http.Header(headers), APIVersionUnknown) if err != nil { return nil, err } @@ -129,8 +129,8 @@ type APIEndpoint struct { } // ToV1Endpoint returns a V1 API endpoint based on the APIEndpoint -func (e APIEndpoint) ToV1Endpoint(metaHeaders http.Header) (*Endpoint, error) { - return newEndpoint(e.URL, e.TLSConfig, metaHeaders) +func (e APIEndpoint) ToV1Endpoint(userAgent string, metaHeaders http.Header) (*Endpoint, error) { + return newEndpoint(e.URL, e.TLSConfig, userAgent, metaHeaders) } // TLSConfig constructs a client TLS configuration based on server defaults From 0e06c1cad1e97fc912c74f9bf2a71c02c2179bac Mon Sep 17 00:00:00 2001 From: Aaron Lehmann Date: Wed, 20 Jan 2016 10:53:41 -0800 Subject: [PATCH 353/375] Clarify error message when a .cert file is missing a corresponding key The daemon uses two similar filename extensions to identify different kinds of certificates. ".crt" files are interpreted as CA certificates, and ".cert" files are interprted as client certificates. If a CA certificate is accidentally given the extension ".cert", it will lead to the following error message: Missing key ca.key for certificate ca.cert To make this slightly less confusing, clarify the error message with a note that CA certificates should use the extension ".crt". Signed-off-by: Aaron Lehmann --- docs/registry.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/registry.go b/docs/registry.go index 643fa56e..bacc4aed 100644 --- a/docs/registry.go +++ b/docs/registry.go @@ -109,7 +109,7 @@ func ReadCertsDirectory(tlsConfig *tls.Config, directory string) error { keyName := certName[:len(certName)-5] + ".key" logrus.Debugf("cert: %s", filepath.Join(directory, f.Name())) if !hasFile(fs, keyName) { - return fmt.Errorf("Missing key %s for certificate %s", keyName, certName) + return fmt.Errorf("Missing key %s for client certificate %s. Note that CA certificates should use the extension .crt.", keyName, certName) } cert, err := tls.LoadX509KeyPair(filepath.Join(directory, certName), filepath.Join(directory, keyName)) if err != nil { @@ -122,7 +122,7 @@ func ReadCertsDirectory(tlsConfig *tls.Config, directory string) error { certName := keyName[:len(keyName)-4] + ".cert" logrus.Debugf("key: %s", filepath.Join(directory, f.Name())) if !hasFile(fs, certName) { - return fmt.Errorf("Missing certificate %s for key %s", certName, keyName) + return fmt.Errorf("Missing client certificate %s for key %s", certName, keyName) } } } From 377f556464608d99712d9921fcec02bd60060016 Mon Sep 17 00:00:00 2001 From: David Calavera Date: Tue, 26 Jan 2016 13:30:58 -0500 Subject: [PATCH 354/375] Respond with 401 when there is an unauthorized error from the registry. Signed-off-by: David Calavera --- docs/registry.go | 1 - docs/session.go | 7 ++++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/docs/registry.go b/docs/registry.go index f135cc1b..6214d41a 100644 --- a/docs/registry.go +++ b/docs/registry.go @@ -28,7 +28,6 @@ var ( // ErrAlreadyExists is an error returned if an image being pushed // already exists on the remote side ErrAlreadyExists = errors.New("Image already exists") - errLoginRequired = errors.New("Authentication is required.") ) func init() { diff --git a/docs/session.go b/docs/session.go index 57acbc0c..4b18d0d1 100644 --- a/docs/session.go +++ b/docs/session.go @@ -19,6 +19,7 @@ import ( "strings" "github.com/Sirupsen/logrus" + "github.com/docker/distribution/registry/api/errcode" "github.com/docker/docker/pkg/httputils" "github.com/docker/docker/pkg/ioutils" "github.com/docker/docker/pkg/stringid" @@ -213,7 +214,7 @@ func (r *Session) GetRemoteHistory(imgID, registry string) ([]string, error) { defer res.Body.Close() if res.StatusCode != 200 { if res.StatusCode == 401 { - return nil, errLoginRequired + return nil, errcode.ErrorCodeUnauthorized.WithArgs() } return nil, httputils.NewHTTPRequestError(fmt.Sprintf("Server error: %d trying to fetch remote history for %s", res.StatusCode, imgID), res) } @@ -427,7 +428,7 @@ func (r *Session) GetRepositoryData(name reference.Named) (*RepositoryData, erro } defer res.Body.Close() if res.StatusCode == 401 { - return nil, errLoginRequired + return nil, errcode.ErrorCodeUnauthorized.WithArgs() } // TODO: Right now we're ignoring checksums in the response body. // In the future, we need to use them to check image validity. @@ -661,7 +662,7 @@ func (r *Session) PushImageJSONIndex(remote reference.Named, imgList []*ImgData, defer res.Body.Close() if res.StatusCode == 401 { - return nil, errLoginRequired + return nil, errcode.ErrorCodeUnauthorized.WithArgs() } var tokens, endpoints []string From 4bb475cd3ce2690339ee19cebf35d9c28dac83e4 Mon Sep 17 00:00:00 2001 From: Aaron Lehmann Date: Thu, 11 Feb 2016 14:08:49 -0800 Subject: [PATCH 355/375] Push/pull errors improvement and cleanup Several improvements to error handling: - Introduce ImageConfigPullError type, wrapping errors related to downloading the image configuration blob in schema2. This allows for a more descriptive error message to be seen by the end user. - Change some logrus.Debugf calls that display errors to logrus.Errorf. Add log lines in the push/pull fallback cases to make sure the errors leading to the fallback are shown. - Move error-related types and functions which are only used by the distribution package out of the registry package. Signed-off-by: Aaron Lehmann --- docs/registry.go | 49 ------------------------------------------------ 1 file changed, 49 deletions(-) diff --git a/docs/registry.go b/docs/registry.go index 6214d41a..9071d9dc 100644 --- a/docs/registry.go +++ b/docs/registry.go @@ -13,13 +13,9 @@ import ( "path/filepath" "runtime" "strings" - "syscall" "time" "github.com/Sirupsen/logrus" - "github.com/docker/distribution/registry/api/errcode" - "github.com/docker/distribution/registry/api/v2" - "github.com/docker/distribution/registry/client" "github.com/docker/distribution/registry/client/transport" "github.com/docker/go-connections/tlsconfig" ) @@ -169,51 +165,6 @@ func addRequiredHeadersToRedirectedRequests(req *http.Request, via []*http.Reque return nil } -// ShouldV2Fallback returns true if this error is a reason to fall back to v1. -func ShouldV2Fallback(err errcode.Error) bool { - switch err.Code { - case errcode.ErrorCodeUnauthorized, v2.ErrorCodeManifestUnknown, v2.ErrorCodeNameUnknown: - return true - } - return false -} - -// ErrNoSupport is an error type used for errors indicating that an operation -// is not supported. It encapsulates a more specific error. -type ErrNoSupport struct{ Err error } - -func (e ErrNoSupport) Error() string { - if e.Err == nil { - return "not supported" - } - return e.Err.Error() -} - -// ContinueOnError returns true if we should fallback to the next endpoint -// as a result of this error. -func ContinueOnError(err error) bool { - switch v := err.(type) { - case errcode.Errors: - if len(v) == 0 { - return true - } - return ContinueOnError(v[0]) - case ErrNoSupport: - return ContinueOnError(v.Err) - case errcode.Error: - return ShouldV2Fallback(v) - case *client.UnexpectedHTTPResponseError: - return true - case error: - return !strings.Contains(err.Error(), strings.ToLower(syscall.ENOSPC.Error())) - } - // let's be nice and fallback if the error is a completely - // unexpected one. - // If new errors have to be handled in some way, please - // add them to the switch above. - return true -} - // NewTransport returns a new HTTP transport. If tlsConfig is nil, it uses the // default TLS configuration. func NewTransport(tlsConfig *tls.Config) *http.Transport { From 9a2cef38e31bbdeaff74c2e724990df64182c1db Mon Sep 17 00:00:00 2001 From: Aaron Lehmann Date: Wed, 17 Feb 2016 16:53:25 -0800 Subject: [PATCH 356/375] Change APIEndpoint to contain the URL in a parsed format This allows easier URL handling in code that uses APIEndpoint. If we continued to store the URL unparsed, it would require redundant parsing whenver we want to extract information from it. Also, parsing the URL earlier should give improve validation. Signed-off-by: Aaron Lehmann --- docs/config.go | 4 ++-- docs/config_unix.go | 16 +++++++++++--- docs/config_windows.go | 13 +++++++++--- docs/endpoint.go | 47 +++++++++++++++++++++++++++--------------- docs/endpoint_test.go | 2 +- docs/registry_test.go | 2 +- docs/service.go | 10 +++------ docs/service_v1.go | 11 ++++++++-- docs/service_v2.go | 22 ++++++++++++++++---- 9 files changed, 87 insertions(+), 40 deletions(-) diff --git a/docs/config.go b/docs/config.go index ec8ec271..ebad6f86 100644 --- a/docs/config.go +++ b/docs/config.go @@ -19,7 +19,7 @@ type Options struct { InsecureRegistries opts.ListOpts } -const ( +var ( // DefaultNamespace is the default namespace DefaultNamespace = "docker.io" // DefaultRegistryVersionHeader is the name of the default HTTP header @@ -27,7 +27,7 @@ const ( DefaultRegistryVersionHeader = "Docker-Distribution-Api-Version" // IndexServer is the v1 registry server used for user auth + account creation - IndexServer = DefaultV1Registry + "/v1/" + IndexServer = DefaultV1Registry.String() + "/v1/" // IndexName is the name of the index IndexName = "docker.io" diff --git a/docs/config_unix.go b/docs/config_unix.go index df970181..c3c19162 100644 --- a/docs/config_unix.go +++ b/docs/config_unix.go @@ -2,12 +2,22 @@ package registry -const ( +import ( + "net/url" +) + +var ( // DefaultV1Registry is the URI of the default v1 registry - DefaultV1Registry = "https://index.docker.io" + DefaultV1Registry = &url.URL{ + Scheme: "https", + Host: "index.docker.io", + } // DefaultV2Registry is the URI of the default v2 registry - DefaultV2Registry = "https://registry-1.docker.io" + DefaultV2Registry = &url.URL{ + Scheme: "https", + Host: "registry-1.docker.io", + } ) var ( diff --git a/docs/config_windows.go b/docs/config_windows.go index d01b2618..f1ee488b 100644 --- a/docs/config_windows.go +++ b/docs/config_windows.go @@ -1,21 +1,28 @@ package registry import ( + "net/url" "os" "path/filepath" "strings" ) -const ( +var ( // DefaultV1Registry is the URI of the default v1 registry - DefaultV1Registry = "https://registry-win-tp3.docker.io" + DefaultV1Registry = &url.URL{ + Scheme: "https", + Host: "registry-win-tp3.docker.io", + } // DefaultV2Registry is the URI of the default (official) v2 registry. // This is the windows-specific endpoint. // // Currently it is a TEMPORARY link that allows Microsoft to continue // development of Docker Engine for Windows. - DefaultV2Registry = "https://registry-win-tp3.docker.io" + DefaultV2Registry = &url.URL{ + Scheme: "https", + Host: "registry-win-tp3.docker.io", + } ) // CertsDir is the directory where certificates are stored diff --git a/docs/endpoint.go b/docs/endpoint.go index ef00431f..b056caf1 100644 --- a/docs/endpoint.go +++ b/docs/endpoint.go @@ -50,10 +50,12 @@ func NewEndpoint(index *registrytypes.IndexInfo, userAgent string, metaHeaders h if err != nil { return nil, err } - endpoint, err := newEndpoint(GetAuthConfigKey(index), tlsConfig, userAgent, metaHeaders) + + endpoint, err := newEndpointFromStr(GetAuthConfigKey(index), tlsConfig, userAgent, metaHeaders) if err != nil { return nil, err } + if v != APIVersionUnknown { endpoint.Version = v } @@ -91,24 +93,14 @@ func validateEndpoint(endpoint *Endpoint) error { return nil } -func newEndpoint(address string, tlsConfig *tls.Config, userAgent string, metaHeaders http.Header) (*Endpoint, error) { - var ( - endpoint = new(Endpoint) - trimmedAddress string - err error - ) - - if !strings.HasPrefix(address, "http") { - address = "https://" + address +func newEndpoint(address url.URL, tlsConfig *tls.Config, userAgent string, metaHeaders http.Header) (*Endpoint, error) { + endpoint := &Endpoint{ + IsSecure: (tlsConfig == nil || !tlsConfig.InsecureSkipVerify), + URL: new(url.URL), + Version: APIVersionUnknown, } - endpoint.IsSecure = (tlsConfig == nil || !tlsConfig.InsecureSkipVerify) - - trimmedAddress, endpoint.Version = scanForAPIVersion(address) - - if endpoint.URL, err = url.Parse(trimmedAddress); err != nil { - return nil, err - } + *endpoint.URL = address // TODO(tiborvass): make sure a ConnectTimeout transport is used tr := NewTransport(tlsConfig) @@ -116,6 +108,27 @@ func newEndpoint(address string, tlsConfig *tls.Config, userAgent string, metaHe return endpoint, nil } +func newEndpointFromStr(address string, tlsConfig *tls.Config, userAgent string, metaHeaders http.Header) (*Endpoint, error) { + if !strings.HasPrefix(address, "http://") && !strings.HasPrefix(address, "https://") { + address = "https://" + address + } + + trimmedAddress, detectedVersion := scanForAPIVersion(address) + + uri, err := url.Parse(trimmedAddress) + if err != nil { + return nil, err + } + + endpoint, err := newEndpoint(*uri, tlsConfig, userAgent, metaHeaders) + if err != nil { + return nil, err + } + + endpoint.Version = detectedVersion + return endpoint, nil +} + // Endpoint stores basic information about a registry endpoint. type Endpoint struct { client *http.Client diff --git a/docs/endpoint_test.go b/docs/endpoint_test.go index 4677e0c9..fa18eea0 100644 --- a/docs/endpoint_test.go +++ b/docs/endpoint_test.go @@ -19,7 +19,7 @@ func TestEndpointParse(t *testing.T) { {"0.0.0.0:5000", "https://0.0.0.0:5000/v0/"}, } for _, td := range testData { - e, err := newEndpoint(td.str, nil, "", nil) + e, err := newEndpointFromStr(td.str, nil, "", nil) if err != nil { t.Errorf("%q: %s", td.str, err) } diff --git a/docs/registry_test.go b/docs/registry_test.go index 98a3aa1c..33d85347 100644 --- a/docs/registry_test.go +++ b/docs/registry_test.go @@ -673,7 +673,7 @@ func TestNewIndexInfo(t *testing.T) { func TestMirrorEndpointLookup(t *testing.T) { containsMirror := func(endpoints []APIEndpoint) bool { for _, pe := range endpoints { - if pe.URL == "my.mirror" { + if pe.URL.Host == "my.mirror" { return true } } diff --git a/docs/service.go b/docs/service.go index 861cdb46..bba1e842 100644 --- a/docs/service.go +++ b/docs/service.go @@ -121,7 +121,7 @@ func (s *Service) ResolveIndex(name string) (*registrytypes.IndexInfo, error) { // APIEndpoint represents a remote API endpoint type APIEndpoint struct { Mirror bool - URL string + URL *url.URL Version APIVersion Official bool TrimHostname bool @@ -130,7 +130,7 @@ type APIEndpoint struct { // ToV1Endpoint returns a V1 API endpoint based on the APIEndpoint func (e APIEndpoint) ToV1Endpoint(userAgent string, metaHeaders http.Header) (*Endpoint, error) { - return newEndpoint(e.URL, e.TLSConfig, userAgent, metaHeaders) + return newEndpoint(*e.URL, e.TLSConfig, userAgent, metaHeaders) } // TLSConfig constructs a client TLS configuration based on server defaults @@ -138,11 +138,7 @@ func (s *Service) TLSConfig(hostname string) (*tls.Config, error) { return newTLSConfig(hostname, isSecureIndex(s.Config, hostname)) } -func (s *Service) tlsConfigForMirror(mirror string) (*tls.Config, error) { - mirrorURL, err := url.Parse(mirror) - if err != nil { - return nil, err - } +func (s *Service) tlsConfigForMirror(mirrorURL *url.URL) (*tls.Config, error) { return s.TLSConfig(mirrorURL.Host) } diff --git a/docs/service_v1.go b/docs/service_v1.go index 340ce957..5328b8f1 100644 --- a/docs/service_v1.go +++ b/docs/service_v1.go @@ -2,6 +2,7 @@ package registry import ( "fmt" + "net/url" "strings" "github.com/docker/docker/reference" @@ -36,7 +37,10 @@ func (s *Service) lookupV1Endpoints(repoName reference.Named) (endpoints []APIEn endpoints = []APIEndpoint{ { - URL: "https://" + hostname, + URL: &url.URL{ + Scheme: "https", + Host: hostname, + }, Version: APIVersion1, TrimHostname: true, TLSConfig: tlsConfig, @@ -45,7 +49,10 @@ func (s *Service) lookupV1Endpoints(repoName reference.Named) (endpoints []APIEn if tlsConfig.InsecureSkipVerify { endpoints = append(endpoints, APIEndpoint{ // or this - URL: "http://" + hostname, + URL: &url.URL{ + Scheme: "http", + Host: hostname, + }, Version: APIVersion1, TrimHostname: true, // used to check if supposed to be secure via InsecureSkipVerify diff --git a/docs/service_v2.go b/docs/service_v2.go index f89326d5..4dbbb9fa 100644 --- a/docs/service_v2.go +++ b/docs/service_v2.go @@ -2,6 +2,7 @@ package registry import ( "fmt" + "net/url" "strings" "github.com/docker/docker/reference" @@ -15,12 +16,19 @@ func (s *Service) lookupV2Endpoints(repoName reference.Named) (endpoints []APIEn if strings.HasPrefix(nameString, DefaultNamespace+"/") { // v2 mirrors for _, mirror := range s.Config.Mirrors { - mirrorTLSConfig, err := s.tlsConfigForMirror(mirror) + if !strings.HasPrefix(mirror, "http://") && !strings.HasPrefix(mirror, "https://") { + mirror = "https://" + mirror + } + mirrorURL, err := url.Parse(mirror) + if err != nil { + return nil, err + } + mirrorTLSConfig, err := s.tlsConfigForMirror(mirrorURL) if err != nil { return nil, err } endpoints = append(endpoints, APIEndpoint{ - URL: mirror, + URL: mirrorURL, // guess mirrors are v2 Version: APIVersion2, Mirror: true, @@ -53,7 +61,10 @@ func (s *Service) lookupV2Endpoints(repoName reference.Named) (endpoints []APIEn endpoints = []APIEndpoint{ { - URL: "https://" + hostname, + URL: &url.URL{ + Scheme: "https", + Host: hostname, + }, Version: APIVersion2, TrimHostname: true, TLSConfig: tlsConfig, @@ -62,7 +73,10 @@ func (s *Service) lookupV2Endpoints(repoName reference.Named) (endpoints []APIEn if tlsConfig.InsecureSkipVerify { endpoints = append(endpoints, APIEndpoint{ - URL: "http://" + hostname, + URL: &url.URL{ + Scheme: "http", + Host: hostname, + }, Version: APIVersion2, TrimHostname: true, // used to check if supposed to be secure via InsecureSkipVerify From e123ca925e4b027a17a1f39a6387a0c4c62a1ffd Mon Sep 17 00:00:00 2001 From: Ken Cochrane Date: Mon, 29 Feb 2016 17:51:36 -0800 Subject: [PATCH 357/375] Remove email address field from login This removes the email prompt when you use docker login, and also removes the ability to register via the docker cli. Docker login, will strictly be used for logging into a registry server. Signed-off-by: Ken Cochrane --- docs/auth.go | 108 +++++++++++----------------------------------- docs/auth_test.go | 17 +++----- docs/session.go | 1 - 3 files changed, 29 insertions(+), 97 deletions(-) diff --git a/docs/auth.go b/docs/auth.go index 7175598c..bd7bd52d 100644 --- a/docs/auth.go +++ b/docs/auth.go @@ -1,7 +1,6 @@ package registry import ( - "encoding/json" "fmt" "io/ioutil" "net/http" @@ -24,11 +23,8 @@ func Login(authConfig *types.AuthConfig, registryEndpoint *Endpoint) (string, er // loginV1 tries to register/login to the v1 registry server. func loginV1(authConfig *types.AuthConfig, registryEndpoint *Endpoint) (string, error) { var ( - status string - respBody []byte - err error - respStatusCode = 0 - serverAddress = authConfig.ServerAddress + err error + serverAddress = authConfig.ServerAddress ) logrus.Debugf("attempting v1 login to registry endpoint %s", registryEndpoint) @@ -39,93 +35,37 @@ func loginV1(authConfig *types.AuthConfig, registryEndpoint *Endpoint) (string, loginAgainstOfficialIndex := serverAddress == IndexServer - // to avoid sending the server address to the server it should be removed before being marshaled - authCopy := *authConfig - authCopy.ServerAddress = "" - - jsonBody, err := json.Marshal(authCopy) + req, err := http.NewRequest("GET", serverAddress+"users/", nil) + req.SetBasicAuth(authConfig.Username, authConfig.Password) + resp, err := registryEndpoint.client.Do(req) if err != nil { - return "", fmt.Errorf("Config Error: %s", err) + return "", err } - - // using `bytes.NewReader(jsonBody)` here causes the server to respond with a 411 status. - b := strings.NewReader(string(jsonBody)) - resp1, err := registryEndpoint.client.Post(serverAddress+"users/", "application/json; charset=utf-8", b) + defer resp.Body.Close() + body, err := ioutil.ReadAll(resp.Body) if err != nil { - return "", fmt.Errorf("Server Error: %s", err) + return "", err } - defer resp1.Body.Close() - respStatusCode = resp1.StatusCode - respBody, err = ioutil.ReadAll(resp1.Body) - if err != nil { - return "", fmt.Errorf("Server Error: [%#v] %s", respStatusCode, err) - } - - if respStatusCode == 201 { + if resp.StatusCode == http.StatusOK { + return "Login Succeeded", nil + } else if resp.StatusCode == http.StatusUnauthorized { if loginAgainstOfficialIndex { - status = "Account created. Please use the confirmation link we sent" + - " to your e-mail to activate it." - } else { - // *TODO: Use registry configuration to determine what this says, if anything? - status = "Account created. Please see the documentation of the registry " + serverAddress + " for instructions how to activate it." + return "", fmt.Errorf("Wrong login/password, please try again. Haven't got a Docker ID? Create one at https://hub.docker.com") } - } else if respStatusCode == 400 { - if string(respBody) == "\"Username or email already exists\"" { - req, err := http.NewRequest("GET", serverAddress+"users/", nil) - req.SetBasicAuth(authConfig.Username, authConfig.Password) - resp, err := registryEndpoint.client.Do(req) - if err != nil { - return "", err - } - defer resp.Body.Close() - body, err := ioutil.ReadAll(resp.Body) - if err != nil { - return "", err - } - if resp.StatusCode == 200 { - return "Login Succeeded", nil - } else if resp.StatusCode == 401 { - return "", fmt.Errorf("Wrong login/password, please try again") - } else if resp.StatusCode == 403 { - if loginAgainstOfficialIndex { - return "", fmt.Errorf("Login: Account is not Active. Please check your e-mail for a confirmation link.") - } - // *TODO: Use registry configuration to determine what this says, if anything? - return "", fmt.Errorf("Login: Account is not Active. Please see the documentation of the registry %s for instructions how to activate it.", serverAddress) - } else if resp.StatusCode == 500 { // Issue #14326 - logrus.Errorf("%s returned status code %d. Response Body :\n%s", req.URL.String(), resp.StatusCode, body) - return "", fmt.Errorf("Internal Server Error") - } - return "", fmt.Errorf("Login: %s (Code: %d; Headers: %s)", body, resp.StatusCode, resp.Header) - } - return "", fmt.Errorf("Registration: %s", respBody) - - } else if respStatusCode == 401 { - // This case would happen with private registries where /v1/users is - // protected, so people can use `docker login` as an auth check. - req, err := http.NewRequest("GET", serverAddress+"users/", nil) - req.SetBasicAuth(authConfig.Username, authConfig.Password) - resp, err := registryEndpoint.client.Do(req) - if err != nil { - return "", err - } - defer resp.Body.Close() - body, err := ioutil.ReadAll(resp.Body) - if err != nil { - return "", err - } - if resp.StatusCode == 200 { - return "Login Succeeded", nil - } else if resp.StatusCode == 401 { - return "", fmt.Errorf("Wrong login/password, please try again") - } else { - return "", fmt.Errorf("Login: %s (Code: %d; Headers: %s)", body, - resp.StatusCode, resp.Header) + return "", fmt.Errorf("Wrong login/password, please try again") + } else if resp.StatusCode == http.StatusForbidden { + if loginAgainstOfficialIndex { + return "", fmt.Errorf("Login: Account is not active. Please check your e-mail for a confirmation link.") } + // *TODO: Use registry configuration to determine what this says, if anything? + return "", fmt.Errorf("Login: Account is not active. Please see the documentation of the registry %s for instructions how to activate it.", serverAddress) + } else if resp.StatusCode == http.StatusInternalServerError { // Issue #14326 + logrus.Errorf("%s returned status code %d. Response Body :\n%s", req.URL.String(), resp.StatusCode, body) + return "", fmt.Errorf("Internal Server Error") } else { - return "", fmt.Errorf("Unexpected status code [%d] : %s", respStatusCode, respBody) + return "", fmt.Errorf("Login: %s (Code: %d; Headers: %s)", body, + resp.StatusCode, resp.Header) } - return status, nil } // loginV2 tries to login to the v2 registry server. The given registry endpoint has been diff --git a/docs/auth_test.go b/docs/auth_test.go index caff8667..eedee44e 100644 --- a/docs/auth_test.go +++ b/docs/auth_test.go @@ -14,7 +14,6 @@ func buildAuthConfigs() map[string]types.AuthConfig { authConfigs[registry] = types.AuthConfig{ Username: "docker-user", Password: "docker-pass", - Email: "docker@docker.io", } } @@ -30,9 +29,6 @@ func TestSameAuthDataPostSave(t *testing.T) { if authConfig.Password != "docker-pass" { t.Fail() } - if authConfig.Email != "docker@docker.io" { - t.Fail() - } if authConfig.Auth != "" { t.Fail() } @@ -62,17 +58,14 @@ func TestResolveAuthConfigFullURL(t *testing.T) { registryAuth := types.AuthConfig{ Username: "foo-user", Password: "foo-pass", - Email: "foo@example.com", } localAuth := types.AuthConfig{ Username: "bar-user", Password: "bar-pass", - Email: "bar@example.com", } officialAuth := types.AuthConfig{ Username: "baz-user", Password: "baz-pass", - Email: "baz@example.com", } authConfigs[IndexServer] = officialAuth @@ -105,7 +98,7 @@ func TestResolveAuthConfigFullURL(t *testing.T) { for configKey, registries := range validRegistries { configured, ok := expectedAuths[configKey] - if !ok || configured.Email == "" { + if !ok { t.Fail() } index := ®istrytypes.IndexInfo{ @@ -114,13 +107,13 @@ func TestResolveAuthConfigFullURL(t *testing.T) { for _, registry := range registries { authConfigs[registry] = configured resolved := ResolveAuthConfig(authConfigs, index) - if resolved.Email != configured.Email { - t.Errorf("%s -> %q != %q\n", registry, resolved.Email, configured.Email) + if resolved.Username != configured.Username || resolved.Password != configured.Password { + t.Errorf("%s -> %v != %v\n", registry, resolved, configured) } delete(authConfigs, registry) resolved = ResolveAuthConfig(authConfigs, index) - if resolved.Email == configured.Email { - t.Errorf("%s -> %q == %q\n", registry, resolved.Email, configured.Email) + if resolved.Username == configured.Username || resolved.Password == configured.Password { + t.Errorf("%s -> %v == %v\n", registry, resolved, configured) } } } diff --git a/docs/session.go b/docs/session.go index 4b18d0d1..daf44982 100644 --- a/docs/session.go +++ b/docs/session.go @@ -752,7 +752,6 @@ func (r *Session) GetAuthConfig(withPasswd bool) *types.AuthConfig { return &types.AuthConfig{ Username: r.authConfig.Username, Password: password, - Email: r.authConfig.Email, } } From 065ddf0186c44d67a5a1de830b58ce74b0d40993 Mon Sep 17 00:00:00 2001 From: Derek McGowan Date: Mon, 29 Feb 2016 23:07:41 -0800 Subject: [PATCH 358/375] Login update and endpoint refactor Further differentiate the APIEndpoint used with V2 with the endpoint type which is only used for v1 registry interactions Rename Endpoint to V1Endpoint and remove version ambiguity Use distribution token handler for login Signed-off-by: Derek McGowan Signed-off-by: Aaron Lehmann --- docs/auth.go | 232 ++++++++++++++++----------- docs/authchallenge.go | 150 ----------------- docs/config.go | 3 + docs/endpoint_test.go | 63 +++----- docs/{endpoint.go => endpoint_v1.go} | 195 ++++++---------------- docs/registry_test.go | 36 ++--- docs/service.go | 55 ++++--- docs/service_v1.go | 14 +- docs/service_v2.go | 13 +- docs/session.go | 16 +- docs/token.go | 81 ---------- docs/types.go | 14 +- 12 files changed, 276 insertions(+), 596 deletions(-) delete mode 100644 docs/authchallenge.go rename docs/{endpoint.go => endpoint_v1.go} (50%) delete mode 100644 docs/token.go diff --git a/docs/auth.go b/docs/auth.go index bd7bd52d..a8fdb675 100644 --- a/docs/auth.go +++ b/docs/auth.go @@ -4,28 +4,25 @@ import ( "fmt" "io/ioutil" "net/http" + "net/url" "strings" + "time" "github.com/Sirupsen/logrus" + "github.com/docker/distribution/registry/client/auth" + "github.com/docker/distribution/registry/client/transport" "github.com/docker/engine-api/types" registrytypes "github.com/docker/engine-api/types/registry" ) -// Login tries to register/login to the registry server. -func Login(authConfig *types.AuthConfig, registryEndpoint *Endpoint) (string, error) { - // Separates the v2 registry login logic from the v1 logic. - if registryEndpoint.Version == APIVersion2 { - return loginV2(authConfig, registryEndpoint, "" /* scope */) - } - return loginV1(authConfig, registryEndpoint) -} - // loginV1 tries to register/login to the v1 registry server. -func loginV1(authConfig *types.AuthConfig, registryEndpoint *Endpoint) (string, error) { - var ( - err error - serverAddress = authConfig.ServerAddress - ) +func loginV1(authConfig *types.AuthConfig, apiEndpoint APIEndpoint, userAgent string) (string, error) { + registryEndpoint, err := apiEndpoint.ToV1Endpoint(userAgent, nil) + if err != nil { + return "", err + } + + serverAddress := registryEndpoint.String() logrus.Debugf("attempting v1 login to registry endpoint %s", registryEndpoint) @@ -36,10 +33,16 @@ func loginV1(authConfig *types.AuthConfig, registryEndpoint *Endpoint) (string, loginAgainstOfficialIndex := serverAddress == IndexServer req, err := http.NewRequest("GET", serverAddress+"users/", nil) + if err != nil { + return "", err + } req.SetBasicAuth(authConfig.Username, authConfig.Password) resp, err := registryEndpoint.client.Do(req) if err != nil { - return "", err + // fallback when request could not be completed + return "", fallbackError{ + err: err, + } } defer resp.Body.Close() body, err := ioutil.ReadAll(resp.Body) @@ -68,97 +71,82 @@ func loginV1(authConfig *types.AuthConfig, registryEndpoint *Endpoint) (string, } } -// loginV2 tries to login to the v2 registry server. The given registry endpoint has been -// pinged or setup with a list of authorization challenges. Each of these challenges are -// tried until one of them succeeds. Currently supported challenge schemes are: -// HTTP Basic Authorization -// Token Authorization with a separate token issuing server -// NOTE: the v2 logic does not attempt to create a user account if one doesn't exist. For -// now, users should create their account through other means like directly from a web page -// served by the v2 registry service provider. Whether this will be supported in the future -// is to be determined. -func loginV2(authConfig *types.AuthConfig, registryEndpoint *Endpoint, scope string) (string, error) { - logrus.Debugf("attempting v2 login to registry endpoint %s", registryEndpoint) - var ( - err error - allErrors []error - ) - - for _, challenge := range registryEndpoint.AuthChallenges { - params := make(map[string]string, len(challenge.Parameters)+1) - for k, v := range challenge.Parameters { - params[k] = v - } - params["scope"] = scope - logrus.Debugf("trying %q auth challenge with params %v", challenge.Scheme, params) - - switch strings.ToLower(challenge.Scheme) { - case "basic": - err = tryV2BasicAuthLogin(authConfig, params, registryEndpoint) - case "bearer": - err = tryV2TokenAuthLogin(authConfig, params, registryEndpoint) - default: - // Unsupported challenge types are explicitly skipped. - err = fmt.Errorf("unsupported auth scheme: %q", challenge.Scheme) - } - - if err == nil { - return "Login Succeeded", nil - } - - logrus.Debugf("error trying auth challenge %q: %s", challenge.Scheme, err) - - allErrors = append(allErrors, err) - } - - return "", fmt.Errorf("no successful auth challenge for %s - errors: %s", registryEndpoint, allErrors) +type loginCredentialStore struct { + authConfig *types.AuthConfig } -func tryV2BasicAuthLogin(authConfig *types.AuthConfig, params map[string]string, registryEndpoint *Endpoint) error { - req, err := http.NewRequest("GET", registryEndpoint.Path(""), nil) +func (lcs loginCredentialStore) Basic(*url.URL) (string, string) { + return lcs.authConfig.Username, lcs.authConfig.Password +} + +type fallbackError struct { + err error +} + +func (err fallbackError) Error() string { + return err.err.Error() +} + +// loginV2 tries to login to the v2 registry server. The given registry +// endpoint will be pinged to get authorization challenges. These challenges +// will be used to authenticate against the registry to validate credentials. +func loginV2(authConfig *types.AuthConfig, endpoint APIEndpoint, userAgent string) (string, error) { + logrus.Debugf("attempting v2 login to registry endpoint %s", endpoint) + + modifiers := DockerHeaders(userAgent, nil) + authTransport := transport.NewTransport(NewTransport(endpoint.TLSConfig), modifiers...) + + challengeManager, foundV2, err := PingV2Registry(endpoint, authTransport) if err != nil { - return err + if !foundV2 { + err = fallbackError{err: err} + } + return "", err } - req.SetBasicAuth(authConfig.Username, authConfig.Password) + creds := loginCredentialStore{ + authConfig: authConfig, + } - resp, err := registryEndpoint.client.Do(req) + tokenHandler := auth.NewTokenHandler(authTransport, creds, "") + basicHandler := auth.NewBasicHandler(creds) + modifiers = append(modifiers, auth.NewAuthorizer(challengeManager, tokenHandler, basicHandler)) + tr := transport.NewTransport(authTransport, modifiers...) + + loginClient := &http.Client{ + Transport: tr, + Timeout: 15 * time.Second, + } + + endpointStr := strings.TrimRight(endpoint.URL.String(), "/") + "/v2/" + req, err := http.NewRequest("GET", endpointStr, nil) if err != nil { - return err + if !foundV2 { + err = fallbackError{err: err} + } + return "", err + } + + resp, err := loginClient.Do(req) + if err != nil { + if !foundV2 { + err = fallbackError{err: err} + } + return "", err } defer resp.Body.Close() if resp.StatusCode != http.StatusOK { - return fmt.Errorf("basic auth attempt to %s realm %q failed with status: %d %s", registryEndpoint, params["realm"], resp.StatusCode, http.StatusText(resp.StatusCode)) + // TODO(dmcgowan): Attempt to further interpret result, status code and error code string + err := fmt.Errorf("login attempt to %s failed with status: %d %s", endpointStr, resp.StatusCode, http.StatusText(resp.StatusCode)) + if !foundV2 { + err = fallbackError{err: err} + } + return "", err } - return nil -} + return "Login Succeeded", nil -func tryV2TokenAuthLogin(authConfig *types.AuthConfig, params map[string]string, registryEndpoint *Endpoint) error { - token, err := getToken(authConfig.Username, authConfig.Password, params, registryEndpoint) - if err != nil { - return err - } - - req, err := http.NewRequest("GET", registryEndpoint.Path(""), nil) - if err != nil { - return err - } - - req.Header.Set("Authorization", fmt.Sprintf("Bearer %s", token)) - - resp, err := registryEndpoint.client.Do(req) - if err != nil { - return err - } - defer resp.Body.Close() - - if resp.StatusCode != http.StatusOK { - return fmt.Errorf("token auth attempt to %s realm %q failed with status: %d %s", registryEndpoint, params["realm"], resp.StatusCode, http.StatusText(resp.StatusCode)) - } - - return nil } // ResolveAuthConfig matches an auth configuration to a server address or a URL @@ -193,3 +181,63 @@ func ResolveAuthConfig(authConfigs map[string]types.AuthConfig, index *registryt // When all else fails, return an empty auth config return types.AuthConfig{} } + +// PingResponseError is used when the response from a ping +// was received but invalid. +type PingResponseError struct { + Err error +} + +func (err PingResponseError) Error() string { + return err.Error() +} + +// PingV2Registry attempts to ping a v2 registry and on success return a +// challenge manager for the supported authentication types and +// whether v2 was confirmed by the response. If a response is received but +// cannot be interpreted a PingResponseError will be returned. +func PingV2Registry(endpoint APIEndpoint, transport http.RoundTripper) (auth.ChallengeManager, bool, error) { + var ( + foundV2 = false + v2Version = auth.APIVersion{ + Type: "registry", + Version: "2.0", + } + ) + + pingClient := &http.Client{ + Transport: transport, + Timeout: 15 * time.Second, + } + endpointStr := strings.TrimRight(endpoint.URL.String(), "/") + "/v2/" + req, err := http.NewRequest("GET", endpointStr, nil) + if err != nil { + return nil, false, err + } + resp, err := pingClient.Do(req) + if err != nil { + return nil, false, err + } + defer resp.Body.Close() + + versions := auth.APIVersions(resp, DefaultRegistryVersionHeader) + for _, pingVersion := range versions { + if pingVersion == v2Version { + // The version header indicates we're definitely + // talking to a v2 registry. So don't allow future + // fallbacks to the v1 protocol. + + foundV2 = true + break + } + } + + challengeManager := auth.NewSimpleChallengeManager() + if err := challengeManager.AddResponse(resp); err != nil { + return nil, foundV2, PingResponseError{ + Err: err, + } + } + + return challengeManager, foundV2, nil +} diff --git a/docs/authchallenge.go b/docs/authchallenge.go deleted file mode 100644 index e300d82a..00000000 --- a/docs/authchallenge.go +++ /dev/null @@ -1,150 +0,0 @@ -package registry - -import ( - "net/http" - "strings" -) - -// Octet types from RFC 2616. -type octetType byte - -// AuthorizationChallenge carries information -// from a WWW-Authenticate response header. -type AuthorizationChallenge struct { - Scheme string - Parameters map[string]string -} - -var octetTypes [256]octetType - -const ( - isToken octetType = 1 << iota - isSpace -) - -func init() { - // OCTET = - // CHAR = - // CTL = - // CR = - // LF = - // SP = - // HT = - // <"> = - // CRLF = CR LF - // LWS = [CRLF] 1*( SP | HT ) - // TEXT = - // separators = "(" | ")" | "<" | ">" | "@" | "," | ";" | ":" | "\" | <"> - // | "/" | "[" | "]" | "?" | "=" | "{" | "}" | SP | HT - // token = 1* - // qdtext = > - - for c := 0; c < 256; c++ { - var t octetType - isCtl := c <= 31 || c == 127 - isChar := 0 <= c && c <= 127 - isSeparator := strings.IndexRune(" \t\"(),/:;<=>?@[]\\{}", rune(c)) >= 0 - if strings.IndexRune(" \t\r\n", rune(c)) >= 0 { - t |= isSpace - } - if isChar && !isCtl && !isSeparator { - t |= isToken - } - octetTypes[c] = t - } -} - -func parseAuthHeader(header http.Header) []*AuthorizationChallenge { - var challenges []*AuthorizationChallenge - for _, h := range header[http.CanonicalHeaderKey("WWW-Authenticate")] { - v, p := parseValueAndParams(h) - if v != "" { - challenges = append(challenges, &AuthorizationChallenge{Scheme: v, Parameters: p}) - } - } - return challenges -} - -func parseValueAndParams(header string) (value string, params map[string]string) { - params = make(map[string]string) - value, s := expectToken(header) - if value == "" { - return - } - value = strings.ToLower(value) - s = "," + skipSpace(s) - for strings.HasPrefix(s, ",") { - var pkey string - pkey, s = expectToken(skipSpace(s[1:])) - if pkey == "" { - return - } - if !strings.HasPrefix(s, "=") { - return - } - var pvalue string - pvalue, s = expectTokenOrQuoted(s[1:]) - if pvalue == "" { - return - } - pkey = strings.ToLower(pkey) - params[pkey] = pvalue - s = skipSpace(s) - } - return -} - -func skipSpace(s string) (rest string) { - i := 0 - for ; i < len(s); i++ { - if octetTypes[s[i]]&isSpace == 0 { - break - } - } - return s[i:] -} - -func expectToken(s string) (token, rest string) { - i := 0 - for ; i < len(s); i++ { - if octetTypes[s[i]]&isToken == 0 { - break - } - } - return s[:i], s[i:] -} - -func expectTokenOrQuoted(s string) (value string, rest string) { - if !strings.HasPrefix(s, "\"") { - return expectToken(s) - } - s = s[1:] - for i := 0; i < len(s); i++ { - switch s[i] { - case '"': - return s[:i], s[i+1:] - case '\\': - p := make([]byte, len(s)-1) - j := copy(p, s[:i]) - escape := true - for i = i + i; i < len(s); i++ { - b := s[i] - switch { - case escape: - escape = false - p[j] = b - j++ - case b == '\\': - escape = true - case b == '"': - return string(p[:j]), s[i+1:] - default: - p[j] = b - j++ - } - } - return "", "" - } - } - return "", "" -} diff --git a/docs/config.go b/docs/config.go index ebad6f86..7d8b6301 100644 --- a/docs/config.go +++ b/docs/config.go @@ -49,6 +49,9 @@ var ( V2Only = false ) +// for mocking in unit tests +var lookupIP = net.LookupIP + // InstallFlags adds command-line options to the top-level flag parser for // the current process. func (options *Options) InstallFlags(cmd *flag.FlagSet, usageFn func(string) string) { diff --git a/docs/endpoint_test.go b/docs/endpoint_test.go index fa18eea0..8451d3f6 100644 --- a/docs/endpoint_test.go +++ b/docs/endpoint_test.go @@ -14,12 +14,13 @@ func TestEndpointParse(t *testing.T) { }{ {IndexServer, IndexServer}, {"http://0.0.0.0:5000/v1/", "http://0.0.0.0:5000/v1/"}, - {"http://0.0.0.0:5000/v2/", "http://0.0.0.0:5000/v2/"}, - {"http://0.0.0.0:5000", "http://0.0.0.0:5000/v0/"}, - {"0.0.0.0:5000", "https://0.0.0.0:5000/v0/"}, + {"http://0.0.0.0:5000", "http://0.0.0.0:5000/v1/"}, + {"0.0.0.0:5000", "https://0.0.0.0:5000/v1/"}, + {"http://0.0.0.0:5000/nonversion/", "http://0.0.0.0:5000/nonversion/v1/"}, + {"http://0.0.0.0:5000/v0/", "http://0.0.0.0:5000/v0/v1/"}, } for _, td := range testData { - e, err := newEndpointFromStr(td.str, nil, "", nil) + e, err := newV1EndpointFromStr(td.str, nil, "", nil) if err != nil { t.Errorf("%q: %s", td.str, err) } @@ -33,21 +34,26 @@ func TestEndpointParse(t *testing.T) { } } +func TestEndpointParseInvalid(t *testing.T) { + testData := []string{ + "http://0.0.0.0:5000/v2/", + } + for _, td := range testData { + e, err := newV1EndpointFromStr(td, nil, "", nil) + if err == nil { + t.Errorf("expected error parsing %q: parsed as %q", td, e) + } + } +} + // Ensure that a registry endpoint that responds with a 401 only is determined -// to be a v1 registry unless it includes a valid v2 API header. -func TestValidateEndpointAmbiguousAPIVersion(t *testing.T) { +// to be a valid v1 registry endpoint +func TestValidateEndpoint(t *testing.T) { requireBasicAuthHandler := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { w.Header().Add("WWW-Authenticate", `Basic realm="localhost"`) w.WriteHeader(http.StatusUnauthorized) }) - requireBasicAuthHandlerV2 := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - // This mock server supports v2.0, v2.1, v42.0, and v100.0 - w.Header().Add("Docker-Distribution-API-Version", "registry/100.0 registry/42.0") - w.Header().Add("Docker-Distribution-API-Version", "registry/2.0 registry/2.1") - requireBasicAuthHandler.ServeHTTP(w, r) - }) - // Make a test server which should validate as a v1 server. testServer := httptest.NewServer(requireBasicAuthHandler) defer testServer.Close() @@ -57,37 +63,16 @@ func TestValidateEndpointAmbiguousAPIVersion(t *testing.T) { t.Fatal(err) } - testEndpoint := Endpoint{ - URL: testServerURL, - Version: APIVersionUnknown, - client: HTTPClient(NewTransport(nil)), + testEndpoint := V1Endpoint{ + URL: testServerURL, + client: HTTPClient(NewTransport(nil)), } if err = validateEndpoint(&testEndpoint); err != nil { t.Fatal(err) } - if testEndpoint.Version != APIVersion1 { - t.Fatalf("expected endpoint to validate to %d, got %d", APIVersion1, testEndpoint.Version) - } - - // Make a test server which should validate as a v2 server. - testServer = httptest.NewServer(requireBasicAuthHandlerV2) - defer testServer.Close() - - testServerURL, err = url.Parse(testServer.URL) - if err != nil { - t.Fatal(err) - } - - testEndpoint.URL = testServerURL - testEndpoint.Version = APIVersionUnknown - - if err = validateEndpoint(&testEndpoint); err != nil { - t.Fatal(err) - } - - if testEndpoint.Version != APIVersion2 { - t.Fatalf("expected endpoint to validate to %d, got %d", APIVersion2, testEndpoint.Version) + if testEndpoint.URL.Scheme != "http" { + t.Fatalf("expecting to validate endpoint as http, got url %s", testEndpoint.String()) } } diff --git a/docs/endpoint.go b/docs/endpoint_v1.go similarity index 50% rename from docs/endpoint.go rename to docs/endpoint_v1.go index b056caf1..58e2600e 100644 --- a/docs/endpoint.go +++ b/docs/endpoint_v1.go @@ -5,60 +5,35 @@ import ( "encoding/json" "fmt" "io/ioutil" - "net" "net/http" "net/url" "strings" "github.com/Sirupsen/logrus" - "github.com/docker/distribution/registry/api/v2" "github.com/docker/distribution/registry/client/transport" registrytypes "github.com/docker/engine-api/types/registry" ) -// for mocking in unit tests -var lookupIP = net.LookupIP - -// scans string for api version in the URL path. returns the trimmed address, if version found, string and API version. -func scanForAPIVersion(address string) (string, APIVersion) { - var ( - chunks []string - apiVersionStr string - ) - - if strings.HasSuffix(address, "/") { - address = address[:len(address)-1] - } - - chunks = strings.Split(address, "/") - apiVersionStr = chunks[len(chunks)-1] - - for k, v := range apiVersions { - if apiVersionStr == v { - address = strings.Join(chunks[:len(chunks)-1], "/") - return address, k - } - } - - return address, APIVersionUnknown +// V1Endpoint stores basic information about a V1 registry endpoint. +type V1Endpoint struct { + client *http.Client + URL *url.URL + IsSecure bool } -// NewEndpoint parses the given address to return a registry endpoint. v can be used to +// NewV1Endpoint parses the given address to return a registry endpoint. v can be used to // specify a specific endpoint version -func NewEndpoint(index *registrytypes.IndexInfo, userAgent string, metaHeaders http.Header, v APIVersion) (*Endpoint, error) { +func NewV1Endpoint(index *registrytypes.IndexInfo, userAgent string, metaHeaders http.Header) (*V1Endpoint, error) { tlsConfig, err := newTLSConfig(index.Name, index.Secure) if err != nil { return nil, err } - endpoint, err := newEndpointFromStr(GetAuthConfigKey(index), tlsConfig, userAgent, metaHeaders) + endpoint, err := newV1EndpointFromStr(GetAuthConfigKey(index), tlsConfig, userAgent, metaHeaders) if err != nil { return nil, err } - if v != APIVersionUnknown { - endpoint.Version = v - } if err := validateEndpoint(endpoint); err != nil { return nil, err } @@ -66,7 +41,7 @@ func NewEndpoint(index *registrytypes.IndexInfo, userAgent string, metaHeaders h return endpoint, nil } -func validateEndpoint(endpoint *Endpoint) error { +func validateEndpoint(endpoint *V1Endpoint) error { logrus.Debugf("pinging registry endpoint %s", endpoint) // Try HTTPS ping to registry @@ -93,11 +68,10 @@ func validateEndpoint(endpoint *Endpoint) error { return nil } -func newEndpoint(address url.URL, tlsConfig *tls.Config, userAgent string, metaHeaders http.Header) (*Endpoint, error) { - endpoint := &Endpoint{ +func newV1Endpoint(address url.URL, tlsConfig *tls.Config, userAgent string, metaHeaders http.Header) (*V1Endpoint, error) { + endpoint := &V1Endpoint{ IsSecure: (tlsConfig == nil || !tlsConfig.InsecureSkipVerify), URL: new(url.URL), - Version: APIVersionUnknown, } *endpoint.URL = address @@ -108,86 +82,69 @@ func newEndpoint(address url.URL, tlsConfig *tls.Config, userAgent string, metaH return endpoint, nil } -func newEndpointFromStr(address string, tlsConfig *tls.Config, userAgent string, metaHeaders http.Header) (*Endpoint, error) { +// trimV1Address trims the version off the address and returns the +// trimmed address or an error if there is a non-V1 version. +func trimV1Address(address string) (string, error) { + var ( + chunks []string + apiVersionStr string + ) + + if strings.HasSuffix(address, "/") { + address = address[:len(address)-1] + } + + chunks = strings.Split(address, "/") + apiVersionStr = chunks[len(chunks)-1] + if apiVersionStr == "v1" { + return strings.Join(chunks[:len(chunks)-1], "/"), nil + } + + for k, v := range apiVersions { + if k != APIVersion1 && apiVersionStr == v { + return "", fmt.Errorf("unsupported V1 version path %s", apiVersionStr) + } + } + + return address, nil +} + +func newV1EndpointFromStr(address string, tlsConfig *tls.Config, userAgent string, metaHeaders http.Header) (*V1Endpoint, error) { if !strings.HasPrefix(address, "http://") && !strings.HasPrefix(address, "https://") { address = "https://" + address } - trimmedAddress, detectedVersion := scanForAPIVersion(address) - - uri, err := url.Parse(trimmedAddress) + address, err := trimV1Address(address) if err != nil { return nil, err } - endpoint, err := newEndpoint(*uri, tlsConfig, userAgent, metaHeaders) + uri, err := url.Parse(address) + if err != nil { + return nil, err + } + + endpoint, err := newV1Endpoint(*uri, tlsConfig, userAgent, metaHeaders) if err != nil { return nil, err } - endpoint.Version = detectedVersion return endpoint, nil } -// Endpoint stores basic information about a registry endpoint. -type Endpoint struct { - client *http.Client - URL *url.URL - Version APIVersion - IsSecure bool - AuthChallenges []*AuthorizationChallenge - URLBuilder *v2.URLBuilder -} - // Get the formatted URL for the root of this registry Endpoint -func (e *Endpoint) String() string { - return fmt.Sprintf("%s/v%d/", e.URL, e.Version) -} - -// VersionString returns a formatted string of this -// endpoint address using the given API Version. -func (e *Endpoint) VersionString(version APIVersion) string { - return fmt.Sprintf("%s/v%d/", e.URL, version) +func (e *V1Endpoint) String() string { + return e.URL.String() + "/v1/" } // Path returns a formatted string for the URL // of this endpoint with the given path appended. -func (e *Endpoint) Path(path string) string { - return fmt.Sprintf("%s/v%d/%s", e.URL, e.Version, path) +func (e *V1Endpoint) Path(path string) string { + return e.URL.String() + "/v1/" + path } -// Ping pings the remote endpoint with v2 and v1 pings to determine the API -// version. It returns a PingResult containing the discovered version. The -// PingResult also indicates whether the registry is standalone or not. -func (e *Endpoint) Ping() (PingResult, error) { - // The ping logic to use is determined by the registry endpoint version. - switch e.Version { - case APIVersion1: - return e.pingV1() - case APIVersion2: - return e.pingV2() - } - - // APIVersionUnknown - // We should try v2 first... - e.Version = APIVersion2 - regInfo, errV2 := e.pingV2() - if errV2 == nil { - return regInfo, nil - } - - // ... then fallback to v1. - e.Version = APIVersion1 - regInfo, errV1 := e.pingV1() - if errV1 == nil { - return regInfo, nil - } - - e.Version = APIVersionUnknown - return PingResult{}, fmt.Errorf("unable to ping registry endpoint %s\nv2 ping attempt failed with error: %s\n v1 ping attempt failed with error: %s", e, errV2, errV1) -} - -func (e *Endpoint) pingV1() (PingResult, error) { +// Ping returns a PingResult which indicates whether the registry is standalone or not. +func (e *V1Endpoint) Ping() (PingResult, error) { logrus.Debugf("attempting v1 ping for registry endpoint %s", e) if e.String() == IndexServer { @@ -240,51 +197,3 @@ func (e *Endpoint) pingV1() (PingResult, error) { logrus.Debugf("PingResult.Standalone: %t", info.Standalone) return info, nil } - -func (e *Endpoint) pingV2() (PingResult, error) { - logrus.Debugf("attempting v2 ping for registry endpoint %s", e) - - req, err := http.NewRequest("GET", e.Path(""), nil) - if err != nil { - return PingResult{}, err - } - - resp, err := e.client.Do(req) - if err != nil { - return PingResult{}, err - } - defer resp.Body.Close() - - // The endpoint may have multiple supported versions. - // Ensure it supports the v2 Registry API. - var supportsV2 bool - -HeaderLoop: - for _, supportedVersions := range resp.Header[http.CanonicalHeaderKey("Docker-Distribution-API-Version")] { - for _, versionName := range strings.Fields(supportedVersions) { - if versionName == "registry/2.0" { - supportsV2 = true - break HeaderLoop - } - } - } - - if !supportsV2 { - return PingResult{}, fmt.Errorf("%s does not appear to be a v2 registry endpoint", e) - } - - if resp.StatusCode == http.StatusOK { - // It would seem that no authentication/authorization is required. - // So we don't need to parse/add any authorization schemes. - return PingResult{Standalone: true}, nil - } - - if resp.StatusCode == http.StatusUnauthorized { - // Parse the WWW-Authenticate Header and store the challenges - // on this endpoint object. - e.AuthChallenges = parseAuthHeader(resp.Header) - return PingResult{}, nil - } - - return PingResult{}, fmt.Errorf("v2 registry endpoint returned status %d: %q", resp.StatusCode, http.StatusText(resp.StatusCode)) -} diff --git a/docs/registry_test.go b/docs/registry_test.go index 33d85347..02eb683d 100644 --- a/docs/registry_test.go +++ b/docs/registry_test.go @@ -25,7 +25,7 @@ const ( func spawnTestRegistrySession(t *testing.T) *Session { authConfig := &types.AuthConfig{} - endpoint, err := NewEndpoint(makeIndex("/v1/"), "", nil, APIVersionUnknown) + endpoint, err := NewV1Endpoint(makeIndex("/v1/"), "", nil) if err != nil { t.Fatal(err) } @@ -53,7 +53,7 @@ func spawnTestRegistrySession(t *testing.T) *Session { func TestPingRegistryEndpoint(t *testing.T) { testPing := func(index *registrytypes.IndexInfo, expectedStandalone bool, assertMessage string) { - ep, err := NewEndpoint(index, "", nil, APIVersionUnknown) + ep, err := NewV1Endpoint(index, "", nil) if err != nil { t.Fatal(err) } @@ -72,8 +72,8 @@ func TestPingRegistryEndpoint(t *testing.T) { func TestEndpoint(t *testing.T) { // Simple wrapper to fail test if err != nil - expandEndpoint := func(index *registrytypes.IndexInfo) *Endpoint { - endpoint, err := NewEndpoint(index, "", nil, APIVersionUnknown) + expandEndpoint := func(index *registrytypes.IndexInfo) *V1Endpoint { + endpoint, err := NewV1Endpoint(index, "", nil) if err != nil { t.Fatal(err) } @@ -82,7 +82,7 @@ func TestEndpoint(t *testing.T) { assertInsecureIndex := func(index *registrytypes.IndexInfo) { index.Secure = true - _, err := NewEndpoint(index, "", nil, APIVersionUnknown) + _, err := NewV1Endpoint(index, "", nil) assertNotEqual(t, err, nil, index.Name+": Expected error for insecure index") assertEqual(t, strings.Contains(err.Error(), "insecure-registry"), true, index.Name+": Expected insecure-registry error for insecure index") index.Secure = false @@ -90,7 +90,7 @@ func TestEndpoint(t *testing.T) { assertSecureIndex := func(index *registrytypes.IndexInfo) { index.Secure = true - _, err := NewEndpoint(index, "", nil, APIVersionUnknown) + _, err := NewV1Endpoint(index, "", nil) assertNotEqual(t, err, nil, index.Name+": Expected cert error for secure index") assertEqual(t, strings.Contains(err.Error(), "certificate signed by unknown authority"), true, index.Name+": Expected cert error for secure index") index.Secure = false @@ -100,51 +100,33 @@ func TestEndpoint(t *testing.T) { index.Name = makeURL("/v1/") endpoint := expandEndpoint(index) assertEqual(t, endpoint.String(), index.Name, "Expected endpoint to be "+index.Name) - if endpoint.Version != APIVersion1 { - t.Fatal("Expected endpoint to be v1") - } assertInsecureIndex(index) index.Name = makeURL("") endpoint = expandEndpoint(index) assertEqual(t, endpoint.String(), index.Name+"/v1/", index.Name+": Expected endpoint to be "+index.Name+"/v1/") - if endpoint.Version != APIVersion1 { - t.Fatal("Expected endpoint to be v1") - } assertInsecureIndex(index) httpURL := makeURL("") index.Name = strings.SplitN(httpURL, "://", 2)[1] endpoint = expandEndpoint(index) assertEqual(t, endpoint.String(), httpURL+"/v1/", index.Name+": Expected endpoint to be "+httpURL+"/v1/") - if endpoint.Version != APIVersion1 { - t.Fatal("Expected endpoint to be v1") - } assertInsecureIndex(index) index.Name = makeHTTPSURL("/v1/") endpoint = expandEndpoint(index) assertEqual(t, endpoint.String(), index.Name, "Expected endpoint to be "+index.Name) - if endpoint.Version != APIVersion1 { - t.Fatal("Expected endpoint to be v1") - } assertSecureIndex(index) index.Name = makeHTTPSURL("") endpoint = expandEndpoint(index) assertEqual(t, endpoint.String(), index.Name+"/v1/", index.Name+": Expected endpoint to be "+index.Name+"/v1/") - if endpoint.Version != APIVersion1 { - t.Fatal("Expected endpoint to be v1") - } assertSecureIndex(index) httpsURL := makeHTTPSURL("") index.Name = strings.SplitN(httpsURL, "://", 2)[1] endpoint = expandEndpoint(index) assertEqual(t, endpoint.String(), httpsURL+"/v1/", index.Name+": Expected endpoint to be "+httpsURL+"/v1/") - if endpoint.Version != APIVersion1 { - t.Fatal("Expected endpoint to be v1") - } assertSecureIndex(index) badEndpoints := []string{ @@ -156,7 +138,7 @@ func TestEndpoint(t *testing.T) { } for _, address := range badEndpoints { index.Name = address - _, err := NewEndpoint(index, "", nil, APIVersionUnknown) + _, err := NewV1Endpoint(index, "", nil) checkNotEqual(t, err, nil, "Expected error while expanding bad endpoint") } } @@ -685,7 +667,7 @@ func TestMirrorEndpointLookup(t *testing.T) { if err != nil { t.Error(err) } - pushAPIEndpoints, err := s.LookupPushEndpoints(imageName) + pushAPIEndpoints, err := s.LookupPushEndpoints(imageName.Hostname()) if err != nil { t.Fatal(err) } @@ -693,7 +675,7 @@ func TestMirrorEndpointLookup(t *testing.T) { t.Fatal("Push endpoint should not contain mirror") } - pullAPIEndpoints, err := s.LookupPullEndpoints(imageName) + pullAPIEndpoints, err := s.LookupPullEndpoints(imageName.Hostname()) if err != nil { t.Fatal(err) } diff --git a/docs/service.go b/docs/service.go index bba1e842..2124da6d 100644 --- a/docs/service.go +++ b/docs/service.go @@ -6,6 +6,7 @@ import ( "net/url" "strings" + "github.com/Sirupsen/logrus" "github.com/docker/docker/reference" "github.com/docker/engine-api/types" registrytypes "github.com/docker/engine-api/types/registry" @@ -28,29 +29,31 @@ func NewService(options *Options) *Service { // Auth contacts the public registry with the provided credentials, // and returns OK if authentication was successful. // It can be used to verify the validity of a client's credentials. -func (s *Service) Auth(authConfig *types.AuthConfig, userAgent string) (string, error) { - addr := authConfig.ServerAddress - if addr == "" { - // Use the official registry address if not specified. - addr = IndexServer - } - index, err := s.ResolveIndex(addr) +func (s *Service) Auth(authConfig *types.AuthConfig, userAgent string) (status string, err error) { + endpoints, err := s.LookupPushEndpoints(authConfig.ServerAddress) if err != nil { return "", err } - endpointVersion := APIVersion(APIVersionUnknown) - if V2Only { - // Override the endpoint to only attempt a v2 ping - endpointVersion = APIVersion2 - } + for _, endpoint := range endpoints { + login := loginV2 + if endpoint.Version == APIVersion1 { + login = loginV1 + } - endpoint, err := NewEndpoint(index, userAgent, nil, endpointVersion) - if err != nil { + status, err = login(authConfig, endpoint, userAgent) + if err == nil { + return + } + if fErr, ok := err.(fallbackError); ok { + err = fErr.err + logrus.Infof("Error logging in to %s endpoint, trying next endpoint: %v", endpoint.Version, err) + continue + } return "", err } - authConfig.ServerAddress = endpoint.String() - return Login(authConfig, endpoint) + + return "", err } // splitReposSearchTerm breaks a search term into an index name and remote name @@ -85,7 +88,7 @@ func (s *Service) Search(term string, authConfig *types.AuthConfig, userAgent st } // *TODO: Search multiple indexes. - endpoint, err := NewEndpoint(index, userAgent, http.Header(headers), APIVersionUnknown) + endpoint, err := NewV1Endpoint(index, userAgent, http.Header(headers)) if err != nil { return nil, err } @@ -129,8 +132,8 @@ type APIEndpoint struct { } // ToV1Endpoint returns a V1 API endpoint based on the APIEndpoint -func (e APIEndpoint) ToV1Endpoint(userAgent string, metaHeaders http.Header) (*Endpoint, error) { - return newEndpoint(*e.URL, e.TLSConfig, userAgent, metaHeaders) +func (e APIEndpoint) ToV1Endpoint(userAgent string, metaHeaders http.Header) (*V1Endpoint, error) { + return newV1Endpoint(*e.URL, e.TLSConfig, userAgent, metaHeaders) } // TLSConfig constructs a client TLS configuration based on server defaults @@ -145,15 +148,15 @@ func (s *Service) tlsConfigForMirror(mirrorURL *url.URL) (*tls.Config, error) { // LookupPullEndpoints creates an list of endpoints to try to pull from, in order of preference. // It gives preference to v2 endpoints over v1, mirrors over the actual // registry, and HTTPS over plain HTTP. -func (s *Service) LookupPullEndpoints(repoName reference.Named) (endpoints []APIEndpoint, err error) { - return s.lookupEndpoints(repoName) +func (s *Service) LookupPullEndpoints(hostname string) (endpoints []APIEndpoint, err error) { + return s.lookupEndpoints(hostname) } // LookupPushEndpoints creates an list of endpoints to try to push to, in order of preference. // It gives preference to v2 endpoints over v1, and HTTPS over plain HTTP. // Mirrors are not included. -func (s *Service) LookupPushEndpoints(repoName reference.Named) (endpoints []APIEndpoint, err error) { - allEndpoints, err := s.lookupEndpoints(repoName) +func (s *Service) LookupPushEndpoints(hostname string) (endpoints []APIEndpoint, err error) { + allEndpoints, err := s.lookupEndpoints(hostname) if err == nil { for _, endpoint := range allEndpoints { if !endpoint.Mirror { @@ -164,8 +167,8 @@ func (s *Service) LookupPushEndpoints(repoName reference.Named) (endpoints []API return endpoints, err } -func (s *Service) lookupEndpoints(repoName reference.Named) (endpoints []APIEndpoint, err error) { - endpoints, err = s.lookupV2Endpoints(repoName) +func (s *Service) lookupEndpoints(hostname string) (endpoints []APIEndpoint, err error) { + endpoints, err = s.lookupV2Endpoints(hostname) if err != nil { return nil, err } @@ -174,7 +177,7 @@ func (s *Service) lookupEndpoints(repoName reference.Named) (endpoints []APIEndp return endpoints, nil } - legacyEndpoints, err := s.lookupV1Endpoints(repoName) + legacyEndpoints, err := s.lookupV1Endpoints(hostname) if err != nil { return nil, err } diff --git a/docs/service_v1.go b/docs/service_v1.go index 5328b8f1..56121eea 100644 --- a/docs/service_v1.go +++ b/docs/service_v1.go @@ -1,19 +1,15 @@ package registry import ( - "fmt" "net/url" - "strings" - "github.com/docker/docker/reference" "github.com/docker/go-connections/tlsconfig" ) -func (s *Service) lookupV1Endpoints(repoName reference.Named) (endpoints []APIEndpoint, err error) { +func (s *Service) lookupV1Endpoints(hostname string) (endpoints []APIEndpoint, err error) { var cfg = tlsconfig.ServerDefault tlsConfig := &cfg - nameString := repoName.FullName() - if strings.HasPrefix(nameString, DefaultNamespace+"/") { + if hostname == DefaultNamespace { endpoints = append(endpoints, APIEndpoint{ URL: DefaultV1Registry, Version: APIVersion1, @@ -24,12 +20,6 @@ func (s *Service) lookupV1Endpoints(repoName reference.Named) (endpoints []APIEn return endpoints, nil } - slashIndex := strings.IndexRune(nameString, '/') - if slashIndex <= 0 { - return nil, fmt.Errorf("invalid repo name: missing '/': %s", nameString) - } - hostname := nameString[:slashIndex] - tlsConfig, err = s.TLSConfig(hostname) if err != nil { return nil, err diff --git a/docs/service_v2.go b/docs/service_v2.go index 4dbbb9fa..9c909f18 100644 --- a/docs/service_v2.go +++ b/docs/service_v2.go @@ -1,19 +1,16 @@ package registry import ( - "fmt" "net/url" "strings" - "github.com/docker/docker/reference" "github.com/docker/go-connections/tlsconfig" ) -func (s *Service) lookupV2Endpoints(repoName reference.Named) (endpoints []APIEndpoint, err error) { +func (s *Service) lookupV2Endpoints(hostname string) (endpoints []APIEndpoint, err error) { var cfg = tlsconfig.ServerDefault tlsConfig := &cfg - nameString := repoName.FullName() - if strings.HasPrefix(nameString, DefaultNamespace+"/") { + if hostname == DefaultNamespace { // v2 mirrors for _, mirror := range s.Config.Mirrors { if !strings.HasPrefix(mirror, "http://") && !strings.HasPrefix(mirror, "https://") { @@ -48,12 +45,6 @@ func (s *Service) lookupV2Endpoints(repoName reference.Named) (endpoints []APIEn return endpoints, nil } - slashIndex := strings.IndexRune(nameString, '/') - if slashIndex <= 0 { - return nil, fmt.Errorf("invalid repo name: missing '/': %s", nameString) - } - hostname := nameString[:slashIndex] - tlsConfig, err = s.TLSConfig(hostname) if err != nil { return nil, err diff --git a/docs/session.go b/docs/session.go index daf44982..bd0dfb2c 100644 --- a/docs/session.go +++ b/docs/session.go @@ -37,7 +37,7 @@ var ( // A Session is used to communicate with a V1 registry type Session struct { - indexEndpoint *Endpoint + indexEndpoint *V1Endpoint client *http.Client // TODO(tiborvass): remove authConfig authConfig *types.AuthConfig @@ -163,7 +163,7 @@ func (tr *authTransport) CancelRequest(req *http.Request) { // NewSession creates a new session // TODO(tiborvass): remove authConfig param once registry client v2 is vendored -func NewSession(client *http.Client, authConfig *types.AuthConfig, endpoint *Endpoint) (r *Session, err error) { +func NewSession(client *http.Client, authConfig *types.AuthConfig, endpoint *V1Endpoint) (r *Session, err error) { r = &Session{ authConfig: authConfig, client: client, @@ -175,7 +175,7 @@ func NewSession(client *http.Client, authConfig *types.AuthConfig, endpoint *End // If we're working with a standalone private registry over HTTPS, send Basic Auth headers // alongside all our requests. - if endpoint.VersionString(1) != IndexServer && endpoint.URL.Scheme == "https" { + if endpoint.String() != IndexServer && endpoint.URL.Scheme == "https" { info, err := endpoint.Ping() if err != nil { return nil, err @@ -405,7 +405,7 @@ func buildEndpointsList(headers []string, indexEp string) ([]string, error) { // GetRepositoryData returns lists of images and endpoints for the repository func (r *Session) GetRepositoryData(name reference.Named) (*RepositoryData, error) { - repositoryTarget := fmt.Sprintf("%srepositories/%s/images", r.indexEndpoint.VersionString(1), name.RemoteName()) + repositoryTarget := fmt.Sprintf("%srepositories/%s/images", r.indexEndpoint.String(), name.RemoteName()) logrus.Debugf("[registry] Calling GET %s", repositoryTarget) @@ -444,7 +444,7 @@ func (r *Session) GetRepositoryData(name reference.Named) (*RepositoryData, erro var endpoints []string if res.Header.Get("X-Docker-Endpoints") != "" { - endpoints, err = buildEndpointsList(res.Header["X-Docker-Endpoints"], r.indexEndpoint.VersionString(1)) + endpoints, err = buildEndpointsList(res.Header["X-Docker-Endpoints"], r.indexEndpoint.String()) if err != nil { return nil, err } @@ -634,7 +634,7 @@ func (r *Session) PushImageJSONIndex(remote reference.Named, imgList []*ImgData, if validate { suffix = "images" } - u := fmt.Sprintf("%srepositories/%s/%s", r.indexEndpoint.VersionString(1), remote.RemoteName(), suffix) + u := fmt.Sprintf("%srepositories/%s/%s", r.indexEndpoint.String(), remote.RemoteName(), suffix) logrus.Debugf("[registry] PUT %s", u) logrus.Debugf("Image list pushed to index:\n%s", imgListJSON) headers := map[string][]string{ @@ -680,7 +680,7 @@ func (r *Session) PushImageJSONIndex(remote reference.Named, imgList []*ImgData, if res.Header.Get("X-Docker-Endpoints") == "" { return nil, fmt.Errorf("Index response didn't contain any endpoints") } - endpoints, err = buildEndpointsList(res.Header["X-Docker-Endpoints"], r.indexEndpoint.VersionString(1)) + endpoints, err = buildEndpointsList(res.Header["X-Docker-Endpoints"], r.indexEndpoint.String()) if err != nil { return nil, err } @@ -722,7 +722,7 @@ func shouldRedirect(response *http.Response) bool { // SearchRepositories performs a search against the remote repository func (r *Session) SearchRepositories(term string) (*registrytypes.SearchResults, error) { logrus.Debugf("Index server: %s", r.indexEndpoint) - u := r.indexEndpoint.VersionString(1) + "search?q=" + url.QueryEscape(term) + u := r.indexEndpoint.String() + "search?q=" + url.QueryEscape(term) req, err := http.NewRequest("GET", u, nil) if err != nil { diff --git a/docs/token.go b/docs/token.go deleted file mode 100644 index d91bd455..00000000 --- a/docs/token.go +++ /dev/null @@ -1,81 +0,0 @@ -package registry - -import ( - "encoding/json" - "errors" - "fmt" - "net/http" - "net/url" - "strings" -) - -type tokenResponse struct { - Token string `json:"token"` -} - -func getToken(username, password string, params map[string]string, registryEndpoint *Endpoint) (string, error) { - realm, ok := params["realm"] - if !ok { - return "", errors.New("no realm specified for token auth challenge") - } - - realmURL, err := url.Parse(realm) - if err != nil { - return "", fmt.Errorf("invalid token auth challenge realm: %s", err) - } - - if realmURL.Scheme == "" { - if registryEndpoint.IsSecure { - realmURL.Scheme = "https" - } else { - realmURL.Scheme = "http" - } - } - - req, err := http.NewRequest("GET", realmURL.String(), nil) - if err != nil { - return "", err - } - - reqParams := req.URL.Query() - service := params["service"] - scope := params["scope"] - - if service != "" { - reqParams.Add("service", service) - } - - for _, scopeField := range strings.Fields(scope) { - reqParams.Add("scope", scopeField) - } - - if username != "" { - reqParams.Add("account", username) - req.SetBasicAuth(username, password) - } - - req.URL.RawQuery = reqParams.Encode() - - resp, err := registryEndpoint.client.Do(req) - if err != nil { - return "", err - } - defer resp.Body.Close() - - if resp.StatusCode != http.StatusOK { - return "", fmt.Errorf("token auth attempt for registry %s: %s request failed with status: %d %s", registryEndpoint, req.URL, resp.StatusCode, http.StatusText(resp.StatusCode)) - } - - decoder := json.NewDecoder(resp.Body) - - tr := new(tokenResponse) - if err = decoder.Decode(tr); err != nil { - return "", fmt.Errorf("unable to decode token response: %s", err) - } - - if tr.Token == "" { - return "", errors.New("authorization server did not include a token in the response") - } - - return tr.Token, nil -} diff --git a/docs/types.go b/docs/types.go index ee88276e..4247fed6 100644 --- a/docs/types.go +++ b/docs/types.go @@ -46,18 +46,18 @@ func (av APIVersion) String() string { return apiVersions[av] } -var apiVersions = map[APIVersion]string{ - 1: "v1", - 2: "v2", -} - // API Version identifiers. const ( - APIVersionUnknown = iota - APIVersion1 + _ = iota + APIVersion1 APIVersion = iota APIVersion2 ) +var apiVersions = map[APIVersion]string{ + APIVersion1: "v1", + APIVersion2: "v2", +} + // RepositoryInfo describes a repository type RepositoryInfo struct { reference.Named From cbd95acbbc556e3da505c95d7b83ffb46742c5ec Mon Sep 17 00:00:00 2001 From: Derek McGowan Date: Tue, 23 Feb 2016 15:18:04 -0800 Subject: [PATCH 359/375] Add support for identity token with token handler Use token handler options for initialization. Update auth endpoint to set identity token in response. Update credential store to match distribution interface changes. Signed-off-by: Derek McGowan (github: dmcgowan) --- docs/auth.go | 65 ++++++++++++++++++++++++++++++---------------- docs/service.go | 22 +++++++++++----- docs/service_v2.go | 2 +- 3 files changed, 59 insertions(+), 30 deletions(-) diff --git a/docs/auth.go b/docs/auth.go index a8fdb675..8351cd91 100644 --- a/docs/auth.go +++ b/docs/auth.go @@ -15,11 +15,16 @@ import ( registrytypes "github.com/docker/engine-api/types/registry" ) +const ( + // AuthClientID is used the ClientID used for the token server + AuthClientID = "docker" +) + // loginV1 tries to register/login to the v1 registry server. -func loginV1(authConfig *types.AuthConfig, apiEndpoint APIEndpoint, userAgent string) (string, error) { +func loginV1(authConfig *types.AuthConfig, apiEndpoint APIEndpoint, userAgent string) (string, string, error) { registryEndpoint, err := apiEndpoint.ToV1Endpoint(userAgent, nil) if err != nil { - return "", err + return "", "", err } serverAddress := registryEndpoint.String() @@ -27,48 +32,47 @@ func loginV1(authConfig *types.AuthConfig, apiEndpoint APIEndpoint, userAgent st logrus.Debugf("attempting v1 login to registry endpoint %s", registryEndpoint) if serverAddress == "" { - return "", fmt.Errorf("Server Error: Server Address not set.") + return "", "", fmt.Errorf("Server Error: Server Address not set.") } loginAgainstOfficialIndex := serverAddress == IndexServer req, err := http.NewRequest("GET", serverAddress+"users/", nil) if err != nil { - return "", err + return "", "", err } req.SetBasicAuth(authConfig.Username, authConfig.Password) resp, err := registryEndpoint.client.Do(req) if err != nil { // fallback when request could not be completed - return "", fallbackError{ + return "", "", fallbackError{ err: err, } } defer resp.Body.Close() body, err := ioutil.ReadAll(resp.Body) if err != nil { - return "", err + return "", "", err } if resp.StatusCode == http.StatusOK { - return "Login Succeeded", nil + return "Login Succeeded", "", nil } else if resp.StatusCode == http.StatusUnauthorized { if loginAgainstOfficialIndex { - return "", fmt.Errorf("Wrong login/password, please try again. Haven't got a Docker ID? Create one at https://hub.docker.com") + return "", "", fmt.Errorf("Wrong login/password, please try again. Haven't got a Docker ID? Create one at https://hub.docker.com") } - return "", fmt.Errorf("Wrong login/password, please try again") + return "", "", fmt.Errorf("Wrong login/password, please try again") } else if resp.StatusCode == http.StatusForbidden { if loginAgainstOfficialIndex { - return "", fmt.Errorf("Login: Account is not active. Please check your e-mail for a confirmation link.") + return "", "", fmt.Errorf("Login: Account is not active. Please check your e-mail for a confirmation link.") } // *TODO: Use registry configuration to determine what this says, if anything? - return "", fmt.Errorf("Login: Account is not active. Please see the documentation of the registry %s for instructions how to activate it.", serverAddress) + return "", "", fmt.Errorf("Login: Account is not active. Please see the documentation of the registry %s for instructions how to activate it.", serverAddress) } else if resp.StatusCode == http.StatusInternalServerError { // Issue #14326 logrus.Errorf("%s returned status code %d. Response Body :\n%s", req.URL.String(), resp.StatusCode, body) - return "", fmt.Errorf("Internal Server Error") - } else { - return "", fmt.Errorf("Login: %s (Code: %d; Headers: %s)", body, - resp.StatusCode, resp.Header) + return "", "", fmt.Errorf("Internal Server Error") } + return "", "", fmt.Errorf("Login: %s (Code: %d; Headers: %s)", body, + resp.StatusCode, resp.Header) } type loginCredentialStore struct { @@ -79,6 +83,14 @@ func (lcs loginCredentialStore) Basic(*url.URL) (string, string) { return lcs.authConfig.Username, lcs.authConfig.Password } +func (lcs loginCredentialStore) RefreshToken(*url.URL, string) string { + return lcs.authConfig.IdentityToken +} + +func (lcs loginCredentialStore) SetRefreshToken(u *url.URL, service, token string) { + lcs.authConfig.IdentityToken = token +} + type fallbackError struct { err error } @@ -90,7 +102,7 @@ func (err fallbackError) Error() string { // loginV2 tries to login to the v2 registry server. The given registry // endpoint will be pinged to get authorization challenges. These challenges // will be used to authenticate against the registry to validate credentials. -func loginV2(authConfig *types.AuthConfig, endpoint APIEndpoint, userAgent string) (string, error) { +func loginV2(authConfig *types.AuthConfig, endpoint APIEndpoint, userAgent string) (string, string, error) { logrus.Debugf("attempting v2 login to registry endpoint %s", endpoint) modifiers := DockerHeaders(userAgent, nil) @@ -101,14 +113,21 @@ func loginV2(authConfig *types.AuthConfig, endpoint APIEndpoint, userAgent strin if !foundV2 { err = fallbackError{err: err} } - return "", err + return "", "", err } + credentialAuthConfig := *authConfig creds := loginCredentialStore{ - authConfig: authConfig, + authConfig: &credentialAuthConfig, } - tokenHandler := auth.NewTokenHandler(authTransport, creds, "") + tokenHandlerOptions := auth.TokenHandlerOptions{ + Transport: authTransport, + Credentials: creds, + OfflineAccess: true, + ClientID: AuthClientID, + } + tokenHandler := auth.NewTokenHandlerWithOptions(tokenHandlerOptions) basicHandler := auth.NewBasicHandler(creds) modifiers = append(modifiers, auth.NewAuthorizer(challengeManager, tokenHandler, basicHandler)) tr := transport.NewTransport(authTransport, modifiers...) @@ -124,7 +143,7 @@ func loginV2(authConfig *types.AuthConfig, endpoint APIEndpoint, userAgent strin if !foundV2 { err = fallbackError{err: err} } - return "", err + return "", "", err } resp, err := loginClient.Do(req) @@ -132,7 +151,7 @@ func loginV2(authConfig *types.AuthConfig, endpoint APIEndpoint, userAgent strin if !foundV2 { err = fallbackError{err: err} } - return "", err + return "", "", err } defer resp.Body.Close() @@ -142,10 +161,10 @@ func loginV2(authConfig *types.AuthConfig, endpoint APIEndpoint, userAgent strin if !foundV2 { err = fallbackError{err: err} } - return "", err + return "", "", err } - return "Login Succeeded", nil + return "Login Succeeded", credentialAuthConfig.IdentityToken, nil } diff --git a/docs/service.go b/docs/service.go index 2124da6d..830c2bf6 100644 --- a/docs/service.go +++ b/docs/service.go @@ -2,6 +2,7 @@ package registry import ( "crypto/tls" + "fmt" "net/http" "net/url" "strings" @@ -29,10 +30,19 @@ func NewService(options *Options) *Service { // Auth contacts the public registry with the provided credentials, // and returns OK if authentication was successful. // It can be used to verify the validity of a client's credentials. -func (s *Service) Auth(authConfig *types.AuthConfig, userAgent string) (status string, err error) { - endpoints, err := s.LookupPushEndpoints(authConfig.ServerAddress) +func (s *Service) Auth(authConfig *types.AuthConfig, userAgent string) (status, token string, err error) { + serverAddress := authConfig.ServerAddress + if !strings.HasPrefix(serverAddress, "https://") && !strings.HasPrefix(serverAddress, "http://") { + serverAddress = "https://" + serverAddress + } + u, err := url.Parse(serverAddress) if err != nil { - return "", err + return "", "", fmt.Errorf("unable to parse server address: %v", err) + } + + endpoints, err := s.LookupPushEndpoints(u.Host) + if err != nil { + return "", "", err } for _, endpoint := range endpoints { @@ -41,7 +51,7 @@ func (s *Service) Auth(authConfig *types.AuthConfig, userAgent string) (status s login = loginV1 } - status, err = login(authConfig, endpoint, userAgent) + status, token, err = login(authConfig, endpoint, userAgent) if err == nil { return } @@ -50,10 +60,10 @@ func (s *Service) Auth(authConfig *types.AuthConfig, userAgent string) (status s logrus.Infof("Error logging in to %s endpoint, trying next endpoint: %v", endpoint.Version, err) continue } - return "", err + return "", "", err } - return "", err + return "", "", err } // splitReposSearchTerm breaks a search term into an index name and remote name diff --git a/docs/service_v2.go b/docs/service_v2.go index 9c909f18..0c8f04c5 100644 --- a/docs/service_v2.go +++ b/docs/service_v2.go @@ -10,7 +10,7 @@ import ( func (s *Service) lookupV2Endpoints(hostname string) (endpoints []APIEndpoint, err error) { var cfg = tlsconfig.ServerDefault tlsConfig := &cfg - if hostname == DefaultNamespace { + if hostname == DefaultNamespace || hostname == DefaultV1Registry.Host { // v2 mirrors for _, mirror := range s.Config.Mirrors { if !strings.HasPrefix(mirror, "http://") && !strings.HasPrefix(mirror, "https://") { From 3e2da4263eef8573c6e8dcef55b7e999d22c4e80 Mon Sep 17 00:00:00 2001 From: allencloud Date: Thu, 10 Mar 2016 00:17:57 +0800 Subject: [PATCH 360/375] fix some typos. Signed-off-by: allencloud --- docs/service.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/service.go b/docs/service.go index 2124da6d..d9ea71b3 100644 --- a/docs/service.go +++ b/docs/service.go @@ -145,14 +145,14 @@ func (s *Service) tlsConfigForMirror(mirrorURL *url.URL) (*tls.Config, error) { return s.TLSConfig(mirrorURL.Host) } -// LookupPullEndpoints creates an list of endpoints to try to pull from, in order of preference. +// LookupPullEndpoints creates a list of endpoints to try to pull from, in order of preference. // It gives preference to v2 endpoints over v1, mirrors over the actual // registry, and HTTPS over plain HTTP. func (s *Service) LookupPullEndpoints(hostname string) (endpoints []APIEndpoint, err error) { return s.lookupEndpoints(hostname) } -// LookupPushEndpoints creates an list of endpoints to try to push to, in order of preference. +// LookupPushEndpoints creates a list of endpoints to try to push to, in order of preference. // It gives preference to v2 endpoints over v1, and HTTPS over plain HTTP. // Mirrors are not included. func (s *Service) LookupPushEndpoints(hostname string) (endpoints []APIEndpoint, err error) { From 7caf33d6c5516f5497401ffe1223b65b377ad20c Mon Sep 17 00:00:00 2001 From: David Calavera Date: Tue, 8 Mar 2016 16:03:37 -0500 Subject: [PATCH 361/375] Move registry service options to the daemon configuration. Allowing to set their values in the daemon configuration file. Signed-off-by: David Calavera --- docs/config.go | 76 ++++++++++++++++++++------------------ docs/registry.go | 7 ---- docs/registry_mock_test.go | 21 +++-------- docs/registry_test.go | 6 +-- docs/service.go | 21 +++++++---- docs/service_v2.go | 2 +- 6 files changed, 62 insertions(+), 71 deletions(-) diff --git a/docs/config.go b/docs/config.go index 7d8b6301..ab6f0715 100644 --- a/docs/config.go +++ b/docs/config.go @@ -13,10 +13,20 @@ import ( registrytypes "github.com/docker/engine-api/types/registry" ) -// Options holds command line options. -type Options struct { - Mirrors opts.ListOpts - InsecureRegistries opts.ListOpts +// ServiceOptions holds command line options. +type ServiceOptions struct { + Mirrors []string `json:"registry-mirrors,omitempty"` + InsecureRegistries []string `json:"insecure-registries,omitempty"` + + // V2Only controls access to legacy registries. If it is set to true via the + // command line flag the daemon will not attempt to contact v1 legacy registries + V2Only bool `json:"disable-legacy-registry,omitempty"` +} + +// serviceConfig holds daemon configuration for the registry service. +type serviceConfig struct { + registrytypes.ServiceConfig + V2Only bool } var ( @@ -42,51 +52,45 @@ var ( // not have the correct form ErrInvalidRepositoryName = errors.New("Invalid repository name (ex: \"registry.domain.tld/myrepos\")") - emptyServiceConfig = NewServiceConfig(nil) - - // V2Only controls access to legacy registries. If it is set to true via the - // command line flag the daemon will not attempt to contact v1 legacy registries - V2Only = false + emptyServiceConfig = newServiceConfig(ServiceOptions{}) ) // for mocking in unit tests var lookupIP = net.LookupIP -// InstallFlags adds command-line options to the top-level flag parser for +// InstallCliFlags adds command-line options to the top-level flag parser for // the current process. -func (options *Options) InstallFlags(cmd *flag.FlagSet, usageFn func(string) string) { - options.Mirrors = opts.NewListOpts(ValidateMirror) - cmd.Var(&options.Mirrors, []string{"-registry-mirror"}, usageFn("Preferred Docker registry mirror")) - options.InsecureRegistries = opts.NewListOpts(ValidateIndexName) - cmd.Var(&options.InsecureRegistries, []string{"-insecure-registry"}, usageFn("Enable insecure registry communication")) - cmd.BoolVar(&V2Only, []string{"-disable-legacy-registry"}, false, usageFn("Do not contact legacy registries")) +func (options *ServiceOptions) InstallCliFlags(cmd *flag.FlagSet, usageFn func(string) string) { + mirrors := opts.NewNamedListOptsRef("registry-mirrors", &options.Mirrors, ValidateMirror) + cmd.Var(mirrors, []string{"-registry-mirror"}, usageFn("Preferred Docker registry mirror")) + + insecureRegistries := opts.NewNamedListOptsRef("insecure-registries", &options.InsecureRegistries, ValidateIndexName) + cmd.Var(insecureRegistries, []string{"-insecure-registry"}, usageFn("Enable insecure registry communication")) + + cmd.BoolVar(&options.V2Only, []string{"-disable-legacy-registry"}, false, usageFn("Do not contact legacy registries")) } -// NewServiceConfig returns a new instance of ServiceConfig -func NewServiceConfig(options *Options) *registrytypes.ServiceConfig { - if options == nil { - options = &Options{ - Mirrors: opts.NewListOpts(nil), - InsecureRegistries: opts.NewListOpts(nil), - } - } - +// newServiceConfig returns a new instance of ServiceConfig +func newServiceConfig(options ServiceOptions) *serviceConfig { // Localhost is by default considered as an insecure registry // This is a stop-gap for people who are running a private registry on localhost (especially on Boot2docker). // // TODO: should we deprecate this once it is easier for people to set up a TLS registry or change // daemon flags on boot2docker? - options.InsecureRegistries.Set("127.0.0.0/8") + options.InsecureRegistries = append(options.InsecureRegistries, "127.0.0.0/8") - config := ®istrytypes.ServiceConfig{ - InsecureRegistryCIDRs: make([]*registrytypes.NetIPNet, 0), - IndexConfigs: make(map[string]*registrytypes.IndexInfo, 0), - // Hack: Bypass setting the mirrors to IndexConfigs since they are going away - // and Mirrors are only for the official registry anyways. - Mirrors: options.Mirrors.GetAll(), + config := &serviceConfig{ + ServiceConfig: registrytypes.ServiceConfig{ + InsecureRegistryCIDRs: make([]*registrytypes.NetIPNet, 0), + IndexConfigs: make(map[string]*registrytypes.IndexInfo, 0), + // Hack: Bypass setting the mirrors to IndexConfigs since they are going away + // and Mirrors are only for the official registry anyways. + Mirrors: options.Mirrors, + }, + V2Only: options.V2Only, } // Split --insecure-registry into CIDR and registry-specific settings. - for _, r := range options.InsecureRegistries.GetAll() { + for _, r := range options.InsecureRegistries { // Check if CIDR was passed to --insecure-registry _, ipnet, err := net.ParseCIDR(r) if err == nil { @@ -125,7 +129,7 @@ func NewServiceConfig(options *Options) *registrytypes.ServiceConfig { // or an IP address. If it is a domain name, then it will be resolved in order to check if the IP is contained // in a subnet. If the resolving is not successful, isSecureIndex will only try to match hostname to any element // of insecureRegistries. -func isSecureIndex(config *registrytypes.ServiceConfig, indexName string) bool { +func isSecureIndex(config *serviceConfig, indexName string) bool { // Check for configured index, first. This is needed in case isSecureIndex // is called from anything besides newIndexInfo, in order to honor per-index configurations. if index, ok := config.IndexConfigs[indexName]; ok { @@ -201,7 +205,7 @@ func validateNoSchema(reposName string) error { } // newIndexInfo returns IndexInfo configuration from indexName -func newIndexInfo(config *registrytypes.ServiceConfig, indexName string) (*registrytypes.IndexInfo, error) { +func newIndexInfo(config *serviceConfig, indexName string) (*registrytypes.IndexInfo, error) { var err error indexName, err = ValidateIndexName(indexName) if err != nil { @@ -233,7 +237,7 @@ func GetAuthConfigKey(index *registrytypes.IndexInfo) string { } // newRepositoryInfo validates and breaks down a repository name into a RepositoryInfo -func newRepositoryInfo(config *registrytypes.ServiceConfig, name reference.Named) (*RepositoryInfo, error) { +func newRepositoryInfo(config *serviceConfig, name reference.Named) (*RepositoryInfo, error) { index, err := newIndexInfo(config, name.Hostname()) if err != nil { return nil, err diff --git a/docs/registry.go b/docs/registry.go index 9071d9dc..8fdfe3b0 100644 --- a/docs/registry.go +++ b/docs/registry.go @@ -11,7 +11,6 @@ import ( "net/http" "os" "path/filepath" - "runtime" "strings" "time" @@ -26,12 +25,6 @@ var ( ErrAlreadyExists = errors.New("Image already exists") ) -func init() { - if runtime.GOOS != "linux" { - V2Only = true - } -} - func newTLSConfig(hostname string, isSecure bool) (*tls.Config, error) { // PreferredServerCipherSuites should have no effect tlsConfig := tlsconfig.ServerDefault diff --git a/docs/registry_mock_test.go b/docs/registry_mock_test.go index 057afac1..828f48fc 100644 --- a/docs/registry_mock_test.go +++ b/docs/registry_mock_test.go @@ -15,7 +15,6 @@ import ( "testing" "time" - "github.com/docker/docker/opts" "github.com/docker/docker/reference" registrytypes "github.com/docker/engine-api/types/registry" "github.com/gorilla/mux" @@ -174,23 +173,13 @@ func makePublicIndex() *registrytypes.IndexInfo { return index } -func makeServiceConfig(mirrors []string, insecureRegistries []string) *registrytypes.ServiceConfig { - options := &Options{ - Mirrors: opts.NewListOpts(nil), - InsecureRegistries: opts.NewListOpts(nil), - } - if mirrors != nil { - for _, mirror := range mirrors { - options.Mirrors.Set(mirror) - } - } - if insecureRegistries != nil { - for _, insecureRegistries := range insecureRegistries { - options.InsecureRegistries.Set(insecureRegistries) - } +func makeServiceConfig(mirrors []string, insecureRegistries []string) *serviceConfig { + options := ServiceOptions{ + Mirrors: mirrors, + InsecureRegistries: insecureRegistries, } - return NewServiceConfig(options) + return newServiceConfig(options) } func writeHeaders(w http.ResponseWriter) { diff --git a/docs/registry_test.go b/docs/registry_test.go index 02eb683d..7f9cc8e4 100644 --- a/docs/registry_test.go +++ b/docs/registry_test.go @@ -523,7 +523,7 @@ func TestParseRepositoryInfo(t *testing.T) { } func TestNewIndexInfo(t *testing.T) { - testIndexInfo := func(config *registrytypes.ServiceConfig, expectedIndexInfos map[string]*registrytypes.IndexInfo) { + testIndexInfo := func(config *serviceConfig, expectedIndexInfos map[string]*registrytypes.IndexInfo) { for indexName, expectedIndexInfo := range expectedIndexInfos { index, err := newIndexInfo(config, indexName) if err != nil { @@ -537,7 +537,7 @@ func TestNewIndexInfo(t *testing.T) { } } - config := NewServiceConfig(nil) + config := newServiceConfig(ServiceOptions{}) noMirrors := []string{} expectedIndexInfos := map[string]*registrytypes.IndexInfo{ IndexName: { @@ -661,7 +661,7 @@ func TestMirrorEndpointLookup(t *testing.T) { } return false } - s := Service{Config: makeServiceConfig([]string{"my.mirror"}, nil)} + s := Service{config: makeServiceConfig([]string{"my.mirror"}, nil)} imageName, err := reference.WithName(IndexName + "/test/image") if err != nil { diff --git a/docs/service.go b/docs/service.go index 2124da6d..47bffed3 100644 --- a/docs/service.go +++ b/docs/service.go @@ -15,17 +15,22 @@ import ( // Service is a registry service. It tracks configuration data such as a list // of mirrors. type Service struct { - Config *registrytypes.ServiceConfig + config *serviceConfig } // NewService returns a new instance of Service ready to be // installed into an engine. -func NewService(options *Options) *Service { +func NewService(options ServiceOptions) *Service { return &Service{ - Config: NewServiceConfig(options), + config: newServiceConfig(options), } } +// ServiceConfig returns the public registry service configuration. +func (s *Service) ServiceConfig() *registrytypes.ServiceConfig { + return &s.config.ServiceConfig +} + // Auth contacts the public registry with the provided credentials, // and returns OK if authentication was successful. // It can be used to verify the validity of a client's credentials. @@ -82,7 +87,7 @@ func (s *Service) Search(term string, authConfig *types.AuthConfig, userAgent st indexName, remoteName := splitReposSearchTerm(term) - index, err := newIndexInfo(s.Config, indexName) + index, err := newIndexInfo(s.config, indexName) if err != nil { return nil, err } @@ -113,12 +118,12 @@ func (s *Service) Search(term string, authConfig *types.AuthConfig, userAgent st // ResolveRepository splits a repository name into its components // and configuration of the associated registry. func (s *Service) ResolveRepository(name reference.Named) (*RepositoryInfo, error) { - return newRepositoryInfo(s.Config, name) + return newRepositoryInfo(s.config, name) } // ResolveIndex takes indexName and returns index info func (s *Service) ResolveIndex(name string) (*registrytypes.IndexInfo, error) { - return newIndexInfo(s.Config, name) + return newIndexInfo(s.config, name) } // APIEndpoint represents a remote API endpoint @@ -138,7 +143,7 @@ func (e APIEndpoint) ToV1Endpoint(userAgent string, metaHeaders http.Header) (*V // TLSConfig constructs a client TLS configuration based on server defaults func (s *Service) TLSConfig(hostname string) (*tls.Config, error) { - return newTLSConfig(hostname, isSecureIndex(s.Config, hostname)) + return newTLSConfig(hostname, isSecureIndex(s.config, hostname)) } func (s *Service) tlsConfigForMirror(mirrorURL *url.URL) (*tls.Config, error) { @@ -173,7 +178,7 @@ func (s *Service) lookupEndpoints(hostname string) (endpoints []APIEndpoint, err return nil, err } - if V2Only { + if s.config.V2Only { return endpoints, nil } diff --git a/docs/service_v2.go b/docs/service_v2.go index 9c909f18..744be691 100644 --- a/docs/service_v2.go +++ b/docs/service_v2.go @@ -12,7 +12,7 @@ func (s *Service) lookupV2Endpoints(hostname string) (endpoints []APIEndpoint, e tlsConfig := &cfg if hostname == DefaultNamespace { // v2 mirrors - for _, mirror := range s.Config.Mirrors { + for _, mirror := range s.config.Mirrors { if !strings.HasPrefix(mirror, "http://") && !strings.HasPrefix(mirror, "https://") { mirror = "https://" + mirror } From 2f170573145d7fb8a24ed8e0df11555aa8336ec0 Mon Sep 17 00:00:00 2001 From: Aaron Lehmann Date: Thu, 10 Mar 2016 12:26:32 -0800 Subject: [PATCH 362/375] Remove Windows-specific default registry definitions Going forward, Docker won't use a different default registry on Windows. This changes Windows to use the standard Docker Hub registry as the default registry. There is a plan in place to migrate existing images from the Windows registry to Hub's normal registry, in advance of the 1.11 release. In the mean time, images on the Windows registry can be accessed by prefixing them with `registry-win-tp3.docker.io/`. Signed-off-by: Aaron Lehmann --- docs/config.go | 12 +++++++++++- docs/config_unix.go | 18 ------------------ docs/config_windows.go | 19 ------------------- 3 files changed, 11 insertions(+), 38 deletions(-) diff --git a/docs/config.go b/docs/config.go index 7d8b6301..561e1d6e 100644 --- a/docs/config.go +++ b/docs/config.go @@ -34,7 +34,17 @@ var ( // NotaryServer is the endpoint serving the Notary trust server NotaryServer = "https://notary.docker.io" - // IndexServer = "https://registry-stage.hub.docker.com/v1/" + // DefaultV1Registry is the URI of the default v1 registry + DefaultV1Registry = &url.URL{ + Scheme: "https", + Host: "index.docker.io", + } + + // DefaultV2Registry is the URI of the default v2 registry + DefaultV2Registry = &url.URL{ + Scheme: "https", + Host: "registry-1.docker.io", + } ) var ( diff --git a/docs/config_unix.go b/docs/config_unix.go index c3c19162..b81d2493 100644 --- a/docs/config_unix.go +++ b/docs/config_unix.go @@ -2,24 +2,6 @@ package registry -import ( - "net/url" -) - -var ( - // DefaultV1Registry is the URI of the default v1 registry - DefaultV1Registry = &url.URL{ - Scheme: "https", - Host: "index.docker.io", - } - - // DefaultV2Registry is the URI of the default v2 registry - DefaultV2Registry = &url.URL{ - Scheme: "https", - Host: "registry-1.docker.io", - } -) - var ( // CertsDir is the directory where certificates are stored CertsDir = "/etc/docker/certs.d" diff --git a/docs/config_windows.go b/docs/config_windows.go index f1ee488b..82bc4afe 100644 --- a/docs/config_windows.go +++ b/docs/config_windows.go @@ -1,30 +1,11 @@ package registry import ( - "net/url" "os" "path/filepath" "strings" ) -var ( - // DefaultV1Registry is the URI of the default v1 registry - DefaultV1Registry = &url.URL{ - Scheme: "https", - Host: "registry-win-tp3.docker.io", - } - - // DefaultV2Registry is the URI of the default (official) v2 registry. - // This is the windows-specific endpoint. - // - // Currently it is a TEMPORARY link that allows Microsoft to continue - // development of Docker Engine for Windows. - DefaultV2Registry = &url.URL{ - Scheme: "https", - Host: "registry-win-tp3.docker.io", - } -) - // CertsDir is the directory where certificates are stored var CertsDir = os.Getenv("programdata") + `\docker\certs.d` From d5160a02110e411447ca154e86875648083cf6ea Mon Sep 17 00:00:00 2001 From: Antonio Murdaca Date: Tue, 15 Mar 2016 17:12:20 +0100 Subject: [PATCH 363/375] daemon: update: check len inside public function Signed-off-by: Antonio Murdaca --- docs/registry_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/registry_test.go b/docs/registry_test.go index 7f9cc8e4..7442ebc0 100644 --- a/docs/registry_test.go +++ b/docs/registry_test.go @@ -171,7 +171,7 @@ func TestGetRemoteImageJSON(t *testing.T) { t.Fatal(err) } assertEqual(t, size, int64(154), "Expected size 154") - if len(json) <= 0 { + if len(json) == 0 { t.Fatal("Expected non-empty json") } From b4d9ae605214569fe535979f5b2e98d377ac71c8 Mon Sep 17 00:00:00 2001 From: Antonio Murdaca Date: Wed, 16 Mar 2016 12:53:07 +0100 Subject: [PATCH 364/375] registry: endpoint_v1: fix outdated comment Signed-off-by: Antonio Murdaca --- docs/endpoint_v1.go | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/docs/endpoint_v1.go b/docs/endpoint_v1.go index 58e2600e..fd81972c 100644 --- a/docs/endpoint_v1.go +++ b/docs/endpoint_v1.go @@ -21,8 +21,7 @@ type V1Endpoint struct { IsSecure bool } -// NewV1Endpoint parses the given address to return a registry endpoint. v can be used to -// specify a specific endpoint version +// NewV1Endpoint parses the given address to return a registry endpoint. func NewV1Endpoint(index *registrytypes.IndexInfo, userAgent string, metaHeaders http.Header) (*V1Endpoint, error) { tlsConfig, err := newTLSConfig(index.Name, index.Secure) if err != nil { From 7f7cb8214961d95283571cba5e23791654bfcd8a Mon Sep 17 00:00:00 2001 From: Antonio Murdaca Date: Wed, 16 Mar 2016 16:38:13 +0100 Subject: [PATCH 365/375] *: fix response body leaks Signed-off-by: Antonio Murdaca --- docs/session.go | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/session.go b/docs/session.go index bd0dfb2c..5647ad28 100644 --- a/docs/session.go +++ b/docs/session.go @@ -284,6 +284,7 @@ func (r *Session) GetRemoteImageLayer(imgID, registry string, imgSize int64) (io res, err = r.client.Do(req) if err != nil { logrus.Debugf("Error contacting registry %s: %v", registry, err) + // the only case err != nil && res != nil is https://golang.org/src/net/http/client.go#L515 if res != nil { if res.Body != nil { res.Body.Close() From 0f4b8d34555803b77901fb8a017d015059641c19 Mon Sep 17 00:00:00 2001 From: Aaron Lehmann Date: Mon, 28 Mar 2016 18:22:24 -0700 Subject: [PATCH 366/375] Correct login debug log message I noticed the following message in a daemon log: ``` attempting v2 login to registry endpoint {%!s(bool=false) https://registry:5000 v2 %!s(bool=false) %!s(bool=true) %!s(*tls.Config=&{ [] map[] 0xc82075c030 [] 0 false [49196 49200 49195 49199 49162 49161 49172 49171 53 47] true false [0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0] 769 0 [] {{0 0} 0} {{0 0} 0 0 0 0} []})} ``` loginV2 tries to log an APIEndpoint as a string, but this struct does not have a String method. Log the actual URL that will be used as the endpoint, instead. Signed-off-by: Aaron Lehmann --- docs/auth.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/auth.go b/docs/auth.go index 8351cd91..c5663f58 100644 --- a/docs/auth.go +++ b/docs/auth.go @@ -29,7 +29,7 @@ func loginV1(authConfig *types.AuthConfig, apiEndpoint APIEndpoint, userAgent st serverAddress := registryEndpoint.String() - logrus.Debugf("attempting v1 login to registry endpoint %s", registryEndpoint) + logrus.Debugf("attempting v1 login to registry endpoint %s", serverAddress) if serverAddress == "" { return "", "", fmt.Errorf("Server Error: Server Address not set.") @@ -103,7 +103,7 @@ func (err fallbackError) Error() string { // endpoint will be pinged to get authorization challenges. These challenges // will be used to authenticate against the registry to validate credentials. func loginV2(authConfig *types.AuthConfig, endpoint APIEndpoint, userAgent string) (string, string, error) { - logrus.Debugf("attempting v2 login to registry endpoint %s", endpoint) + logrus.Debugf("attempting v2 login to registry endpoint %s", strings.TrimRight(endpoint.URL.String(), "/")+"/v2/") modifiers := DockerHeaders(userAgent, nil) authTransport := transport.NewTransport(NewTransport(endpoint.TLSConfig), modifiers...) From 53a8806b40437074e0bdcd29425c54a87df7e0c0 Mon Sep 17 00:00:00 2001 From: allencloud Date: Tue, 29 Mar 2016 14:36:38 +0800 Subject: [PATCH 367/375] 1.change validateNoSchema into validateNoScheme 2.change schema into scheme in docs and some annotations. Signed-off-by: allencloud --- docs/config.go | 2 +- docs/service.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/config.go b/docs/config.go index 50061329..51302d11 100644 --- a/docs/config.go +++ b/docs/config.go @@ -206,7 +206,7 @@ func ValidateIndexName(val string) (string, error) { return val, nil } -func validateNoSchema(reposName string) error { +func validateNoScheme(reposName string) error { if strings.Contains(reposName, "://") { // It cannot contain a scheme! return ErrInvalidRepositoryName diff --git a/docs/service.go b/docs/service.go index acafc34b..c27f9b1c 100644 --- a/docs/service.go +++ b/docs/service.go @@ -91,7 +91,7 @@ func splitReposSearchTerm(reposName string) (string, string) { // Search queries the public registry for images matching the specified // search terms, and returns the results. func (s *Service) Search(term string, authConfig *types.AuthConfig, userAgent string, headers map[string][]string) (*registrytypes.SearchResults, error) { - if err := validateNoSchema(term); err != nil { + if err := validateNoScheme(term); err != nil { return nil, err } From 56480ce80ac6d16e44cc99afec83c5f1914c0fb8 Mon Sep 17 00:00:00 2001 From: Yong Tang Date: Fri, 22 Apr 2016 20:00:47 -0700 Subject: [PATCH 368/375] Add default `serveraddress` value in remote API `/auth` This fix tries to address the issue in #22244 where the remote API `/auth` will not set the default value of `serveraddress` if not provided. This behavior happens after only in 1.11.0 and is a regression as in 1.10.3 `serveraddress` will be assigned with `IndexServer` if no value is provided. The default value `IndexServer` is assigned to `serveraddress` if no value provided in this fix. An integration test `TestAuthApi` has been added to cover this change This fix fixes #22244. Signed-off-by: Yong Tang --- docs/service.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/docs/service.go b/docs/service.go index c27f9b1c..3006e8ab 100644 --- a/docs/service.go +++ b/docs/service.go @@ -37,6 +37,9 @@ func (s *Service) ServiceConfig() *registrytypes.ServiceConfig { // It can be used to verify the validity of a client's credentials. func (s *Service) Auth(authConfig *types.AuthConfig, userAgent string) (status, token string, err error) { serverAddress := authConfig.ServerAddress + if serverAddress == "" { + serverAddress = IndexServer + } if !strings.HasPrefix(serverAddress, "https://") && !strings.HasPrefix(serverAddress, "http://") { serverAddress = "https://" + serverAddress } From c4778ea1bea152515d2fbd82bf0085a6ce9af663 Mon Sep 17 00:00:00 2001 From: Brett Higgins Date: Mon, 25 Apr 2016 07:54:48 -0400 Subject: [PATCH 369/375] Respect ALL_PROXY during registry operations Use sockets.DialerFromEnvironment, as is done in other places, to transparently support SOCKS proxy config from ALL_PROXY environment variable. Requires the *engine* have the ALL_PROXY env var set, which doesn't seem ideal. Maybe it should be a CLI option somehow? Only tested with push and a v2 registry so far. I'm happy to look further into testing more broadly, but I wanted to get feedback on the general idea first. Signed-off-by: Brett Higgins --- docs/registry.go | 24 +++++++++++++++++------- 1 file changed, 17 insertions(+), 7 deletions(-) diff --git a/docs/registry.go b/docs/registry.go index 8fdfe3b0..0b5a070e 100644 --- a/docs/registry.go +++ b/docs/registry.go @@ -16,6 +16,7 @@ import ( "github.com/Sirupsen/logrus" "github.com/docker/distribution/registry/client/transport" + "github.com/docker/go-connections/sockets" "github.com/docker/go-connections/tlsconfig" ) @@ -165,16 +166,25 @@ func NewTransport(tlsConfig *tls.Config) *http.Transport { var cfg = tlsconfig.ServerDefault tlsConfig = &cfg } - return &http.Transport{ - Proxy: http.ProxyFromEnvironment, - Dial: (&net.Dialer{ - Timeout: 30 * time.Second, - KeepAlive: 30 * time.Second, - DualStack: true, - }).Dial, + + direct := &net.Dialer{ + Timeout: 30 * time.Second, + KeepAlive: 30 * time.Second, + DualStack: true, + } + + base := &http.Transport{ + Proxy: http.ProxyFromEnvironment, + Dial: direct.Dial, TLSHandshakeTimeout: 10 * time.Second, TLSClientConfig: tlsConfig, // TODO(dmcgowan): Call close idle connections when complete and use keep alive DisableKeepAlives: true, } + + proxyDialer, err := sockets.DialerFromEnvironment(direct) + if err == nil { + base.Dial = proxyDialer.Dial + } + return base } From 04476ff5a9e638b793a3701e951e80ab97c5c6f1 Mon Sep 17 00:00:00 2001 From: Vincent Demeester Date: Sat, 21 May 2016 16:00:28 +0200 Subject: [PATCH 370/375] =?UTF-8?q?Add=20Unit=20test=20to=20daemon.SearchR?= =?UTF-8?q?egistryForImages=E2=80=A6?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit … and refactor a little bit some daemon on the way. - Move `SearchRegistryForImages` to a new file (`daemon/search.go`) as `daemon.go` is getting pretty big. - `registry.Service` is now an interface (allowing us to decouple it a little bit and thus unit test easily). - Add some unit test for `SearchRegistryForImages`. - Use UniqueExactMatch for search filters - And use empty restore id for now in client.ContainerStart. Signed-off-by: Vincent Demeester --- docs/registry_test.go | 2 +- docs/service.go | 46 +++++++++++++++++++++++++++++-------------- docs/service_v1.go | 2 +- docs/service_v2.go | 2 +- 4 files changed, 34 insertions(+), 18 deletions(-) diff --git a/docs/registry_test.go b/docs/registry_test.go index 7442ebc0..39a01bcd 100644 --- a/docs/registry_test.go +++ b/docs/registry_test.go @@ -661,7 +661,7 @@ func TestMirrorEndpointLookup(t *testing.T) { } return false } - s := Service{config: makeServiceConfig([]string{"my.mirror"}, nil)} + s := DefaultService{config: makeServiceConfig([]string{"my.mirror"}, nil)} imageName, err := reference.WithName(IndexName + "/test/image") if err != nil { diff --git a/docs/service.go b/docs/service.go index 3006e8ab..d48063cd 100644 --- a/docs/service.go +++ b/docs/service.go @@ -7,35 +7,50 @@ import ( "net/url" "strings" + "golang.org/x/net/context" + "github.com/Sirupsen/logrus" "github.com/docker/docker/reference" "github.com/docker/engine-api/types" registrytypes "github.com/docker/engine-api/types/registry" ) -// Service is a registry service. It tracks configuration data such as a list +// Service is the interface defining what a registry service should implement. +type Service interface { + Auth(ctx context.Context, authConfig *types.AuthConfig, userAgent string) (status, token string, err error) + LookupPullEndpoints(hostname string) (endpoints []APIEndpoint, err error) + LookupPushEndpoints(hostname string) (endpoints []APIEndpoint, err error) + ResolveRepository(name reference.Named) (*RepositoryInfo, error) + ResolveIndex(name string) (*registrytypes.IndexInfo, error) + Search(ctx context.Context, term string, authConfig *types.AuthConfig, userAgent string, headers map[string][]string) (*registrytypes.SearchResults, error) + ServiceConfig() *registrytypes.ServiceConfig + TLSConfig(hostname string) (*tls.Config, error) +} + +// DefaultService is a registry service. It tracks configuration data such as a list // of mirrors. -type Service struct { +type DefaultService struct { config *serviceConfig } -// NewService returns a new instance of Service ready to be +// NewService returns a new instance of DefaultService ready to be // installed into an engine. -func NewService(options ServiceOptions) *Service { - return &Service{ +func NewService(options ServiceOptions) *DefaultService { + return &DefaultService{ config: newServiceConfig(options), } } // ServiceConfig returns the public registry service configuration. -func (s *Service) ServiceConfig() *registrytypes.ServiceConfig { +func (s *DefaultService) ServiceConfig() *registrytypes.ServiceConfig { return &s.config.ServiceConfig } // Auth contacts the public registry with the provided credentials, // and returns OK if authentication was successful. // It can be used to verify the validity of a client's credentials. -func (s *Service) Auth(authConfig *types.AuthConfig, userAgent string) (status, token string, err error) { +func (s *DefaultService) Auth(ctx context.Context, authConfig *types.AuthConfig, userAgent string) (status, token string, err error) { + // TODO Use ctx when searching for repositories serverAddress := authConfig.ServerAddress if serverAddress == "" { serverAddress = IndexServer @@ -93,7 +108,8 @@ func splitReposSearchTerm(reposName string) (string, string) { // Search queries the public registry for images matching the specified // search terms, and returns the results. -func (s *Service) Search(term string, authConfig *types.AuthConfig, userAgent string, headers map[string][]string) (*registrytypes.SearchResults, error) { +func (s *DefaultService) Search(ctx context.Context, term string, authConfig *types.AuthConfig, userAgent string, headers map[string][]string) (*registrytypes.SearchResults, error) { + // TODO Use ctx when searching for repositories if err := validateNoScheme(term); err != nil { return nil, err } @@ -130,12 +146,12 @@ func (s *Service) Search(term string, authConfig *types.AuthConfig, userAgent st // ResolveRepository splits a repository name into its components // and configuration of the associated registry. -func (s *Service) ResolveRepository(name reference.Named) (*RepositoryInfo, error) { +func (s *DefaultService) ResolveRepository(name reference.Named) (*RepositoryInfo, error) { return newRepositoryInfo(s.config, name) } // ResolveIndex takes indexName and returns index info -func (s *Service) ResolveIndex(name string) (*registrytypes.IndexInfo, error) { +func (s *DefaultService) ResolveIndex(name string) (*registrytypes.IndexInfo, error) { return newIndexInfo(s.config, name) } @@ -155,25 +171,25 @@ func (e APIEndpoint) ToV1Endpoint(userAgent string, metaHeaders http.Header) (*V } // TLSConfig constructs a client TLS configuration based on server defaults -func (s *Service) TLSConfig(hostname string) (*tls.Config, error) { +func (s *DefaultService) TLSConfig(hostname string) (*tls.Config, error) { return newTLSConfig(hostname, isSecureIndex(s.config, hostname)) } -func (s *Service) tlsConfigForMirror(mirrorURL *url.URL) (*tls.Config, error) { +func (s *DefaultService) tlsConfigForMirror(mirrorURL *url.URL) (*tls.Config, error) { return s.TLSConfig(mirrorURL.Host) } // LookupPullEndpoints creates a list of endpoints to try to pull from, in order of preference. // It gives preference to v2 endpoints over v1, mirrors over the actual // registry, and HTTPS over plain HTTP. -func (s *Service) LookupPullEndpoints(hostname string) (endpoints []APIEndpoint, err error) { +func (s *DefaultService) LookupPullEndpoints(hostname string) (endpoints []APIEndpoint, err error) { return s.lookupEndpoints(hostname) } // LookupPushEndpoints creates a list of endpoints to try to push to, in order of preference. // It gives preference to v2 endpoints over v1, and HTTPS over plain HTTP. // Mirrors are not included. -func (s *Service) LookupPushEndpoints(hostname string) (endpoints []APIEndpoint, err error) { +func (s *DefaultService) LookupPushEndpoints(hostname string) (endpoints []APIEndpoint, err error) { allEndpoints, err := s.lookupEndpoints(hostname) if err == nil { for _, endpoint := range allEndpoints { @@ -185,7 +201,7 @@ func (s *Service) LookupPushEndpoints(hostname string) (endpoints []APIEndpoint, return endpoints, err } -func (s *Service) lookupEndpoints(hostname string) (endpoints []APIEndpoint, err error) { +func (s *DefaultService) lookupEndpoints(hostname string) (endpoints []APIEndpoint, err error) { endpoints, err = s.lookupV2Endpoints(hostname) if err != nil { return nil, err diff --git a/docs/service_v1.go b/docs/service_v1.go index 56121eea..5d7e8989 100644 --- a/docs/service_v1.go +++ b/docs/service_v1.go @@ -6,7 +6,7 @@ import ( "github.com/docker/go-connections/tlsconfig" ) -func (s *Service) lookupV1Endpoints(hostname string) (endpoints []APIEndpoint, err error) { +func (s *DefaultService) lookupV1Endpoints(hostname string) (endpoints []APIEndpoint, err error) { var cfg = tlsconfig.ServerDefault tlsConfig := &cfg if hostname == DefaultNamespace { diff --git a/docs/service_v2.go b/docs/service_v2.go index 4113d57d..5e62f8ff 100644 --- a/docs/service_v2.go +++ b/docs/service_v2.go @@ -7,7 +7,7 @@ import ( "github.com/docker/go-connections/tlsconfig" ) -func (s *Service) lookupV2Endpoints(hostname string) (endpoints []APIEndpoint, err error) { +func (s *DefaultService) lookupV2Endpoints(hostname string) (endpoints []APIEndpoint, err error) { var cfg = tlsconfig.ServerDefault tlsConfig := &cfg if hostname == DefaultNamespace || hostname == DefaultV1Registry.Host { From d265da7356ae8c272d73bb7d1d321935dc7bf9a2 Mon Sep 17 00:00:00 2001 From: allencloud Date: Sun, 8 May 2016 09:36:10 +0800 Subject: [PATCH 371/375] fix typos Signed-off-by: allencloud --- docs/registry.go | 2 +- docs/session.go | 2 +- docs/types.go | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/registry.go b/docs/registry.go index 0b5a070e..973bff9f 100644 --- a/docs/registry.go +++ b/docs/registry.go @@ -114,7 +114,7 @@ func DockerHeaders(userAgent string, metaHeaders http.Header) []transport.Reques return modifiers } -// HTTPClient returns a HTTP client structure which uses the given transport +// HTTPClient returns an HTTP client structure which uses the given transport // and contains the necessary headers for redirected requests func HTTPClient(transport http.RoundTripper) *http.Client { return &http.Client{ diff --git a/docs/session.go b/docs/session.go index 5647ad28..82593cd7 100644 --- a/docs/session.go +++ b/docs/session.go @@ -95,7 +95,7 @@ func cloneRequest(r *http.Request) *http.Request { return r2 } -// RoundTrip changes a HTTP request's headers to add the necessary +// RoundTrip changes an HTTP request's headers to add the necessary // authentication-related headers func (tr *authTransport) RoundTrip(orig *http.Request) (*http.Response, error) { // Authorization should not be set on 302 redirect for untrusted locations. diff --git a/docs/types.go b/docs/types.go index 4247fed6..601fa09e 100644 --- a/docs/types.go +++ b/docs/types.go @@ -29,7 +29,7 @@ type ImgData struct { // indicates the registry's version and whether the registry claims to be a // standalone registry. type PingResult struct { - // Version is the registry version supplied by the registry in a HTTP + // Version is the registry version supplied by the registry in an HTTP // header Version string `json:"version"` // Standalone is set to true if the registry indicates it is a From 08426ad10debbc46e922334c3890f76950024713 Mon Sep 17 00:00:00 2001 From: Yong Tang Date: Wed, 1 Jun 2016 13:38:14 -0700 Subject: [PATCH 372/375] Add `--limit` option to `docker search` This fix tries to address the issue raised in #23055. Currently `docker search` result caps at 25 and there is no way to allow getting more results (if exist). This fix adds the flag `--limit` so that it is possible to return more results from the `docker search`. Related documentation has been updated. Additional tests have been added to cover the changes. This fix fixes #23055. Signed-off-by: Yong Tang --- docs/registry_test.go | 2 +- docs/service.go | 13 +++++++++---- docs/session.go | 7 +++++-- 3 files changed, 15 insertions(+), 7 deletions(-) diff --git a/docs/registry_test.go b/docs/registry_test.go index 39a01bcd..9927af32 100644 --- a/docs/registry_test.go +++ b/docs/registry_test.go @@ -730,7 +730,7 @@ func TestPushImageJSONIndex(t *testing.T) { func TestSearchRepositories(t *testing.T) { r := spawnTestRegistrySession(t) - results, err := r.SearchRepositories("fakequery") + results, err := r.SearchRepositories("fakequery", 25) if err != nil { t.Fatal(err) } diff --git a/docs/service.go b/docs/service.go index d48063cd..25b4990e 100644 --- a/docs/service.go +++ b/docs/service.go @@ -15,6 +15,11 @@ import ( registrytypes "github.com/docker/engine-api/types/registry" ) +const ( + // DefaultSearchLimit is the default value for maximum number of returned search results. + DefaultSearchLimit = 25 +) + // Service is the interface defining what a registry service should implement. type Service interface { Auth(ctx context.Context, authConfig *types.AuthConfig, userAgent string) (status, token string, err error) @@ -22,7 +27,7 @@ type Service interface { LookupPushEndpoints(hostname string) (endpoints []APIEndpoint, err error) ResolveRepository(name reference.Named) (*RepositoryInfo, error) ResolveIndex(name string) (*registrytypes.IndexInfo, error) - Search(ctx context.Context, term string, authConfig *types.AuthConfig, userAgent string, headers map[string][]string) (*registrytypes.SearchResults, error) + Search(ctx context.Context, term string, limit int, authConfig *types.AuthConfig, userAgent string, headers map[string][]string) (*registrytypes.SearchResults, error) ServiceConfig() *registrytypes.ServiceConfig TLSConfig(hostname string) (*tls.Config, error) } @@ -108,7 +113,7 @@ func splitReposSearchTerm(reposName string) (string, string) { // Search queries the public registry for images matching the specified // search terms, and returns the results. -func (s *DefaultService) Search(ctx context.Context, term string, authConfig *types.AuthConfig, userAgent string, headers map[string][]string) (*registrytypes.SearchResults, error) { +func (s *DefaultService) Search(ctx context.Context, term string, limit int, authConfig *types.AuthConfig, userAgent string, headers map[string][]string) (*registrytypes.SearchResults, error) { // TODO Use ctx when searching for repositories if err := validateNoScheme(term); err != nil { return nil, err @@ -139,9 +144,9 @@ func (s *DefaultService) Search(ctx context.Context, term string, authConfig *ty localName = strings.SplitN(localName, "/", 2)[1] } - return r.SearchRepositories(localName) + return r.SearchRepositories(localName, limit) } - return r.SearchRepositories(remoteName) + return r.SearchRepositories(remoteName, limit) } // ResolveRepository splits a repository name into its components diff --git a/docs/session.go b/docs/session.go index 82593cd7..140c458e 100644 --- a/docs/session.go +++ b/docs/session.go @@ -721,9 +721,12 @@ func shouldRedirect(response *http.Response) bool { } // SearchRepositories performs a search against the remote repository -func (r *Session) SearchRepositories(term string) (*registrytypes.SearchResults, error) { +func (r *Session) SearchRepositories(term string, limit int) (*registrytypes.SearchResults, error) { + if limit < 1 || limit > 100 { + return nil, fmt.Errorf("Limit %d is outside the range of [1, 100]", limit) + } logrus.Debugf("Index server: %s", r.indexEndpoint) - u := r.indexEndpoint.String() + "search?q=" + url.QueryEscape(term) + u := r.indexEndpoint.String() + "search?q=" + url.QueryEscape(term) + "&n=" + url.QueryEscape(fmt.Sprintf("%d", limit)) req, err := http.NewRequest("GET", u, nil) if err != nil { From a58c74303c6a527dedccb5720822ece24bee085f Mon Sep 17 00:00:00 2001 From: Yong Tang Date: Sat, 11 Jun 2016 13:16:55 -0700 Subject: [PATCH 373/375] Fix logrus formatting This fix tries to fix logrus formatting by removing `f` from `logrus.[Error|Warn|Debug|Fatal|Panic|Info]f` when formatting string is not present. This fix fixes #23459. Signed-off-by: Yong Tang --- docs/session.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/session.go b/docs/session.go index 140c458e..bb51c7eb 100644 --- a/docs/session.go +++ b/docs/session.go @@ -302,10 +302,10 @@ func (r *Session) GetRemoteImageLayer(imgID, registry string, imgSize int64) (io } if res.Header.Get("Accept-Ranges") == "bytes" && imgSize > 0 { - logrus.Debugf("server supports resume") + logrus.Debug("server supports resume") return httputils.ResumableRequestReaderWithInitialResponse(r.client, req, 5, imgSize, res), nil } - logrus.Debugf("server doesn't support resume") + logrus.Debug("server doesn't support resume") return res.Body, nil } From 93f029e87c97935a14e85df41c1a1819a5e17a03 Mon Sep 17 00:00:00 2001 From: Derek McGowan Date: Wed, 13 Jul 2016 13:30:24 -0700 Subject: [PATCH 374/375] Allow v1 search to use v2 auth with identity token Updates the v1 search endpoint to also support v2 auth when an identity token is given. Only search v1 endpoint is supported since there is not v2 search currently defined to replace it. Signed-off-by: Derek McGowan (cherry picked from commit 19d48f0b8ba59eea9f2cac4ad1c7977712a6b7ac) Signed-off-by: Tibor Vass --- docs/auth.go | 86 +++++++++++++++++++++++++++++++++++-------------- docs/service.go | 40 +++++++++++++++++++++-- docs/session.go | 36 +++++++++++++-------- 3 files changed, 122 insertions(+), 40 deletions(-) diff --git a/docs/auth.go b/docs/auth.go index c5663f58..0b525718 100644 --- a/docs/auth.go +++ b/docs/auth.go @@ -91,6 +91,35 @@ func (lcs loginCredentialStore) SetRefreshToken(u *url.URL, service, token strin lcs.authConfig.IdentityToken = token } +type staticCredentialStore struct { + auth *types.AuthConfig +} + +// NewStaticCredentialStore returns a credential store +// which always returns the same credential values. +func NewStaticCredentialStore(auth *types.AuthConfig) auth.CredentialStore { + return staticCredentialStore{ + auth: auth, + } +} + +func (scs staticCredentialStore) Basic(*url.URL) (string, string) { + if scs.auth == nil { + return "", "" + } + return scs.auth.Username, scs.auth.Password +} + +func (scs staticCredentialStore) RefreshToken(*url.URL, string) string { + if scs.auth == nil { + return "" + } + return scs.auth.IdentityToken +} + +func (scs staticCredentialStore) SetRefreshToken(*url.URL, string, string) { +} + type fallbackError struct { err error } @@ -108,33 +137,14 @@ func loginV2(authConfig *types.AuthConfig, endpoint APIEndpoint, userAgent strin modifiers := DockerHeaders(userAgent, nil) authTransport := transport.NewTransport(NewTransport(endpoint.TLSConfig), modifiers...) - challengeManager, foundV2, err := PingV2Registry(endpoint, authTransport) - if err != nil { - if !foundV2 { - err = fallbackError{err: err} - } - return "", "", err - } - credentialAuthConfig := *authConfig creds := loginCredentialStore{ authConfig: &credentialAuthConfig, } - tokenHandlerOptions := auth.TokenHandlerOptions{ - Transport: authTransport, - Credentials: creds, - OfflineAccess: true, - ClientID: AuthClientID, - } - tokenHandler := auth.NewTokenHandlerWithOptions(tokenHandlerOptions) - basicHandler := auth.NewBasicHandler(creds) - modifiers = append(modifiers, auth.NewAuthorizer(challengeManager, tokenHandler, basicHandler)) - tr := transport.NewTransport(authTransport, modifiers...) - - loginClient := &http.Client{ - Transport: tr, - Timeout: 15 * time.Second, + loginClient, foundV2, err := v2AuthHTTPClient(endpoint.URL, authTransport, modifiers, creds, nil) + if err != nil { + return "", "", err } endpointStr := strings.TrimRight(endpoint.URL.String(), "/") + "/v2/" @@ -168,6 +178,34 @@ func loginV2(authConfig *types.AuthConfig, endpoint APIEndpoint, userAgent strin } +func v2AuthHTTPClient(endpoint *url.URL, authTransport http.RoundTripper, modifiers []transport.RequestModifier, creds auth.CredentialStore, scopes []auth.Scope) (*http.Client, bool, error) { + challengeManager, foundV2, err := PingV2Registry(endpoint, authTransport) + if err != nil { + if !foundV2 { + err = fallbackError{err: err} + } + return nil, foundV2, err + } + + tokenHandlerOptions := auth.TokenHandlerOptions{ + Transport: authTransport, + Credentials: creds, + OfflineAccess: true, + ClientID: AuthClientID, + Scopes: scopes, + } + tokenHandler := auth.NewTokenHandlerWithOptions(tokenHandlerOptions) + basicHandler := auth.NewBasicHandler(creds) + modifiers = append(modifiers, auth.NewAuthorizer(challengeManager, tokenHandler, basicHandler)) + tr := transport.NewTransport(authTransport, modifiers...) + + return &http.Client{ + Transport: tr, + Timeout: 15 * time.Second, + }, foundV2, nil + +} + // ResolveAuthConfig matches an auth configuration to a server address or a URL func ResolveAuthConfig(authConfigs map[string]types.AuthConfig, index *registrytypes.IndexInfo) types.AuthConfig { configKey := GetAuthConfigKey(index) @@ -215,7 +253,7 @@ func (err PingResponseError) Error() string { // challenge manager for the supported authentication types and // whether v2 was confirmed by the response. If a response is received but // cannot be interpreted a PingResponseError will be returned. -func PingV2Registry(endpoint APIEndpoint, transport http.RoundTripper) (auth.ChallengeManager, bool, error) { +func PingV2Registry(endpoint *url.URL, transport http.RoundTripper) (auth.ChallengeManager, bool, error) { var ( foundV2 = false v2Version = auth.APIVersion{ @@ -228,7 +266,7 @@ func PingV2Registry(endpoint APIEndpoint, transport http.RoundTripper) (auth.Cha Transport: transport, Timeout: 15 * time.Second, } - endpointStr := strings.TrimRight(endpoint.URL.String(), "/") + "/v2/" + endpointStr := strings.TrimRight(endpoint.String(), "/") + "/v2/" req, err := http.NewRequest("GET", endpointStr, nil) if err != nil { return nil, false, err diff --git a/docs/service.go b/docs/service.go index 25b4990e..dbc16284 100644 --- a/docs/service.go +++ b/docs/service.go @@ -10,6 +10,7 @@ import ( "golang.org/x/net/context" "github.com/Sirupsen/logrus" + "github.com/docker/distribution/registry/client/auth" "github.com/docker/docker/reference" "github.com/docker/engine-api/types" registrytypes "github.com/docker/engine-api/types/registry" @@ -132,11 +133,44 @@ func (s *DefaultService) Search(ctx context.Context, term string, limit int, aut return nil, err } - r, err := NewSession(endpoint.client, authConfig, endpoint) - if err != nil { - return nil, err + var client *http.Client + if authConfig != nil && authConfig.IdentityToken != "" && authConfig.Username != "" { + creds := NewStaticCredentialStore(authConfig) + scopes := []auth.Scope{ + auth.RegistryScope{ + Name: "catalog", + Actions: []string{"search"}, + }, + } + + modifiers := DockerHeaders(userAgent, nil) + v2Client, foundV2, err := v2AuthHTTPClient(endpoint.URL, endpoint.client.Transport, modifiers, creds, scopes) + if err != nil { + if fErr, ok := err.(fallbackError); ok { + logrus.Errorf("Cannot use identity token for search, v2 auth not supported: %v", fErr.err) + } else { + return nil, err + } + } else if foundV2 { + // Copy non transport http client features + v2Client.Timeout = endpoint.client.Timeout + v2Client.CheckRedirect = endpoint.client.CheckRedirect + v2Client.Jar = endpoint.client.Jar + + logrus.Debugf("using v2 client for search to %s", endpoint.URL) + client = v2Client + } } + if client == nil { + client = endpoint.client + if err := authorizeClient(client, authConfig, endpoint); err != nil { + return nil, err + } + } + + r := newSession(client, authConfig, endpoint) + if index.Official { localName := remoteName if strings.HasPrefix(localName, "library/") { diff --git a/docs/session.go b/docs/session.go index bb51c7eb..d48b9e8d 100644 --- a/docs/session.go +++ b/docs/session.go @@ -161,16 +161,7 @@ func (tr *authTransport) CancelRequest(req *http.Request) { } } -// NewSession creates a new session -// TODO(tiborvass): remove authConfig param once registry client v2 is vendored -func NewSession(client *http.Client, authConfig *types.AuthConfig, endpoint *V1Endpoint) (r *Session, err error) { - r = &Session{ - authConfig: authConfig, - client: client, - indexEndpoint: endpoint, - id: stringid.GenerateRandomID(), - } - +func authorizeClient(client *http.Client, authConfig *types.AuthConfig, endpoint *V1Endpoint) error { var alwaysSetBasicAuth bool // If we're working with a standalone private registry over HTTPS, send Basic Auth headers @@ -178,7 +169,7 @@ func NewSession(client *http.Client, authConfig *types.AuthConfig, endpoint *V1E if endpoint.String() != IndexServer && endpoint.URL.Scheme == "https" { info, err := endpoint.Ping() if err != nil { - return nil, err + return err } if info.Standalone && authConfig != nil { logrus.Debugf("Endpoint %s is eligible for private registry. Enabling decorator.", endpoint.String()) @@ -192,11 +183,30 @@ func NewSession(client *http.Client, authConfig *types.AuthConfig, endpoint *V1E jar, err := cookiejar.New(nil) if err != nil { - return nil, errors.New("cookiejar.New is not supposed to return an error") + return errors.New("cookiejar.New is not supposed to return an error") } client.Jar = jar - return r, nil + return nil +} + +func newSession(client *http.Client, authConfig *types.AuthConfig, endpoint *V1Endpoint) *Session { + return &Session{ + authConfig: authConfig, + client: client, + indexEndpoint: endpoint, + id: stringid.GenerateRandomID(), + } +} + +// NewSession creates a new session +// TODO(tiborvass): remove authConfig param once registry client v2 is vendored +func NewSession(client *http.Client, authConfig *types.AuthConfig, endpoint *V1Endpoint) (*Session, error) { + if err := authorizeClient(client, authConfig, endpoint); err != nil { + return nil, err + } + + return newSession(client, authConfig, endpoint), nil } // ID returns this registry session's ID. From 8d287d4332e68b6f9a60897053a37996e961ff79 Mon Sep 17 00:00:00 2001 From: Sebastiaan van Stijn Date: Sat, 16 Jul 2016 01:52:59 +0200 Subject: [PATCH 375/375] Improve flag help consistency, and update docs This adds the `--live-restore` option to the documentation. Also synched usage description in the documentation with the actual description, and re-phrased some flag descriptions to be a bit more consistent. Signed-off-by: Sebastiaan van Stijn (cherry picked from commit 64a8317a5a306dffd0ec080d9ec5b4ceb2479a01) Signed-off-by: Tibor Vass --- docs/config.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/config.go b/docs/config.go index 51302d11..e349660e 100644 --- a/docs/config.go +++ b/docs/config.go @@ -77,7 +77,7 @@ func (options *ServiceOptions) InstallCliFlags(cmd *flag.FlagSet, usageFn func(s insecureRegistries := opts.NewNamedListOptsRef("insecure-registries", &options.InsecureRegistries, ValidateIndexName) cmd.Var(insecureRegistries, []string{"-insecure-registry"}, usageFn("Enable insecure registry communication")) - cmd.BoolVar(&options.V2Only, []string{"-disable-legacy-registry"}, false, usageFn("Do not contact legacy registries")) + cmd.BoolVar(&options.V2Only, []string{"-disable-legacy-registry"}, false, usageFn("Disable contacting legacy registries")) } // newServiceConfig returns a new instance of ServiceConfig