diff --git a/vendor.conf b/vendor.conf index 631391bf..84728809 100644 --- a/vendor.conf +++ b/vendor.conf @@ -4,7 +4,7 @@ k8s.io/apimachinery release-1.7 https://github.com/kubernetes/apimachinery k8s.io/apiserver release-1.7 https://github.com/kubernetes/apiserver # github.com/sirupsen/logrus v1.0.0 -github.com/containers/image d17474f39dae1da15ab9ae033d57ebefcf62f77a +github.com/containers/image 57b257d128d6075ea3287991ee408d24c7bd2758 github.com/docker/docker-credential-helpers d68f9aeca33f5fd3f08eeae5e9d175edf4e731d1 github.com/ostreedev/ostree-go master github.com/containers/storage 64bf27465d0d1edd89e7a4ce49866fea01145782 diff --git a/vendor/github.com/containers/image/copy/copy.go b/vendor/github.com/containers/image/copy/copy.go index c7af5ed1..590b3787 100644 --- a/vendor/github.com/containers/image/copy/copy.go +++ b/vendor/github.com/containers/image/copy/copy.go @@ -581,7 +581,7 @@ func (ic *imageCopier) copyBlobFromStream(srcStream io.Reader, srcInfo types.Blo bar.ShowPercent = false bar.Start() destStream = bar.NewProxyReader(destStream) - defer fmt.Fprint(ic.reportWriter, "\n") + defer bar.Finish() // === Send a copy of the original, uncompressed, stream, to a separate path if necessary. var originalLayerReader io.Reader // DO NOT USE this other than to drain the input if no other consumer in the pipeline has done so. diff --git a/vendor/github.com/containers/image/docker/docker_client.go b/vendor/github.com/containers/image/docker/docker_client.go index 4c3d8b9f..24b82d6f 100644 --- a/vendor/github.com/containers/image/docker/docker_client.go +++ b/vendor/github.com/containers/image/docker/docker_client.go @@ -3,12 +3,10 @@ package docker import ( "context" "crypto/tls" - "encoding/base64" "encoding/json" "fmt" "io" "io/ioutil" - "net" "net/http" "os" "path/filepath" @@ -16,11 +14,10 @@ import ( "time" "github.com/containers/image/docker/reference" + "github.com/containers/image/pkg/docker/config" + "github.com/containers/image/pkg/tlsclientconfig" "github.com/containers/image/types" - "github.com/containers/storage/pkg/homedir" "github.com/docker/distribution/registry/client" - helperclient "github.com/docker/docker-credential-helpers/client" - "github.com/docker/go-connections/sockets" "github.com/docker/go-connections/tlsconfig" "github.com/opencontainers/go-digest" "github.com/pkg/errors" @@ -28,13 +25,8 @@ import ( ) const ( - dockerHostname = "docker.io" - dockerRegistry = "registry-1.docker.io" - dockerAuthRegistry = "https://index.docker.io/v1/" - - dockerCfg = ".docker" - dockerCfgFileName = "config.json" - dockerCfgObsolete = ".dockercfg" + dockerHostname = "docker.io" + dockerRegistry = "registry-1.docker.io" systemPerHostCertDirPath = "/etc/docker/certs.d" @@ -52,9 +44,13 @@ const ( extensionSignatureTypeAtomic = "atomic" // extensionSignature.Type ) -// ErrV1NotSupported is returned when we're trying to talk to a -// docker V1 registry. -var ErrV1NotSupported = errors.New("can't talk to a V1 docker registry") +var ( + // ErrV1NotSupported is returned when we're trying to talk to a + // docker V1 registry. + ErrV1NotSupported = errors.New("can't talk to a V1 docker registry") + // ErrUnauthorizedForCredentials is returned when the status code returned is 401 + ErrUnauthorizedForCredentials = errors.New("unable to retrieve auth token: invalid username/password") +) // extensionSignature and extensionSignatureList come from github.com/openshift/origin/pkg/dockerregistry/server/signaturedispatcher.go: // signature represents a Docker image signature. @@ -113,27 +109,7 @@ func serverDefault() *tls.Config { } } -func newTransport() *http.Transport { - direct := &net.Dialer{ - Timeout: 30 * time.Second, - KeepAlive: 30 * time.Second, - DualStack: true, - } - tr := &http.Transport{ - Proxy: http.ProxyFromEnvironment, - Dial: direct.Dial, - TLSHandshakeTimeout: 10 * time.Second, - // TODO(dmcgowan): Call close idle connections when complete and use keep alive - DisableKeepAlives: true, - } - proxyDialer, err := sockets.DialerFromEnvironment(direct) - if err == nil { - tr.Dial = proxyDialer.Dial - } - return tr -} - -// dockerCertDir returns a path to a directory to be consumed by setupCertificates() depending on ctx and hostPort. +// dockerCertDir returns a path to a directory to be consumed by tlsclientconfig.SetupCertificates() depending on ctx and hostPort. func dockerCertDir(ctx *types.SystemContext, hostPort string) string { if ctx != nil && ctx.DockerCertPath != "" { return ctx.DockerCertPath @@ -212,52 +188,84 @@ func hasFile(files []os.FileInfo, name string) bool { return false } -// newDockerClient returns a new dockerClient instance for refHostname (a host a specified in the Docker image reference, not canonicalized to dockerRegistry) +// newDockerClientFromRef returns a new dockerClient instance for refHostname (a host a specified in the Docker image reference, not canonicalized to dockerRegistry) // “write” specifies whether the client will be used for "write" access (in particular passed to lookaside.go:toplevelFromSection) -func newDockerClient(ctx *types.SystemContext, ref dockerReference, write bool, actions string) (*dockerClient, error) { +func newDockerClientFromRef(ctx *types.SystemContext, ref dockerReference, write bool, actions string) (*dockerClient, error) { registry := reference.Domain(ref.ref) - if registry == dockerHostname { - registry = dockerRegistry + username, password, err := config.GetAuthentication(ctx, reference.Domain(ref.ref)) + if err != nil { + return nil, errors.Wrapf(err, "error getting username and password") } - username, password, err := getAuth(ctx, reference.Domain(ref.ref)) + sigBase, err := configuredSignatureStorageBase(ctx, ref, write) if err != nil { return nil, err } - tr := newTransport() + remoteName := reference.Path(ref.ref) + + return newDockerClientWithDetails(ctx, registry, username, password, actions, sigBase, remoteName) +} + +// newDockerClientWithDetails returns a new dockerClient instance for the given parameters +func newDockerClientWithDetails(ctx *types.SystemContext, registry, username, password, actions string, sigBase signatureStorageBase, remoteName string) (*dockerClient, error) { + hostName := registry + if registry == dockerHostname { + registry = dockerRegistry + } + tr := tlsclientconfig.NewTransport() tr.TLSClientConfig = serverDefault() + // It is undefined whether the host[:port] string for dockerHostname should be dockerHostname or dockerRegistry, // because docker/docker does not read the certs.d subdirectory at all in that case. We use the user-visible // dockerHostname here, because it is more symmetrical to read the configuration in that case as well, and because // generally the UI hides the existence of the different dockerRegistry. But note that this behavior is // undocumented and may change if docker/docker changes. - certDir := dockerCertDir(ctx, reference.Domain(ref.ref)) - if err := setupCertificates(certDir, tr.TLSClientConfig); err != nil { + certDir := dockerCertDir(ctx, hostName) + if err := tlsclientconfig.SetupCertificates(certDir, tr.TLSClientConfig); err != nil { return nil, err } + if ctx != nil && ctx.DockerInsecureSkipTLSVerify { tr.TLSClientConfig.InsecureSkipVerify = true } - client := &http.Client{Transport: tr} - - sigBase, err := configuredSignatureStorageBase(ctx, ref, write) - if err != nil { - return nil, err - } return &dockerClient{ ctx: ctx, registry: registry, username: username, password: password, - client: client, + client: &http.Client{Transport: tr}, signatureBase: sigBase, scope: authScope{ actions: actions, - remoteName: reference.Path(ref.ref), + remoteName: remoteName, }, }, nil } +// CheckAuth validates the credentials by attempting to log into the registry +// returns an error if an error occcured while making the http request or the status code received was 401 +func CheckAuth(ctx context.Context, sCtx *types.SystemContext, username, password, registry string) error { + newLoginClient, err := newDockerClientWithDetails(sCtx, registry, username, password, "", nil, "") + if err != nil { + return errors.Wrapf(err, "error creating new docker client") + } + + resp, err := newLoginClient.makeRequest(ctx, "GET", "/v2/", nil, nil) + if err != nil { + return err + } + defer resp.Body.Close() + + switch resp.StatusCode { + case http.StatusOK: + return nil + case http.StatusUnauthorized: + return ErrUnauthorizedForCredentials + default: + return errors.Errorf("error occured with status code %q", resp.StatusCode) + } +} + // makeRequest creates and executes a http.Request with the specified parameters, adding authentication and TLS options for the Docker client. // The host name and schema is taken from the client or autodetected, and the path is relative to it, i.e. the path usually starts with /v2/. func (c *dockerClient) makeRequest(ctx context.Context, method, path string, headers map[string][]string, stream io.Reader) (*http.Response, error) { @@ -329,7 +337,10 @@ func (c *dockerClient) setupRequestAuth(req *http.Request) error { return errors.Errorf("missing realm in bearer auth challenge") } service, _ := challenge.Parameters["service"] // Will be "" if not present - scope := fmt.Sprintf("repository:%s:%s", c.scope.remoteName, c.scope.actions) + var scope string + if c.scope.remoteName != "" && c.scope.actions != "" { + scope = fmt.Sprintf("repository:%s:%s", c.scope.remoteName, c.scope.actions) + } token, err := c.getBearerToken(req.Context(), realm, service, scope) if err != nil { return err @@ -364,7 +375,7 @@ func (c *dockerClient) getBearerToken(ctx context.Context, realm, service, scope if c.username != "" && c.password != "" { authReq.SetBasicAuth(c.username, c.password) } - tr := newTransport() + tr := tlsclientconfig.NewTransport() // TODO(runcom): insecure for now to contact the external token service tr.TLSClientConfig = &tls.Config{InsecureSkipVerify: true} client := &http.Client{Transport: tr} @@ -375,7 +386,7 @@ func (c *dockerClient) getBearerToken(ctx context.Context, realm, service, scope defer res.Body.Close() switch res.StatusCode { case http.StatusUnauthorized: - return nil, errors.Errorf("unable to retrieve auth token: 401 unauthorized") + return nil, ErrUnauthorizedForCredentials case http.StatusOK: break default: @@ -399,65 +410,6 @@ func (c *dockerClient) getBearerToken(ctx context.Context, realm, service, scope return &token, nil } -func getAuth(ctx *types.SystemContext, registry string) (string, string, error) { - if ctx != nil && ctx.DockerAuthConfig != nil { - return ctx.DockerAuthConfig.Username, ctx.DockerAuthConfig.Password, nil - } - var dockerAuth dockerConfigFile - dockerCfgPath := filepath.Join(getDefaultConfigDir(".docker"), dockerCfgFileName) - if _, err := os.Stat(dockerCfgPath); err == nil { - j, err := ioutil.ReadFile(dockerCfgPath) - if err != nil { - return "", "", err - } - if err := json.Unmarshal(j, &dockerAuth); err != nil { - return "", "", err - } - - } else if os.IsNotExist(err) { - // try old config path - oldDockerCfgPath := filepath.Join(getDefaultConfigDir(dockerCfgObsolete)) - if _, err := os.Stat(oldDockerCfgPath); err != nil { - if os.IsNotExist(err) { - return "", "", nil - } - return "", "", errors.Wrap(err, oldDockerCfgPath) - } - - j, err := ioutil.ReadFile(oldDockerCfgPath) - if err != nil { - return "", "", err - } - if err := json.Unmarshal(j, &dockerAuth.AuthConfigs); err != nil { - return "", "", err - } - - } else if err != nil { - return "", "", errors.Wrap(err, dockerCfgPath) - } - - // First try cred helpers. They should always be normalized. - if ch, exists := dockerAuth.CredHelpers[registry]; exists { - return getAuthFromCredHelper(ch, registry) - } - - // I'm feeling lucky. - if c, exists := dockerAuth.AuthConfigs[registry]; exists { - return decodeDockerAuth(c.Auth) - } - - // bad luck; let's normalize the entries first - registry = normalizeRegistry(registry) - normalizedAuths := map[string]dockerAuthConfig{} - for k, v := range dockerAuth.AuthConfigs { - normalizedAuths[normalizeRegistry(k)] = v - } - if c, exists := normalizedAuths[registry]; exists { - return decodeDockerAuth(c.Auth) - } - return "", "", nil -} - // detectProperties detects various properties of the registry. // See the dockerClient documentation for members which are affected by this. func (c *dockerClient) detectProperties(ctx context.Context) error { @@ -540,67 +492,3 @@ func (c *dockerClient) getExtensionsSignatures(ctx context.Context, ref dockerRe } return &parsedBody, nil } - -func getDefaultConfigDir(confPath string) string { - return filepath.Join(homedir.Get(), confPath) -} - -type dockerAuthConfig struct { - Auth string `json:"auth,omitempty"` -} - -type dockerConfigFile struct { - AuthConfigs map[string]dockerAuthConfig `json:"auths"` - CredHelpers map[string]string `json:"credHelpers,omitempty"` -} - -func getAuthFromCredHelper(credHelper, registry string) (string, string, error) { - helperName := fmt.Sprintf("docker-credential-%s", credHelper) - p := helperclient.NewShellProgramFunc(helperName) - creds, err := helperclient.Get(p, registry) - if err != nil { - return "", "", err - } - - return creds.Username, creds.Secret, nil -} - -func decodeDockerAuth(s string) (string, string, error) { - decoded, err := base64.StdEncoding.DecodeString(s) - if err != nil { - return "", "", err - } - parts := strings.SplitN(string(decoded), ":", 2) - if len(parts) != 2 { - // if it's invalid just skip, as docker does - return "", "", nil - } - user := parts[0] - password := strings.Trim(parts[1], "\x00") - return user, password, nil -} - -// convertToHostname converts a registry url which has http|https prepended -// to just an hostname. -// Copied from github.com/docker/docker/registry/auth.go -func convertToHostname(url string) string { - stripped := url - if strings.HasPrefix(url, "http://") { - stripped = strings.TrimPrefix(url, "http://") - } else if strings.HasPrefix(url, "https://") { - stripped = strings.TrimPrefix(url, "https://") - } - - nameParts := strings.SplitN(stripped, "/", 2) - - return nameParts[0] -} - -func normalizeRegistry(registry string) string { - normalized := convertToHostname(registry) - switch normalized { - case "registry-1.docker.io", "docker.io": - return "index.docker.io" - } - return normalized -} diff --git a/vendor/github.com/containers/image/docker/docker_image_dest.go b/vendor/github.com/containers/image/docker/docker_image_dest.go index ee2af92b..32d5a18b 100644 --- a/vendor/github.com/containers/image/docker/docker_image_dest.go +++ b/vendor/github.com/containers/image/docker/docker_image_dest.go @@ -34,7 +34,7 @@ type dockerImageDestination struct { // newImageDestination creates a new ImageDestination for the specified image reference. func newImageDestination(ctx *types.SystemContext, ref dockerReference) (types.ImageDestination, error) { - c, err := newDockerClient(ctx, ref, true, "pull,push") + c, err := newDockerClientFromRef(ctx, ref, true, "pull,push") if err != nil { return nil, err } diff --git a/vendor/github.com/containers/image/docker/docker_image_src.go b/vendor/github.com/containers/image/docker/docker_image_src.go index e574305b..232c3cf9 100644 --- a/vendor/github.com/containers/image/docker/docker_image_src.go +++ b/vendor/github.com/containers/image/docker/docker_image_src.go @@ -31,7 +31,7 @@ type dockerImageSource struct { // newImageSource creates a new ImageSource for the specified image reference. // The caller must call .Close() on the returned ImageSource. func newImageSource(ctx *types.SystemContext, ref dockerReference) (*dockerImageSource, error) { - c, err := newDockerClient(ctx, ref, false, "pull") + c, err := newDockerClientFromRef(ctx, ref, false, "pull") if err != nil { return nil, err } @@ -140,6 +140,7 @@ func (s *dockerImageSource) getExternalBlob(urls []string) (io.ReadCloser, int64 logrus.Debug(err) continue } + break } } if resp.Body != nil && err == nil { @@ -297,7 +298,7 @@ func (s *dockerImageSource) getSignaturesFromAPIExtension(ctx context.Context) ( // deleteImage deletes the named image from the registry, if supported. func deleteImage(ctx *types.SystemContext, ref dockerReference) error { - c, err := newDockerClient(ctx, ref, true, "push") + c, err := newDockerClientFromRef(ctx, ref, true, "push") if err != nil { return err } diff --git a/vendor/github.com/containers/image/image/oci.go b/vendor/github.com/containers/image/image/oci.go index 7ea5d2bb..5f7c0728 100644 --- a/vendor/github.com/containers/image/image/oci.go +++ b/vendor/github.com/containers/image/image/oci.go @@ -109,7 +109,7 @@ func (m *manifestOCI1) OCIConfig() (*imgspecv1.Image, error) { func (m *manifestOCI1) LayerInfos() []types.BlobInfo { blobs := []types.BlobInfo{} for _, layer := range m.LayersDescriptors { - blobs = append(blobs, types.BlobInfo{Digest: layer.Digest, Size: layer.Size, Annotations: layer.Annotations}) + blobs = append(blobs, types.BlobInfo{Digest: layer.Digest, Size: layer.Size, Annotations: layer.Annotations, URLs: layer.URLs}) } return blobs } @@ -160,6 +160,7 @@ func (m *manifestOCI1) UpdatedImage(options types.ManifestUpdateOptions) (types. copy.LayersDescriptors[i].Digest = info.Digest copy.LayersDescriptors[i].Size = info.Size copy.LayersDescriptors[i].Annotations = info.Annotations + copy.LayersDescriptors[i].URLs = info.URLs } } // Ignore options.EmbeddedDockerReference: it may be set when converting from schema1, but we really don't care. diff --git a/vendor/github.com/containers/image/oci/archive/oci_dest.go b/vendor/github.com/containers/image/oci/archive/oci_dest.go index b2c4bb63..52e99a43 100644 --- a/vendor/github.com/containers/image/oci/archive/oci_dest.go +++ b/vendor/github.com/containers/image/oci/archive/oci_dest.go @@ -106,11 +106,7 @@ func (d *ociArchiveImageDestination) Commit() error { src := d.tempDirRef.tempDirectory // path to save tarred up file dst := d.ref.resolvedFile - if err := tarDirectory(src, dst); err != nil { - return err - } - - return nil + return tarDirectory(src, dst) } // tar converts the directory at src and saves it to dst diff --git a/vendor/github.com/containers/image/oci/layout/oci_dest.go b/vendor/github.com/containers/image/oci/layout/oci_dest.go index c4801e34..ce1e0c3e 100644 --- a/vendor/github.com/containers/image/oci/layout/oci_dest.go +++ b/vendor/github.com/containers/image/oci/layout/oci_dest.go @@ -66,7 +66,7 @@ func (d *ociImageDestination) ShouldCompressLayers() bool { // AcceptsForeignLayerURLs returns false iff foreign layers in manifest should be actually // uploaded to the image destination, true otherwise. func (d *ociImageDestination) AcceptsForeignLayerURLs() bool { - return false + return true } // MustMatchRuntimeOS returns true iff the destination can store only images targeted for the current runtime OS. False otherwise. diff --git a/vendor/github.com/containers/image/oci/layout/oci_src.go b/vendor/github.com/containers/image/oci/layout/oci_src.go index 99b9f208..be8a2aa7 100644 --- a/vendor/github.com/containers/image/oci/layout/oci_src.go +++ b/vendor/github.com/containers/image/oci/layout/oci_src.go @@ -4,25 +4,43 @@ import ( "context" "io" "io/ioutil" + "net/http" "os" + "strconv" + "github.com/containers/image/pkg/tlsclientconfig" "github.com/containers/image/types" + "github.com/docker/go-connections/tlsconfig" "github.com/opencontainers/go-digest" imgspecv1 "github.com/opencontainers/image-spec/specs-go/v1" + "github.com/pkg/errors" ) type ociImageSource struct { ref ociReference descriptor imgspecv1.Descriptor + client *http.Client } // newImageSource returns an ImageSource for reading from an existing directory. -func newImageSource(ref ociReference) (types.ImageSource, error) { +func newImageSource(ctx *types.SystemContext, ref ociReference) (types.ImageSource, error) { + tr := tlsclientconfig.NewTransport() + tr.TLSClientConfig = tlsconfig.ServerDefault() + + if ctx != nil && ctx.OCICertPath != "" { + if err := tlsclientconfig.SetupCertificates(ctx.OCICertPath, tr.TLSClientConfig); err != nil { + return nil, err + } + tr.TLSClientConfig.InsecureSkipVerify = ctx.OCIInsecureSkipTLSVerify + } + + client := &http.Client{} + client.Transport = tr descriptor, err := ref.getManifestDescriptor() if err != nil { return nil, err } - return &ociImageSource{ref: ref, descriptor: descriptor}, nil + return &ociImageSource{ref: ref, descriptor: descriptor, client: client}, nil } // Reference returns the reference used to set up this source. @@ -70,6 +88,10 @@ func (s *ociImageSource) GetTargetManifest(digest digest.Digest) ([]byte, string // GetBlob returns a stream for the specified blob, and the blob's size. func (s *ociImageSource) GetBlob(info types.BlobInfo) (io.ReadCloser, int64, error) { + if len(info.URLs) != 0 { + return s.getExternalBlob(info.URLs) + } + path, err := s.ref.blobPath(info.Digest) if err != nil { return nil, 0, err @@ -89,3 +111,32 @@ func (s *ociImageSource) GetBlob(info types.BlobInfo) (io.ReadCloser, int64, err func (s *ociImageSource) GetSignatures(context.Context) ([][]byte, error) { return [][]byte{}, nil } + +func (s *ociImageSource) getExternalBlob(urls []string) (io.ReadCloser, int64, error) { + errWrap := errors.New("failed fetching external blob from all urls") + for _, url := range urls { + resp, err := s.client.Get(url) + if err != nil { + errWrap = errors.Wrapf(errWrap, "fetching %s failed %s", url, err.Error()) + continue + } + + if resp.StatusCode != http.StatusOK { + resp.Body.Close() + errWrap = errors.Wrapf(errWrap, "fetching %s failed, response code not 200", url) + continue + } + + return resp.Body, getBlobSize(resp), nil + } + + return nil, 0, errWrap +} + +func getBlobSize(resp *http.Response) int64 { + size, err := strconv.ParseInt(resp.Header.Get("Content-Length"), 10, 64) + if err != nil { + size = -1 + } + return size +} diff --git a/vendor/github.com/containers/image/oci/layout/oci_transport.go b/vendor/github.com/containers/image/oci/layout/oci_transport.go index 7fd826f4..312bc0e4 100644 --- a/vendor/github.com/containers/image/oci/layout/oci_transport.go +++ b/vendor/github.com/containers/image/oci/layout/oci_transport.go @@ -182,7 +182,7 @@ func (ref ociReference) PolicyConfigurationNamespaces() []string { // NOTE: If any kind of signature verification should happen, build an UnparsedImage from the value returned by NewImageSource, // verify that UnparsedImage, and convert it into a real Image via image.FromUnparsedImage. func (ref ociReference) NewImage(ctx *types.SystemContext) (types.Image, error) { - src, err := newImageSource(ref) + src, err := newImageSource(ctx, ref) if err != nil { return nil, err } @@ -244,7 +244,7 @@ func LoadManifestDescriptor(imgRef types.ImageReference) (imgspecv1.Descriptor, // NewImageSource returns a types.ImageSource for this reference. // The caller must call .Close() on the returned ImageSource. func (ref ociReference) NewImageSource(ctx *types.SystemContext) (types.ImageSource, error) { - return newImageSource(ref) + return newImageSource(ctx, ref) } // NewImageDestination returns a types.ImageDestination for this reference. diff --git a/vendor/github.com/containers/image/ostree/ostree_dest.go b/vendor/github.com/containers/image/ostree/ostree_dest.go index 885297bd..26137431 100644 --- a/vendor/github.com/containers/image/ostree/ostree_dest.go +++ b/vendor/github.com/containers/image/ostree/ostree_dest.go @@ -46,6 +46,7 @@ type ostreeImageDestination struct { schema manifestSchema tmpDirPath string blobs map[string]*blobToImport + digest digest.Digest } // newImageDestination returns an ImageDestination for writing to an existing ostree. @@ -54,7 +55,7 @@ func newImageDestination(ref ostreeReference, tmpDirPath string) (types.ImageDes if err := ensureDirectoryExists(tmpDirPath); err != nil { return nil, err } - return &ostreeImageDestination{ref, "", manifestSchema{}, tmpDirPath, map[string]*blobToImport{}}, nil + return &ostreeImageDestination{ref, "", manifestSchema{}, tmpDirPath, map[string]*blobToImport{}, ""}, nil } // Reference returns the reference used to set up this destination. Note that this should directly correspond to user's intent, @@ -238,10 +239,10 @@ func (d *ostreeImageDestination) ReapplyBlob(info types.BlobInfo) (types.BlobInf // FIXME? This should also receive a MIME type if known, to differentiate between schema versions. // If the destination is in principle available, refuses this manifest type (e.g. it does not recognize the schema), // but may accept a different manifest type, the returned error must be an ManifestTypeRejectedError. -func (d *ostreeImageDestination) PutManifest(manifest []byte) error { - d.manifest = string(manifest) +func (d *ostreeImageDestination) PutManifest(manifestBlob []byte) error { + d.manifest = string(manifestBlob) - if err := json.Unmarshal(manifest, &d.schema); err != nil { + if err := json.Unmarshal(manifestBlob, &d.schema); err != nil { return err } @@ -250,7 +251,13 @@ func (d *ostreeImageDestination) PutManifest(manifest []byte) error { return err } - return ioutil.WriteFile(manifestPath, manifest, 0644) + digest, err := manifest.Digest(manifestBlob) + if err != nil { + return err + } + d.digest = digest + + return ioutil.WriteFile(manifestPath, manifestBlob, 0644) } func (d *ostreeImageDestination) PutSignatures(signatures [][]byte) error { @@ -304,7 +311,7 @@ func (d *ostreeImageDestination) Commit() error { manifestPath := filepath.Join(d.tmpDirPath, "manifest") - metadata := []string{fmt.Sprintf("docker.manifest=%s", string(d.manifest))} + metadata := []string{fmt.Sprintf("docker.manifest=%s", string(d.manifest)), fmt.Sprintf("docker.digest=%s", string(d.digest))} err = d.ostreeCommit(repo, fmt.Sprintf("ociimage/%s", d.ref.branchName), manifestPath, metadata) _, err = repo.CommitTransaction() diff --git a/vendor/github.com/containers/image/pkg/docker/config/config.go b/vendor/github.com/containers/image/pkg/docker/config/config.go new file mode 100644 index 00000000..fd0ae7d8 --- /dev/null +++ b/vendor/github.com/containers/image/pkg/docker/config/config.go @@ -0,0 +1,295 @@ +package config + +import ( + "encoding/base64" + "encoding/json" + "fmt" + "io/ioutil" + "os" + "path/filepath" + "strconv" + "strings" + + "github.com/containers/image/types" + helperclient "github.com/docker/docker-credential-helpers/client" + "github.com/docker/docker-credential-helpers/credentials" + "github.com/docker/docker/pkg/homedir" + "github.com/pkg/errors" +) + +type dockerAuthConfig struct { + Auth string `json:"auth,omitempty"` +} + +type dockerConfigFile struct { + AuthConfigs map[string]dockerAuthConfig `json:"auths"` + CredHelpers map[string]string `json:"credHelpers,omitempty"` +} + +const ( + defaultPath = "/run/user" + authCfg = "containers" + authCfgFileName = "auth.json" + dockerCfg = ".docker" + dockerCfgFileName = "config.json" + dockerLegacyCfg = ".dockercfg" +) + +var ( + // ErrNotLoggedIn is returned for users not logged into a registry + // that they are trying to logout of + ErrNotLoggedIn = errors.New("not logged in") +) + +// SetAuthentication stores the username and password in the auth.json file +func SetAuthentication(ctx *types.SystemContext, registry, username, password string) error { + return modifyJSON(ctx, func(auths *dockerConfigFile) (bool, error) { + if ch, exists := auths.CredHelpers[registry]; exists { + return false, setAuthToCredHelper(ch, registry, username, password) + } + + creds := base64.StdEncoding.EncodeToString([]byte(username + ":" + password)) + newCreds := dockerAuthConfig{Auth: creds} + auths.AuthConfigs[registry] = newCreds + return true, nil + }) +} + +// GetAuthentication returns the registry credentials stored in +// either auth.json file or .docker/config.json +// If an entry is not found empty strings are returned for the username and password +func GetAuthentication(ctx *types.SystemContext, registry string) (string, string, error) { + if ctx != nil && ctx.DockerAuthConfig != nil { + return ctx.DockerAuthConfig.Username, ctx.DockerAuthConfig.Password, nil + } + + dockerLegacyPath := filepath.Join(homedir.Get(), dockerLegacyCfg) + paths := [3]string{getPathToAuth(ctx), filepath.Join(homedir.Get(), dockerCfg, dockerCfgFileName), dockerLegacyPath} + + for _, path := range paths { + legacyFormat := path == dockerLegacyPath + username, password, err := findAuthentication(registry, path, legacyFormat) + if err != nil { + return "", "", err + } + if username != "" && password != "" { + return username, password, nil + } + } + return "", "", nil +} + +// GetUserLoggedIn returns the username logged in to registry from either +// auth.json or XDG_RUNTIME_DIR +// Used to tell the user if someone is logged in to the registry when logging in +func GetUserLoggedIn(ctx *types.SystemContext, registry string) string { + path := getPathToAuth(ctx) + username, _, _ := findAuthentication(registry, path, false) + if username != "" { + return username + } + return "" +} + +// RemoveAuthentication deletes the credentials stored in auth.json +func RemoveAuthentication(ctx *types.SystemContext, registry string) error { + return modifyJSON(ctx, func(auths *dockerConfigFile) (bool, error) { + // First try cred helpers. + if ch, exists := auths.CredHelpers[registry]; exists { + return false, deleteAuthFromCredHelper(ch, registry) + } + + if _, ok := auths.AuthConfigs[registry]; ok { + delete(auths.AuthConfigs, registry) + } else if _, ok := auths.AuthConfigs[normalizeRegistry(registry)]; ok { + delete(auths.AuthConfigs, normalizeRegistry(registry)) + } else { + return false, ErrNotLoggedIn + } + return true, nil + }) +} + +// RemoveAllAuthentication deletes all the credentials stored in auth.json +func RemoveAllAuthentication(ctx *types.SystemContext) error { + return modifyJSON(ctx, func(auths *dockerConfigFile) (bool, error) { + auths.CredHelpers = make(map[string]string) + auths.AuthConfigs = make(map[string]dockerAuthConfig) + return true, nil + }) +} + +// getPath gets the path of the auth.json file +// The path can be overriden by the user if the overwrite-path flag is set +// If the flag is not set and XDG_RUNTIME_DIR is ser, the auth.json file is saved in XDG_RUNTIME_DIR/containers +// Otherwise, the auth.json file is stored in /run/user/UID/containers +func getPathToAuth(ctx *types.SystemContext) string { + if ctx != nil { + if ctx.AuthFilePath != "" { + return ctx.AuthFilePath + } + if ctx.RootForImplicitAbsolutePaths != "" { + return filepath.Join(ctx.RootForImplicitAbsolutePaths, defaultPath, strconv.Itoa(os.Getuid()), authCfg, authCfgFileName) + } + } + runtimeDir := os.Getenv("XDG_RUNTIME_DIR") + if runtimeDir == "" { + runtimeDir = filepath.Join(defaultPath, strconv.Itoa(os.Getuid())) + } + return filepath.Join(runtimeDir, authCfg, authCfgFileName) +} + +// readJSONFile unmarshals the authentications stored in the auth.json file and returns it +// or returns an empty dockerConfigFile data structure if auth.json does not exist +// if the file exists and is empty, readJSONFile returns an error +func readJSONFile(path string, legacyFormat bool) (dockerConfigFile, error) { + var auths dockerConfigFile + + raw, err := ioutil.ReadFile(path) + if os.IsNotExist(err) { + auths.AuthConfigs = map[string]dockerAuthConfig{} + return auths, nil + } + + if legacyFormat { + if err = json.Unmarshal(raw, &auths.AuthConfigs); err != nil { + return dockerConfigFile{}, errors.Wrapf(err, "error unmarshaling JSON at %q", path) + } + return auths, nil + } + + if err = json.Unmarshal(raw, &auths); err != nil { + return dockerConfigFile{}, errors.Wrapf(err, "error unmarshaling JSON at %q", path) + } + + return auths, nil +} + +// modifyJSON writes to auth.json if the dockerConfigFile has been updated +func modifyJSON(ctx *types.SystemContext, editor func(auths *dockerConfigFile) (bool, error)) error { + path := getPathToAuth(ctx) + dir := filepath.Dir(path) + if _, err := os.Stat(dir); os.IsNotExist(err) { + if err = os.Mkdir(dir, 0700); err != nil { + return errors.Wrapf(err, "error creating directory %q", dir) + } + } + + auths, err := readJSONFile(path, false) + if err != nil { + return errors.Wrapf(err, "error reading JSON file %q", path) + } + + updated, err := editor(&auths) + if err != nil { + return errors.Wrapf(err, "error updating %q", path) + } + if updated { + newData, err := json.MarshalIndent(auths, "", "\t") + if err != nil { + return errors.Wrapf(err, "error marshaling JSON %q", path) + } + + if err = ioutil.WriteFile(path, newData, 0755); err != nil { + return errors.Wrapf(err, "error writing to file %q", path) + } + } + + return nil +} + +func getAuthFromCredHelper(credHelper, registry string) (string, string, error) { + helperName := fmt.Sprintf("docker-credential-%s", credHelper) + p := helperclient.NewShellProgramFunc(helperName) + creds, err := helperclient.Get(p, registry) + if err != nil { + return "", "", err + } + return creds.Username, creds.Secret, nil +} + +func setAuthToCredHelper(credHelper, registry, username, password string) error { + helperName := fmt.Sprintf("docker-credential-%s", credHelper) + p := helperclient.NewShellProgramFunc(helperName) + creds := &credentials.Credentials{ + ServerURL: registry, + Username: username, + Secret: password, + } + return helperclient.Store(p, creds) +} + +func deleteAuthFromCredHelper(credHelper, registry string) error { + helperName := fmt.Sprintf("docker-credential-%s", credHelper) + p := helperclient.NewShellProgramFunc(helperName) + return helperclient.Erase(p, registry) +} + +// findAuthentication looks for auth of registry in path +func findAuthentication(registry, path string, legacyFormat bool) (string, string, error) { + auths, err := readJSONFile(path, legacyFormat) + if err != nil { + return "", "", errors.Wrapf(err, "error reading JSON file %q", path) + } + + // First try cred helpers. They should always be normalized. + if ch, exists := auths.CredHelpers[registry]; exists { + return getAuthFromCredHelper(ch, registry) + } + + // I'm feeling lucky + if val, exists := auths.AuthConfigs[registry]; exists { + return decodeDockerAuth(val.Auth) + } + + // bad luck; let's normalize the entries first + registry = normalizeRegistry(registry) + normalizedAuths := map[string]dockerAuthConfig{} + for k, v := range auths.AuthConfigs { + normalizedAuths[normalizeRegistry(k)] = v + } + if val, exists := normalizedAuths[registry]; exists { + return decodeDockerAuth(val.Auth) + } + return "", "", nil +} + +func decodeDockerAuth(s string) (string, string, error) { + decoded, err := base64.StdEncoding.DecodeString(s) + if err != nil { + return "", "", err + } + parts := strings.SplitN(string(decoded), ":", 2) + if len(parts) != 2 { + // if it's invalid just skip, as docker does + return "", "", nil + } + user := parts[0] + password := strings.Trim(parts[1], "\x00") + return user, password, nil +} + +// convertToHostname converts a registry url which has http|https prepended +// to just an hostname. +// Copied from github.com/docker/docker/registry/auth.go +func convertToHostname(url string) string { + stripped := url + if strings.HasPrefix(url, "http://") { + stripped = strings.TrimPrefix(url, "http://") + } else if strings.HasPrefix(url, "https://") { + stripped = strings.TrimPrefix(url, "https://") + } + + nameParts := strings.SplitN(stripped, "/", 2) + + return nameParts[0] +} + +func normalizeRegistry(registry string) string { + normalized := convertToHostname(registry) + switch normalized { + case "registry-1.docker.io", "docker.io": + return "index.docker.io" + } + return normalized +} diff --git a/vendor/github.com/containers/image/pkg/tlsclientconfig/tlsclientconfig.go b/vendor/github.com/containers/image/pkg/tlsclientconfig/tlsclientconfig.go new file mode 100644 index 00000000..0a32861c --- /dev/null +++ b/vendor/github.com/containers/image/pkg/tlsclientconfig/tlsclientconfig.go @@ -0,0 +1,102 @@ +package tlsclientconfig + +import ( + "crypto/tls" + "io/ioutil" + "net" + "net/http" + "os" + "path/filepath" + "strings" + "time" + + "github.com/docker/go-connections/sockets" + "github.com/docker/go-connections/tlsconfig" + "github.com/pkg/errors" + "github.com/sirupsen/logrus" +) + +// SetupCertificates opens all .crt, .cert, and .key files in dir and appends / loads certs and key pairs as appropriate to tlsc +func SetupCertificates(dir string, tlsc *tls.Config) error { + logrus.Debugf("Looking for TLS certificates and private keys in %s", dir) + fs, err := ioutil.ReadDir(dir) + if err != nil { + if os.IsNotExist(err) { + return nil + } + if os.IsPermission(err) { + logrus.Debugf("Skipping scan of %s due to permission error: %v", dir, err) + return nil + } + return err + } + + for _, f := range fs { + fullPath := filepath.Join(dir, f.Name()) + if strings.HasSuffix(f.Name(), ".crt") { + systemPool, err := tlsconfig.SystemCertPool() + if err != nil { + return errors.Wrap(err, "unable to get system cert pool") + } + tlsc.RootCAs = systemPool + logrus.Debugf(" crt: %s", fullPath) + data, err := ioutil.ReadFile(fullPath) + if err != nil { + return err + } + tlsc.RootCAs.AppendCertsFromPEM(data) + } + if strings.HasSuffix(f.Name(), ".cert") { + certName := f.Name() + keyName := certName[:len(certName)-5] + ".key" + logrus.Debugf(" cert: %s", fullPath) + if !hasFile(fs, keyName) { + return errors.Errorf("missing key %s for client certificate %s. Note that CA certificates should use the extension .crt", keyName, certName) + } + cert, err := tls.LoadX509KeyPair(filepath.Join(dir, certName), filepath.Join(dir, keyName)) + if err != nil { + return err + } + tlsc.Certificates = append(tlsc.Certificates, cert) + } + if strings.HasSuffix(f.Name(), ".key") { + keyName := f.Name() + certName := keyName[:len(keyName)-4] + ".cert" + logrus.Debugf(" key: %s", fullPath) + if !hasFile(fs, certName) { + return errors.Errorf("missing client certificate %s for key %s", certName, keyName) + } + } + } + return nil +} + +func hasFile(files []os.FileInfo, name string) bool { + for _, f := range files { + if f.Name() == name { + return true + } + } + return false +} + +// NewTransport Creates a default transport +func NewTransport() *http.Transport { + direct := &net.Dialer{ + Timeout: 30 * time.Second, + KeepAlive: 30 * time.Second, + DualStack: true, + } + tr := &http.Transport{ + Proxy: http.ProxyFromEnvironment, + Dial: direct.Dial, + TLSHandshakeTimeout: 10 * time.Second, + // TODO(dmcgowan): Call close idle connections when complete and use keep alive + DisableKeepAlives: true, + } + proxyDialer, err := sockets.DialerFromEnvironment(direct) + if err == nil { + tr.Dial = proxyDialer.Dial + } + return tr +} diff --git a/vendor/github.com/containers/image/signature/signature.go b/vendor/github.com/containers/image/signature/signature.go index f6219bec..41f13f72 100644 --- a/vendor/github.com/containers/image/signature/signature.go +++ b/vendor/github.com/containers/image/signature/signature.go @@ -180,13 +180,9 @@ func (s *untrustedSignature) strictUnmarshalJSON(data []byte) error { } s.UntrustedDockerManifestDigest = digest.Digest(digestString) - if err := paranoidUnmarshalJSONObjectExactFields(identity, map[string]interface{}{ + return paranoidUnmarshalJSONObjectExactFields(identity, map[string]interface{}{ "docker-reference": &s.UntrustedDockerReference, - }); err != nil { - return err - } - - return nil + }) } // Sign formats the signature and returns a blob signed using mech and keyIdentity diff --git a/vendor/github.com/containers/image/types/types.go b/vendor/github.com/containers/image/types/types.go index a0424106..4ede907b 100644 --- a/vendor/github.com/containers/image/types/types.go +++ b/vendor/github.com/containers/image/types/types.go @@ -304,6 +304,16 @@ type SystemContext struct { RegistriesDirPath string // Path to the system-wide registries configuration file SystemRegistriesConfPath string + // If not "", overrides the default path for the authentication file + AuthFilePath string + + // === OCI.Transport overrides === + // If not "", a directory containing a CA certificate (ending with ".crt"), + // a client certificate (ending with ".cert") and a client ceritificate key + // (ending with ".key") used when downloading OCI image layers. + OCICertPath string + // Allow downloading OCI image layers over HTTP, or HTTPS with failed TLS verification. Note that this does not affect other TLS connections. + OCIInsecureSkipTLSVerify bool // === docker.Transport overrides === // If not "", a directory containing a CA certificate (ending with ".crt"), diff --git a/vendor/github.com/containers/storage/pkg/archive/example_changes.go b/vendor/github.com/containers/storage/pkg/archive/example_changes.go deleted file mode 100644 index 70f9c556..00000000 --- a/vendor/github.com/containers/storage/pkg/archive/example_changes.go +++ /dev/null @@ -1,97 +0,0 @@ -// +build ignore - -// Simple tool to create an archive stream from an old and new directory -// -// By default it will stream the comparison of two temporary directories with junk files -package main - -import ( - "flag" - "fmt" - "io" - "io/ioutil" - "os" - "path" - - "github.com/containers/storage/pkg/archive" - "github.com/sirupsen/logrus" -) - -var ( - flDebug = flag.Bool("D", false, "debugging output") - flNewDir = flag.String("newdir", "", "") - flOldDir = flag.String("olddir", "", "") - log = logrus.New() -) - -func main() { - flag.Usage = func() { - fmt.Println("Produce a tar from comparing two directory paths. By default a demo tar is created of around 200 files (including hardlinks)") - fmt.Printf("%s [OPTIONS]\n", os.Args[0]) - flag.PrintDefaults() - } - flag.Parse() - log.Out = os.Stderr - if (len(os.Getenv("DEBUG")) > 0) || *flDebug { - logrus.SetLevel(logrus.DebugLevel) - } - var newDir, oldDir string - - if len(*flNewDir) == 0 { - var err error - newDir, err = ioutil.TempDir("", "storage-test-newDir") - if err != nil { - log.Fatal(err) - } - defer os.RemoveAll(newDir) - if _, err := prepareUntarSourceDirectory(100, newDir, true); err != nil { - log.Fatal(err) - } - } else { - newDir = *flNewDir - } - - if len(*flOldDir) == 0 { - oldDir, err := ioutil.TempDir("", "storage-test-oldDir") - if err != nil { - log.Fatal(err) - } - defer os.RemoveAll(oldDir) - } else { - oldDir = *flOldDir - } - - changes, err := archive.ChangesDirs(newDir, oldDir) - if err != nil { - log.Fatal(err) - } - - a, err := archive.ExportChanges(newDir, changes) - if err != nil { - log.Fatal(err) - } - defer a.Close() - - i, err := io.Copy(os.Stdout, a) - if err != nil && err != io.EOF { - log.Fatal(err) - } - fmt.Fprintf(os.Stderr, "wrote archive of %d bytes", i) -} - -func prepareUntarSourceDirectory(numberOfFiles int, targetPath string, makeLinks bool) (int, error) { - fileData := []byte("fooo") - for n := 0; n < numberOfFiles; n++ { - fileName := fmt.Sprintf("file-%d", n) - if err := ioutil.WriteFile(path.Join(targetPath, fileName), fileData, 0700); err != nil { - return 0, err - } - if makeLinks { - if err := os.Link(path.Join(targetPath, fileName), path.Join(targetPath, fileName+"-link")); err != nil { - return 0, err - } - } - } - totalSize := numberOfFiles * len(fileData) - return totalSize, nil -}