diff --git a/vendor.conf b/vendor.conf index b37e2f40..557701e9 100644 --- a/vendor.conf +++ b/vendor.conf @@ -4,7 +4,7 @@ k8s.io/apimachinery release-1.7 https://github.com/kubernetes/apimachinery k8s.io/apiserver release-1.7 https://github.com/kubernetes/apiserver # github.com/sirupsen/logrus v1.0.0 -github.com/containers/image d17474f39dae1da15ab9ae033d57ebefcf62f77a +github.com/containers/image 57b257d128d6075ea3287991ee408d24c7bd2758 github.com/docker/docker-credential-helpers d68f9aeca33f5fd3f08eeae5e9d175edf4e731d1 github.com/ostreedev/ostree-go master github.com/containers/storage 64bf27465d0d1edd89e7a4ce49866fea01145782 diff --git a/vendor/github.com/containers/image/copy/copy.go b/vendor/github.com/containers/image/copy/copy.go index c7af5ed1..590b3787 100644 --- a/vendor/github.com/containers/image/copy/copy.go +++ b/vendor/github.com/containers/image/copy/copy.go @@ -581,7 +581,7 @@ func (ic *imageCopier) copyBlobFromStream(srcStream io.Reader, srcInfo types.Blo bar.ShowPercent = false bar.Start() destStream = bar.NewProxyReader(destStream) - defer fmt.Fprint(ic.reportWriter, "\n") + defer bar.Finish() // === Send a copy of the original, uncompressed, stream, to a separate path if necessary. var originalLayerReader io.Reader // DO NOT USE this other than to drain the input if no other consumer in the pipeline has done so. diff --git a/vendor/github.com/containers/image/docker/docker_client.go b/vendor/github.com/containers/image/docker/docker_client.go index 57d1c879..24b82d6f 100644 --- a/vendor/github.com/containers/image/docker/docker_client.go +++ b/vendor/github.com/containers/image/docker/docker_client.go @@ -7,7 +7,6 @@ import ( "fmt" "io" "io/ioutil" - "net" "net/http" "os" "path/filepath" @@ -16,9 +15,9 @@ import ( "github.com/containers/image/docker/reference" "github.com/containers/image/pkg/docker/config" + "github.com/containers/image/pkg/tlsclientconfig" "github.com/containers/image/types" "github.com/docker/distribution/registry/client" - "github.com/docker/go-connections/sockets" "github.com/docker/go-connections/tlsconfig" "github.com/opencontainers/go-digest" "github.com/pkg/errors" @@ -110,27 +109,7 @@ func serverDefault() *tls.Config { } } -func newTransport() *http.Transport { - direct := &net.Dialer{ - Timeout: 30 * time.Second, - KeepAlive: 30 * time.Second, - DualStack: true, - } - tr := &http.Transport{ - Proxy: http.ProxyFromEnvironment, - Dial: direct.Dial, - TLSHandshakeTimeout: 10 * time.Second, - // TODO(dmcgowan): Call close idle connections when complete and use keep alive - DisableKeepAlives: true, - } - proxyDialer, err := sockets.DialerFromEnvironment(direct) - if err == nil { - tr.Dial = proxyDialer.Dial - } - return tr -} - -// dockerCertDir returns a path to a directory to be consumed by setupCertificates() depending on ctx and hostPort. +// dockerCertDir returns a path to a directory to be consumed by tlsclientconfig.SetupCertificates() depending on ctx and hostPort. func dockerCertDir(ctx *types.SystemContext, hostPort string) string { if ctx != nil && ctx.DockerCertPath != "" { return ctx.DockerCertPath @@ -232,7 +211,7 @@ func newDockerClientWithDetails(ctx *types.SystemContext, registry, username, pa if registry == dockerHostname { registry = dockerRegistry } - tr := newTransport() + tr := tlsclientconfig.NewTransport() tr.TLSClientConfig = serverDefault() // It is undefined whether the host[:port] string for dockerHostname should be dockerHostname or dockerRegistry, @@ -241,7 +220,7 @@ func newDockerClientWithDetails(ctx *types.SystemContext, registry, username, pa // generally the UI hides the existence of the different dockerRegistry. But note that this behavior is // undocumented and may change if docker/docker changes. certDir := dockerCertDir(ctx, hostName) - if err := setupCertificates(certDir, tr.TLSClientConfig); err != nil { + if err := tlsclientconfig.SetupCertificates(certDir, tr.TLSClientConfig); err != nil { return nil, err } @@ -396,7 +375,7 @@ func (c *dockerClient) getBearerToken(ctx context.Context, realm, service, scope if c.username != "" && c.password != "" { authReq.SetBasicAuth(c.username, c.password) } - tr := newTransport() + tr := tlsclientconfig.NewTransport() // TODO(runcom): insecure for now to contact the external token service tr.TLSClientConfig = &tls.Config{InsecureSkipVerify: true} client := &http.Client{Transport: tr} diff --git a/vendor/github.com/containers/image/docker/docker_image_src.go b/vendor/github.com/containers/image/docker/docker_image_src.go index 3e9472c4..232c3cf9 100644 --- a/vendor/github.com/containers/image/docker/docker_image_src.go +++ b/vendor/github.com/containers/image/docker/docker_image_src.go @@ -140,6 +140,7 @@ func (s *dockerImageSource) getExternalBlob(urls []string) (io.ReadCloser, int64 logrus.Debug(err) continue } + break } } if resp.Body != nil && err == nil { diff --git a/vendor/github.com/containers/image/image/oci.go b/vendor/github.com/containers/image/image/oci.go index 7ea5d2bb..5f7c0728 100644 --- a/vendor/github.com/containers/image/image/oci.go +++ b/vendor/github.com/containers/image/image/oci.go @@ -109,7 +109,7 @@ func (m *manifestOCI1) OCIConfig() (*imgspecv1.Image, error) { func (m *manifestOCI1) LayerInfos() []types.BlobInfo { blobs := []types.BlobInfo{} for _, layer := range m.LayersDescriptors { - blobs = append(blobs, types.BlobInfo{Digest: layer.Digest, Size: layer.Size, Annotations: layer.Annotations}) + blobs = append(blobs, types.BlobInfo{Digest: layer.Digest, Size: layer.Size, Annotations: layer.Annotations, URLs: layer.URLs}) } return blobs } @@ -160,6 +160,7 @@ func (m *manifestOCI1) UpdatedImage(options types.ManifestUpdateOptions) (types. copy.LayersDescriptors[i].Digest = info.Digest copy.LayersDescriptors[i].Size = info.Size copy.LayersDescriptors[i].Annotations = info.Annotations + copy.LayersDescriptors[i].URLs = info.URLs } } // Ignore options.EmbeddedDockerReference: it may be set when converting from schema1, but we really don't care. diff --git a/vendor/github.com/containers/image/oci/archive/oci_dest.go b/vendor/github.com/containers/image/oci/archive/oci_dest.go index b2c4bb63..52e99a43 100644 --- a/vendor/github.com/containers/image/oci/archive/oci_dest.go +++ b/vendor/github.com/containers/image/oci/archive/oci_dest.go @@ -106,11 +106,7 @@ func (d *ociArchiveImageDestination) Commit() error { src := d.tempDirRef.tempDirectory // path to save tarred up file dst := d.ref.resolvedFile - if err := tarDirectory(src, dst); err != nil { - return err - } - - return nil + return tarDirectory(src, dst) } // tar converts the directory at src and saves it to dst diff --git a/vendor/github.com/containers/image/oci/layout/oci_dest.go b/vendor/github.com/containers/image/oci/layout/oci_dest.go index c4801e34..ce1e0c3e 100644 --- a/vendor/github.com/containers/image/oci/layout/oci_dest.go +++ b/vendor/github.com/containers/image/oci/layout/oci_dest.go @@ -66,7 +66,7 @@ func (d *ociImageDestination) ShouldCompressLayers() bool { // AcceptsForeignLayerURLs returns false iff foreign layers in manifest should be actually // uploaded to the image destination, true otherwise. func (d *ociImageDestination) AcceptsForeignLayerURLs() bool { - return false + return true } // MustMatchRuntimeOS returns true iff the destination can store only images targeted for the current runtime OS. False otherwise. diff --git a/vendor/github.com/containers/image/oci/layout/oci_src.go b/vendor/github.com/containers/image/oci/layout/oci_src.go index 99b9f208..be8a2aa7 100644 --- a/vendor/github.com/containers/image/oci/layout/oci_src.go +++ b/vendor/github.com/containers/image/oci/layout/oci_src.go @@ -4,25 +4,43 @@ import ( "context" "io" "io/ioutil" + "net/http" "os" + "strconv" + "github.com/containers/image/pkg/tlsclientconfig" "github.com/containers/image/types" + "github.com/docker/go-connections/tlsconfig" "github.com/opencontainers/go-digest" imgspecv1 "github.com/opencontainers/image-spec/specs-go/v1" + "github.com/pkg/errors" ) type ociImageSource struct { ref ociReference descriptor imgspecv1.Descriptor + client *http.Client } // newImageSource returns an ImageSource for reading from an existing directory. -func newImageSource(ref ociReference) (types.ImageSource, error) { +func newImageSource(ctx *types.SystemContext, ref ociReference) (types.ImageSource, error) { + tr := tlsclientconfig.NewTransport() + tr.TLSClientConfig = tlsconfig.ServerDefault() + + if ctx != nil && ctx.OCICertPath != "" { + if err := tlsclientconfig.SetupCertificates(ctx.OCICertPath, tr.TLSClientConfig); err != nil { + return nil, err + } + tr.TLSClientConfig.InsecureSkipVerify = ctx.OCIInsecureSkipTLSVerify + } + + client := &http.Client{} + client.Transport = tr descriptor, err := ref.getManifestDescriptor() if err != nil { return nil, err } - return &ociImageSource{ref: ref, descriptor: descriptor}, nil + return &ociImageSource{ref: ref, descriptor: descriptor, client: client}, nil } // Reference returns the reference used to set up this source. @@ -70,6 +88,10 @@ func (s *ociImageSource) GetTargetManifest(digest digest.Digest) ([]byte, string // GetBlob returns a stream for the specified blob, and the blob's size. func (s *ociImageSource) GetBlob(info types.BlobInfo) (io.ReadCloser, int64, error) { + if len(info.URLs) != 0 { + return s.getExternalBlob(info.URLs) + } + path, err := s.ref.blobPath(info.Digest) if err != nil { return nil, 0, err @@ -89,3 +111,32 @@ func (s *ociImageSource) GetBlob(info types.BlobInfo) (io.ReadCloser, int64, err func (s *ociImageSource) GetSignatures(context.Context) ([][]byte, error) { return [][]byte{}, nil } + +func (s *ociImageSource) getExternalBlob(urls []string) (io.ReadCloser, int64, error) { + errWrap := errors.New("failed fetching external blob from all urls") + for _, url := range urls { + resp, err := s.client.Get(url) + if err != nil { + errWrap = errors.Wrapf(errWrap, "fetching %s failed %s", url, err.Error()) + continue + } + + if resp.StatusCode != http.StatusOK { + resp.Body.Close() + errWrap = errors.Wrapf(errWrap, "fetching %s failed, response code not 200", url) + continue + } + + return resp.Body, getBlobSize(resp), nil + } + + return nil, 0, errWrap +} + +func getBlobSize(resp *http.Response) int64 { + size, err := strconv.ParseInt(resp.Header.Get("Content-Length"), 10, 64) + if err != nil { + size = -1 + } + return size +} diff --git a/vendor/github.com/containers/image/oci/layout/oci_transport.go b/vendor/github.com/containers/image/oci/layout/oci_transport.go index 7fd826f4..312bc0e4 100644 --- a/vendor/github.com/containers/image/oci/layout/oci_transport.go +++ b/vendor/github.com/containers/image/oci/layout/oci_transport.go @@ -182,7 +182,7 @@ func (ref ociReference) PolicyConfigurationNamespaces() []string { // NOTE: If any kind of signature verification should happen, build an UnparsedImage from the value returned by NewImageSource, // verify that UnparsedImage, and convert it into a real Image via image.FromUnparsedImage. func (ref ociReference) NewImage(ctx *types.SystemContext) (types.Image, error) { - src, err := newImageSource(ref) + src, err := newImageSource(ctx, ref) if err != nil { return nil, err } @@ -244,7 +244,7 @@ func LoadManifestDescriptor(imgRef types.ImageReference) (imgspecv1.Descriptor, // NewImageSource returns a types.ImageSource for this reference. // The caller must call .Close() on the returned ImageSource. func (ref ociReference) NewImageSource(ctx *types.SystemContext) (types.ImageSource, error) { - return newImageSource(ref) + return newImageSource(ctx, ref) } // NewImageDestination returns a types.ImageDestination for this reference. diff --git a/vendor/github.com/containers/image/ostree/ostree_dest.go b/vendor/github.com/containers/image/ostree/ostree_dest.go index 885297bd..26137431 100644 --- a/vendor/github.com/containers/image/ostree/ostree_dest.go +++ b/vendor/github.com/containers/image/ostree/ostree_dest.go @@ -46,6 +46,7 @@ type ostreeImageDestination struct { schema manifestSchema tmpDirPath string blobs map[string]*blobToImport + digest digest.Digest } // newImageDestination returns an ImageDestination for writing to an existing ostree. @@ -54,7 +55,7 @@ func newImageDestination(ref ostreeReference, tmpDirPath string) (types.ImageDes if err := ensureDirectoryExists(tmpDirPath); err != nil { return nil, err } - return &ostreeImageDestination{ref, "", manifestSchema{}, tmpDirPath, map[string]*blobToImport{}}, nil + return &ostreeImageDestination{ref, "", manifestSchema{}, tmpDirPath, map[string]*blobToImport{}, ""}, nil } // Reference returns the reference used to set up this destination. Note that this should directly correspond to user's intent, @@ -238,10 +239,10 @@ func (d *ostreeImageDestination) ReapplyBlob(info types.BlobInfo) (types.BlobInf // FIXME? This should also receive a MIME type if known, to differentiate between schema versions. // If the destination is in principle available, refuses this manifest type (e.g. it does not recognize the schema), // but may accept a different manifest type, the returned error must be an ManifestTypeRejectedError. -func (d *ostreeImageDestination) PutManifest(manifest []byte) error { - d.manifest = string(manifest) +func (d *ostreeImageDestination) PutManifest(manifestBlob []byte) error { + d.manifest = string(manifestBlob) - if err := json.Unmarshal(manifest, &d.schema); err != nil { + if err := json.Unmarshal(manifestBlob, &d.schema); err != nil { return err } @@ -250,7 +251,13 @@ func (d *ostreeImageDestination) PutManifest(manifest []byte) error { return err } - return ioutil.WriteFile(manifestPath, manifest, 0644) + digest, err := manifest.Digest(manifestBlob) + if err != nil { + return err + } + d.digest = digest + + return ioutil.WriteFile(manifestPath, manifestBlob, 0644) } func (d *ostreeImageDestination) PutSignatures(signatures [][]byte) error { @@ -304,7 +311,7 @@ func (d *ostreeImageDestination) Commit() error { manifestPath := filepath.Join(d.tmpDirPath, "manifest") - metadata := []string{fmt.Sprintf("docker.manifest=%s", string(d.manifest))} + metadata := []string{fmt.Sprintf("docker.manifest=%s", string(d.manifest)), fmt.Sprintf("docker.digest=%s", string(d.digest))} err = d.ostreeCommit(repo, fmt.Sprintf("ociimage/%s", d.ref.branchName), manifestPath, metadata) _, err = repo.CommitTransaction() diff --git a/vendor/github.com/containers/image/pkg/tlsclientconfig/tlsclientconfig.go b/vendor/github.com/containers/image/pkg/tlsclientconfig/tlsclientconfig.go new file mode 100644 index 00000000..0a32861c --- /dev/null +++ b/vendor/github.com/containers/image/pkg/tlsclientconfig/tlsclientconfig.go @@ -0,0 +1,102 @@ +package tlsclientconfig + +import ( + "crypto/tls" + "io/ioutil" + "net" + "net/http" + "os" + "path/filepath" + "strings" + "time" + + "github.com/docker/go-connections/sockets" + "github.com/docker/go-connections/tlsconfig" + "github.com/pkg/errors" + "github.com/sirupsen/logrus" +) + +// SetupCertificates opens all .crt, .cert, and .key files in dir and appends / loads certs and key pairs as appropriate to tlsc +func SetupCertificates(dir string, tlsc *tls.Config) error { + logrus.Debugf("Looking for TLS certificates and private keys in %s", dir) + fs, err := ioutil.ReadDir(dir) + if err != nil { + if os.IsNotExist(err) { + return nil + } + if os.IsPermission(err) { + logrus.Debugf("Skipping scan of %s due to permission error: %v", dir, err) + return nil + } + return err + } + + for _, f := range fs { + fullPath := filepath.Join(dir, f.Name()) + if strings.HasSuffix(f.Name(), ".crt") { + systemPool, err := tlsconfig.SystemCertPool() + if err != nil { + return errors.Wrap(err, "unable to get system cert pool") + } + tlsc.RootCAs = systemPool + logrus.Debugf(" crt: %s", fullPath) + data, err := ioutil.ReadFile(fullPath) + if err != nil { + return err + } + tlsc.RootCAs.AppendCertsFromPEM(data) + } + if strings.HasSuffix(f.Name(), ".cert") { + certName := f.Name() + keyName := certName[:len(certName)-5] + ".key" + logrus.Debugf(" cert: %s", fullPath) + if !hasFile(fs, keyName) { + return errors.Errorf("missing key %s for client certificate %s. Note that CA certificates should use the extension .crt", keyName, certName) + } + cert, err := tls.LoadX509KeyPair(filepath.Join(dir, certName), filepath.Join(dir, keyName)) + if err != nil { + return err + } + tlsc.Certificates = append(tlsc.Certificates, cert) + } + if strings.HasSuffix(f.Name(), ".key") { + keyName := f.Name() + certName := keyName[:len(keyName)-4] + ".cert" + logrus.Debugf(" key: %s", fullPath) + if !hasFile(fs, certName) { + return errors.Errorf("missing client certificate %s for key %s", certName, keyName) + } + } + } + return nil +} + +func hasFile(files []os.FileInfo, name string) bool { + for _, f := range files { + if f.Name() == name { + return true + } + } + return false +} + +// NewTransport Creates a default transport +func NewTransport() *http.Transport { + direct := &net.Dialer{ + Timeout: 30 * time.Second, + KeepAlive: 30 * time.Second, + DualStack: true, + } + tr := &http.Transport{ + Proxy: http.ProxyFromEnvironment, + Dial: direct.Dial, + TLSHandshakeTimeout: 10 * time.Second, + // TODO(dmcgowan): Call close idle connections when complete and use keep alive + DisableKeepAlives: true, + } + proxyDialer, err := sockets.DialerFromEnvironment(direct) + if err == nil { + tr.Dial = proxyDialer.Dial + } + return tr +} diff --git a/vendor/github.com/containers/image/signature/signature.go b/vendor/github.com/containers/image/signature/signature.go index f6219bec..41f13f72 100644 --- a/vendor/github.com/containers/image/signature/signature.go +++ b/vendor/github.com/containers/image/signature/signature.go @@ -180,13 +180,9 @@ func (s *untrustedSignature) strictUnmarshalJSON(data []byte) error { } s.UntrustedDockerManifestDigest = digest.Digest(digestString) - if err := paranoidUnmarshalJSONObjectExactFields(identity, map[string]interface{}{ + return paranoidUnmarshalJSONObjectExactFields(identity, map[string]interface{}{ "docker-reference": &s.UntrustedDockerReference, - }); err != nil { - return err - } - - return nil + }) } // Sign formats the signature and returns a blob signed using mech and keyIdentity diff --git a/vendor/github.com/containers/image/types/types.go b/vendor/github.com/containers/image/types/types.go index 4ac5e08d..4ede907b 100644 --- a/vendor/github.com/containers/image/types/types.go +++ b/vendor/github.com/containers/image/types/types.go @@ -307,6 +307,14 @@ type SystemContext struct { // If not "", overrides the default path for the authentication file AuthFilePath string + // === OCI.Transport overrides === + // If not "", a directory containing a CA certificate (ending with ".crt"), + // a client certificate (ending with ".cert") and a client ceritificate key + // (ending with ".key") used when downloading OCI image layers. + OCICertPath string + // Allow downloading OCI image layers over HTTP, or HTTPS with failed TLS verification. Note that this does not affect other TLS connections. + OCIInsecureSkipTLSVerify bool + // === docker.Transport overrides === // If not "", a directory containing a CA certificate (ending with ".crt"), // a client certificate (ending with ".cert") and a client ceritificate key