diff --git a/archive/tar.go b/archive/tar.go new file mode 100644 index 0000000..75b74e0 --- /dev/null +++ b/archive/tar.go @@ -0,0 +1,490 @@ +package archive + +import ( + "archive/tar" + "context" + "fmt" + "io" + "io/ioutil" + "os" + "path/filepath" + "runtime" + "strings" + "sync" + "syscall" + "time" + + "github.com/docker/containerd/fs" + "github.com/docker/containerd/log" + "github.com/pkg/errors" +) + +var ( + bufferPool = &sync.Pool{ + New: func() interface{} { + return make([]byte, 32*1024) + }, + } + + breakoutError = errors.New("file name outside of root") +) + +// Diff returns a tar stream of the computed filesystem +// difference between the provided directories. +// +// Produces a tar using OCI style file markers for deletions. Deleted +// files will be prepended with the prefix ".wh.". This style is +// based off AUFS whiteouts. +// See https://github.com/opencontainers/image-spec/blob/master/layer.md +func Diff(ctx context.Context, a, b string) io.ReadCloser { + r, w := io.Pipe() + + go func() { + var err error + cw := newChangeWriter(w, b) + if err = fs.Changes(ctx, a, b, cw.HandleChange); err != nil { + err = errors.Wrap(err, "failed to create diff tar stream") + } else { + err = cw.Close() + } + if err = w.CloseWithError(err); err != nil { + log.G(ctx).WithError(err).Debugf("closing tar pipe failed") + } + }() + + return r +} + +const ( + // whiteoutPrefix prefix means file is a whiteout. If this is followed by a + // filename this means that file has been removed from the base layer. + // See https://github.com/opencontainers/image-spec/blob/master/layer.md#whiteouts + whiteoutPrefix = ".wh." + + // whiteoutMetaPrefix prefix means whiteout has a special meaning and is not + // for removing an actual file. Normally these files are excluded from exported + // archives. + whiteoutMetaPrefix = whiteoutPrefix + whiteoutPrefix + + // whiteoutLinkDir is a directory AUFS uses for storing hardlink links to other + // layers. Normally these should not go into exported archives and all changed + // hardlinks should be copied to the top layer. + whiteoutLinkDir = whiteoutMetaPrefix + "plnk" + + // whiteoutOpaqueDir file means directory has been made opaque - meaning + // readdir calls to this directory do not follow to lower layers. + whiteoutOpaqueDir = whiteoutMetaPrefix + ".opq" +) + +// Apply applies a tar stream of an OCI style diff tar. +// See https://github.com/opencontainers/image-spec/blob/master/layer.md#applying-changesets +func Apply(ctx context.Context, root string, r io.Reader) (int64, error) { + root = filepath.Clean(root) + fn := prepareApply() + defer fn() + + var ( + tr = tar.NewReader(r) + size int64 + dirs []*tar.Header + + // Used for handling opaque directory markers which + // may occur out of order + unpackedPaths = make(map[string]struct{}) + + // Used for aufs plink directory + aufsTempdir = "" + aufsHardlinks = make(map[string]*tar.Header) + ) + + // Iterate through the files in the archive. + for { + hdr, err := tr.Next() + if err == io.EOF { + // end of tar archive + break + } + if err != nil { + return 0, err + } + + size += hdr.Size + + // Normalize name, for safety and for a simple is-root check + hdr.Name = filepath.Clean(hdr.Name) + + if skipFile(hdr) { + log.G(ctx).Warnf("file %q ignored: archive may not be supported on system", hdr.Name) + continue + } + + // Note as these operations are platform specific, so must the slash be. + if !strings.HasSuffix(hdr.Name, string(os.PathSeparator)) { + // Not the root directory, ensure that the parent directory exists. + // This happened in some tests where an image had a tarfile without any + // parent directories. + parent := filepath.Dir(hdr.Name) + parentPath := filepath.Join(root, parent) + + if _, err := os.Lstat(parentPath); err != nil && os.IsNotExist(err) { + err = mkdirAll(parentPath, 0600) + if err != nil { + return 0, err + } + } + } + + // Skip AUFS metadata dirs + if strings.HasPrefix(hdr.Name, whiteoutMetaPrefix) { + // Regular files inside /.wh..wh.plnk can be used as hardlink targets + // We don't want this directory, but we need the files in them so that + // such hardlinks can be resolved. + if strings.HasPrefix(hdr.Name, whiteoutLinkDir) && hdr.Typeflag == tar.TypeReg { + basename := filepath.Base(hdr.Name) + aufsHardlinks[basename] = hdr + if aufsTempdir == "" { + if aufsTempdir, err = ioutil.TempDir("", "dockerplnk"); err != nil { + return 0, err + } + defer os.RemoveAll(aufsTempdir) + } + if err := createTarFile(ctx, filepath.Join(aufsTempdir, basename), root, hdr, tr); err != nil { + return 0, err + } + } + + if hdr.Name != whiteoutOpaqueDir { + continue + } + } + + path := filepath.Join(root, hdr.Name) + rel, err := filepath.Rel(root, path) + if err != nil { + return 0, err + } + + // Note as these operations are platform specific, so must the slash be. + if strings.HasPrefix(rel, ".."+string(os.PathSeparator)) { + return 0, errors.Wrapf(breakoutError, "%q is outside of %q", hdr.Name, root) + } + base := filepath.Base(path) + + if strings.HasPrefix(base, whiteoutPrefix) { + dir := filepath.Dir(path) + if base == whiteoutOpaqueDir { + _, err := os.Lstat(dir) + if err != nil { + return 0, err + } + err = filepath.Walk(dir, func(path string, info os.FileInfo, err error) error { + if err != nil { + if os.IsNotExist(err) { + err = nil // parent was deleted + } + return err + } + if path == dir { + return nil + } + if _, exists := unpackedPaths[path]; !exists { + err := os.RemoveAll(path) + return err + } + return nil + }) + if err != nil { + return 0, err + } + continue + } + + originalBase := base[len(whiteoutPrefix):] + originalPath := filepath.Join(dir, originalBase) + if err := os.RemoveAll(originalPath); err != nil { + return 0, err + } + continue + } + // If path exits we almost always just want to remove and replace it. + // The only exception is when it is a directory *and* the file from + // the layer is also a directory. Then we want to merge them (i.e. + // just apply the metadata from the layer). + if fi, err := os.Lstat(path); err == nil { + if !(fi.IsDir() && hdr.Typeflag == tar.TypeDir) { + if err := os.RemoveAll(path); err != nil { + return 0, err + } + } + } + + srcData := io.Reader(tr) + srcHdr := hdr + + // Hard links into /.wh..wh.plnk don't work, as we don't extract that directory, so + // we manually retarget these into the temporary files we extracted them into + if hdr.Typeflag == tar.TypeLink && strings.HasPrefix(filepath.Clean(hdr.Linkname), whiteoutLinkDir) { + linkBasename := filepath.Base(hdr.Linkname) + srcHdr = aufsHardlinks[linkBasename] + if srcHdr == nil { + return 0, fmt.Errorf("Invalid aufs hardlink") + } + tmpFile, err := os.Open(filepath.Join(aufsTempdir, linkBasename)) + if err != nil { + return 0, err + } + defer tmpFile.Close() + srcData = tmpFile + } + + if err := createTarFile(ctx, path, root, srcHdr, srcData); err != nil { + return 0, err + } + + // Directory mtimes must be handled at the end to avoid further + // file creation in them to modify the directory mtime + if hdr.Typeflag == tar.TypeDir { + dirs = append(dirs, hdr) + } + unpackedPaths[path] = struct{}{} + } + + for _, hdr := range dirs { + path := filepath.Join(root, hdr.Name) + if err := chtimes(path, boundTime(latestTime(hdr.AccessTime, hdr.ModTime)), boundTime(hdr.ModTime)); err != nil { + return 0, err + } + } + + return size, nil +} + +type changeWriter struct { + tw *tar.Writer + source string + whiteoutT time.Time + inodeCache map[uint64]string +} + +func newChangeWriter(w io.Writer, source string) *changeWriter { + return &changeWriter{ + tw: tar.NewWriter(w), + source: source, + whiteoutT: time.Now(), + inodeCache: map[uint64]string{}, + } +} + +func (cw *changeWriter) HandleChange(k fs.ChangeKind, p string, f os.FileInfo, err error) error { + if err != nil { + return err + } + if k == fs.ChangeKindDelete { + whiteOutDir := filepath.Dir(p) + whiteOutBase := filepath.Base(p) + whiteOut := filepath.Join(whiteOutDir, whiteoutPrefix+whiteOutBase) + hdr := &tar.Header{ + Name: whiteOut[1:], + Size: 0, + ModTime: cw.whiteoutT, + AccessTime: cw.whiteoutT, + ChangeTime: cw.whiteoutT, + } + if err := cw.tw.WriteHeader(hdr); err != nil { + errors.Wrap(err, "failed to write whiteout header") + } + } else { + var ( + link string + err error + source = filepath.Join(cw.source, p) + ) + + if f.Mode()&os.ModeSymlink != 0 { + if link, err = os.Readlink(source); err != nil { + return err + } + } + + hdr, err := tar.FileInfoHeader(f, link) + if err != nil { + return err + } + + hdr.Mode = int64(chmodTarEntry(os.FileMode(hdr.Mode))) + + name := p + if strings.HasPrefix(name, string(filepath.Separator)) { + name, err = filepath.Rel(string(filepath.Separator), name) + if err != nil { + return errors.Wrap(err, "failed to make path relative") + } + } + name, err = tarName(name) + if err != nil { + return errors.Wrap(err, "cannot canonicalize path") + } + // suffix with '/' for directories + if f.IsDir() && !strings.HasSuffix(name, "/") { + name += "/" + } + hdr.Name = name + + if err := setHeaderForSpecialDevice(hdr, name, f); err != nil { + return errors.Wrap(err, "failed to set device headers") + } + + linkname, err := fs.GetLinkSource(name, f, cw.inodeCache) + if err != nil { + return errors.Wrap(err, "failed to get hardlink") + } + + if linkname != "" { + hdr.Typeflag = tar.TypeLink + hdr.Linkname = linkname + hdr.Size = 0 + } + + if capability, err := getxattr(source, "security.capability"); err != nil { + return errors.Wrap(err, "failed to get capabilities xattr") + } else if capability != nil { + hdr.Xattrs = map[string]string{ + "security.capability": string(capability), + } + } + + if err := cw.tw.WriteHeader(hdr); err != nil { + return errors.Wrap(err, "failed to write file header") + } + + if hdr.Typeflag == tar.TypeReg && hdr.Size > 0 { + file, err := open(source) + if err != nil { + return errors.Wrapf(err, "failed to open path: %v", source) + } + defer file.Close() + + buf := bufferPool.Get().([]byte) + n, err := io.CopyBuffer(cw.tw, file, buf) + bufferPool.Put(buf) + if err != nil { + return errors.Wrap(err, "failed to copy") + } + if n != hdr.Size { + return errors.New("short write copying file") + } + } + } + return nil +} + +func (cw *changeWriter) Close() error { + if err := cw.tw.Close(); err != nil { + return errors.Wrap(err, "failed to close tar writer") + } + return nil +} + +func createTarFile(ctx context.Context, path, extractDir string, hdr *tar.Header, reader io.Reader) error { + // hdr.Mode is in linux format, which we can use for sycalls, + // but for os.Foo() calls we need the mode converted to os.FileMode, + // so use hdrInfo.Mode() (they differ for e.g. setuid bits) + hdrInfo := hdr.FileInfo() + + switch hdr.Typeflag { + case tar.TypeDir: + // Create directory unless it exists as a directory already. + // In that case we just want to merge the two + if fi, err := os.Lstat(path); !(err == nil && fi.IsDir()) { + if err := os.Mkdir(path, hdrInfo.Mode()); err != nil { + return err + } + } + + case tar.TypeReg, tar.TypeRegA: + file, err := openFile(path, os.O_CREATE|os.O_WRONLY, hdrInfo.Mode()) + if err != nil { + return err + } + buf := bufferPool.Get().([]byte) + _, err = io.CopyBuffer(file, reader, buf) + if err1 := file.Close(); err == nil { + err = err1 + } + if err != nil { + return err + } + + case tar.TypeBlock, tar.TypeChar: + // Handle this is an OS-specific way + if err := handleTarTypeBlockCharFifo(hdr, path); err != nil { + return err + } + + case tar.TypeFifo: + // Handle this is an OS-specific way + if err := handleTarTypeBlockCharFifo(hdr, path); err != nil { + return err + } + + case tar.TypeLink: + targetPath := filepath.Join(extractDir, hdr.Linkname) + // check for hardlink breakout + if !strings.HasPrefix(targetPath, extractDir) { + return errors.Wrapf(breakoutError, "invalid hardlink %q -> %q", targetPath, hdr.Linkname) + } + if err := os.Link(targetPath, path); err != nil { + return err + } + + case tar.TypeSymlink: + // path -> hdr.Linkname = targetPath + // e.g. /extractDir/path/to/symlink -> ../2/file = /extractDir/path/2/file + targetPath := filepath.Join(filepath.Dir(path), hdr.Linkname) + + // the reason we don't need to check symlinks in the path (with FollowSymlinkInScope) is because + // that symlink would first have to be created, which would be caught earlier, at this very check: + if !strings.HasPrefix(targetPath, extractDir) { + return errors.Wrapf(breakoutError, "invalid symlink %q -> %q", path, hdr.Linkname) + } + if err := os.Symlink(hdr.Linkname, path); err != nil { + return err + } + + case tar.TypeXGlobalHeader: + log.G(ctx).Debug("PAX Global Extended Headers found and ignored") + return nil + + default: + return errors.Errorf("unhandled tar header type %d\n", hdr.Typeflag) + } + + // Lchown is not supported on Windows. + if runtime.GOOS != "windows" { + if err := os.Lchown(path, hdr.Uid, hdr.Gid); err != nil { + return err + } + } + + for key, value := range hdr.Xattrs { + if err := setxattr(path, key, value); err != nil { + if errors.Cause(err) == syscall.ENOTSUP { + log.G(ctx).WithError(err).Warnf("ignored xattr %s in archive", key) + continue + } + return err + } + } + + // There is no LChmod, so ignore mode for symlink. Also, this + // must happen after chown, as that can modify the file mode + if err := handleLChmod(hdr, path, hdrInfo); err != nil { + return err + } + + if err := chtimes(path, boundTime(latestTime(hdr.AccessTime, hdr.ModTime)), boundTime(hdr.ModTime)); err != nil { + return err + } + + return nil +} diff --git a/archive/tar_linux.go b/archive/tar_linux.go new file mode 100644 index 0000000..3b855af --- /dev/null +++ b/archive/tar_linux.go @@ -0,0 +1,134 @@ +package archive + +import ( + "archive/tar" + "os" + "sync" + "syscall" + + "github.com/opencontainers/runc/libcontainer/system" + "github.com/pkg/errors" + "github.com/stevvooe/continuity/sysx" +) + +func tarName(p string) (string, error) { + return p, nil +} + +func chmodTarEntry(perm os.FileMode) os.FileMode { + return perm +} + +func setHeaderForSpecialDevice(hdr *tar.Header, name string, fi os.FileInfo) error { + s, ok := fi.Sys().(*syscall.Stat_t) + if !ok { + return errors.New("unsupported stat type") + } + + // Currently go does not fill in the major/minors + if s.Mode&syscall.S_IFBLK != 0 || + s.Mode&syscall.S_IFCHR != 0 { + hdr.Devmajor = int64(major(uint64(s.Rdev))) + hdr.Devminor = int64(minor(uint64(s.Rdev))) + } + + return nil +} + +func major(device uint64) uint64 { + return (device >> 8) & 0xfff +} + +func minor(device uint64) uint64 { + return (device & 0xff) | ((device >> 12) & 0xfff00) +} + +func mkdev(major int64, minor int64) uint32 { + return uint32(((minor & 0xfff00) << 12) | ((major & 0xfff) << 8) | (minor & 0xff)) +} + +func open(p string) (*os.File, error) { + return os.Open(p) +} + +func openFile(name string, flag int, perm os.FileMode) (*os.File, error) { + return os.OpenFile(name, flag, perm) +} + +func mkdirAll(path string, perm os.FileMode) error { + return os.MkdirAll(path, perm) +} + +func prepareApply() func() { + // Unset unmask before doing an apply operation, + // restore unmask when complete + oldmask := syscall.Umask(0) + return func() { + syscall.Umask(oldmask) + } +} + +func skipFile(*tar.Header) bool { + return false +} + +var ( + inUserNS bool + nsOnce sync.Once +) + +func setInUserNS() { + inUserNS = system.RunningInUserNS() +} + +// handleTarTypeBlockCharFifo is an OS-specific helper function used by +// createTarFile to handle the following types of header: Block; Char; Fifo +func handleTarTypeBlockCharFifo(hdr *tar.Header, path string) error { + nsOnce.Do(setInUserNS) + if inUserNS { + // cannot create a device if running in user namespace + return nil + } + + mode := uint32(hdr.Mode & 07777) + switch hdr.Typeflag { + case tar.TypeBlock: + mode |= syscall.S_IFBLK + case tar.TypeChar: + mode |= syscall.S_IFCHR + case tar.TypeFifo: + mode |= syscall.S_IFIFO + } + + if err := syscall.Mknod(path, mode, int(mkdev(hdr.Devmajor, hdr.Devminor))); err != nil { + return err + } + return nil +} + +func handleLChmod(hdr *tar.Header, path string, hdrInfo os.FileInfo) error { + if hdr.Typeflag == tar.TypeLink { + if fi, err := os.Lstat(hdr.Linkname); err == nil && (fi.Mode()&os.ModeSymlink == 0) { + if err := os.Chmod(path, hdrInfo.Mode()); err != nil { + return err + } + } + } else if hdr.Typeflag != tar.TypeSymlink { + if err := os.Chmod(path, hdrInfo.Mode()); err != nil { + return err + } + } + return nil +} + +func getxattr(path, attr string) ([]byte, error) { + b, err := sysx.LGetxattr(path, attr) + if err == syscall.ENOTSUP || err == syscall.ENODATA { + return nil, nil + } + return b, err +} + +func setxattr(path, key, value string) error { + return sysx.LSetxattr(path, key, []byte(value), 0) +} diff --git a/archive/tar_test.go b/archive/tar_test.go new file mode 100644 index 0000000..1245baa --- /dev/null +++ b/archive/tar_test.go @@ -0,0 +1,240 @@ +package archive + +import ( + "context" + "io/ioutil" + "os" + "os/exec" + "testing" + "time" + + _ "crypto/sha256" + + "github.com/docker/containerd/fs" + "github.com/docker/containerd/fs/fstest" + "github.com/pkg/errors" +) + +const tarCmd = "tar" + +// baseApplier creates a basic filesystem layout +// with multiple types of files for basic tests. +var baseApplier = fstest.Apply( + fstest.CreateDir("/etc/", 0755), + fstest.CreateFile("/etc/hosts", []byte("127.0.0.1 localhost"), 0644), + fstest.Link("/etc/hosts", "/etc/hosts.allow"), + fstest.CreateDir("/usr/local/lib", 0755), + fstest.CreateFile("/usr/local/lib/libnothing.so", []byte{0x00, 0x00}, 0755), + fstest.Symlink("libnothing.so", "/usr/local/lib/libnothing.so.2"), + fstest.CreateDir("/home", 0755), + fstest.CreateDir("/home/derek", 0700), +) + +func TestUnpack(t *testing.T) { + requireTar(t) + + if err := testApply(baseApplier); err != nil { + t.Fatalf("Test apply failed: %+v", err) + } +} + +func TestBaseDiff(t *testing.T) { + requireTar(t) + + if err := testBaseDiff(baseApplier); err != nil { + t.Fatalf("Test base diff failed: %+v", err) + } +} + +func TestDiffApply(t *testing.T) { + as := []fstest.Applier{ + baseApplier, + fstest.Apply( + fstest.CreateFile("/etc/hosts", []byte("127.0.0.1 localhost.localdomain"), 0644), + fstest.CreateFile("/etc/fstab", []byte("/dev/sda1\t/\text4\tdefaults 1 1\n"), 0600), + fstest.CreateFile("/etc/badfile", []byte(""), 0666), + fstest.CreateFile("/home/derek/.zshrc", []byte("#ZSH is just better\n"), 0640), + ), + fstest.Apply( + fstest.RemoveFile("/etc/badfile"), + fstest.Rename("/home/derek", "/home/notderek"), + ), + fstest.Apply( + fstest.RemoveFile("/usr"), + fstest.RemoveFile("/etc/hosts.allow"), + ), + fstest.Apply( + fstest.RemoveFile("/home"), + fstest.CreateDir("/home/derek", 0700), + fstest.CreateFile("/home/derek/.bashrc", []byte("#not going away\n"), 0640), + // "/etc/hosts" must be touched to be hardlinked in same layer + fstest.Chtime("/etc/hosts", time.Now()), + fstest.Link("/etc/hosts", "/etc/hosts.allow"), + ), + } + + if err := testDiffApply(as...); err != nil { + t.Fatalf("Test diff apply failed: %+v", err) + } +} + +// TestDiffApplyDeletion checks various deletion scenarios to ensure +// deletions are properly picked up and applied +func TestDiffApplyDeletion(t *testing.T) { + as := []fstest.Applier{ + fstest.Apply( + fstest.CreateDir("/test/somedir", 0755), + fstest.CreateDir("/lib", 0700), + fstest.CreateFile("/lib/hidden", []byte{}, 0644), + ), + fstest.Apply( + fstest.CreateFile("/test/a", []byte{}, 0644), + fstest.CreateFile("/test/b", []byte{}, 0644), + fstest.CreateDir("/test/otherdir", 0755), + fstest.CreateFile("/test/otherdir/.empty", []byte{}, 0644), + fstest.RemoveFile("/lib"), + fstest.CreateDir("/lib", 0700), + fstest.CreateFile("/lib/not-hidden", []byte{}, 0644), + ), + fstest.Apply( + fstest.RemoveFile("/test/a"), + fstest.RemoveFile("/test/b"), + fstest.RemoveFile("/test/otherdir"), + fstest.CreateFile("/lib/newfile", []byte{}, 0644), + ), + } + + if err := testDiffApply(as...); err != nil { + t.Fatalf("Test diff apply failed: %+v", err) + } +} + +func testApply(a fstest.Applier) error { + td, err := ioutil.TempDir("", "test-apply-") + if err != nil { + return errors.Wrap(err, "failed to create temp dir") + } + defer os.RemoveAll(td) + dest, err := ioutil.TempDir("", "test-apply-dest-") + if err != nil { + return errors.Wrap(err, "failed to create temp dir") + } + defer os.RemoveAll(dest) + + if err := a.Apply(td); err != nil { + return errors.Wrap(err, "failed to apply filesystem changes") + } + + tarArgs := []string{"c", "-C", td} + names, err := readDirNames(td) + if err != nil { + return errors.Wrap(err, "failed to read directory names") + } + tarArgs = append(tarArgs, names...) + + cmd := exec.Command(tarCmd, tarArgs...) + + arch, err := cmd.StdoutPipe() + if err != nil { + return errors.Wrap(err, "failed to create stdout pipe") + } + + if err := cmd.Start(); err != nil { + return errors.Wrap(err, "failed to start command") + } + + if _, err := Apply(context.Background(), dest, arch); err != nil { + return errors.Wrap(err, "failed to apply tar stream") + } + + return fstest.CheckDirectoryEqual(td, dest) +} + +func testBaseDiff(a fstest.Applier) error { + td, err := ioutil.TempDir("", "test-base-diff-") + if err != nil { + return errors.Wrap(err, "failed to create temp dir") + } + defer os.RemoveAll(td) + dest, err := ioutil.TempDir("", "test-base-diff-dest-") + if err != nil { + return errors.Wrap(err, "failed to create temp dir") + } + defer os.RemoveAll(dest) + + if err := a.Apply(td); err != nil { + return errors.Wrap(err, "failed to apply filesystem changes") + } + + arch := Diff(context.Background(), "", td) + + cmd := exec.Command(tarCmd, "x", "-C", dest) + cmd.Stdin = arch + if err := cmd.Run(); err != nil { + return errors.Wrap(err, "tar command failed") + } + + return fstest.CheckDirectoryEqual(td, dest) +} + +func testDiffApply(as ...fstest.Applier) error { + base, err := ioutil.TempDir("", "test-diff-apply-base-") + if err != nil { + return errors.Wrap(err, "failed to create temp dir") + } + defer os.RemoveAll(base) + dest, err := ioutil.TempDir("", "test-diff-apply-dest-") + if err != nil { + return errors.Wrap(err, "failed to create temp dir") + } + defer os.RemoveAll(dest) + + ctx := context.Background() + for i, a := range as { + if err := diffApply(ctx, a, base, dest); err != nil { + return errors.Wrapf(err, "diff apply failed at layer %d", i) + } + } + return nil +} + +// diffApply applies the given changes on the base and +// computes the diff and applies to the dest. +func diffApply(ctx context.Context, a fstest.Applier, base, dest string) error { + baseCopy, err := ioutil.TempDir("", "test-diff-apply-copy-") + if err != nil { + return errors.Wrap(err, "failed to create temp dir") + } + defer os.RemoveAll(baseCopy) + if err := fs.CopyDir(baseCopy, base); err != nil { + return errors.Wrap(err, "failed to copy base") + } + + if err := a.Apply(base); err != nil { + return errors.Wrap(err, "failed to apply changes to base") + } + + if _, err := Apply(ctx, dest, Diff(ctx, baseCopy, base)); err != nil { + return errors.Wrap(err, "failed to apply tar stream") + } + + return fstest.CheckDirectoryEqual(base, dest) +} + +func readDirNames(p string) ([]string, error) { + fis, err := ioutil.ReadDir(p) + if err != nil { + return nil, err + } + names := make([]string, len(fis)) + for i, fi := range fis { + names[i] = fi.Name() + } + return names, nil +} + +func requireTar(t *testing.T) { + if _, err := exec.LookPath(tarCmd); err != nil { + t.Skipf("%s not found, skipping", tarCmd) + } +} diff --git a/archive/tar_windows.go b/archive/tar_windows.go new file mode 100644 index 0000000..1c52f3a --- /dev/null +++ b/archive/tar_windows.go @@ -0,0 +1,104 @@ +package archive + +import ( + "archive/tar" + "errors" + "fmt" + "os" + "strings" + + "github.com/docker/docker/pkg/system" +) + +// canonicalTarNameForPath returns platform-specific filepath +// to canonical posix-style path for tar archival. p is relative +// path. +func tarName(p string) (string, error) { + // windows: convert windows style relative path with backslashes + // into forward slashes. Since windows does not allow '/' or '\' + // in file names, it is mostly safe to replace however we must + // check just in case + if strings.Contains(p, "/") { + return "", fmt.Errorf("Windows path contains forward slash: %s", p) + } + + return strings.Replace(p, string(os.PathSeparator), "/", -1), nil +} + +// chmodTarEntry is used to adjust the file permissions used in tar header based +// on the platform the archival is done. +func chmodTarEntry(perm os.FileMode) os.FileMode { + perm &= 0755 + // Add the x bit: make everything +x from windows + perm |= 0111 + + return perm +} + +func setHeaderForSpecialDevice(*tar.Header, string, os.FileInfo) error { + // do nothing. no notion of Rdev, Inode, Nlink in stat on Windows + return nil +} + +func open(p string) (*os.File, error) { + // We use system.OpenSequential to ensure we use sequential file + // access on Windows to avoid depleting the standby list. + return system.OpenSequential(p) +} + +func openFile(name string, flag int, perm os.FileMode) (*os.File, error) { + // Source is regular file. We use system.OpenFileSequential to use sequential + // file access to avoid depleting the standby list on Windows. + return system.OpenFileSequential(name, flag, perm) +} + +func mkdirAll(path string, perm os.FileMode) error { + return system.MkdirAll(path, perm) +} + +func prepareApply() func() { + // No umask or filesystem changes needed before apply + return func() {} +} + +func skipFile(hdr *tar.Header) bool { + // Windows does not support filenames with colons in them. Ignore + // these files. This is not a problem though (although it might + // appear that it is). Let's suppose a client is running docker pull. + // The daemon it points to is Windows. Would it make sense for the + // client to be doing a docker pull Ubuntu for example (which has files + // with colons in the name under /usr/share/man/man3)? No, absolutely + // not as it would really only make sense that they were pulling a + // Windows image. However, for development, it is necessary to be able + // to pull Linux images which are in the repository. + // + // TODO Windows. Once the registry is aware of what images are Windows- + // specific or Linux-specific, this warning should be changed to an error + // to cater for the situation where someone does manage to upload a Linux + // image but have it tagged as Windows inadvertently. + if strings.Contains(hdr.Name, ":") { + return true + } + + return false +} + +// handleTarTypeBlockCharFifo is an OS-specific helper function used by +// createTarFile to handle the following types of header: Block; Char; Fifo +func handleTarTypeBlockCharFifo(hdr *tar.Header, path string) error { + return nil +} + +func handleLChmod(hdr *tar.Header, path string, hdrInfo os.FileInfo) error { + return nil +} + +func getxattr(path, attr string) ([]byte, error) { + return nil, nil +} + +func setxattr(path, key, value string) error { + // Return not support error, do not wrap underlying not supported + // since xattrs should not exist in windows diff archives + return errors.New("xattrs not supported on Windows") +} diff --git a/archive/time.go b/archive/time.go new file mode 100644 index 0000000..4e9ae95 --- /dev/null +++ b/archive/time.go @@ -0,0 +1,38 @@ +package archive + +import ( + "syscall" + "time" + "unsafe" +) + +var ( + minTime = time.Unix(0, 0) + maxTime time.Time +) + +func init() { + if unsafe.Sizeof(syscall.Timespec{}.Nsec) == 8 { + // This is a 64 bit timespec + // os.Chtimes limits time to the following + maxTime = time.Unix(0, 1<<63-1) + } else { + // This is a 32 bit timespec + maxTime = time.Unix(1<<31-1, 0) + } +} + +func boundTime(t time.Time) time.Time { + if t.Before(minTime) || t.After(maxTime) { + return minTime + } + + return t +} + +func latestTime(t1, t2 time.Time) time.Time { + if t1.Before(t2) { + return t2 + } + return t1 +} diff --git a/archive/time_linux.go b/archive/time_linux.go new file mode 100644 index 0000000..4863608 --- /dev/null +++ b/archive/time_linux.go @@ -0,0 +1,21 @@ +package archive + +import ( + "time" + + "golang.org/x/sys/unix" + + "github.com/pkg/errors" +) + +func chtimes(path string, atime, mtime time.Time) error { + var utimes [2]unix.Timespec + utimes[0] = unix.NsecToTimespec(atime.UnixNano()) + utimes[1] = unix.NsecToTimespec(mtime.UnixNano()) + + if err := unix.UtimesNanoAt(unix.AT_FDCWD, path, utimes[0:], unix.AT_SYMLINK_NOFOLLOW); err != nil { + return errors.Wrap(err, "failed call to UtimesNanoAt") + } + + return nil +} diff --git a/archive/time_windows.go b/archive/time_windows.go new file mode 100644 index 0000000..a17f6ac --- /dev/null +++ b/archive/time_windows.go @@ -0,0 +1,25 @@ +package archive + +import ( + "syscall" + "time" +) + +// chtimes will set the create time on a file using the given modtime. +// This requires calling SetFileTime and explicitly including the create time. +func chtimes(path string, atime, mtime time.Time) error { + ctimespec := syscall.NsecToTimespec(mtime.UnixNano()) + pathp, e := syscall.UTF16PtrFromString(path) + if e != nil { + return e + } + h, e := syscall.CreateFile(pathp, + syscall.FILE_WRITE_ATTRIBUTES, syscall.FILE_SHARE_WRITE, nil, + syscall.OPEN_EXISTING, syscall.FILE_FLAG_BACKUP_SEMANTICS, 0) + if e != nil { + return e + } + defer syscall.Close(h) + c := syscall.NsecToFiletime(syscall.TimespecToNsec(ctimespec)) + return syscall.SetFileTime(h, &c, nil, nil) +} diff --git a/cmd/dist/apply.go b/cmd/dist/apply.go index bb5858c..598c722 100644 --- a/cmd/dist/apply.go +++ b/cmd/dist/apply.go @@ -4,8 +4,9 @@ import ( contextpkg "context" "os" + "github.com/docker/containerd/archive" "github.com/docker/containerd/log" - "github.com/docker/docker/pkg/archive" + dockerarchive "github.com/docker/docker/pkg/archive" "github.com/urfave/cli" ) @@ -21,7 +22,13 @@ var applyCommand = cli.Command{ ) log.G(ctx).Info("applying layer from stdin") - if _, err := archive.ApplyLayer(dir, os.Stdin); err != nil { + + rd, err := dockerarchive.DecompressStream(os.Stdin) + if err != nil { + return err + } + + if _, err := archive.Apply(ctx, dir, rd); err != nil { return err } diff --git a/rootfs/apply.go b/rootfs/apply.go index 5824384..74b7d6a 100644 --- a/rootfs/apply.go +++ b/rootfs/apply.go @@ -1,13 +1,15 @@ package rootfs import ( + "context" "io" "io/ioutil" "github.com/docker/containerd" + "github.com/docker/containerd/archive" "github.com/docker/containerd/log" "github.com/docker/containerd/snapshot" - "github.com/docker/docker/pkg/archive" + dockerarchive "github.com/docker/docker/pkg/archive" "github.com/opencontainers/go-digest" "github.com/opencontainers/image-spec/identity" ocispec "github.com/opencontainers/image-spec/specs-go/v1" @@ -56,7 +58,12 @@ func ApplyLayer(snapshots snapshot.Snapshotter, mounter Mounter, rd io.Reader, p } defer mounter.Unmount(mounts...) - if _, err := archive.ApplyLayer(key, rd); err != nil { + rd, err = dockerarchive.DecompressStream(rd) + if err != nil { + return "", err + } + + if _, err := archive.Apply(context.Background(), key, rd); err != nil { return "", err }