From 9bb13f8f5177c2aa8d0813338d69fb597793cc59 Mon Sep 17 00:00:00 2001 From: Derek McGowan Date: Wed, 6 Apr 2016 15:07:29 -0700 Subject: [PATCH] Update archive package to support overlay whiteouts Signed-off-by: Derek McGowan (github: dmcgowan) --- archive/archive.go | 109 ++++++++++++++------------------------- archive/archive_linux.go | 89 ++++++++++++++++++++++++++++++++ archive/archive_other.go | 7 +++ archive/changes.go | 46 ++++++++++++++--- archive/changes_linux.go | 27 ++++++++++ 5 files changed, 200 insertions(+), 78 deletions(-) create mode 100644 archive/archive_linux.go create mode 100644 archive/archive_other.go diff --git a/archive/archive.go b/archive/archive.go index f198198..5219153 100644 --- a/archive/archive.go +++ b/archive/archive.go @@ -33,6 +33,8 @@ type ( Reader io.Reader // Compression is the state represents if compressed or not. Compression int + // WhiteoutFormat is the format of whiteouts unpacked + WhiteoutFormat int // TarChownOptions wraps the chown options UID and GID. TarChownOptions struct { UID, GID int @@ -47,9 +49,10 @@ type ( GIDMaps []idtools.IDMap ChownOpts *TarChownOptions IncludeSourceDir bool - // When unpacking convert whiteouts and opaque dirs from aufs format to overlayfs format - // When packing convert whiteouts and opaque dirs from overlayfs format to aufs format - OverlayFormat bool + // WhiteoutFormat is the expected on disk format for whiteout files. + // This format will be converted to the standard format on pack + // and from the standard format on unpack. + WhiteoutFormat WhiteoutFormat // When unpacking, specifies whether overwriting a directory with a // non-directory is allowed and vice versa. NoOverwriteDirNonDir bool @@ -96,6 +99,14 @@ const ( Xz ) +const ( + // AUFSWhiteoutFormat is the default format for whitesouts + AUFSWhiteoutFormat WhiteoutFormat = iota + // OverlayWhiteoutFormat formats whiteout according to the overlay + // standard. + OverlayWhiteoutFormat +) + // IsArchive checks for the magic bytes of a tar or any supported compression // algorithm. func IsArchive(header []byte) bool { @@ -231,6 +242,11 @@ func (compression *Compression) Extension() string { return "" } +type tarWhiteoutConverter interface { + ConvertWrite(*tar.Header, string, os.FileInfo) error + ConvertRead(*tar.Header, string) (bool, error) +} + type tarAppender struct { TarWriter *tar.Writer Buffer *bufio.Writer @@ -240,10 +256,11 @@ type tarAppender struct { UIDMaps []idtools.IDMap GIDMaps []idtools.IDMap - // `overlayFormat` controls whether to interpret character devices with numbers 0,0 - // and directories with the attribute `trusted.overlay.opaque` using their overlayfs - // meanings and remap them to AUFS format - OverlayFormat bool + // For packing and unpacking whiteout files in the + // non standard format. The whiteout files defined + // by the AUFS standard are used as the tar whiteout + // standard. + WhiteoutConverter tarWhiteoutConverter } // canonicalTarName provides a platform-independent and consistent posix-style @@ -332,13 +349,9 @@ func (ta *tarAppender) addTarFile(path, name string) error { hdr.Gid = xGID } - if ta.OverlayFormat { - // convert whiteouts to AUFS format - if fi.Mode()&os.ModeCharDevice != 0 && hdr.Devmajor == 0 && hdr.Devminor == 0 { - // we just rename the file and make it normal - hdr.Name = WhiteoutPrefix + hdr.Name - hdr.Mode = 0600 - hdr.Typeflag = tar.TypeReg + if ta.WhiteoutConverter != nil { + if err := ta.WhiteoutConverter.ConvertWrite(hdr, path, fi); err != nil { + return err } } @@ -365,30 +378,6 @@ func (ta *tarAppender) addTarFile(path, name string) error { } } - if ta.OverlayFormat { - // convert opaque dirs to AUFS format by writing an empty file with the prefix - opaque, _ := system.Lgetxattr(path, "trusted.overlay.opaque") - if opaque != nil && len(opaque) == 1 && opaque[0] == 'y' { - // create a header for the whiteout file - // it should inherit some properties from the parent, but be a regular file - whHdr := &tar.Header{ - Typeflag: tar.TypeReg, - Mode: hdr.Mode & int64(os.ModePerm), - Name: filepath.Join(name, WhiteoutOpaqueDir), - Size: 0, - Uid: hdr.Uid, - Uname: hdr.Uname, - Gid: hdr.Gid, - Gname: hdr.Gname, - AccessTime: hdr.AccessTime, - ChangeTime: hdr.ChangeTime, - } - if err := ta.TarWriter.WriteHeader(whHdr); err != nil { - return err - } - } - } - return nil } @@ -544,12 +533,12 @@ func TarWithOptions(srcPath string, options *TarOptions) (io.ReadCloser, error) go func() { ta := &tarAppender{ - TarWriter: tar.NewWriter(compressWriter), - Buffer: pools.BufioWriter32KPool.Get(nil), - SeenFiles: make(map[uint64]string), - UIDMaps: options.UIDMaps, - GIDMaps: options.GIDMaps, - OverlayFormat: options.OverlayFormat, + TarWriter: tar.NewWriter(compressWriter), + Buffer: pools.BufioWriter32KPool.Get(nil), + SeenFiles: make(map[uint64]string), + UIDMaps: options.UIDMaps, + GIDMaps: options.GIDMaps, + WhiteoutConverter: getWhiteoutConverter(options.WhiteoutFormat), } defer func() { @@ -711,6 +700,7 @@ func Unpack(decompressedArchive io.Reader, dest string, options *TarOptions) err if err != nil { return err } + whiteoutConverter := getWhiteoutConverter(options.WhiteoutFormat) // Iterate through the files in the archive. loop: @@ -810,33 +800,12 @@ loop: hdr.Gid = xGID } - base := filepath.Base(path) - dir := filepath.Dir(path) - - if options.OverlayFormat { - // if a directory is marked as opaque by the AUFS special file, we need to translate that to overlay - if base == WhiteoutOpaqueDir { - if err := syscall.Setxattr(dir, "trusted.overlay.opaque", []byte{'y'}, 0); err != nil { - return err - } - - // don't write the file itself - continue + if whiteoutConverter != nil { + writeFile, err := whiteoutConverter.ConvertRead(hdr, path) + if err != nil { + return err } - - // if a file was deleted and we are using overlay, we need to create a character device - if strings.HasPrefix(base, WhiteoutPrefix) { - originalBase := base[len(WhiteoutPrefix):] - originalPath := filepath.Join(dir, originalBase) - - if err := syscall.Mknod(originalPath, syscall.S_IFCHR, 0); err != nil { - return err - } - if err := os.Chown(originalPath, hdr.Uid, hdr.Gid); err != nil { - return err - } - - // don't write the file itself + if !writeFile { continue } } diff --git a/archive/archive_linux.go b/archive/archive_linux.go new file mode 100644 index 0000000..86d2c7f --- /dev/null +++ b/archive/archive_linux.go @@ -0,0 +1,89 @@ +package archive + +import ( + "archive/tar" + "os" + "path/filepath" + "strings" + "syscall" + + "github.com/docker/docker/pkg/system" +) + +func getWhiteoutConverter(format WhiteoutFormat) tarWhiteoutConverter { + if format == OverlayWhiteoutFormat { + return overlayWhiteoutConverter{} + } + return nil +} + +type overlayWhiteoutConverter struct{} + +func (overlayWhiteoutConverter) ConvertWrite(hdr *tar.Header, path string, fi os.FileInfo) error { + // convert whiteouts to AUFS format + if fi.Mode()&os.ModeCharDevice != 0 && hdr.Devmajor == 0 && hdr.Devminor == 0 { + // we just rename the file and make it normal + hdr.Name = WhiteoutPrefix + hdr.Name + hdr.Mode = 0600 + hdr.Typeflag = tar.TypeReg + } + + if fi.Mode()&os.ModeDir != 0 { + // convert opaque dirs to AUFS format by writing an empty file with the prefix + opaque, err := system.Lgetxattr(path, "trusted.overlay.opaque") + if err != nil { + return err + } + if opaque != nil && len(opaque) == 1 && opaque[0] == 'y' { + // create a header for the whiteout file + // it should inherit some properties from the parent, but be a regular file + *hdr = tar.Header{ + Typeflag: tar.TypeReg, + Mode: hdr.Mode & int64(os.ModePerm), + Name: filepath.Join(hdr.Name, WhiteoutOpaqueDir), + Size: 0, + Uid: hdr.Uid, + Uname: hdr.Uname, + Gid: hdr.Gid, + Gname: hdr.Gname, + AccessTime: hdr.AccessTime, + ChangeTime: hdr.ChangeTime, + } + } + } + + return nil +} + +func (overlayWhiteoutConverter) ConvertRead(hdr *tar.Header, path string) (bool, error) { + base := filepath.Base(path) + dir := filepath.Dir(path) + + // if a directory is marked as opaque by the AUFS special file, we need to translate that to overlay + if base == WhiteoutOpaqueDir { + if err := syscall.Setxattr(dir, "trusted.overlay.opaque", []byte{'y'}, 0); err != nil { + return false, err + } + + // don't write the file itself + return false, nil + } + + // if a file was deleted and we are using overlay, we need to create a character device + if strings.HasPrefix(base, WhiteoutPrefix) { + originalBase := base[len(WhiteoutPrefix):] + originalPath := filepath.Join(dir, originalBase) + + if err := syscall.Mknod(originalPath, syscall.S_IFCHR, 0); err != nil { + return false, err + } + if err := os.Chown(originalPath, hdr.Uid, hdr.Gid); err != nil { + return false, err + } + + // don't write the file itself + return false, nil + } + + return true, nil +} diff --git a/archive/archive_other.go b/archive/archive_other.go new file mode 100644 index 0000000..54acbf2 --- /dev/null +++ b/archive/archive_other.go @@ -0,0 +1,7 @@ +// +build !linux + +package archive + +func getWhiteoutConverter(format WhiteoutFormat) tarWhiteoutConverter { + return nil +} diff --git a/archive/changes.go b/archive/changes.go index 81651c6..4e2d8e5 100644 --- a/archive/changes.go +++ b/archive/changes.go @@ -81,6 +81,33 @@ func sameFsTimeSpec(a, b syscall.Timespec) bool { // Changes walks the path rw and determines changes for the files in the path, // with respect to the parent layers func Changes(layers []string, rw string) ([]Change, error) { + return changes(layers, rw, aufsDeletedFile, aufsMetadataSkip) +} + +func aufsMetadataSkip(path string) (skip bool, err error) { + skip, err = filepath.Match(string(os.PathSeparator)+WhiteoutMetaPrefix+"*", path) + if err != nil { + skip = true + } + return +} + +func aufsDeletedFile(root, path string, fi os.FileInfo) (string, error) { + f := filepath.Base(path) + + // If there is a whiteout, then the file was removed + if strings.HasPrefix(f, WhiteoutPrefix) { + originalFile := f[len(WhiteoutPrefix):] + return filepath.Join(filepath.Dir(path), originalFile), nil + } + + return "", nil +} + +type skipChange func(string) (bool, error) +type deleteChange func(string, string, os.FileInfo) (string, error) + +func changes(layers []string, rw string, dc deleteChange, sc skipChange) ([]Change, error) { var ( changes []Change changedDirs = make(map[string]struct{}) @@ -105,21 +132,24 @@ func Changes(layers []string, rw string) ([]Change, error) { return nil } - // Skip AUFS metadata - if matched, err := filepath.Match(string(os.PathSeparator)+WhiteoutMetaPrefix+"*", path); err != nil || matched { - return err + if sc != nil { + if skip, err := sc(path); skip { + return err + } } change := Change{ Path: path, } + deletedFile, err := dc(rw, path, f) + if err != nil { + return err + } + // Find out what kind of modification happened - file := filepath.Base(path) - // If there is a whiteout, then the file was removed - if strings.HasPrefix(file, WhiteoutPrefix) { - originalFile := file[len(WhiteoutPrefix):] - change.Path = filepath.Join(filepath.Dir(path), originalFile) + if deletedFile != "" { + change.Path = deletedFile change.Kind = ChangeDelete } else { // Otherwise, the file was added diff --git a/archive/changes_linux.go b/archive/changes_linux.go index dee8b7c..a4cc0c6 100644 --- a/archive/changes_linux.go +++ b/archive/changes_linux.go @@ -283,3 +283,30 @@ func clen(n []byte) int { } return len(n) } + +// OverlayChanges walks the path rw and determines changes for the files in the path, +// with respect to the parent layers +func OverlayChanges(layers []string, rw string) ([]Change, error) { + return changes(layers, rw, overlayDeletedFile, nil) +} + +func overlayDeletedFile(root, path string, fi os.FileInfo) (string, error) { + if fi.Mode()&os.ModeCharDevice != 0 { + s := fi.Sys().(*syscall.Stat_t) + if major(uint64(s.Rdev)) == 0 && minor(uint64(s.Rdev)) == 0 { + return path, nil + } + } + if fi.Mode()&os.ModeDir != 0 { + opaque, err := system.Lgetxattr(filepath.Join(root, path), "trusted.overlay.opaque") + if err != nil { + return "", err + } + if opaque != nil && len(opaque) == 1 && opaque[0] == 'y' { + return path, nil + } + } + + return "", nil + +}